aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-21 12:00:43 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-22 14:54:35 -0400
commit774302487c8595ec4383558e9d8387a6219ef2ed (patch)
tree8a2001340c39e50f2f85b26cb5fabfb939c2b8a0
parentdigests: db digest taken once on file completion (diff)
test db: multilingual database test
one file holding several languages. Both assumed <uid>.<lang>.ocda.db beside <uid>.<lang>.ssp, one for one. - test-abstraction-db.sh takes the language from the .ssp name, the database from the stem, and scopes every query by the doc_id that language resolves to: objects by doc_id, the four child tables through their objects, metadata by doc_id, schema.version by doc_id is null. It also checks what a per language loop cannot see, that the languages a database holds are exactly the languages there are .ssp files for. - test-abstraction-db-roundtrip.sh asks for each language by name and holds it against its own .ssp, and requires a database of more than one language to refuse a round trip that names none: a reader quietly emitting the first language would otherwise pass against the first .ssp and say nothing about the other nine. Its language list comes from the .ssp filenames, so it still needs no sqlite3. databases (re-)emitting all .ssp files exactly. (assisted by Claude-Code)
-rw-r--r--org/tests_for_document_abstraction_shell_scripts.org161
-rwxr-xr-xtest/run-tests.sh2
-rwxr-xr-xtest/test-abstraction-db-roundtrip.sh76
-rwxr-xr-xtest/test-abstraction-db.sh79
4 files changed, 240 insertions, 78 deletions
diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org
index e321286..0f41ca6 100644
--- a/org/tests_for_document_abstraction_shell_scripts.org
+++ b/org/tests_for_document_abstraction_shell_scripts.org
@@ -30,7 +30,7 @@
# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \
# ./test/run-tests.sh ./bin/spine-ldc
#
-# sqlite3 is needed by two of the four. If it is not on PATH the script says
+# sqlite3 is needed by one of the five. If it is not on PATH the script says
# so and re-runs itself inside a nix shell that has it, so this works:
# SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc
#
@@ -533,23 +533,38 @@ lev4_subtoc|object_subtoc
report() { echo " MISMATCH $1: .ssp $2, db $3"; }
-# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the
-# database beside it (pod/<uid>.ocda.db)
+# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), one
+# per language, and the database beside it (pod/<uid>.ocda.db), one per
+# document holding every language of it. So the .ssp name carries the
+# language and the database name does not: <uid>.<lang>.ssp beside
+# <uid>.ocda.db, and inside the database the language's rows are the ones
+# whose doc_id is the id documents gives that language.
for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
[ -f "$ssp" ] || continue
base=$(basename "$ssp" .ssp)
- db="$OUT_DIR/pod/$base.ocda.db"
+ lang="${base##*.}"
+ stem="${base%.*}"
+ db="$OUT_DIR/pod/$stem.ocda.db"
if [ ! -f "$db" ]; then
echo "MISSING db for $base"
FAILURES=$((FAILURES + 1))
continue
fi
+ did=$(sqlite3 "$db" "select id from documents where lang='$lang';" 2>/dev/null || echo '')
+ if [ -z "$did" ]; then
+ echo "MISSING $lang in $(basename "$db"); it holds:" \
+ "$(sqlite3 "$db" "select group_concat(lang,' ') from documents;" 2>/dev/null)"
+ FAILURES=$((FAILURES + 1))
+ continue
+ fi
+ # the objects of this one language, for the child tables to key on
+ mine="object_id in (select id from objects where doc_id=$did)"
DOCS=$((DOCS + 1))
doc_fail=0
# object population
s_objs=$(grep -c '^\[' "$ssp" || true)
- d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR)
+ d_objs=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did;" 2>/dev/null || echo ERR)
if [ "$s_objs" != "$d_objs" ]; then
echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1
fi
@@ -557,7 +572,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
# per section
for sec in head toc body endnotes glossary bibliography bookindex blurb; do
s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp")
- d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR)
+ d_sec=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and section='$sec';" 2>/dev/null || echo ERR)
if [ "$s_sec" != "$d_sec" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
report "section $sec" "$s_sec" "$d_sec"; doc_fail=1
@@ -568,10 +583,10 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
echo "$CHECKS" | while IFS='|' read -r prop pred; do
[ -z "$prop" ] && continue
case "$prop" in
- ,*": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;;
- ,*) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;;
+ *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;;
+ *) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;;
esac
- d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR)
+ d_n=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and ($pred);" 2>/dev/null || echo ERR)
if [ "$s_n" != "$d_n" ]; then
echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n"
fi
@@ -587,7 +602,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
echo "$LISTS" | while IFS='|' read -r prop table; do
[ -z "$prop" ] && continue
s_n=$(grep -c "^\.$prop: " "$ssp" || true)
- d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR)
+ d_n=$(sqlite3 "$db" "select count(*) from $table where $mine;" 2>/dev/null || echo ERR)
if [ "$s_n" != "$d_n" ]; then
echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n"
fi
@@ -599,10 +614,15 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
fi
rm -f "$OUT_DIR/.lists.$$"
- # every image the abstraction referred to and found is carried in the file
- want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR)
+ # every image the abstraction referred to and found is carried in the file.
+ # the files table is shared: the languages of a document refer to the same
+ # images and one copy serves them all, so what is counted here is the names
+ # this language referred to and the stored files that answer them.
+ want=$(sqlite3 "$db" "select count(distinct name) from object_images
+ where missing = 0 and $mine;" 2>/dev/null || echo ERR)
have=$(sqlite3 "$db" "select count(*) from files f where f.role='image'
- and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR)
+ and exists (select 1 from object_images i where i.name = f.name
+ and i.missing = 0 and i.$mine);" 2>/dev/null || echo ERR)
if [ "$want" != "$have" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
report "images carried" "$want referenced" "$have stored"; doc_fail=1
@@ -610,9 +630,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
# the text itself, on the first and last citable object
for pick in "min" "max"; do
- ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "")
+ ocn=$(sqlite3 "$db" "select $pick(ocn) from objects
+ where doc_id=$did and ocn > 0 and section='body';" 2>/dev/null || echo "")
[ -z "$ocn" ] && continue
- sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true
+ sqlite3 "$db" "select text from objects
+ where doc_id=$did and ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true
awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \
"$ssp" > "$OUT_DIR/.ssptxt.$$"
if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then
@@ -638,8 +660,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
}
' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$"
sqlite3 -separator "$(printf '\t')" "$db" \
- "select key, value from metadata where key not in
- ('schema.name','schema.version','source.filename','source.ssp_digest');" \
+ "select key, value from metadata where doc_id=$did and key not in
+ ('source.filename','source.ssp_digest');" \
2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$"
if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then
[ $doc_fail -eq 0 ] && echo "$base"
@@ -650,7 +672,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
rm -f "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$"
# the source filename, and the version the two share
- sf=$(sqlite3 "$db" "select value from metadata where key='source.filename';" 2>/dev/null || echo '')
+ sf=$(sqlite3 "$db" "select value from metadata
+ where doc_id=$did and key='source.filename';" 2>/dev/null || echo '')
sf_ssp=$(sed -n 's/^% Source: //p' "$ssp" | head -1)
if [ "$sf" != "$sf_ssp" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
@@ -658,7 +681,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
fi
sv_ssp=$(sed -n 's/^% SiSU Document Abstraction v//p' "$ssp" | head -1)
# the file says what it is
- sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '')
+ sv=$(sqlite3 "$db" "select value from metadata
+ where doc_id is null and key='schema.version';" 2>/dev/null || echo '')
if [ -z "$sv" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
echo " MISSING metadata schema.version"; doc_fail=1
@@ -670,10 +694,31 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
FAILURES=$((FAILURES + doc_fail))
done
+# one database per document, holding every language of it: the languages the
+# documents table names are exactly the languages there are .ssp files for.
+# A per language loop cannot see this - it checks the languages it finds and
+# would not notice one the database had gained or lost - so it is checked per
+# database, once, here.
+DBS=0
+for db in $(find "$OUT_DIR/pod" -maxdepth 1 -name "*.ocda.db" 2>/dev/null | sort); do
+ [ -f "$db" ] || continue
+ DBS=$((DBS + 1))
+ stem=$(basename "$db" .ocda.db)
+ d_langs=$(sqlite3 "$db" "select lang from documents order by lang;" 2>/dev/null \
+ | tr '\n' ' ' | sed 's/ $//')
+ s_langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \
+ | sed "s|.*/$stem\.||; s|\.ssp$||" | sort | tr '\n' ' ' | sed 's/ $//')
+ if [ "$d_langs" != "$s_langs" ]; then
+ echo "$stem"
+ echo " MISMATCH languages: db '$d_langs', .ssp files '$s_langs'"
+ FAILURES=$((FAILURES + 1))
+ fi
+done
+
rm -rf "$OUT_DIR"
if [ "$FAILURES" -eq 0 ]; then
- echo "PASS: .ssp and .ocda.db agree for all $DOCS documents"
+ echo "PASS: .ssp and .ocda.db agree for all $DOCS documents in $DBS databases"
exit 0
else
echo "FAIL: $FAILURES document(s) disagree"
@@ -691,7 +736,10 @@ fi
#
# Reads each <doc>.ocda.db back into the abstraction, emits it as
# .ssp, and requires the result to be byte identical to the .ssp written
-# from the same document.
+# from the same document. A database holds every language of its document,
+# so each language is asked for by name (--lang) and held against its own
+# .ssp, and a database of more than one language is required to refuse a
+# round trip that names none.
#
# What that proves: the two serialisations really do carry the same thing.
# test-abstraction-db.sh counts fields and compares them; this reconstructs
@@ -749,37 +797,70 @@ $SPINE_BIN --show-abstraction --ocda-db --skip-output \
--output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1
COUNT=0
+DBS=0
FAILURES=0
-# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp inside it
-# (pod/<doc>/media/abstraction/<uid>.ssp)
+# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp files inside
+# it (pod/<doc>/media/abstraction/<uid>.<lang>.ssp).
+#
+# A database holds every language of its document, so one database answers
+# for as many .ssp files as the document has languages, and each has to be
+# asked for by name: --lang says which language to emit. With no language
+# named a database holding more than one refuses to choose, which is checked
+# too - a round trip that quietly emitted the first language would pass
+# against the first .ssp and say nothing about the other nine.
for db in "$OUT_DIR"/pod/*.ocda.db; do
[ -f "$db" ] || continue
- base=$(basename "$db" .ocda.db)
- ssp=$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)
- COUNT=$((COUNT + 1))
- if [ ! -f "$ssp" ]; then
- echo "MISSING .ssp for $base"
+ DBS=$((DBS + 1))
+ stem=$(basename "$db" .ocda.db)
+ # the languages come from the .ssp files rather than from the database, so
+ # that this script needs no sqlite3: each .ssp is a language the database
+ # has to be able to re-emit. A language in the database with no .ssp beside
+ # it is the other direction, and test-abstraction-db.sh checks that.
+ langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \
+ | sed "s|.*/$stem\.||; s|\.ssp$||" | sort)
+ if [ -z "$langs" ]; then
+ echo "MISSING .ssp for $stem"
FAILURES=$((FAILURES + 1))
continue
fi
- if ! $SPINE_BIN --db-round-trip="$db" > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then
- echo "ERROR: $base (spine exited non-zero)"
- head -3 "$OUT_DIR/from-db.err"
- FAILURES=$((FAILURES + 1))
- continue
- fi
- if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then
- echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)"
- diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12
- echo " ..."
- FAILURES=$((FAILURES + 1))
+ n_langs=$(printf '%s\n' "$langs" | grep -c .)
+ for lang in $langs; do
+ base="$stem.$lang"
+ ssp="$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)"
+ COUNT=$((COUNT + 1))
+ if [ ! -f "$ssp" ]; then
+ echo "MISSING .ssp for $base"
+ FAILURES=$((FAILURES + 1))
+ continue
+ fi
+ if ! $SPINE_BIN --db-round-trip="$db" --lang="$lang" \
+ > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then
+ echo "ERROR: $base (spine exited non-zero)"
+ head -3 "$OUT_DIR/from-db.err"
+ FAILURES=$((FAILURES + 1))
+ continue
+ fi
+ if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then
+ echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)"
+ diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12
+ echo " ..."
+ FAILURES=$((FAILURES + 1))
+ fi
+ done
+ # more than one language and none named: it must refuse rather than pick
+ if [ "$n_langs" -gt 1 ]; then
+ if $SPINE_BIN --db-round-trip="$db" > /dev/null 2> /dev/null; then
+ echo "$stem"
+ echo " ACCEPTED a round trip with no language named, holding $n_langs"
+ FAILURES=$((FAILURES + 1))
+ fi
fi
done
rm -rf "$OUT_DIR"
if [ "$FAILURES" -eq 0 ]; then
- echo "PASS: all $COUNT databases re-emit their document's .ssp exactly"
+ echo "PASS: $DBS databases re-emit all $COUNT of their documents' .ssp exactly"
exit 0
else
echo "FAIL: $FAILURES of $COUNT do not"
diff --git a/test/run-tests.sh b/test/run-tests.sh
index 1c8ac07..2f27892 100755
--- a/test/run-tests.sh
+++ b/test/run-tests.sh
@@ -7,7 +7,7 @@
# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \
# ./test/run-tests.sh ./bin/spine-ldc
#
-# sqlite3 is needed by two of the four. If it is not on PATH the script says
+# sqlite3 is needed by one of the five. If it is not on PATH the script says
# so and re-runs itself inside a nix shell that has it, so this works:
# SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc
#
diff --git a/test/test-abstraction-db-roundtrip.sh b/test/test-abstraction-db-roundtrip.sh
index 338f375..0ec7847 100755
--- a/test/test-abstraction-db-roundtrip.sh
+++ b/test/test-abstraction-db-roundtrip.sh
@@ -3,7 +3,10 @@
#
# Reads each <doc>.ocda.db back into the abstraction, emits it as
# .ssp, and requires the result to be byte identical to the .ssp written
-# from the same document.
+# from the same document. A database holds every language of its document,
+# so each language is asked for by name (--lang) and held against its own
+# .ssp, and a database of more than one language is required to refuse a
+# round trip that names none.
#
# What that proves: the two serialisations really do carry the same thing.
# test-abstraction-db.sh counts fields and compares them; this reconstructs
@@ -61,37 +64,70 @@ $SPINE_BIN --show-abstraction --ocda-db --skip-output \
--output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1
COUNT=0
+DBS=0
FAILURES=0
-# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp inside it
-# (pod/<doc>/media/abstraction/<uid>.ssp)
+# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp files inside
+# it (pod/<doc>/media/abstraction/<uid>.<lang>.ssp).
+#
+# A database holds every language of its document, so one database answers
+# for as many .ssp files as the document has languages, and each has to be
+# asked for by name: --lang says which language to emit. With no language
+# named a database holding more than one refuses to choose, which is checked
+# too - a round trip that quietly emitted the first language would pass
+# against the first .ssp and say nothing about the other nine.
for db in "$OUT_DIR"/pod/*.ocda.db; do
[ -f "$db" ] || continue
- base=$(basename "$db" .ocda.db)
- ssp=$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)
- COUNT=$((COUNT + 1))
- if [ ! -f "$ssp" ]; then
- echo "MISSING .ssp for $base"
+ DBS=$((DBS + 1))
+ stem=$(basename "$db" .ocda.db)
+ # the languages come from the .ssp files rather than from the database, so
+ # that this script needs no sqlite3: each .ssp is a language the database
+ # has to be able to re-emit. A language in the database with no .ssp beside
+ # it is the other direction, and test-abstraction-db.sh checks that.
+ langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \
+ | sed "s|.*/$stem\.||; s|\.ssp$||" | sort)
+ if [ -z "$langs" ]; then
+ echo "MISSING .ssp for $stem"
FAILURES=$((FAILURES + 1))
continue
fi
- if ! $SPINE_BIN --db-round-trip="$db" > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then
- echo "ERROR: $base (spine exited non-zero)"
- head -3 "$OUT_DIR/from-db.err"
- FAILURES=$((FAILURES + 1))
- continue
- fi
- if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then
- echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)"
- diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12
- echo " ..."
- FAILURES=$((FAILURES + 1))
+ n_langs=$(printf '%s\n' "$langs" | grep -c .)
+ for lang in $langs; do
+ base="$stem.$lang"
+ ssp="$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)"
+ COUNT=$((COUNT + 1))
+ if [ ! -f "$ssp" ]; then
+ echo "MISSING .ssp for $base"
+ FAILURES=$((FAILURES + 1))
+ continue
+ fi
+ if ! $SPINE_BIN --db-round-trip="$db" --lang="$lang" \
+ > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then
+ echo "ERROR: $base (spine exited non-zero)"
+ head -3 "$OUT_DIR/from-db.err"
+ FAILURES=$((FAILURES + 1))
+ continue
+ fi
+ if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then
+ echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)"
+ diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12
+ echo " ..."
+ FAILURES=$((FAILURES + 1))
+ fi
+ done
+ # more than one language and none named: it must refuse rather than pick
+ if [ "$n_langs" -gt 1 ]; then
+ if $SPINE_BIN --db-round-trip="$db" > /dev/null 2> /dev/null; then
+ echo "$stem"
+ echo " ACCEPTED a round trip with no language named, holding $n_langs"
+ FAILURES=$((FAILURES + 1))
+ fi
fi
done
rm -rf "$OUT_DIR"
if [ "$FAILURES" -eq 0 ]; then
- echo "PASS: all $COUNT databases re-emit their document's .ssp exactly"
+ echo "PASS: $DBS databases re-emit all $COUNT of their documents' .ssp exactly"
exit 0
else
echo "FAIL: $FAILURES of $COUNT do not"
diff --git a/test/test-abstraction-db.sh b/test/test-abstraction-db.sh
index 6a0e4e6..2fac414 100755
--- a/test/test-abstraction-db.sh
+++ b/test/test-abstraction-db.sh
@@ -121,23 +121,38 @@ lev4_subtoc|object_subtoc
report() { echo " MISMATCH $1: .ssp $2, db $3"; }
-# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the
-# database beside it (pod/<uid>.ocda.db)
+# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), one
+# per language, and the database beside it (pod/<uid>.ocda.db), one per
+# document holding every language of it. So the .ssp name carries the
+# language and the database name does not: <uid>.<lang>.ssp beside
+# <uid>.ocda.db, and inside the database the language's rows are the ones
+# whose doc_id is the id documents gives that language.
for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
[ -f "$ssp" ] || continue
base=$(basename "$ssp" .ssp)
- db="$OUT_DIR/pod/$base.ocda.db"
+ lang="${base##*.}"
+ stem="${base%.*}"
+ db="$OUT_DIR/pod/$stem.ocda.db"
if [ ! -f "$db" ]; then
echo "MISSING db for $base"
FAILURES=$((FAILURES + 1))
continue
fi
+ did=$(sqlite3 "$db" "select id from documents where lang='$lang';" 2>/dev/null || echo '')
+ if [ -z "$did" ]; then
+ echo "MISSING $lang in $(basename "$db"); it holds:" \
+ "$(sqlite3 "$db" "select group_concat(lang,' ') from documents;" 2>/dev/null)"
+ FAILURES=$((FAILURES + 1))
+ continue
+ fi
+ # the objects of this one language, for the child tables to key on
+ mine="object_id in (select id from objects where doc_id=$did)"
DOCS=$((DOCS + 1))
doc_fail=0
# object population
s_objs=$(grep -c '^\[' "$ssp" || true)
- d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR)
+ d_objs=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did;" 2>/dev/null || echo ERR)
if [ "$s_objs" != "$d_objs" ]; then
echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1
fi
@@ -145,7 +160,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
# per section
for sec in head toc body endnotes glossary bibliography bookindex blurb; do
s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp")
- d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR)
+ d_sec=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and section='$sec';" 2>/dev/null || echo ERR)
if [ "$s_sec" != "$d_sec" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
report "section $sec" "$s_sec" "$d_sec"; doc_fail=1
@@ -159,7 +174,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
*": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;;
*) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;;
esac
- d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR)
+ d_n=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and ($pred);" 2>/dev/null || echo ERR)
if [ "$s_n" != "$d_n" ]; then
echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n"
fi
@@ -175,7 +190,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
echo "$LISTS" | while IFS='|' read -r prop table; do
[ -z "$prop" ] && continue
s_n=$(grep -c "^\.$prop: " "$ssp" || true)
- d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR)
+ d_n=$(sqlite3 "$db" "select count(*) from $table where $mine;" 2>/dev/null || echo ERR)
if [ "$s_n" != "$d_n" ]; then
echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n"
fi
@@ -187,10 +202,15 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
fi
rm -f "$OUT_DIR/.lists.$$"
- # every image the abstraction referred to and found is carried in the file
- want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR)
+ # every image the abstraction referred to and found is carried in the file.
+ # the files table is shared: the languages of a document refer to the same
+ # images and one copy serves them all, so what is counted here is the names
+ # this language referred to and the stored files that answer them.
+ want=$(sqlite3 "$db" "select count(distinct name) from object_images
+ where missing = 0 and $mine;" 2>/dev/null || echo ERR)
have=$(sqlite3 "$db" "select count(*) from files f where f.role='image'
- and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR)
+ and exists (select 1 from object_images i where i.name = f.name
+ and i.missing = 0 and i.$mine);" 2>/dev/null || echo ERR)
if [ "$want" != "$have" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
report "images carried" "$want referenced" "$have stored"; doc_fail=1
@@ -198,9 +218,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
# the text itself, on the first and last citable object
for pick in "min" "max"; do
- ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "")
+ ocn=$(sqlite3 "$db" "select $pick(ocn) from objects
+ where doc_id=$did and ocn > 0 and section='body';" 2>/dev/null || echo "")
[ -z "$ocn" ] && continue
- sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true
+ sqlite3 "$db" "select text from objects
+ where doc_id=$did and ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true
awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \
"$ssp" > "$OUT_DIR/.ssptxt.$$"
if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then
@@ -226,8 +248,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
}
' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$"
sqlite3 -separator "$(printf '\t')" "$db" \
- "select key, value from metadata where key not in
- ('schema.name','schema.version','source.filename','source.ssp_digest');" \
+ "select key, value from metadata where doc_id=$did and key not in
+ ('source.filename','source.ssp_digest');" \
2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$"
if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then
[ $doc_fail -eq 0 ] && echo "$base"
@@ -238,7 +260,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
rm -f "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$"
# the source filename, and the version the two share
- sf=$(sqlite3 "$db" "select value from metadata where key='source.filename';" 2>/dev/null || echo '')
+ sf=$(sqlite3 "$db" "select value from metadata
+ where doc_id=$did and key='source.filename';" 2>/dev/null || echo '')
sf_ssp=$(sed -n 's/^% Source: //p' "$ssp" | head -1)
if [ "$sf" != "$sf_ssp" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
@@ -246,7 +269,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
fi
sv_ssp=$(sed -n 's/^% SiSU Document Abstraction v//p' "$ssp" | head -1)
# the file says what it is
- sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '')
+ sv=$(sqlite3 "$db" "select value from metadata
+ where doc_id is null and key='schema.version';" 2>/dev/null || echo '')
if [ -z "$sv" ]; then
[ $doc_fail -eq 0 ] && echo "$base"
echo " MISSING metadata schema.version"; doc_fail=1
@@ -258,10 +282,31 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do
FAILURES=$((FAILURES + doc_fail))
done
+# one database per document, holding every language of it: the languages the
+# documents table names are exactly the languages there are .ssp files for.
+# A per language loop cannot see this - it checks the languages it finds and
+# would not notice one the database had gained or lost - so it is checked per
+# database, once, here.
+DBS=0
+for db in $(find "$OUT_DIR/pod" -maxdepth 1 -name "*.ocda.db" 2>/dev/null | sort); do
+ [ -f "$db" ] || continue
+ DBS=$((DBS + 1))
+ stem=$(basename "$db" .ocda.db)
+ d_langs=$(sqlite3 "$db" "select lang from documents order by lang;" 2>/dev/null \
+ | tr '\n' ' ' | sed 's/ $//')
+ s_langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \
+ | sed "s|.*/$stem\.||; s|\.ssp$||" | sort | tr '\n' ' ' | sed 's/ $//')
+ if [ "$d_langs" != "$s_langs" ]; then
+ echo "$stem"
+ echo " MISMATCH languages: db '$d_langs', .ssp files '$s_langs'"
+ FAILURES=$((FAILURES + 1))
+ fi
+done
+
rm -rf "$OUT_DIR"
if [ "$FAILURES" -eq 0 ]; then
- echo "PASS: .ssp and .ocda.db agree for all $DOCS documents"
+ echo "PASS: .ssp and .ocda.db agree for all $DOCS documents in $DBS databases"
exit 0
else
echo "FAIL: $FAILURES document(s) disagree"