diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-21 12:00:43 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-22 14:54:35 -0400 |
| commit | 774302487c8595ec4383558e9d8387a6219ef2ed (patch) | |
| tree | 8a2001340c39e50f2f85b26cb5fabfb939c2b8a0 /test/test-abstraction-db.sh | |
| parent | digests: db digest taken once on file completion (diff) | |
test db: multilingual database test
one file holding several languages.
Both assumed <uid>.<lang>.ocda.db beside <uid>.<lang>.ssp, one for
one.
- test-abstraction-db.sh takes the language from the .ssp name, the
database from the stem, and scopes every query by the doc_id that
language resolves to: objects by doc_id, the four child tables
through their objects, metadata by doc_id, schema.version by doc_id
is null. It also checks what a per language loop cannot see, that the
languages a database holds are exactly the languages there are .ssp
files for.
- test-abstraction-db-roundtrip.sh asks for each language by name and
holds it against its own .ssp, and requires a database of more than
one language to refuse a round trip that names none: a reader quietly
emitting the first language would otherwise pass against the first
.ssp and say nothing about the other nine. Its language list comes
from the .ssp filenames, so it still needs no sqlite3.
databases (re-)emitting all .ssp files exactly.
(assisted by Claude-Code)
Diffstat (limited to 'test/test-abstraction-db.sh')
| -rwxr-xr-x | test/test-abstraction-db.sh | 79 |
1 files changed, 62 insertions, 17 deletions
diff --git a/test/test-abstraction-db.sh b/test/test-abstraction-db.sh index 6a0e4e6..2fac414 100755 --- a/test/test-abstraction-db.sh +++ b/test/test-abstraction-db.sh @@ -121,23 +121,38 @@ lev4_subtoc|object_subtoc report() { echo " MISMATCH $1: .ssp $2, db $3"; } -# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the -# database beside it (pod/<uid>.ocda.db) +# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), one +# per language, and the database beside it (pod/<uid>.ocda.db), one per +# document holding every language of it. So the .ssp name carries the +# language and the database name does not: <uid>.<lang>.ssp beside +# <uid>.ocda.db, and inside the database the language's rows are the ones +# whose doc_id is the id documents gives that language. for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do [ -f "$ssp" ] || continue base=$(basename "$ssp" .ssp) - db="$OUT_DIR/pod/$base.ocda.db" + lang="${base##*.}" + stem="${base%.*}" + db="$OUT_DIR/pod/$stem.ocda.db" if [ ! -f "$db" ]; then echo "MISSING db for $base" FAILURES=$((FAILURES + 1)) continue fi + did=$(sqlite3 "$db" "select id from documents where lang='$lang';" 2>/dev/null || echo '') + if [ -z "$did" ]; then + echo "MISSING $lang in $(basename "$db"); it holds:" \ + "$(sqlite3 "$db" "select group_concat(lang,' ') from documents;" 2>/dev/null)" + FAILURES=$((FAILURES + 1)) + continue + fi + # the objects of this one language, for the child tables to key on + mine="object_id in (select id from objects where doc_id=$did)" DOCS=$((DOCS + 1)) doc_fail=0 # object population s_objs=$(grep -c '^\[' "$ssp" || true) - d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR) + d_objs=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did;" 2>/dev/null || echo ERR) if [ "$s_objs" != "$d_objs" ]; then echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1 fi @@ -145,7 +160,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # per section for sec in head toc body endnotes glossary bibliography bookindex blurb; do s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp") - d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR) + d_sec=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and section='$sec';" 2>/dev/null || echo ERR) if [ "$s_sec" != "$d_sec" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "section $sec" "$s_sec" "$d_sec"; doc_fail=1 @@ -159,7 +174,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; *) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; esac - d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and ($pred);" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n" fi @@ -175,7 +190,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do echo "$LISTS" | while IFS='|' read -r prop table; do [ -z "$prop" ] && continue s_n=$(grep -c "^\.$prop: " "$ssp" || true) - d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from $table where $mine;" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n" fi @@ -187,10 +202,15 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi rm -f "$OUT_DIR/.lists.$$" - # every image the abstraction referred to and found is carried in the file - want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR) + # every image the abstraction referred to and found is carried in the file. + # the files table is shared: the languages of a document refer to the same + # images and one copy serves them all, so what is counted here is the names + # this language referred to and the stored files that answer them. + want=$(sqlite3 "$db" "select count(distinct name) from object_images + where missing = 0 and $mine;" 2>/dev/null || echo ERR) have=$(sqlite3 "$db" "select count(*) from files f where f.role='image' - and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR) + and exists (select 1 from object_images i where i.name = f.name + and i.missing = 0 and i.$mine);" 2>/dev/null || echo ERR) if [ "$want" != "$have" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "images carried" "$want referenced" "$have stored"; doc_fail=1 @@ -198,9 +218,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # the text itself, on the first and last citable object for pick in "min" "max"; do - ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "") + ocn=$(sqlite3 "$db" "select $pick(ocn) from objects + where doc_id=$did and ocn > 0 and section='body';" 2>/dev/null || echo "") [ -z "$ocn" ] && continue - sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true + sqlite3 "$db" "select text from objects + where doc_id=$did and ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \ "$ssp" > "$OUT_DIR/.ssptxt.$$" if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then @@ -226,8 +248,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do } ' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$" sqlite3 -separator "$(printf '\t')" "$db" \ - "select key, value from metadata where key not in - ('schema.name','schema.version','source.filename','source.ssp_digest');" \ + "select key, value from metadata where doc_id=$did and key not in + ('source.filename','source.ssp_digest');" \ 2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$" if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then [ $doc_fail -eq 0 ] && echo "$base" @@ -238,7 +260,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do rm -f "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" # the source filename, and the version the two share - sf=$(sqlite3 "$db" "select value from metadata where key='source.filename';" 2>/dev/null || echo '') + sf=$(sqlite3 "$db" "select value from metadata + where doc_id=$did and key='source.filename';" 2>/dev/null || echo '') sf_ssp=$(sed -n 's/^% Source: //p' "$ssp" | head -1) if [ "$sf" != "$sf_ssp" ]; then [ $doc_fail -eq 0 ] && echo "$base" @@ -246,7 +269,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi sv_ssp=$(sed -n 's/^% SiSU Document Abstraction v//p' "$ssp" | head -1) # the file says what it is - sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '') + sv=$(sqlite3 "$db" "select value from metadata + where doc_id is null and key='schema.version';" 2>/dev/null || echo '') if [ -z "$sv" ]; then [ $doc_fail -eq 0 ] && echo "$base" echo " MISSING metadata schema.version"; doc_fail=1 @@ -258,10 +282,31 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do FAILURES=$((FAILURES + doc_fail)) done +# one database per document, holding every language of it: the languages the +# documents table names are exactly the languages there are .ssp files for. +# A per language loop cannot see this - it checks the languages it finds and +# would not notice one the database had gained or lost - so it is checked per +# database, once, here. +DBS=0 +for db in $(find "$OUT_DIR/pod" -maxdepth 1 -name "*.ocda.db" 2>/dev/null | sort); do + [ -f "$db" ] || continue + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + d_langs=$(sqlite3 "$db" "select lang from documents order by lang;" 2>/dev/null \ + | tr '\n' ' ' | sed 's/ $//') + s_langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort | tr '\n' ' ' | sed 's/ $//') + if [ "$d_langs" != "$s_langs" ]; then + echo "$stem" + echo " MISMATCH languages: db '$d_langs', .ssp files '$s_langs'" + FAILURES=$((FAILURES + 1)) + fi +done + rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: .ssp and .ocda.db agree for all $DOCS documents" + echo "PASS: .ssp and .ocda.db agree for all $DOCS documents in $DBS databases" exit 0 else echo "FAIL: $FAILURES document(s) disagree" |
