aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/sisudoc/ocda
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-21 13:33:44 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-22 16:17:59 -0400
commit4943c93919f349e2d47e6a74f498779fb13f7862 (patch)
treee54696dbe4b8299b76f383641a75f95dd9379a36 /src/sisudoc/ocda
parentocda db: materialise a pod from carried source (diff)
ocda db: carry catalogues & text blobs compressed
The pod carries tools/po4a and the database (until now) did not, so a pod written back out of a database came back a lossy copy, without its translation catalogues. the database now carries them, walked and name-checked exactly as the pod writer walks and checks them, once on the last language. They are compressed to save space, the test sample carrying 6.25 MB of catalogue as read would have made that one document's database two thirds larger. Files gains a compression column: NULL or 'none' for the bytes as read, 'zstd' for a frame. What to compress is decided by role, not by size (source, conf, manifest and tools are text and are compressed; images being compressed already are not). The same kind of file is always stored the same way. Document objects are untouched and stay raw: objects_fts is external content over objects.text and reads that column directly. bytes and sha256 remain those of the original file, so every digest check, digests.txt line and source.digest rebuilt from these rows works unchanged, and a reader that does not decompress can still say what it is looking at. dbReadFiles decompresses, so no caller learns how a blob is stored. It asks pragma_table_info whether the column is there at all: a database written before it reads as raw. A row that claims zstd and will not decompress is named and left out, rather than handed back as a frame where markup should be. live-manual: 9,490,432 bytes without the catalogues, 9,908,224 with them. Output from the materialised pod is identical to output from the original pod, 679 files each side, no differences. (assisted by Claude-Code)
Diffstat (limited to 'src/sisudoc/ocda')
-rw-r--r--src/sisudoc/ocda/abstraction/db_in.d43
-rw-r--r--src/sisudoc/ocda/abstraction/pod_from_db.d6
2 files changed, 44 insertions, 5 deletions
diff --git a/src/sisudoc/ocda/abstraction/db_in.d b/src/sisudoc/ocda/abstraction/db_in.d
index 53ac95d..0a01364 100644
--- a/src/sisudoc/ocda/abstraction/db_in.d
+++ b/src/sisudoc/ocda/abstraction/db_in.d
@@ -380,7 +380,13 @@ template spineAbstractionDbRead() {
a .ocda.db holds its images as blobs, which is what lets it travel on
its own where a .ssp cannot: a .ssp describes its images by name,
size, pixels and sha256, but does not carry them. The role column
- says what a file is; today only 'image' is written.
+ says what a file is: 'image', and the markup, configuration, manifest
+ and tools carried with it.
+ .
+ data comes back as the original file, decompressed where the row says
+ it was compressed, so that no caller has to know how a blob is stored.
+ bytes and sha256 are the original's either way, and are what a caller
+ checks the data it is given against.
+/
struct ST_ArtefactFile {
string role;
@@ -390,21 +396,52 @@ template spineAbstractionDbRead() {
ubyte[] data;
}
@trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") {
+ import sisudoc.ocda.zstd : zstdDecompress;
ST_ArtefactFile[] _out;
if (!db_file.exists) { return _out; }
auto db = Database(db_file, SQLITE_OPEN_READONLY);
+ /+ ↓ whether this file has a compression column at all. A database
+ written before the column has none, and selecting it would throw
+ rather than answer; everything in such a file is stored as read.
+ +/
+ bool _has_compression = false;
foreach (row; db.execute(
- "SELECT role, name, bytes, sha256, data FROM files WHERE role = '" ~ role
- ~ "' ORDER BY name")
+ "SELECT count(*) AS n FROM pragma_table_info('files')"
+ ~ " WHERE name = 'compression'")
) {
+ _has_compression = (row["n"].as!long > 0);
+ }
+ auto _stmt = db.prepare(
+ "SELECT role, name, bytes, sha256, "
+ ~ (_has_compression ? "compression" : "NULL AS compression")
+ ~ ", data FROM files WHERE role = :role ORDER BY name"
+ );
+ _stmt.bind(":role", role);
+ foreach (row; _stmt.execute()) {
ST_ArtefactFile _f;
_f.role = row["role"].as!string;
_f.name = row["name"].as!string;
_f.bytes = row["bytes"].as!ulong;
_f.sha256 = row["sha256"].as!string;
_f.data = row["data"].as!(ubyte[]);
+ string _compression = row["compression"].as!string;
+ if (_compression == "zstd") {
+ try {
+ _f.data = zstdDecompress(_f.data);
+ } catch (Exception ex) {
+ /+ ↓ a row that says it is compressed and will not decompress is
+ not usable, and handing back the frame as if it were the file
+ would put a zstd frame where markup should be. Named, and
+ left out.
+ +/
+ stderr.writeln("WARNING: ", db_file.baseName, " carries ", _f.name,
+ " as zstd and it will not decompress: ", ex.msg, "; left out");
+ continue;
+ }
+ }
_out ~= _f;
}
+ _stmt.finalize();
return _out;
}
}
diff --git a/src/sisudoc/ocda/abstraction/pod_from_db.d b/src/sisudoc/ocda/abstraction/pod_from_db.d
index 0bdb3bc..2e231d3 100644
--- a/src/sisudoc/ocda/abstraction/pod_from_db.d
+++ b/src/sisudoc/ocda/abstraction/pod_from_db.d
@@ -74,6 +74,7 @@ module sisudoc.ocda.abstraction.pod_from_db;
<dest>/<pod>/conf/document_make role='conf'
<dest>/<pod>/media/text/<lang>/<file> role='source'
<dest>/<pod>/media/image/<file> role='image'
+ <dest>/<pod>/tools/po4a/... role='tools'
.
The pod's name comes from the database's own filename: <doc>.ocda.db is
named by doc_uid_out_no_lang, which is the pod name and the document's
@@ -136,7 +137,8 @@ template spinePodFromDb() {
case "image": return "media/image/" ~ _name;
case "source":
case "conf":
- case "manifest": return _name;
+ case "manifest":
+ case "tools": return _name;
default: return "";
}
}
@@ -153,7 +155,7 @@ template spinePodFromDb() {
return _out;
}
_dbr.ST_ArtefactFile[] _files;
- foreach (_role; ["manifest", "conf", "source", "image"]) {
+ foreach (_role; ["manifest", "conf", "source", "image", "tools"]) {
_files ~= _dbr.dbReadFiles(_db_file, _role);
}
if (_files.length == 0) {