diff options
Diffstat (limited to 'src/sisudoc/outputs')
| -rw-r--r-- | src/sisudoc/outputs/io_out/sqlite_ocda_db.d | 105 |
1 files changed, 88 insertions, 17 deletions
diff --git a/src/sisudoc/outputs/io_out/sqlite_ocda_db.d b/src/sisudoc/outputs/io_out/sqlite_ocda_db.d index d30295a..f312d91 100644 --- a/src/sisudoc/outputs/io_out/sqlite_ocda_db.d +++ b/src/sisudoc/outputs/io_out/sqlite_ocda_db.d @@ -62,6 +62,7 @@ template spineAbstractionDb() { import sisudoc.outputs.io_out.paths_output; import sisudoc.ocda.io_in.paths_source; import sisudoc.ocda.abstraction.ssp : ssp_format_version; + import sisudoc.ocda.zstd : zstdCompress; /+ ↓ the document is passed in rather than taken from doc, because the caller hands over one that has been through the .ssp: written as text and read back. so the database is a function of the .ssp and cannot @@ -251,20 +252,34 @@ template spineAbstractionDb() { -- materialised from this database has to put them. -- conf conf/document_make, the document's own configuration. -- manifest pod.manifest, as the author wrote it. + -- tools tools/po4a, the translation catalogues, the same tree + -- the pod carries: a pod written back out of this file is + -- then the pod that went in, and not most of it. -- -- source, conf and manifest are what make this a document source and -- not only a serialised abstraction: with them a pod can be written -- back out of the file and rebuilt from the markup, rather than only -- rendered from the objects. + -- bytes and sha256 are always those of the ORIGINAL file, whatever + -- compression says, so that a digest check, a digests.txt line and a + -- source.digest rebuilt from these rows all keep working, and a reader + -- that does not decompress can still say what it is looking at. + -- + -- compression says how data is stored: NULL or 'none' for the bytes as + -- read, 'zstd' for a zstd frame. It is decided by role and not by size, + -- so the same kind of file always stores the same way and no reader has + -- to measure anything to predict it: the text roles are compressed and + -- images are not, png and jpeg being compressed already. CREATE TABLE IF NOT EXISTS files ( - id INTEGER PRIMARY KEY, - role TEXT NOT NULL, - name TEXT NOT NULL, - bytes INTEGER NOT NULL, - sha256 TEXT, - width INTEGER, - height INTEGER, - data BLOB, + id INTEGER PRIMARY KEY, + role TEXT NOT NULL, + name TEXT NOT NULL, + bytes INTEGER NOT NULL, + sha256 TEXT, + width INTEGER, + height INTEGER, + compression TEXT, + data BLOB, UNIQUE(role, name) ); @@ -704,8 +719,9 @@ template spineAbstractionDb() { import std.digest.sha : sha256Of; auto file_stmt = db.prepare( "INSERT OR IGNORE INTO files" - ~ " (role, name, bytes, sha256, width, height, data)" - ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height, :data)" + ~ " (role, name, bytes, sha256, width, height, compression, data)" + ~ " VALUES (:role, :name, :bytes, :sha256, :width, :height," + ~ " :compression, :data)" ); int[string] _w, _h; foreach (section; section_order) { @@ -745,6 +761,7 @@ template spineAbstractionDb() { import std.typecons : Nullable; file_stmt.bind(":height", Nullable!int()); } + file_stmt.bind(":compression", "none"); file_stmt.bind(":data", _bytes); file_stmt.execute(); file_stmt.reset(); @@ -776,13 +793,35 @@ template spineAbstractionDb() { } return; } - file_stmt.bind(":role", _role); - file_stmt.bind(":name", _name); - file_stmt.bind(":bytes", cast(long) _bytes.length); - file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); - file_stmt.bind(":width", Nullable!int()); - file_stmt.bind(":height", Nullable!int()); - file_stmt.bind(":data", _bytes); + /+ ↓ the text roles are stored compressed, the images are not. + By role and not by size: the same kind of file then always + stores the same way, and nobody has to measure a document to + know how its database will look. The bytes and the digest + recorded stay those of the original file either way. + . + A tiny file can come out a few bytes larger, a zstd frame + having some overhead. Left alone: a rule to avoid that would + be the size threshold this deliberately does not have. + +/ + ubyte[] _stored = _bytes; + string _compression = "none"; + try { + _stored = zstdCompress(_bytes, doc_matters.opt.action.pod_compression); + _compression = "zstd"; + } catch (Exception ex) { + stderr.writeln("WARNING: could not compress ", _name, ": ", ex.msg, + "; stored as it stands"); + _stored = _bytes; + _compression = "none"; + } + file_stmt.bind(":role", _role); + file_stmt.bind(":name", _name); + file_stmt.bind(":bytes", cast(long) _bytes.length); + file_stmt.bind(":sha256", sha256Of(_bytes).toHexString.to!string); + file_stmt.bind(":width", Nullable!int()); + file_stmt.bind(":height", Nullable!int()); + file_stmt.bind(":compression", _compression); + file_stmt.bind(":data", _stored); file_stmt.execute(); file_stmt.reset(); } @@ -808,6 +847,38 @@ template spineAbstractionDb() { _carry("manifest", "pod.manifest", doc_matters.pod.manifest_file_with_path); } + /+ ↓ the translation catalogues, the same tools/po4a tree the pod + writer carries and by the same rule: a directory walk, because a + catalogue set is whatever the translator has, with every name + checked before it is taken. Carrying less than the pod does + would make a materialised pod a lossy copy of the original, + which is the one thing this is for. + . + Done once, on the last language, as the pod writer does it (the + tree is not language specific, and a pass per language would + re-read all of it (5.5 MB ten times over, for live-manual) to + insert rows the first pass already wrote). + +/ + if (doc_matters.src.is_pod + && doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1] + ) { + import sisudoc.ocda.io_in.carried_names; + mixin spineCarriedNames; + enum string _po4a_root = "tools/po4a"; + string _tools_in = doc_matters.pod.manifest_path ~ "/" ~ _po4a_root; + if (exists(_tools_in) && _tools_in.isDir) { + foreach (string _f; dirEntries(_tools_in, SpanMode.depth)) { + if (!(_f.isFile)) { continue; } + string _rel = _po4a_root ~ "/" ~ _f[(_tools_in.length + 1) .. $]; + string _bad = validateCarriedPath(_rel); + if (_bad.length > 0) { + writeln("WARNING not carried into the database: ", _bad); + continue; + } + _carry("tools", _rel, _f); + } + } + } } file_stmt.finalize(); } |
