/+ - Name: SisuDoc Spine - Description: part of SiSU spine (Doc Reform): for document structuring, processing, publishing and search - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later, for everything in this repository unless a file or a directory specifies otherwise. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License (AGPLv3) as published by the Free Software Foundation (FSF), either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at: [https://www.gnu.org/licenses/agpl-3.0.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.sisudoc.org] [https://www.doc-reform.org] - Git [https://git.sisudoc.org/] +/ module sisudoc.outputs.io_out.source_pod; @system: // is not @safe: use: @system: or @trusted: template spinePod() { import std.digest.sha; // import std.digest : toHexString; import std.file; import std.outbuffer; import std.zip; import std.conv : to; import sisudoc.ocda.meta.rgx_files; import sisudoc.outputs.io_out; import sisudoc.ocda.io_in.carried_names; import sisudoc.outputs.io_out.zip_pod; import sisudoc.outputs.io_out.xmls; void spinePod(T)(T doc_matters) { debug(asserts) { // static assert(is(typeof(doc_matters) == tuple)); } mixin spineRgxFiles; string pwd = doc_matters.env.pwd; auto pths_pod = spinePathsPods!()(doc_matters); mixin spineLanguageCodes; auto lang = Lang(); static auto rgx_files = RgxFiles(); assert (doc_matters.src.filename.match(rgx_files.src_fn)); if (doc_matters.opt.action.source_or_pod) { try { /+ ↓ clean slate: remove the per-document pod directory before regeneration, but only on the first language of a multi-language document, so that subsequent languages' files are not wiped. . media/abstraction/ is left standing: the .ssp files there are written by the abstraction stage, which runs before this, and for a multi-language document they accumulate one language at a time. What is pruned from it instead is any .ssp that does not belong to a language this document has, which would otherwise linger from an earlier run and be bundled. That is done here rather than in the writer because pod actions run serially and the manifest's list of languages is to hand, where the writer may be one of several languages running at once. +/ if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0]) { string doc_pod_dir = pths_pod.base_filesystem_(doc_matters.src.filename); string keep_dir = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string; if (exists(doc_pod_dir) && doc_pod_dir.isDir) { foreach (string _e; dirEntries(doc_pod_dir, SpanMode.shallow)) { if (_e.isDir) { if (keep_dir.startsWith(_e)) { // media/, which holds abstraction/ foreach (string _m; dirEntries(_e, SpanMode.shallow)) { if (_m == keep_dir) { continue; } if (_m.isDir) { _m.rmdirRecurse; } else { _m.remove; } } } else { _e.rmdirRecurse; } } else { _e.remove; } } } if (exists(keep_dir) && keep_dir.isDir) { string _doc_uid_base = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length]; foreach (string _f; dirEntries(keep_dir, "*.ssp", SpanMode.shallow)) { bool _wanted = false; foreach (_lang; doc_matters.pod.manifest_list_of_languages) { if (_f.baseName == (_doc_uid_base ~ _lang ~ ".ssp")) { _wanted = true; break; } } if (!_wanted) { _f.remove; } } } } { podArchive_directory_tree(doc_matters, pths_pod); } { struct STsrcDigests { std.zip.ZipArchive zip; string fn_pod; string[string][string] digests; } STsrcDigests _st; _st = pod_zipMakeReady(doc_matters, pths_pod, _st); { zipArchive(doc_matters, _st.fn_pod, _st.zip); if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { zipArchiveDigest(doc_matters, _st.fn_pod, _st.digests); } } } } catch (ErrnoException ex) { // Handle error } } } void podArchive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure if (!exists(pths_pod.pod_dir_())) { // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) pths_pod.pod_dir_().mkdirRecurse; } if (doc_matters.opt.action.source_or_pod) { // if (doc_matters.opt.action.vox_gt_1) { writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); } if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.media_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.css(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.css(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (doc_matters.opt.action.pod2) { if (!exists(pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } } if (!exists(pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } if (!exists(pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod)) { pths_pod.doc_lng(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.mkdirRecurse; } } if (!exists(pths_pod.pod_dir_() ~ "/index.html")) { import sisudoc.outputs.io_out.html_snippet; mixin htmlSnippet; auto f = File(pths_pod.pod_dir_() ~"/index.html", "w"); f.writeln(format_html_blank_page_guide_home( "../../css/html_scroll.css", (doc_matters.opt.action.webserver_url_doc_root.length > 0) ? doc_matters.opt.action.webserver_url_doc_root : doc_matters.conf_make_meta.conf.w_srv_data_root_url, "../../index.html", )); } } // ↓ - object digest & length struct ST_file_stat { ubyte[32] sha256hash; ulong fileSize; } ST_file_stat getFileStat(F)(F filepath) @trusted { auto file = File(filepath, "r"); ubyte[32] file_sha256 = digest!SHA256(file.byChunk(4096 * 1024)); ulong fileSize = file.size; ST_file_stat _file; _file.sha256hash = file_sha256; _file.fileSize = fileSize; return _file; } // ↑ - object digest & length auto pod_zipMakeReady(M,P,S)(M doc_matters, P pths_pod, S _st) { auto pth_dr_doc_src = doc_matters.src_path_info; if (doc_matters.opt.action.vox_gt_3) { // correct writeln(__LINE__, ":", __FILE__, ":\n", doc_matters.src.filename, " -> ", pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod ); } auto zip = new ZipArchive(); // needed auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod; string[string][string] _digests; { // bundle images - get digest foreach (image; doc_matters.srcs.image_list) { if (doc_matters.opt.action.vox_gt_3) { writeln( pth_dr_doc_src.image_root.to!string, "/", image, " -> ", pths_pod.image_root(doc_matters.src.filename).zpod, "/", image ); } auto fn_src_in = doc_matters.src.image_dir_path ~ "/" ~ image; auto fn_src_out_pod_zip_base = pths_pod.image_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ image; auto fn_src_out_filesystem = pths_pod.image_root(doc_matters.src.filename).filesystem_open_zpod.to!string ~ "/" ~ image; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } { // take DIGEST write to pod file digests.txt ST_file_stat file_ = getFileStat(fn_src_in); _digests["shared"]["images"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ image ~ "\n"; } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.source_or_pod) { zip = podArchive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src out NOT found (image): ", fn_src_in); } } } } { // the abstraction .ssp, written in place by --show-abstraction (only for --pod2) if (doc_matters.opt.action.pod2) { if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssp written /+ the .ssp is written straight into pod//media/abstraction/, so there is nothing to copy here: it is digested and added to the zip where it stands. doc_uid_out for any language follows the same pattern, differing only in the trailing ".{lng}". Strip the current language to reuse the base across all languages. +/ string _doc_uid_base = doc_matters.src.doc_uid_out[0 .. $ - doc_matters.src.lng.length]; foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions string ssp_filename = _doc_uid_base ~ _lang ~ ".ssp"; auto fn_src_out_pod_zip_base = pths_pod.abstraction_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ ssp_filename; auto fn_src_in = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string ~ "/" ~ ssp_filename; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } { // take DIGEST write to pod file digests.txt ST_file_stat file_ = getFileStat(fn_src_in); _digests[_lang]["ssp"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ ssp_filename ~ "\n"; } zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src out NOT found (abstraction): ", fn_src_in); } } } /+ ↓ the .ocda.db sits beside the pod rather than inside it: it carries the images itself, and putting it in the zip would store them twice. digested here so that digests.txt covers every artefact the pod directory holds for this document. One database holds every language of the document, so it is digested once, when the last language has been written into it, and listed as shared rather than per language. +/ { string db_filename = doc_matters.src.doc_uid_out_no_lang ~ ".ocda.db"; string fn_db = pths_pod.pod_dir_() ~ "/" ~ db_filename; if (exists(fn_db)) { ST_file_stat file_ = getFileStat(fn_db); _digests["shared"]["ocda_db"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ db_filename ~ "\n"; } } } } } { // bundle conf/document_make auto fn_src_in = ((doc_matters.src.is_pod) ? doc_matters.src.conf_dir_path : pth_dr_doc_src.conf_root).to!string ~ "/" ~ "document_make"; auto fn_src_out_pod_zip_base = pths_pod.conf_root(doc_matters.src.filename).zpod.to!string ~ "/" ~ "document_make"; auto fn_src_out_filesystem = pths_pod.conf_root(doc_matters.src.filename).filesystem_open_zpod.to!string ~ "/" ~ "document_make"; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.source_or_pod) { zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); } } } { /+ ↓ bundle tools/po4a, the translation working set . A directory walk, where every other thing the pod writer carries is named. A catalogue set is whatever the translator has, and naming .pot and .po files would mean the writer knowing the document's languages a second time, from a second place, to say what it already reads from the manifest. . Done once, on the last language, as the markup blocks are. . Every name is checked before it is carried, by the rule the pod reader applies to an archive it is handed. The writer is reading a directory someone else may have filled. +/ if (doc_matters.src.is_pod && doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1] ) { mixin spineCarriedNames; enum string _po4a_root = "tools/po4a"; string _tools_in = doc_matters.pod.manifest_path ~ "/" ~ _po4a_root; if (exists(_tools_in) && _tools_in.isDir) { foreach (string _f; dirEntries(_tools_in, SpanMode.depth)) { if (!(_f.isFile)) { continue; } string _rel = _po4a_root ~ "/" ~ _f[(_tools_in.length + 1) .. $]; string _bad = validateCarriedPath(_rel); if (_bad.length > 0) { writeln("WARNING not carried into the pod: ", _bad); continue; } if (doc_matters.opt.action.source_or_pod) { string _out_fs = pths_pod.pod_root(doc_matters.src.filename) .filesystem_open_zpod.to!string ~ "/" ~ _rel; string _parent = _out_fs.dirName; if (!exists(_parent)) { _parent.mkdirRecurse; } _f.copy(_out_fs); zip = podArchive("file_path_bin", _f, "pod/" ~ _rel, zip); } } } } } { // pod manifest auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; auto fn_src_out_pod_zip_base = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; auto fn_src_out_filesystem = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path auto fn_src_out_inside_pod = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path string[] filelist_src_out_pod_arr; string[] filelist_src_zpod_arr; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src in found: ", fn_src_in); } filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; filelist_src_zpod_arr ~= fn_src_out_inside_pod; { import dyaml; auto pod_filelist_yaml_string = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); Node _pmy; string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { try { _pmy = Loader.fromString(_pm).load(); } catch (ErrnoException ex) { } catch (Throwable) { writeln("ERROR failed to read config file content, not parsed as yaml"); } writeln("pod filename: ", _pmy["doc"]["filename"].get!string); writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); writeln("pod languages: ", doc_matters.src.language); // foreach(string _l; _pmy["doc"]["language"]) { // writeln("language: ", _l); // } } if (doc_matters.opt.action.source_or_pod) { pod_filelist_yaml_string.writeln(_pm); } if (doc_matters.opt.action.source_or_pod) { zip = podArchive("string", _pm, fn_src_out_pod_zip_base, zip); } } } } { // bundle primary file (.ssm/.sst) - get digest auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; auto fn_src_out_pod_zip_base = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; auto fn_src_out_filesystem = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: auto fn_src_out_inside_pod = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: string[] filelist_src_out_pod_arr; string[] filelist_src_zpod_arr; if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssm parsed foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions string fn_src_out_filesystem_lng = pths_pod.fn_doc(doc_matters.src.filename, _lang).filesystem_open_zpod.to!string; string _sstm = (doc_matters.pod.manifest_path ~ "/media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename); string _pth_file_sstm; if (exists(_sstm)) { _pth_file_sstm = _sstm; } else if (exists(fn_src_in)) { _pth_file_sstm = fn_src_in; } if (exists(_pth_file_sstm)) { // what of language? debug(io) { writeln("(io debug) src in found: ", _pth_file_sstm); } { // take DIGEST write to pod file digests.txt ST_file_stat file_ = getFileStat(_pth_file_sstm); _digests[_lang]["sstm"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ doc_matters.src.filename ~ " - [" ~ _lang ~ "]"; } filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; filelist_src_zpod_arr ~= fn_src_out_inside_pod; string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename; if (doc_matters.opt.action.source_or_pod) { _pth_file_sstm.copy(fn_src_out_filesystem_lng); } if (doc_matters.opt.action.source_or_pod) { auto _rgx_sstm = regex(r"(?P\S+?)(?P[a-z_-]+)/(?Pmedia/text/)(?P\S+?)/(?P\S+?\.ss[mt])"); if (auto _x = _pth_file_sstm.match(_rgx_sstm)){ if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { // again wait until all language versions of .ssm parsed string _path_to_pod = _x.captures["path_to_pod"]; string _podname = _x.captures["podname"]; string _root_to_lang = _x.captures["from_root"]; string _language = _x.captures["language"]; // .ssi inserts expected to have same name across languages string _filename = _x.captures["filename"]; foreach (_lang1; doc_matters.pod.manifest_list_of_languages) { // do for all language versions string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename; string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename; // writeln("\nin: ", _pth_mkup_src_in, "\nout: ", _pth_mkup_src_out); // DEBUG, REMOVE zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); } } } else { zip = podArchive("file_path_text", _pth_file_sstm, fn_src_out_pod_zip_base, zip); } } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src in NOT found (markup source): \n", _sstm, "or in", fn_src_in); } } } } } { // bundle insert files (.ssi) - get digest if (doc_matters.srcs.file_insert_list.length > 0) { auto _rgx_ssi = regex(r"(?P\S+?)(?P[a-z_-]+)/(?Pmedia/text/)(?P\S+?)/(?P\S+?\.ss[i])"); foreach (insert_file; doc_matters.srcs.file_insert_list) { debug(pod) { writeln( insert_file, " -> ", pths_pod.fn_doc_insert( doc_matters.src.filename, insert_file, doc_matters.src.language, ).zpod ); } if (auto _x = insert_file.match(_rgx_ssi)){ if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { string _path_to_pod = _x.captures["path_to_pod"]; string _podname = _x.captures["podname"]; string _root_to_lang = _x.captures["from_root"]; string _filename = _x.captures["filename"]; /+ ↓ The digest key is the language being digested (as the sstm digest above already does), not the language captured from the resolved insert path. +/ foreach (_lang; doc_matters.pod.manifest_list_of_languages) { string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; /+ ↓ the digest must be taken inside this guard, not before it (getFileStat opens the file) +/ if (exists(_pth_mkup_src_in)) { { // take DIGEST write to pod file digests.txt ST_file_stat file_ = getFileStat(_pth_mkup_src_in); _digests[_lang]["ssi"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ _filename ~ " - [" ~ _lang ~ "]" ~ "\n"; } if (doc_matters.opt.action.source_or_pod) { auto fn_src_out_filesystem // you need to change language sources = pths_pod.fn_doc_insert( doc_matters.src.filename, // doc_matters.src.filename _pth_mkup_src_in, // insert_file _lang, ).filesystem_open_zpod.to!string; _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere } if (doc_matters.opt.action.source_or_pod) { zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); } } } } } else { auto fn_src_in = insert_file; /+ ↓ keyed on the document's own language, not on hard-coded "en". The emit loop looks each key up in the manifest's language list. . Named as the pod carries it, by filename: insert_file is a path here, where every other line in digests.txt is a filename, and a digest line naming a file that is not in the pod under that name cannot be checked against it. The import is local, not added to this template's imports, which are visible in every scope it is mixed into. +/ if (exists(fn_src_in)) { // take DIGEST write to pod file digests.txt import std.path : baseName; ST_file_stat file_ = getFileStat(fn_src_in); _digests[doc_matters.src.language]["ssi"] ~= file_.sha256hash.toHexString ~ "::" ~ file_.fileSize.to!string ~ " - " ~ insert_file.baseName ~ "\n"; } auto fn_src_out_pod_zip_base = pths_pod.fn_doc_insert( doc_matters.src.filename, insert_file, doc_matters.src.language, ).zpod.to!string; auto fn_src_out_filesystem = pths_pod.fn_doc_insert( doc_matters.src.filename, insert_file, doc_matters.src.language, ).filesystem_open_zpod.to!string; if (exists(fn_src_in)) { debug(io) { writeln("(io debug) src out found: ", fn_src_in); } if (doc_matters.opt.action.source_or_pod) { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.source_or_pod) { zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt_2) { writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); } } } } } } { _st.zip = zip; _st.fn_pod = fn_pod; _st.digests = _digests; } return _st; } @system auto podArchive(Z)( string _source_type, string _data_in, string _pth_out, Z zip ) { auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = _pth_out; auto zip_data = new OutBuffer(); switch (_source_type) { case "file_path_bin": zip_data.write(cast(char[]) ((_data_in).read)); goto default; case "file_path_text": zip_data.write((_data_in).readText); goto default; case "string": zip_data.write(_data_in); goto default; default: zip_arc_member_file.expandedData = zip_data.toBytes(); zip.addMember(zip_arc_member_file); } return zip; } void zipArchive(M,F,Z)(M doc_matters, F fn_pod, Z zip) { import sisudoc.ocda.zstd : zstdCompress, ZstdException; import sisudoc.ocda.io_in.carried_names : MAX_POD_ENTRY_COUNT; auto fn_src_in = doc_matters.src.filename; if (doc_matters.opt.action.pod) { /+ ↓ the writer is held to the number the reader enforces. Previously it was not: a document with enough parts produced an archive spine would write and then refused to read, reporting too many entries, so a good file looked corrupt. The count is a judgement call; a writer able to emit what the reader rejects is a fault whatever the number is. . An error rather than a warning, because the artefact would be unreadable: writing it and reporting success is the one outcome worth ruling out. The message names the count, the limit and the document, so that "this document has too many parts for one archive" is legible without a debugger. +/ if (zip.directory.length > MAX_POD_ENTRY_COUNT) { writeln("ERROR >> pod archive not written: ", fn_pod); writeln(" ", doc_matters.src.filename_base, " would hold ", zip.directory.length, " members, over the ", MAX_POD_ENTRY_COUNT, " a pod archive may carry."); writeln(" a document's members are its languages times its markup", " and catalogue files; spine would not be able to read back what", " it wrote."); /+ ↓ an archive left by an earlier pass is removed. A missing artefact is an error anyone will notice; a quietly truncated one is not. +/ if (exists(fn_pod)) { try { fn_pod.remove; writeln(" the incomplete archive left by an earlier language", " has been removed."); } catch (Exception ex) { writeln("WARNING could not remove the incomplete archive: ", fn_pod, " - ", ex.msg); } } return; } if (exists(doc_matters.src.file_with_absolute_path)) { try { /+ ↓ the archive is built exactly as before and then wrapped in one zstd frame. The wrap is here and not in createZipFile, which also writes every epub and every odt: those are real zips by their own specifications. . The compression gain is in whole-stream rather than per-member compression. A zip deflates each member on its own with a 32 KB window, which sees none of the redundancy between multiple language versions of a document and the catalogues that may repeat the English (original language): (live-manual is 10.31x smaller this way, where per-member deflate manages about 3x). +/ auto _archive = cast(ubyte[]) zip.build(); createZipFile!()(fn_pod, cast(void[]) zstdCompress(_archive, doc_matters.opt.action.pod_compression)); } catch (ZstdException ex) { writeln("WARNING failed to compress pod archive: ", fn_pod, " - ", ex.msg); } catch (ErrnoException ex) { // Handle errors } } else { writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); } if (!(exists(fn_pod))) { writeln("WARNING failed to create pod archive: ", fn_pod); } } } void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) { import sisudoc.outputs.io_out.paths_output; auto pths_pod = spinePathsPods!()(doc_matters); char[] _zip_digest; try { if (!exists(pths_pod.pod_dir_())) { // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) pths_pod.pod_dir_().mkdirRecurse; } } catch (ErrnoException ex) { // Handle error } try { // if (doc_matters.opt.action.vox_gt_2) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); } string _digest_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt"; // if (doc_matters.opt.action.vox_gt_2) { writeln(_digest_fn); } auto f = File(_digest_fn, "w"); if (exists(fn_pod)) { try { auto data = (cast(byte[]) (fn_pod).read); // if (doc_matters.opt.action.vox_gt_2) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, pod_archive_suffix); } if (doc_matters.opt.action.pod) { _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ pod_archive_suffix); if (doc_matters.opt.action.vox_gt_1) { writeln(" ", _zip_digest); } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, "/"); } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", _digest_fn); } f.writeln(_zip_digest); } } catch (ErrnoException ex) { // Handle errors } } foreach (_lang; doc_matters.pod.manifest_list_of_languages) { if (_lang in _digests) { if (("sstm" in _digests[_lang]) && (_digests[_lang]["sstm"].length > 0)) { // if (doc_matters.opt.action.vox_gt_2) { writeln(_digests[_lang]["sstm"]); } f.writeln(_digests[_lang]["sstm"]); } if (("ssi" in _digests[_lang]) && (_digests[_lang]["ssi"].length > 0)) { // if (doc_matters.opt.action.vox_gt_2) { writeln(_digests[_lang]["ssi"]); } f.writeln(_digests[_lang]["ssi"]); } if (("ssp" in _digests[_lang]) && (_digests[_lang]["ssp"].length > 0)) { f.writeln(_digests[_lang]["ssp"].strip); } } } if ("shared" in _digests) { if (("ocda_db" in _digests["shared"]) && (_digests["shared"]["ocda_db"].length > 0)) { f.writeln(_digests["shared"]["ocda_db"].strip); } if (("images" in _digests["shared"]) && (_digests["shared"]["images"].length > 0)) { // if (doc_matters.opt.action.vox_gt_2) { writeln(_digests["shared"]["images"]); } f.writeln(_digests["shared"]["images"]); } } } catch (ErrnoException ex) { // Handle error } } }