From f41bf1cc525c70cced764a5e4df70d499d82f9c9 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 11 Sep 2026 13:57:47 -0400 Subject: spine: process a document from .ocda.db or .ssp Process a document from its .ocda.db or .ssp abstraction. A .ssp or a .ocda.db given as an argument is now a document like any other: read rather than parsed, and handed to the output writers as the same doc they already take. The 35 document sample collection built from its .ssp files is byte identical to the same collection built from markup: 1457 files, no differences. Three things had to be rebuilt rather than read, all of them a value parsed rather than a structure derived: - classify_topic_register_arr and its expanded twin. The split is not a plain one, so the rule moved to sisudoc.ocda.meta.topic_register and both the yaml reader and this one call it. - creator_author_arr, which is creator.author split on the ", " it was joined with. - title_sub, a copy of title_subtitle made where the header is read, and what epub3 puts in dc:title id="subtitle". Fixed ordering bug found by the acceptance test. A heading's own anchor can be a bare number taken from its text and this can collide with the ocn of an unrelated object. Whole output comparison found that before the fix there was one wrong link in one epub's table of contents. From a .ocda.db, 1446 of 1457 files are identical. The 11 that are not are the images: a database carries its own image blobs and nothing yet extracts them, so the five sisu_markup images are not copied and the epub that embeds them differs. That is the next step and is not a defect in this one. --source and --pod2 are refused with a warning rather than half done, no artefact carrying the markup. (assisted by Claude-Code) --- src/sisudoc/ocda/abstraction/doc_from_artefact.d | 279 +++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 src/sisudoc/ocda/abstraction/doc_from_artefact.d (limited to 'src/sisudoc/ocda/abstraction/doc_from_artefact.d') diff --git a/src/sisudoc/ocda/abstraction/doc_from_artefact.d b/src/sisudoc/ocda/abstraction/doc_from_artefact.d new file mode 100644 index 0000000..e45cbf0 --- /dev/null +++ b/src/sisudoc/ocda/abstraction/doc_from_artefact.d @@ -0,0 +1,279 @@ +/+ +- Name: SisuDoc Spine, Doc Reform [a part of] + - Description: documents, structuring, processing, publishing, search + - static content generator + + - Author: Ralph Amissah + [ralph.amissah@gmail.com] + + - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. + + - License: AGPL 3 or later: + + Spine (SiSU), a framework for document structuring, publishing and + search + + Copyright (C) Ralph Amissah + + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU AFERO General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + You should have received a copy of the GNU General Public License along with + this program. If not, see [https://www.gnu.org/licenses/]. + + If you have Internet connection, the latest version of the AGPL should be + available at these locations: + [https://www.fsf.org/licensing/licenses/agpl.html] + [https://www.gnu.org/licenses/agpl.html] + + - Spine (by Doc Reform, related to SiSU) uses standard: + - docReform markup syntax + - standard SiSU markup syntax with modified headers and minor modifications + - docReform object numbering + - standard SiSU object citation numbering & system + + - Homepages: + [https://www.sisudoc.org] + [https://www.doc-reform.org] + + - Git + [https://git.sisudoc.org/] + ++/ +module sisudoc.ocda.abstraction.doc_from_artefact; +@safe: +/+ ↓ a document, from an artefact rather than from markup + + spineAbstraction() parses markup and hands the output writers a doc: + an abstraction and a doc_matters. This does the same from a .ssp or a + .ocda.db, filling the same docMattersMake() arguments, so that what + comes out is the value the writers already take and they never learn + which of the two made it. + + Three of the seven arguments come from the run and are the same either + way: program_info, opt_action, and the conf half of conf_make_meta, + which is site and run scoped (urls, papersize, the search database) and + has no business in a document's own artefact. The other four come from + the artefact: + + conf_make_meta meta and make from the @meta and @make blocks, over + the site config the run assembled + doc_has docHasFromAbstraction, over the objects + doc_digest the markup digest from @source + manifest synthesised from the artefact's path and the + document's own name and language + + What is not here, deliberately: insert_file_list, which is read only by + source_pod.d for --source and --pod2. No artefact carries the markup, + so a pod cannot be rebuilt from one, and the empty list is the truthful + answer rather than a gap. ++/ +template spineDocFromArtefact() { + import std.algorithm : endsWith; + import std.array; + import std.conv : to; + import std.path; + import std.regex; + import std.stdio; + import std.string; + import sisudoc.ocda.meta.conf_make_meta_structs; + import sisudoc.ocda.meta.doc_matters; + import sisudoc.ocda.io_in.paths_source; + import sisudoc.ocda.abstraction.doc_has; + import sisudoc.ocda.abstraction.load; + import sisudoc.ocda.meta.topic_register; + mixin spineDocMatters; + mixin spineDocHasFromAbstraction; + mixin spineAbstractionLoad; + mixin spineTopicRegister; + /+ ↓ a .ssp header key from a field name: the first underscore becomes the + dot that separates the group, so title_main is title.main and + rights_copyright_text is rights.copyright_text. The writer's keys are + formed the same way, which is what lets this be read off the struct + rather than kept as a second list to fall out of step with it. + +/ + private string _sspKeyOf(string _member) { + foreach (i, c; _member) { + if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; } + } + return _member; + } + /+ ↓ the document's own header, over the site config the run assembled. + + every string field of MetaComposite is looked for under its own key, + so a field added there and emitted by the writer arrives here without + anything being added. The arr fields are split from the string they + come from, as agreed: that is parsing a value, not deriving a + structure. + +/ + ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) { + ConfComposite _out = _site_conf; // conf: site and run scoped, kept + static foreach (_m; __traits(allMembers, MetaComposite)) { + static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{ + enum string _key = _sspKeyOf(_m); + if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; } + }} + } + /+ ↓ title_sub is a copy of title_subtitle, made where the header is + read, and is what epub3 puts in dc:title id="subtitle". It is not + emitted, being the same string twice; it is remade here the way the + yaml reader makes it +/ + _out.meta.title_sub = _out.meta.title_subtitle; + /+ ↓ creator.author is the author list joined with ", ", so splitting on + it gives the list back +/ + if (_out.meta.creator_author.length > 0) { + foreach (_a; _out.meta.creator_author.split(", ")) { + if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; } + } + } + /+ ↓ the topic register arrays, by the one rule the yaml reader uses: + the split is not a plain one and deriving it twice would be two + answers to the same question +/ + { + auto _tr = topicRegisterArrays(_out.meta.classify_topic_register); + _out.meta.classify_topic_register_arr = _tr.arr; + _out.meta.classify_topic_register_expanded_arr = _tr.expanded; + } + /+ ↓ make: the keys are the field names, with no group to separate +/ + if (auto _v = "doc_type" in _ssp.make) { + _out.make.doc_type = *_v; _out.make_str.doc_type = *_v; + } + if (auto _v = "auto_num_top_at_level" in _ssp.make) { + _out.make.auto_num_top_at_level = *_v; + } + if (auto _v = "auto_num_top_lv" in _ssp.make) { + try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {} + } + if (auto _v = "auto_num_depth" in _ssp.make) { + try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {} + } + if (auto _v = "breaks" in _ssp.make) { + _out.make.breaks = *_v; _out.make_str.breaks = *_v; + } + if (auto _v = "home_button_text" in _ssp.make) { + _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v; + } + /+ ↓ footer is a list, joined with the separator the format escapes +/ + if (auto _v = "footer" in _ssp.make) { + _out.make.footer = sspUnescapeSplit(*_v); + _out.make_str.footer = _out.make.footer; + } + return _out; + } + /+ ↓ where a document loaded from an artefact says it is. + + an artefact names its document (% Source:) and its language (@source + language) but not where the pod holding its images sits, so that is + taken from where the artefact itself was found: + + pod//media/abstraction/.ssp -> pod// + pod/.ocda.db -> pod// + + which is where --pod2 puts them both. The images beside a .ssp are + then found where html.d already looks; a .ocda.db carries its own and + does not need the directory to exist. + +/ + struct ST_ArtefactPaths { + string pod_dir; + string src_file_with_path; + } + ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) { + ST_ArtefactPaths _out; + string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en"; + string _dir = _artefact.dirName; + if (_artefact.endsWith(".ssp")) { + /+ ↓ /media/abstraction/.ssp: two directories up is the pod +/ + _out.pod_dir = (_dir.baseName == "abstraction") + ? _dir.dirName.dirName : _dir; + } else { + /+ ↓ /.ocda.db, the pod tree beside it named for the document +/ + string _uid = _artefact.baseName; + foreach (_sfx; [".ocda.db", ".db"]) { + if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; } + } + if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; } + _out.pod_dir = (_dir.chainPath(_uid).array).to!string; + } + _out.src_file_with_path = (_out.pod_dir + .chainPath("media").chainPath("text").chainPath(_lang) + .chainPath(_ssp.source).array).to!string; + return _out; + } + /+ ↓ the doc the output writers take, from an artefact. + + the reading is done here rather than by the caller, and that is not + only tidiness: main() is @system, so a template mixed in there infers + @system for everything it declares, and the output writers are @safe + and will not take objects of such a type. Doing it inside this module, + which is @safe:, is what makes the loaded objects the same kind of + thing the parser's are. Called as spineDocFromArtefact!()(...), the + way spineAbstraction!() is, rather than mixed in. + +/ + auto spineDocFromArtefact(P,O,E)( + string _artefact, + P program_info, + O _opt_action, + E _env, + ConfComposite _site_conf, + ) { + auto _loaded = abstractionLoad(_artefact); + auto _ssp = _loaded.doc; + auto _pths = artefactPaths(_artefact, _ssp); + auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); + auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); + auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); + /+ ↓ the digests: the markup digest is what @source carries, and it is + the one that says which source this abstraction was made from. The + header and text digests are of parts an artefact does not keep + separately, and are left unset rather than guessed at + +/ + ST_ArtefactDigest _dig; + if (auto _v = "digest" in _ssp.source_info) { + _dig.markup_doc = _hexToDigest(*_v); + } + auto _abstraction = _ssp.abstraction; + auto _matters = docMattersMake( + program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init, + ); + /+ ↓ the same shape spineAbstraction() returns, built the same way: a + struct of accessors over the locals, with whether the artefact + could be read at all, which the caller has to know +/ + auto theDOC() { + struct ST_DOC { + const auto abstraction() { return _abstraction; } + auto matters() { return _matters; } + bool loaded() { return _loaded.loaded; } + string note() { return _loaded.note; } + string source_name() { return abstractionSourceName(_loaded.source); } + } + return ST_DOC(); + } + return theDOC(); + } + /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from + an artefact, @source having recorded it; the header and text digests + are of parts no artefact keeps separately and are left unset rather + than guessed at. Shaped to match ST_doc_digest, which is declared + inside spineRawMarkupContent and reachable only by parsing a file. + +/ + struct ST_ArtefactDigest { + ubyte[32] markup_doc; + ubyte[32] header; + ubyte[32] text; + } + private ubyte[32] _hexToDigest(string _hex) { + ubyte[32] _out; + if (_hex.length < 64) { return _out; } + foreach (i; 0..32) { + try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {} + } + return _out; + } +} -- cgit v1.2.3