/+ - Name: SisuDoc Spine, Doc Reform [a part of] - Description: documents, structuring, processing, publishing, search - static content generator - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: Spine (SiSU), a framework for document structuring, publishing and search Copyright (C) Ralph Amissah This program is free software: you can redistribute it and/or modify it under the terms of the GNU AFERO General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at these locations: [https://www.fsf.org/licensing/licenses/agpl.html] [https://www.gnu.org/licenses/agpl.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.sisudoc.org] [https://www.doc-reform.org] - Git [https://git.sisudoc.org/] +/ module sisudoc.ocda.abstraction.ssp; @safe: /+ ↓ write document abstraction as human-readable .ssp text file +/ /+ ↓ escaping for values that are folded onto one line a .ssp property value runs to the end of its line, and .heading_ancestors_text joins its eight slots with "|", so a slot that itself contains "|", a backslash or a line break has to be escaped or the value cannot be read back. the pair here is the whole of the rule, used by the writer and by the reader. +/ template sspEscape() { import std.array : replace; string sspEscapeValue(string _s) { return _s .replace("\\", "\\\\") .replace("|", "\\|") .replace("\n", "\\n") .replace("\r", "\\r"); } string[] sspUnescapeSplit(string _s) { string[] _out; string _cur; bool _esc; foreach (c; _s) { if (_esc) { switch (c) { case 'n': _cur ~= '\n'; break; case 'r': _cur ~= '\r'; break; default: _cur ~= c; break; // covers \\ and \| } _esc = false; } else if (c == '\\') { _esc = true; } else if (c == '|') { _out ~= _cur; _cur = ""; } else { _cur ~= c; } } _out ~= _cur; return _out; } } /+ ↓ one object record: the single definition of what an object looks like in a .ssp file. the writer emits with it, and the reader is checked against it by round trip, so the two cannot drift +/ template sspObjectRecord() { import std.conv : to; import std.digest : toHexString; import std.stdio; import std.string; import std.array; import sisudoc.ocda.io_in.paths_source; mixin sspEscape; string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) { string[] output; /+ ↓ object declaration line. a heading carries its identifier here, on the declaration line, where it is not the ocn; every other object carries it as .identifier below, on the same condition +/ string obj_decl = "[" ~ obj.metainfo.ocn.to!string ~ "] "; if (obj.metainfo.is_a == "heading") { string lev = obj.metainfo.marked_up_level; obj_decl ~= "heading :" ~ lev; if (obj.metainfo.identifier.length > 0 && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { obj_decl ~= " " ~ obj.metainfo.identifier; } } else { obj_decl ~= obj.metainfo.is_a; } output ~= obj_decl; /+ ↓ properties (only non-default values) +/ if (obj.metainfo.is_a != "heading" && obj.metainfo.identifier.length > 0 && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { output ~= ".identifier: " ~ obj.metainfo.identifier; } if (obj.metainfo.is_of_part.length > 0) { output ~= ".part: " ~ obj.metainfo.is_of_part; } if (obj.metainfo.is_of_section.length > 0 && obj.metainfo.is_of_section != section) { output ~= ".section: " ~ obj.metainfo.is_of_section; } /+ ↓ parent: emitted for every object except: - the document root, the one object that has no parent - toc entries, generated furniture rather than authored objects, whose place is given by their indent level; the toc heading itself does carry its parent (the document root) A parent ocn of 0 is not "no parent recorded", it is a parent heading the author has made non-substantive (ocn suppressed, or a dummy heading introduced to segment an output); it is structurally truthful and is stated explicitly rather than omitted so that the two cases remain distinguishable in the file +/ if (!(obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup == 0) && obj.metainfo.is_a != "toc" ) { output ~= ".parent: " ~ obj.metainfo.parent_ocn.to!string; } if (obj.metainfo.last_descendant_ocn != 0) { output ~= ".last_descendant: " ~ obj.metainfo.last_descendant_ocn.to!string; } /+ ↓ child headings (from pre-computed map) +/ if (obj.metainfo.children_headings.length > 0) { string[] ch; foreach (c; obj.metainfo.children_headings) { ch ~= c.to!string; } output ~= ".children: " ~ ch.join(" "); } /+ ↓ ancestors (only if non-zero) +/ { bool has_anc = false; foreach (a; obj.metainfo.markedup_ancestors) { if (a != 0) { has_anc = true; break; } } if (has_anc) { string anc; foreach (i, a; obj.metainfo.markedup_ancestors) { if (i > 0) { anc ~= " "; } anc ~= a.to!string; } output ~= ".ancestors: " ~ anc; } } /+ ↓ collapsed ancestors (only if non-zero) +/ { bool has_anc_c = false; foreach (a; obj.metainfo.collapsed_ancestors) { if (a != 0) { has_anc_c = true; break; } } if (has_anc_c) { string anc; foreach (i, a; obj.metainfo.collapsed_ancestors) { if (i > 0) anc ~= " "; anc ~= a.to!string; } output ~= ".ancestors_collapsed: " ~ anc; } } /+ ↓ dom structure status (only if non-zero) +/ { bool has_dom = false; foreach (d; obj.metainfo.dom_structure_markedup_tags_status) { if (d != 0) { has_dom = true; break; } } if (has_dom) { string ds; foreach (i, d; obj.metainfo.dom_structure_markedup_tags_status) { if (i > 0) { ds ~= " "; } ds ~= d.to!string; } output ~= ".dom_status: " ~ ds; } } { bool has_dom_c = false; foreach (d; obj.metainfo.dom_structure_collapsed_tags_status) { if (d != 0) { has_dom_c = true; break; } } if (has_dom_c) { string ds; foreach (i, d; obj.metainfo.dom_structure_collapsed_tags_status) { if (i > 0) { ds ~= " "; } ds ~= d.to!string; } output ~= ".dom_status_collapsed: " ~ ds; } } if (obj.metainfo.heading_lev_collapsed < 9) { output ~= ".heading_lev_collapsed: " ~ obj.metainfo.heading_lev_collapsed.to!string; } if (obj.metainfo.parent_lev_markup != 0) { output ~= ".parent_lev: " ~ obj.metainfo.parent_lev_markup.to!string; } if (obj.metainfo.dummy_heading) { output ~= ".dummy: true"; } if (obj.metainfo.object_number_off) { output ~= ".ocn_off: true"; } if (obj.metainfo.is_of_type.length > 0) { output ~= ".is_of_type: " ~ obj.metainfo.is_of_type; } if (obj.metainfo.attrib.length > 0) { output ~= ".attrib: " ~ obj.metainfo.attrib; } if (obj.metainfo.lang.length > 0) { output ~= ".meta_lang: " ~ obj.metainfo.lang; } if (obj.metainfo.syntax.length > 0) { output ~= ".meta_syntax: " ~ obj.metainfo.syntax; } /+ ↓ sha256 digest +/ { bool has_sha = false; foreach (b; obj.metainfo.sha256.text) { if (b != 0) { has_sha = true; break; } } if (has_sha) { if (_vox) { writeln(obj.metainfo.sha256.summary); } output ~= ".sha256: " ~ obj.metainfo.sha256.text.toHexString.to!string; } } /+ ↓ images, repeatable record, one per image in order of appearance: .image: sha256: bytes: px:x .image: missing:true filename first (the object's handle on the image), remaining fields self-describing key:value tokens so that further fields can be added later without invalidating existing readers. px: is the file's own pixel size, not the display dimensions (w##h##) which the object text already carries +/ if (obj.metainfo.sha256.images.length > 0) { foreach (i; obj.metainfo.sha256.images) { output ~= (i.fileMissing) ? ".image: " ~ i.fileName ~ " missing:true" : ".image: " ~ i.fileName ~ " sha256:" ~ i.fileHash_sha256.toHexString.to!string ~ " bytes:" ~ i.fileSize.to!string ~ ((i.imageWidth > 0 && i.imageHeight > 0) ? " px:" ~ i.imageWidth.to!string ~ "x" ~ i.imageHeight.to!string : ""); } } /+ ↓ text attributes +/ if (obj.attrib.indent_base != 0 || obj.attrib.indent_hang != 0) { output ~= ".indent: " ~ obj.attrib.indent_base.to!string ~ " " ~ obj.attrib.indent_hang.to!string; } if (obj.attrib.bullet) { output ~= ".bullet: true"; } if (obj.attrib.language.length > 0) { output ~= ".lang: " ~ obj.attrib.language; } /+ ↓ has flags +/ { string[] has_flags; if (obj.has.inline_links) { has_flags ~= "links"; } if (obj.has.inline_notes_reg) { has_flags ~= "notes_reg"; } if (obj.has.inline_notes_star) { has_flags ~= "notes_star"; } if (obj.has.images) { has_flags ~= "images"; } if (obj.has.image_without_dimensions) { has_flags ~= "images_no_dim"; } if (has_flags.length > 0) { output ~= ".has: " ~ has_flags.join(" "); } } /+ ↓ table properties +/ if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { output ~= ".table_cols: " ~ obj.table.number_of_columns.to!string; if (obj.table.column_widths.length > 0) { string[] ws; foreach (w; obj.table.column_widths) ws ~= w.to!string; output ~= ".table_widths: " ~ ws.join(" "); } if (obj.table.column_aligns.length > 0) { output ~= ".table_aligns: " ~ obj.table.column_aligns.join(" "); } if (obj.table.heading) { output ~= ".table_header: true"; } } /+ ↓ code block properties +/ if (obj.metainfo.is_a == "code") { if (obj.code_block.linenumbers) { output ~= ".code_linenumbers: true"; } } /+ ↓ stow (extracted links) +/ if (obj.stow.link.length > 0) { foreach (lnk; obj.stow.link) { if (lnk.length > 0) { output ~= ".stow_link: " ~ lnk; } } } /+ ↓ tag properties +/ if (obj.tags.in_segment_html.length > 0) { output ~= ".segment: " ~ obj.tags.in_segment_html; } if (obj.tags.anchor_tag_html.length > 0 && obj.tags.anchor_tag_html != obj.tags.in_segment_html) { output ~= ".anchor: " ~ obj.tags.anchor_tag_html; } if (obj.tags.segname_prev.length > 0) { output ~= ".segment_prev: " ~ obj.tags.segname_prev; } if (obj.tags.segname_next.length > 0) { output ~= ".segment_next: " ~ obj.tags.segname_next; } if (obj.tags.heading_lev_anchor_tag.length > 0) { output ~= ".heading_lev_anchor: " ~ obj.tags.heading_lev_anchor_tag; } if (obj.tags.segment_anchor_tag_epub.length > 0) { output ~= ".segment_epub: " ~ obj.tags.segment_anchor_tag_epub; } /+ ↓ the segment an object belongs to, as opposed to the segment a heading opens. .segment (in_segment_html) is set on headings only, so without these two a reader cannot tell which segment a paragraph is in; the sqlite search writer reads the html one +/ if (obj.tags.html_segment_anchor_tag_is.length > 0) { output ~= ".segment_html_is: " ~ obj.tags.html_segment_anchor_tag_is; } if (obj.tags.epub_segment_anchor_tag_is.length > 0) { output ~= ".segment_epub_is: " ~ obj.tags.epub_segment_anchor_tag_is; } /+ ↓ heading ancestors text +/ { bool has_hat = false; foreach (h; obj.tags.heading_ancestors_text) { if (h.length > 0) { has_hat = true; break; } } if (has_hat) { string[] _hat; foreach (h; obj.tags.heading_ancestors_text) { _hat ~= sspEscapeValue(h.to!string); } output ~= ".heading_ancestors_text: " ~ _hat.join("|"); } } /+ ↓ lev4 subtoc +/ if (obj.tags.lev4_subtoc.length > 0) { foreach (st; obj.tags.lev4_subtoc) { if (st.length > 0) { output ~= ".lev4_subtoc: " ~ st; } } } /+ ↓ anchor tags +/ if (obj.tags.anchor_tags.length > 0) { foreach (at; obj.tags.anchor_tags) { if (at.length > 0) { output ~= ".anchor_tag: " ~ at; } } } /+ ↓ text content +/ if (obj.text.length > 0) { foreach (line; obj.text.split("\n")) { output ~= "| " ~ line; } } output ~= ""; return output; } } template spineAbstractionTxt() { import std.conv : to; import std.digest : toHexString; import std.file; import std.path; import std.stdio; import std.string; import std.array; import sisudoc.ocda.io_in.paths_source; mixin sspObjectRecord; /+ ↓ the whole .ssp for a document, as lines. the file writer below emits these, and the abstraction db is built by reading them back, so the database cannot carry anything the .ssp does not +/ string[] sspDocumentLines(D)(D doc) { auto doc_abstraction = doc.abstraction; auto doc_matters = doc.matters; string[] output; /+ ↓ header comment +/ output ~= "% SiSU Document Abstraction v0.1"; output ~= "% Source: " ~ doc_matters.src.filename; output ~= ""; /+ ↓ @meta block +/ output ~= "@meta {"; auto meta = doc_matters.conf_make_meta.meta; if (meta.title_main.length > 0) { output ~= " title.main: " ~ meta.title_main; } if (meta.title_subtitle.length > 0) { output ~= " title.subtitle: " ~ meta.title_subtitle; } if (meta.title_full.length > 0) { output ~= " title.full: " ~ meta.title_full; } if (meta.title_language.length > 0) { output ~= " title.language: " ~ meta.title_language; } if (meta.creator_author.length > 0) { output ~= " creator.author: " ~ meta.creator_author; } if (meta.creator_author_surname.length > 0) { output ~= " creator.author_surname: " ~ meta.creator_author_surname; } if (meta.creator_author_surname_fn.length > 0) { output ~= " creator.author_surname_fn: " ~ meta.creator_author_surname_fn; } if (meta.creator_author_email.length > 0) { output ~= " creator.author_email: " ~ meta.creator_author_email; } if (meta.creator_illustrator.length > 0) { output ~= " creator.illustrator: " ~ meta.creator_illustrator; } if (meta.creator_translator.length > 0) { output ~= " creator.translator: " ~ meta.creator_translator; } if (meta.date_published.length > 0) { output ~= " date.published: " ~ meta.date_published; } if (meta.date_created.length > 0) { output ~= " date.created: " ~ meta.date_created; } if (meta.date_issued.length > 0) { output ~= " date.issued: " ~ meta.date_issued; } if (meta.date_available.length > 0) { output ~= " date.available: " ~ meta.date_available; } if (meta.date_modified.length > 0) { output ~= " date.modified: " ~ meta.date_modified; } if (meta.date_valid.length > 0) { output ~= " date.valid: " ~ meta.date_valid; } if (meta.rights_copyright.length > 0) { output ~= " rights.copyright: " ~ meta.rights_copyright; } if (meta.rights_license.length > 0) { output ~= " rights.license: " ~ meta.rights_license; } if (meta.classify_topic_register.length > 0) { output ~= " classify.topic_register: " ~ meta.classify_topic_register; } if (meta.classify_subject.length > 0) { output ~= " classify.subject: " ~ meta.classify_subject; } if (meta.classify_keywords.length > 0) { output ~= " classify.keywords: " ~ meta.classify_keywords; } if (meta.classify_loc.length > 0) { output ~= " classify.loc: " ~ meta.classify_loc; } if (meta.classify_dewey.length > 0) { output ~= " classify.dewey: " ~ meta.classify_dewey; } if (meta.identifier_isbn.length > 0) { output ~= " identifier.isbn: " ~ meta.identifier_isbn; } if (meta.identifier_oclc.length > 0) { output ~= " identifier.oclc: " ~ meta.identifier_oclc; } if (meta.language_document.length > 0) { output ~= " language.document: " ~ meta.language_document; } if (meta.notes_abstract.length > 0) { output ~= " notes.abstract: " ~ meta.notes_abstract; } if (meta.notes_description.length > 0) { output ~= " notes.description: " ~ meta.notes_description; } if (meta.notes_summary.length > 0) { output ~= " notes.summary: " ~ meta.notes_summary; } // check output ~= "}"; output ~= ""; /+ ↓ @make block +/ output ~= "@make {"; auto make = doc_matters.conf_make_meta.make; if (make.doc_type.length > 0) { output ~= " doc_type: " ~ make.doc_type; } if (make.auto_num_top_at_level.length > 0) { output ~= " auto_num_top_at_level: " ~ make.auto_num_top_at_level; } // check output ~= " auto_num_top_lv: " ~ make.auto_num_top_lv.to!string; output ~= " auto_num_depth: " ~ make.auto_num_depth.to!string; output ~= "}"; output ~= ""; /+ ↓ @doc_has block +/ output ~= "@doc_has {"; output ~= " inline_links: " ~ doc_matters.has.inline_links.to!string; output ~= " inline_notes_reg: " ~ doc_matters.has.inline_notes_reg.to!string; output ~= " inline_notes_star: " ~ doc_matters.has.inline_notes_star.to!string; output ~= " tables: " ~ doc_matters.has.tables.to!string; output ~= " codeblocks: " ~ doc_matters.has.codeblocks.to!string; output ~= " images: " ~ doc_matters.has.images.to!string; output ~= " poems: " ~ doc_matters.has.poems.to!string; output ~= " groups: " ~ doc_matters.has.groups.to!string; output ~= " blocks: " ~ doc_matters.has.blocks.to!string; output ~= " quotes: " ~ doc_matters.has.quotes.to!string; output ~= "}"; output ~= ""; /+ ↓ document sections +/ string[] section_order = ["head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb"]; foreach (section; section_order) { if (section !in doc_abstraction) { continue; } auto section_objs = doc_abstraction[section]; if (section_objs.length == 0) { continue; } output ~= "@" ~ section ~ " {"; output ~= ""; foreach (obj; section_objs) { output ~= sspObjectRecord(obj, section, doc_matters.opt.action.vox_gt_3); } output ~= "}"; output ~= ""; } return output; } @trusted void spineAbstractionTxt(D)(D doc) { auto doc_matters = doc.matters; string[] output = sspDocumentLines(doc); /+ ↓ write to file, into the pod's own tree: pod//media/abstraction/.ssp . where it sits beside pod//media/image/, which is what a .ssp needs to be a document source: the file describes its images by name, size, pixels and digest, and they have to be findable. Written here once; --pod2 then digests and zips it where it stands rather than copying it in from an output directory of its own. +/ auto pths_pod = spinePathsPods!()(doc_matters); string base_pth = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string; try { if (!exists(base_pth)) { base_pth.mkdirRecurse; } else if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0] ) { /+ ↓ this directory is the one part of the pod written before the pod is built, so the pod builder leaves it alone. It therefore clears itself here, on the first language of a run, or a .ssp for a language the document no longer has would linger and be bundled +/ foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) { _f.remove; } } } catch (Exception ex) { } string out_file = ((base_pth.chainPath( doc_matters.src.doc_uid_out ~ ".ssp")).asNormalizedPath).array; if (doc_matters.opt.action.vox_gt_1) { writeln(" ", out_file); } auto f = File(out_file, "w"); foreach (line; output) { f.writeln(line); } } }