/+ - Name: SisuDoc Spine - Description: part of SiSU spine (Doc Reform): for document structuring, processing, publishing and search - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later, for everything in this repository unless a file or a directory specifies otherwise. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License (AGPLv3) as published by the Free Software Foundation (FSF), either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at: [https://www.gnu.org/licenses/agpl-3.0.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.sisudoc.org] [https://www.doc-reform.org] - Git [https://git.sisudoc.org/] +/ module sisudoc.ocda.abstraction.ssp_in; @safe: /+ ↓ read a .ssp file back into the document abstraction . the reverse of sisudoc.ocda.abstraction.ssp. what it returns is the same value the parser produces, ObjGenericComposite[][string], so anything that consumes the abstraction can be fed from a .ssp instead of from markup. . the check that it is faithful is a round trip: load a .ssp, emit each object again with sspObjectRecord (the writer's own definition of a record) and require the result to be byte identical to the file read. +/ template spineAbstractionRead() { import std.algorithm : startsWith, findSplit; import std.array; import std.conv : to; import std.file; import std.json : JSONValue; import std.stdio; import std.string; import sisudoc.ocda.meta.metadoc_object_setter; import sisudoc.ocda.abstraction.ssp : sspEscape; mixin sspEscape; mixin ObjectSetter; /+ ↓ what a .ssp file holds: the three header blocks as key/value, and the object sections in the order the file gives them +/ struct SSPdocument { string format; // "% SiSU Document Abstraction v2.1" string source; // "% Source: ..." string[string] source_info; // the @source block string[] source_info_order; string ssp_digest; // sha256 of the .ssp read, hex string[string] meta; string[] meta_order; string[string] make; string[] make_order; string[string] doc_has; string[] doc_has_order; string[] section_order; // as found in the file ObjGenericComposite[][string] abstraction; } private int _levOfMarkedUp(string lev) { switch (lev) { case "A": return 0; case "B": return 1; case "C": return 2; case "D": return 3; case "1": return 4; case "2": return 5; case "3": return 6; case "4": return 7; default: return 9; } } private int[] _ints(string val) { int[] _out; foreach (f; val.split(" ")) { if (f.length > 0) { _out ~= f.to!int; } } return _out; } /+ ↓ the eight slot arrays are int[] with a literal default, so every default constructed object shares one array; write into a fresh one +/ private int[8] _eight(string val) { int[8] _out; foreach (i, v; _ints(val)) { if (i < _out.length) { _out[i] = v; } } return _out; } private ubyte[32] _hexToBytes(string hex) { ubyte[32] _out; if (hex.length < 64) { return _out; } foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } return _out; } SSPdocument sspRead(string[] lines) { SSPdocument doc; string section; // the @section block currently open string block; // "meta", "make", "doc_has", or "" for objects bool in_object; string[] text_lines; ObjGenericComposite obj; void _closeObject() { if (!in_object) { return; } obj.text = text_lines.join("\n"); doc.abstraction[section] ~= obj; obj = ObjGenericComposite(); text_lines = []; in_object = false; } foreach (line; lines) { if (line.length == 0) { continue; } if (line.startsWith("% SiSU Document Abstraction")) { doc.format = line; continue; } if (line.startsWith("% Source: ")) { doc.source = line["% Source: ".length .. $]; continue; } if (line.startsWith("@") && line.endsWith(" {")) { _closeObject(); string _name = line[1 .. $-2]; switch (_name) { case "source": case "meta": case "make": case "doc_has": block = _name; section = ""; break; default: block = ""; section = _name; doc.section_order ~= _name; doc.abstraction[_name] = []; break; } continue; } if (line == "}") { _closeObject(); block = ""; section = ""; continue; } if (block.length > 0) { // a header block key: value line auto _kv = line.stripLeft.findSplit(": "); // leading only: a value may end in a space if (_kv[1].length > 0) { final switch (block) { case "source": doc.source_info[_kv[0].to!string] = _kv[2].to!string; doc.source_info_order ~= _kv[0].to!string; break; case "meta": doc.meta[_kv[0].to!string] = _kv[2].to!string; doc.meta_order ~= _kv[0].to!string; break; case "make": doc.make[_kv[0].to!string] = _kv[2].to!string; doc.make_order ~= _kv[0].to!string; break; case "doc_has": doc.doc_has[_kv[0].to!string] = _kv[2].to!string; doc.doc_has_order ~= _kv[0].to!string; break; } } continue; } if (line.startsWith("[")) { // object declaration line _closeObject(); in_object = true; auto _close = line.indexOf("] "); obj.metainfo.ocn = line[1 .. _close].to!int; string _rest = line[_close + 2 .. $]; if (_rest.startsWith("heading :")) { obj.metainfo.is_a = "heading"; string _tail = _rest["heading :".length .. $]; auto _sp = _tail.indexOf(" "); if (_sp > 0) { obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail[0 .. _sp]); obj.metainfo.identifier = _tail[_sp + 1 .. $]; } else { obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail); /+ ↓ omitted means it equals the ocn, and for an object with no ocn it was empty: "a"~N identifiers are always written +/ obj.metainfo.identifier = (obj.metainfo.ocn != 0) ? obj.metainfo.ocn.to!string : ""; } } else { obj.metainfo.is_a = _rest; obj.metainfo.identifier = (obj.metainfo.ocn != 0) ? obj.metainfo.ocn.to!string : ""; } obj.metainfo.is_of_section = section; continue; } if (line.startsWith("| ")) { text_lines ~= line[2 .. $]; continue; } if (!line.startsWith(".")) { continue; } auto _p = line[1 .. $].findSplit(": "); string _key = (_p[1].length > 0) ? _p[0].to!string : line[1 .. $].to!string; string _val = (_p[1].length > 0) ? _p[2].to!string : ""; switch (_key) { case "identifier": obj.metainfo.identifier = _val; break; case "part": obj.metainfo.is_of_part = _val; break; case "section": obj.metainfo.is_of_section = _val; break; case "parent": obj.metainfo.parent_ocn = _val.to!int; break; case "last_descendant": obj.metainfo.last_descendant_ocn = _val.to!int; break; case "ocn_range": { auto _r = _ints(_val); if (_r.length == 2) { obj.metainfo.ocn_range_first = _r[0]; obj.metainfo.ocn_range_last = _r[1]; } break; } case "children": obj.metainfo.children_headings = _ints(_val); break; case "ancestors": obj.metainfo.markedup_ancestors = _eight(_val); break; case "ancestors_collapsed": obj.metainfo.collapsed_ancestors = _eight(_val); break; case "dom_status": obj.metainfo.dom_structure_markedup_tags_status = _eight(_val); break; case "dom_status_collapsed": obj.metainfo.dom_structure_collapsed_tags_status = _eight(_val); break; case "heading_lev_collapsed": obj.metainfo.heading_lev_collapsed = _val.to!int; break; case "parent_lev": obj.metainfo.parent_lev_markup = _val.to!int; break; case "dummy": obj.metainfo.dummy_heading = true; break; case "ocn_off": obj.metainfo.object_number_off = true; break; case "is_of_type": obj.metainfo.is_of_type = _val; break; case "attrib": obj.metainfo.attrib = _val; break; case "meta_lang": obj.metainfo.lang = _val; break; case "meta_syntax": obj.metainfo.syntax = _val; break; case "sha256": obj.metainfo.sha256.text = _hexToBytes(_val); break; case "image": { ST_file_name_hash_size_ _img; auto _fields = _val.split(" "); if (_fields.length > 0) { _img.fileName = _fields[0].to!string; } foreach (f; _fields[1 .. $]) { if (f == "missing:true") { _img.fileMissing = true; } else if (f.startsWith("sha256:")) { _img.fileHash_sha256 = _hexToBytes(f[7 .. $].to!string); } else if (f.startsWith("bytes:")) { _img.fileSize = f[6 .. $].to!ulong; } else if (f.startsWith("px:")) { auto _wh = f[3 .. $].findSplit("x"); if (_wh[1].length > 0) { _img.imageWidth = _wh[0].to!int; _img.imageHeight = _wh[2].to!int; } } } obj.metainfo.sha256.images ~= _img; } break; case "indent": { auto _i = _ints(_val); if (_i.length > 0) { obj.attrib.indent_base = _i[0]; } if (_i.length > 1) { obj.attrib.indent_hang = _i[1]; } } break; case "bullet": obj.attrib.bullet = true; break; case "lang": obj.attrib.language = _val; break; case "has": foreach (f; _val.split(" ")) { switch (f) { case "links": obj.has.inline_links = true; break; case "notes_reg": obj.has.inline_notes_reg = true; break; case "notes_star": obj.has.inline_notes_star = true; break; case "images": obj.has.images = true; break; case "images_no_dim": obj.has.image_without_dimensions = true; break; default: break; } } break; case "table_cols": obj.table.number_of_columns = _val.to!int; break; case "table_widths": foreach (w; _val.split(" ")) { obj.table.column_widths ~= w.to!double; } break; case "table_aligns": foreach (a; _val.split(" ")) { obj.table.column_aligns ~= a.to!string; } break; case "table_header": obj.table.heading = true; break; case "code_linenumbers": obj.code_block.linenumbers = true; break; case "stow_link": obj.stow.link ~= _val; break; case "segment": obj.tags.in_segment_html = _val; break; case "anchor": obj.tags.anchor_tag_html = _val; break; case "segment_prev": obj.tags.segname_prev = _val; break; case "segment_next": obj.tags.segname_next = _val; break; case "heading_lev_anchor": obj.tags.heading_lev_anchor_tag = _val; break; case "segment_epub": obj.tags.segment_anchor_tag_epub = _val; break; case "segment_html_is": obj.tags.html_segment_anchor_tag_is = _val; break; case "segment_epub_is": obj.tags.epub_segment_anchor_tag_is = _val; break; case "segment_lv4_is": obj.tags.segment_lv4_is = _val; break; case "heading_ancestors_text": { string[8] _h; foreach (i, h; sspUnescapeSplit(_val)) { if (i < _h.length) { _h[i] = h; } } obj.tags.heading_ancestors_text = _h; } break; case "lev4_subtoc": obj.tags.lev4_subtoc ~= _val; break; case "anchor_tag": obj.tags.anchor_tags ~= _val; break; default: break; } } _closeObject(); /+ ↓ .anchor is written only when it differs from .segment, and .section only when it differs from the enclosing block; restore both +/ foreach (sect; doc.section_order) { foreach (ref o; doc.abstraction[sect]) { if (o.tags.anchor_tag_html.length == 0 && o.tags.in_segment_html.length > 0) { o.tags.anchor_tag_html = o.tags.in_segment_html; } } } return doc; } /+ ↓ the whole document as it survives a trip through the .ssp text: emitted as lines by the writer, read straight back. what a consumer built from this cannot see, the .ssp does not carry, which is what makes a second artefact built from it unable to drift. the header blocks come back with it, so the database is filled from these and not from a second list read off doc_matters +/ SSPdocument sspRoundTripDocument(D)(D doc) { import sisudoc.ocda.abstraction.ssp; mixin spineAbstractionTxt; string[] _lines = sspDocumentLines(doc); auto _out = sspRead(_lines); /+ ↓ the writer emits one line per entry, each with its newline, so this is the digest of the file that is written beside the pod +/ _out.ssp_digest = sspDigestOfText(_lines.join("\n") ~ "\n"); return _out; } /+ ↓ sha256 of a .ssp, upper case hex, as digests.txt gives it +/ string sspDigestOfText(string _text) { import std.digest : toHexString; import std.digest.sha : sha256Of; return _text.sha256Of.toHexString.to!string; } auto sspRoundTripAbstraction(D)(D doc) { return sspRoundTripDocument(doc).abstraction; } /+ ↓ the version a reader will accept. the major part must match: a format whose properties have changed meaning is not readable by guesswork, and loading it half populated in silence is the failure this check exists to stop. a newer minor version is accepted, since a minor bump only adds properties and an unknown property line is ignored. +/ bool sspFormatVersionOk(string _format_line, string _what) { import sisudoc.ocda.abstraction.ssp : ssp_format_version, ssp_format_version_major; enum _prefix = "% SiSU Document Abstraction v"; if (_format_line.length == 0) { writeln("ERROR: ", _what, ": no format line, not a .ssp"); return false; } if (!_format_line.startsWith(_prefix)) { writeln("ERROR: ", _what, ": unrecognised format line: ", _format_line); return false; } string _ver = _format_line[_prefix.length .. $].strip; string _major = _ver.findSplit(".")[0]; if (_major != ssp_format_version_major) { writeln("ERROR: ", _what, ": .ssp format v", _ver, ", this spine reads v", ssp_format_version_major, ".x (v", ssp_format_version, "). Regenerate the abstraction from its markup."); return false; } return true; } /+ ↓ with_digest is off by default: hashing the file is only wanted where the digest is going to be recorded, and on the largest document in the sample set it is a tenth of what reading the file costs +/ @trusted SSPdocument sspReadFile(string file_path, bool with_digest = false) { string _text; string[] _lines; try { _text = (cast(char[]) file_path.read).to!string; _lines = _text.split("\n"); } catch (Exception ex) { writeln("ERROR: could not read ", file_path, ": ", ex.msg); return SSPdocument(); } auto _doc = sspRead(_lines); if (!sspFormatVersionOk(_doc.format, file_path)) { return SSPdocument(); } if (with_digest) { _doc.ssp_digest = sspDigestOfText(_text); } return _doc; } }