-*- mode: org -*- #+TITLE: sisudoc spine (doc_reform) output pod source abstraction peg text #+DESCRIPTION: documents - structuring, publishing in multiple formats & search #+FILETAGS: :spine:output:source:pod: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code #+PROPERTY: header-args+ :noweb yes :padline no #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - magic single double-quote → " ← FIX changes hilighting behavior (occuring after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] * (Object-Centric) Document Abstraction PEG text - Process markup document, create document abstraction ** _module template_ :module:metadoc_from_src: rename source_abstraction_peg_txt.d #+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> module sisudoc.ocda.abstraction.ssp; @safe: /+ ↓ write document abstraction as human-readable .ssp text file +/ /+ ↓ escaping for values that are folded onto one line a .ssp property value runs to the end of its line, and .heading_ancestors_text joins its eight slots with "|", so a slot that itself contains "|", a backslash or a line break has to be escaped or the value cannot be read back. the pair here is the whole of the rule, used by the writer and by the reader. +/ template sspEscape() { import std.array : replace; string sspEscapeValue(string _s) { return _s .replace("\\", "\\\\") .replace("|", "\\|") .replace("\n", "\\n") .replace("\r", "\\r"); } string[] sspUnescapeSplit(string _s) { string[] _out; string _cur; bool _esc; foreach (c; _s) { if (_esc) { switch (c) { case 'n': _cur ~= '\n'; break; case 'r': _cur ~= '\r'; break; default: _cur ~= c; break; // covers \\ and \| } _esc = false; } else if (c == '\\') { _esc = true; } else if (c == '|') { _out ~= _cur; _cur = ""; } else { _cur ~= c; } } _out ~= _cur; return _out; } } /+ ↓ one object record: the single definition of what an object looks like in a .ssp file. the writer emits with it, and the reader is checked against it by round trip, so the two cannot drift +/ template sspObjectRecord() { import std.conv : to; import std.digest : toHexString; import std.stdio; import std.string; import std.array; import sisudoc.ocda.io_in.paths_source; mixin sspEscape; string[] sspObjectRecord(O)(O obj, string section, bool _vox = false) { string[] output; /+ ↓ object declaration line. a heading carries its identifier here, on the declaration line, where it is not the ocn; every other object carries it as .identifier below, on the same condition +/ string obj_decl = "[" ~ obj.metainfo.ocn.to!string ~ "] "; if (obj.metainfo.is_a == "heading") { string lev = obj.metainfo.marked_up_level; obj_decl ~= "heading :" ~ lev; if (obj.metainfo.identifier.length > 0 && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { obj_decl ~= " " ~ obj.metainfo.identifier; } } else { obj_decl ~= obj.metainfo.is_a; } output ~= obj_decl; /+ ↓ properties (only non-default values) +/ if (obj.metainfo.is_a != "heading" && obj.metainfo.identifier.length > 0 && obj.metainfo.identifier != obj.metainfo.ocn.to!string) { output ~= ".identifier: " ~ obj.metainfo.identifier; } if (obj.metainfo.is_of_part.length > 0) { output ~= ".part: " ~ obj.metainfo.is_of_part; } if (obj.metainfo.is_of_section.length > 0 && obj.metainfo.is_of_section != section) { output ~= ".section: " ~ obj.metainfo.is_of_section; } /+ ↓ parent: emitted for every object except: - the document root, the one object that has no parent - toc entries, generated furniture rather than authored objects, whose place is given by their indent level; the toc heading itself does carry its parent (the document root) A parent ocn of 0 is not "no parent recorded", it is a parent heading the author has made non-substantive (ocn suppressed, or a dummy heading introduced to segment an output); it is structurally truthful and is stated explicitly rather than omitted so that the two cases remain distinguishable in the file +/ if (!(obj.metainfo.is_a == "heading" && obj.metainfo.heading_lev_markup == 0) && obj.metainfo.is_a != "toc" ) { output ~= ".parent: " ~ obj.metainfo.parent_ocn.to!string; } if (obj.metainfo.last_descendant_ocn != 0) { output ~= ".last_descendant: " ~ obj.metainfo.last_descendant_ocn.to!string; } /+ ↓ child headings (from pre-computed map) +/ if (obj.metainfo.children_headings.length > 0) { string[] ch; foreach (c; obj.metainfo.children_headings) { ch ~= c.to!string; } output ~= ".children: " ~ ch.join(" "); } /+ ↓ ancestors (only if non-zero) +/ { bool has_anc = false; foreach (a; obj.metainfo.markedup_ancestors) { if (a != 0) { has_anc = true; break; } } if (has_anc) { string anc; foreach (i, a; obj.metainfo.markedup_ancestors) { if (i > 0) { anc ~= " "; } anc ~= a.to!string; } output ~= ".ancestors: " ~ anc; } } /+ ↓ collapsed ancestors (only if non-zero) +/ { bool has_anc_c = false; foreach (a; obj.metainfo.collapsed_ancestors) { if (a != 0) { has_anc_c = true; break; } } if (has_anc_c) { string anc; foreach (i, a; obj.metainfo.collapsed_ancestors) { if (i > 0) anc ~= " "; anc ~= a.to!string; } output ~= ".ancestors_collapsed: " ~ anc; } } /+ ↓ dom structure status (only if non-zero) +/ { bool has_dom = false; foreach (d; obj.metainfo.dom_structure_markedup_tags_status) { if (d != 0) { has_dom = true; break; } } if (has_dom) { string ds; foreach (i, d; obj.metainfo.dom_structure_markedup_tags_status) { if (i > 0) { ds ~= " "; } ds ~= d.to!string; } output ~= ".dom_status: " ~ ds; } } { bool has_dom_c = false; foreach (d; obj.metainfo.dom_structure_collapsed_tags_status) { if (d != 0) { has_dom_c = true; break; } } if (has_dom_c) { string ds; foreach (i, d; obj.metainfo.dom_structure_collapsed_tags_status) { if (i > 0) { ds ~= " "; } ds ~= d.to!string; } output ~= ".dom_status_collapsed: " ~ ds; } } if (obj.metainfo.heading_lev_collapsed < 9) { output ~= ".heading_lev_collapsed: " ~ obj.metainfo.heading_lev_collapsed.to!string; } if (obj.metainfo.parent_lev_markup != 0) { output ~= ".parent_lev: " ~ obj.metainfo.parent_lev_markup.to!string; } if (obj.metainfo.dummy_heading) { output ~= ".dummy: true"; } if (obj.metainfo.object_number_off) { output ~= ".ocn_off: true"; } if (obj.metainfo.is_of_type.length > 0) { output ~= ".is_of_type: " ~ obj.metainfo.is_of_type; } if (obj.metainfo.attrib.length > 0) { output ~= ".attrib: " ~ obj.metainfo.attrib; } if (obj.metainfo.lang.length > 0) { output ~= ".meta_lang: " ~ obj.metainfo.lang; } if (obj.metainfo.syntax.length > 0) { output ~= ".meta_syntax: " ~ obj.metainfo.syntax; } /+ ↓ sha256 digest +/ { bool has_sha = false; foreach (b; obj.metainfo.sha256.text) { if (b != 0) { has_sha = true; break; } } if (has_sha) { if (_vox) { writeln(obj.metainfo.sha256.summary); } output ~= ".sha256: " ~ obj.metainfo.sha256.text.toHexString.to!string; } } /+ ↓ images, repeatable record, one per image in order of appearance: .image: sha256: bytes: px:x .image: missing:true filename first (the object's handle on the image), remaining fields self-describing key:value tokens so that further fields can be added later without invalidating existing readers. px: is the file's own pixel size, not the display dimensions (w##h##) which the object text already carries +/ if (obj.metainfo.sha256.images.length > 0) { foreach (i; obj.metainfo.sha256.images) { output ~= (i.fileMissing) ? ".image: " ~ i.fileName ~ " missing:true" : ".image: " ~ i.fileName ~ " sha256:" ~ i.fileHash_sha256.toHexString.to!string ~ " bytes:" ~ i.fileSize.to!string ~ ((i.imageWidth > 0 && i.imageHeight > 0) ? " px:" ~ i.imageWidth.to!string ~ "x" ~ i.imageHeight.to!string : ""); } } /+ ↓ text attributes +/ if (obj.attrib.indent_base != 0 || obj.attrib.indent_hang != 0) { output ~= ".indent: " ~ obj.attrib.indent_base.to!string ~ " " ~ obj.attrib.indent_hang.to!string; } if (obj.attrib.bullet) { output ~= ".bullet: true"; } if (obj.attrib.language.length > 0) { output ~= ".lang: " ~ obj.attrib.language; } /+ ↓ has flags +/ { string[] has_flags; if (obj.has.inline_links) { has_flags ~= "links"; } if (obj.has.inline_notes_reg) { has_flags ~= "notes_reg"; } if (obj.has.inline_notes_star) { has_flags ~= "notes_star"; } if (obj.has.images) { has_flags ~= "images"; } if (obj.has.image_without_dimensions) { has_flags ~= "images_no_dim"; } if (has_flags.length > 0) { output ~= ".has: " ~ has_flags.join(" "); } } /+ ↓ table properties +/ if (obj.metainfo.is_a == "table" && obj.table.number_of_columns > 0) { output ~= ".table_cols: " ~ obj.table.number_of_columns.to!string; if (obj.table.column_widths.length > 0) { string[] ws; foreach (w; obj.table.column_widths) ws ~= w.to!string; output ~= ".table_widths: " ~ ws.join(" "); } if (obj.table.column_aligns.length > 0) { output ~= ".table_aligns: " ~ obj.table.column_aligns.join(" "); } if (obj.table.heading) { output ~= ".table_header: true"; } } /+ ↓ code block properties +/ if (obj.metainfo.is_a == "code") { if (obj.code_block.linenumbers) { output ~= ".code_linenumbers: true"; } } /+ ↓ stow (extracted links) +/ if (obj.stow.link.length > 0) { foreach (lnk; obj.stow.link) { if (lnk.length > 0) { output ~= ".stow_link: " ~ lnk; } } } /+ ↓ tag properties +/ if (obj.tags.in_segment_html.length > 0) { output ~= ".segment: " ~ obj.tags.in_segment_html; } if (obj.tags.anchor_tag_html.length > 0 && obj.tags.anchor_tag_html != obj.tags.in_segment_html) { output ~= ".anchor: " ~ obj.tags.anchor_tag_html; } if (obj.tags.segname_prev.length > 0) { output ~= ".segment_prev: " ~ obj.tags.segname_prev; } if (obj.tags.segname_next.length > 0) { output ~= ".segment_next: " ~ obj.tags.segname_next; } if (obj.tags.heading_lev_anchor_tag.length > 0) { output ~= ".heading_lev_anchor: " ~ obj.tags.heading_lev_anchor_tag; } if (obj.tags.segment_anchor_tag_epub.length > 0) { output ~= ".segment_epub: " ~ obj.tags.segment_anchor_tag_epub; } /+ ↓ the segment an object belongs to, as opposed to the segment a heading opens. .segment (in_segment_html) is set on headings only, so without these two a reader cannot tell which segment a paragraph is in; the sqlite search writer reads the html one +/ if (obj.tags.html_segment_anchor_tag_is.length > 0) { output ~= ".segment_html_is: " ~ obj.tags.html_segment_anchor_tag_is; } if (obj.tags.epub_segment_anchor_tag_is.length > 0) { output ~= ".segment_epub_is: " ~ obj.tags.epub_segment_anchor_tag_is; } /+ ↓ heading ancestors text +/ { bool has_hat = false; foreach (h; obj.tags.heading_ancestors_text) { if (h.length > 0) { has_hat = true; break; } } if (has_hat) { string[] _hat; foreach (h; obj.tags.heading_ancestors_text) { _hat ~= sspEscapeValue(h.to!string); } output ~= ".heading_ancestors_text: " ~ _hat.join("|"); } } /+ ↓ lev4 subtoc +/ if (obj.tags.lev4_subtoc.length > 0) { foreach (st; obj.tags.lev4_subtoc) { if (st.length > 0) { output ~= ".lev4_subtoc: " ~ st; } } } /+ ↓ anchor tags +/ if (obj.tags.anchor_tags.length > 0) { foreach (at; obj.tags.anchor_tags) { if (at.length > 0) { output ~= ".anchor_tag: " ~ at; } } } /+ ↓ text content +/ if (obj.text.length > 0) { foreach (line; obj.text.split("\n")) { output ~= "| " ~ line; } } output ~= ""; return output; } } template spineAbstractionTxt() { import std.conv : to; import std.digest : toHexString; import std.file; import std.path; import std.stdio; import std.string; import std.array; import sisudoc.ocda.io_in.paths_source; mixin sspObjectRecord; /+ ↓ the whole .ssp for a document, as lines. the file writer below emits these, and the abstraction db is built by reading them back, so the database cannot carry anything the .ssp does not +/ string[] sspDocumentLines(D)(D doc) { auto doc_abstraction = doc.abstraction; auto doc_matters = doc.matters; string[] output; /+ ↓ header comment +/ output ~= "% SiSU Document Abstraction v0.1"; output ~= "% Source: " ~ doc_matters.src.filename; output ~= ""; /+ ↓ @meta block +/ output ~= "@meta {"; auto meta = doc_matters.conf_make_meta.meta; if (meta.title_main.length > 0) { output ~= " title.main: " ~ meta.title_main; } if (meta.title_subtitle.length > 0) { output ~= " title.subtitle: " ~ meta.title_subtitle; } if (meta.title_full.length > 0) { output ~= " title.full: " ~ meta.title_full; } if (meta.title_language.length > 0) { output ~= " title.language: " ~ meta.title_language; } if (meta.creator_author.length > 0) { output ~= " creator.author: " ~ meta.creator_author; } if (meta.creator_author_surname.length > 0) { output ~= " creator.author_surname: " ~ meta.creator_author_surname; } if (meta.creator_author_surname_fn.length > 0) { output ~= " creator.author_surname_fn: " ~ meta.creator_author_surname_fn; } if (meta.creator_author_email.length > 0) { output ~= " creator.author_email: " ~ meta.creator_author_email; } if (meta.creator_illustrator.length > 0) { output ~= " creator.illustrator: " ~ meta.creator_illustrator; } if (meta.creator_translator.length > 0) { output ~= " creator.translator: " ~ meta.creator_translator; } if (meta.date_published.length > 0) { output ~= " date.published: " ~ meta.date_published; } if (meta.date_created.length > 0) { output ~= " date.created: " ~ meta.date_created; } if (meta.date_issued.length > 0) { output ~= " date.issued: " ~ meta.date_issued; } if (meta.date_available.length > 0) { output ~= " date.available: " ~ meta.date_available; } if (meta.date_modified.length > 0) { output ~= " date.modified: " ~ meta.date_modified; } if (meta.date_valid.length > 0) { output ~= " date.valid: " ~ meta.date_valid; } if (meta.rights_copyright.length > 0) { output ~= " rights.copyright: " ~ meta.rights_copyright; } if (meta.rights_license.length > 0) { output ~= " rights.license: " ~ meta.rights_license; } if (meta.classify_topic_register.length > 0) { output ~= " classify.topic_register: " ~ meta.classify_topic_register; } if (meta.classify_subject.length > 0) { output ~= " classify.subject: " ~ meta.classify_subject; } if (meta.classify_keywords.length > 0) { output ~= " classify.keywords: " ~ meta.classify_keywords; } if (meta.classify_loc.length > 0) { output ~= " classify.loc: " ~ meta.classify_loc; } if (meta.classify_dewey.length > 0) { output ~= " classify.dewey: " ~ meta.classify_dewey; } if (meta.identifier_isbn.length > 0) { output ~= " identifier.isbn: " ~ meta.identifier_isbn; } if (meta.identifier_oclc.length > 0) { output ~= " identifier.oclc: " ~ meta.identifier_oclc; } if (meta.language_document.length > 0) { output ~= " language.document: " ~ meta.language_document; } if (meta.notes_abstract.length > 0) { output ~= " notes.abstract: " ~ meta.notes_abstract; } if (meta.notes_description.length > 0) { output ~= " notes.description: " ~ meta.notes_description; } if (meta.notes_summary.length > 0) { output ~= " notes.summary: " ~ meta.notes_summary; } // check output ~= "}"; output ~= ""; /+ ↓ @make block +/ output ~= "@make {"; auto make = doc_matters.conf_make_meta.make; if (make.doc_type.length > 0) { output ~= " doc_type: " ~ make.doc_type; } if (make.auto_num_top_at_level.length > 0) { output ~= " auto_num_top_at_level: " ~ make.auto_num_top_at_level; } // check output ~= " auto_num_top_lv: " ~ make.auto_num_top_lv.to!string; output ~= " auto_num_depth: " ~ make.auto_num_depth.to!string; output ~= "}"; output ~= ""; /+ ↓ @doc_has block +/ output ~= "@doc_has {"; output ~= " inline_links: " ~ doc_matters.has.inline_links.to!string; output ~= " inline_notes_reg: " ~ doc_matters.has.inline_notes_reg.to!string; output ~= " inline_notes_star: " ~ doc_matters.has.inline_notes_star.to!string; output ~= " tables: " ~ doc_matters.has.tables.to!string; output ~= " codeblocks: " ~ doc_matters.has.codeblocks.to!string; output ~= " images: " ~ doc_matters.has.images.to!string; output ~= " poems: " ~ doc_matters.has.poems.to!string; output ~= " groups: " ~ doc_matters.has.groups.to!string; output ~= " blocks: " ~ doc_matters.has.blocks.to!string; output ~= " quotes: " ~ doc_matters.has.quotes.to!string; output ~= "}"; output ~= ""; /+ ↓ document sections +/ string[] section_order = ["head", "toc", "body", "endnotes", "glossary", "bibliography", "bookindex", "blurb"]; foreach (section; section_order) { if (section !in doc_abstraction) { continue; } auto section_objs = doc_abstraction[section]; if (section_objs.length == 0) { continue; } output ~= "@" ~ section ~ " {"; output ~= ""; foreach (obj; section_objs) { output ~= sspObjectRecord(obj, section, doc_matters.opt.action.vox_gt_3); } output ~= "}"; output ~= ""; } return output; } @trusted void spineAbstractionTxt(D)(D doc) { auto doc_matters = doc.matters; string[] output = sspDocumentLines(doc); /+ ↓ write to file, into the pod's own tree: pod//media/abstraction/.ssp . where it sits beside pod//media/image/, which is what a .ssp needs to be a document source: the file describes its images by name, size, pixels and digest, and they have to be findable. Written here once; --pod2 then digests and zips it where it stands rather than copying it in from an output directory of its own. +/ auto pths_pod = spinePathsPods!()(doc_matters); string base_pth = pths_pod.abstraction_root(doc_matters.src.filename).filesystem_open_zpod.to!string; try { if (!exists(base_pth)) { base_pth.mkdirRecurse; } else if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[0] ) { /+ ↓ this directory is the one part of the pod written before the pod is built, so the pod builder leaves it alone. It therefore clears itself here, on the first language of a run, or a .ssp for a language the document no longer has would linger and be bundled +/ foreach (string _f; dirEntries(base_pth, "*.ssp", SpanMode.shallow)) { _f.remove; } } } catch (Exception ex) { } string out_file = ((base_pth.chainPath( doc_matters.src.doc_uid_out ~ ".ssp")).asNormalizedPath).array; if (doc_matters.opt.action.vox_gt_1) { writeln(" ", out_file); } auto f = File(out_file, "w"); foreach (line; output) { f.writeln(line); } } } #+END_SRC * org includes ** project version #+NAME: spine_version #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> #+END_SRC ** year #+NAME: year #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> #+END_SRC ** document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC * __END__