-*- mode: org -*- #+TITLE: sisudoc spine (doc_reform) markup source raw #+DESCRIPTION: documents - structuring, publishing in multiple formats & search #+FILETAGS: :spine:sourcefile:read: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args+ :eval never-export :exports code #+PROPERTY: header-args+ :noweb yes :padline no #+PROPERTY: header-args+ :results silent :cache no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - magic single double-quote → " ← FIX changes hilighting behavior (occuring after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - [[./doc-reform.org][doc-reform.org]] [[./][org/]] * A. get _config file_, read in (.readText) [#A] ** _module template_ :module:config_files: #+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> /++ read configuration files
- read config files
meta_config_files.d +/ module sisudoc.ocda.io_in.read_config_files; @safe: import std.file; import std.path; import sisudoc.ocda.meta; import sisudoc.ocda.io_in.paths_source; import sisudoc.ocda.meta.rgx_files; import sisudoc.ocda.meta.rgx; <> <> <> #+END_SRC *** read config files (config local site & dr document make) (yaml) **** site configuration SEE NOTES on configuration hierarchy in spine.org #+NAME: meta_config_file_hub_read_site_config #+BEGIN_SRC d template readConfigSite() { @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) { mixin spineRgxIn; static auto rgx = RgxI(); string conf_filename = "NONE"; string config_file_str; string default_config_file_str = format(q"┃ flag: act0: "--html" act1: "--html --epub" output: path: "%s" default: language: "en" papersize: "a4" text_wrap: "80" digest: "sha256" webserv: http: "%s" host: "%s" data_http: "%s" data_host: "%s" data_root_url: "%s" data_root_path: "%s" data_root_part: "" images_root_part: "image" cgi_search_form_title: "%s" cgi_http: "%s" cgi_host: "%s" cgi_bin_url: "%s" cgi_bin_subpath: "%s" cgi_bin_path: "%s" cgi_search_script: "%s" cgi_port: "" cgi_user: "" cgi_action: "%s" db_sqlite_path: "%s" db_sqlite_filename: "%s" db_pg_table: "" db_pg_user: "" ┃", _cfg.processing_path_doc_root, // doc root _cfg.http_request_type, // http _cfg.http_host, // host / domain _cfg.http_request_type, // data "http" or "https" _cfg.http_host, // data domain "localhost" _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org" _cfg.processing_path_doc_root, // data root path _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search" _cfg.http_request_type, // cgi http _cfg.http_host, // cgi host _cfg.cgi_url_root, // cgi bin url _cfg.cgi_bin_subpath, // cgi bin path _cfg.cgi_bin_root, // cgi bin path _cfg.cgi_filename, // cgi filename _cfg.cgi_url_action, // cgi action _cfg.db_sqlite_path, // sqlite db path _cfg.db_sqlite_filename, // sqlite db filename ); foreach(conf_fn; [_conf_file_details.config_filename_site]) { foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) { char[] conf_file; conf_filename = conf_fn; if (exists(pth)) { auto f_attrib = pth.getLinkAttributes; if ( _conf_file_details.possible_config_path_locations.config_local_site.length == 1 && f_attrib.attrIsFile ) { conf_file = pth.to!(char[]); conf_filename = pth.baseName; } else if (f_attrib.attrIsDir) { conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; conf_filename = conf_fn; } try { if (exists(conf_file)) { if (conf_file.getLinkAttributes.attrIsFile) { if (_opt_action.vox_gt_2 || _opt_action.debug_do) { writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)"); } config_file_str = conf_file.readText; break; } } } catch (ErrnoException ex) { } catch (FileException ex) { } } } if (config_file_str.length > 0) { break; } } if (config_file_str.length > 0) { import dyaml; Node yaml_root; try { yaml_root = Loader.fromString(config_file_str).load(); } catch (Throwable) { import std.stdio; writeln("ERROR failed to read config file content, not parsed as yaml, program default used"); conf_filename = "VIRTUAL"; config_file_str = default_config_file_str; } } if (config_file_str.length == 0) { /+ use dummy default config file +/ // writeln("WARNING config file NOT found, default provided"); conf_filename = "VIRTUAL"; config_file_str = default_config_file_str; } struct _ConfContent { string filename() { return conf_filename; } string filetype() { string _ft = ""; if (content.match(rgx.yaml_config)) { _ft = "yaml"; } return _ft; } string content() { return config_file_str; } } return _ConfContent(); } } #+END_SRC **** document make/config #+NAME: meta_config_file_hub_read_document_config #+HEADER: :noweb yes #+BEGIN_SRC d static template readConfigDoc() { import std.file; import std.path; import sisudoc.ocda.meta; import sisudoc.ocda.io_in.paths_source; import sisudoc.ocda.meta.rgx_files; import sisudoc.ocda.meta.rgx; @system final auto readConfigDoc(M,E)(M _manifested, E _env) { mixin spineRgxIn; static auto rgx = RgxI(); mixin spineRgxFiles; static auto rgx_files = RgxFiles(); string config_file_str; string conf_filename = "NONE"; auto _conf_file_details = configFilePaths!()(_manifested, _env); string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; foreach(conf_fn; [_conf_file_details.config_filename_document]) { foreach(pth; possible_config_path_locations) { char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; conf_filename = conf_fn; if (config_file_str.length > 0) { break; } try { if (exists(conf_file)) { if (conf_file.getLinkAttributes.attrIsFile) { config_file_str = conf_file.readText; break; } } } catch (ErrnoException ex) { } catch (FileException ex) { } } if (config_file_str.length > 0) { break; } } struct _ConfContent { string filename() { return conf_filename; } string content() { return config_file_str; } string filetype() { string _ft = ""; if (content.match(rgx.yaml_config)) { _ft = "yaml"; } return _ft; } } return _ConfContent(); } } #+END_SRC *** YAML config (config local site & dr document make) :file:config:hub: #+NAME: meta_config_file_hub_read_site_yaml #+HEADER: :noweb yes #+BEGIN_SRC d static template configReadSiteYAML() { import std.file; import std.path; import sisudoc.ocda.meta; import sisudoc.ocda.io_in.paths_source; import sisudoc.ocda.meta.rgx_files; import sisudoc.ocda.meta.rgx; final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) { string _configuration = configReadInSiteYAML!()(_manifested, _env); auto _conf_file_details = configFilePaths!()(_manifested, _env); string _conf_yaml_fn = _conf_file_details.config_filename_site; YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); return _yaml_conf; } } static template configReadDocYAML() { import std.file; import std.path; import sisudoc.ocda.meta; import sisudoc.ocda.io_in.paths_source; final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) { string _configuration = configReadInDocYAML!()(_manifested, _env); auto _conf_file_details = configFilePaths!()(_manifested, _env); string _conf_yaml_fn = _conf_file_details.config_filename_document; YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn); return _yaml_conf; } } #+END_SRC * B. get _markup source_, read file (.readText) [#A]:module:source_files: ** _module template_ #+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> /++ module source_read_source_files;
- open markup files
- if master file scan for addional files to import/insert +/ module sisudoc.ocda.io_in.read_source_files; @safe: template spineRawMarkupContent() { import std.digest.sha; import std.file; import std.path; import sisudoc.ocda.meta; import sisudoc.ocda.io_in.paths_source; import sisudoc.ocda.meta.rgx_files; import sisudoc.ocda.meta.rgx; mixin spineRgxIn; static auto rgx = RgxI(); mixin spineRgxFiles; static auto rgx_files = RgxFiles(); string[] _images=[]; string[] _extract_images(S)(S content_block) { string[] images_; string _content_block = content_block.to!string; if (auto m = _content_block.matchAll(rgx.image)) { images_ ~= m.captures[1].to!string; } return images_; } auto rawsrc = RawMarkupContent(); struct ST_contents_inserts_images { char[][] contents; string[] insert_files; string[] images; } struct ST_header_content_inserts_images { char[] header; char[][] src_txt; string[] insert_files; string[] images; } struct ST_doc_digest { ubyte[32] markup_doc; ubyte[32] header; ubyte[32] text; } struct ST_doc_parts { char[] header_raw; char[][] sourcefile_body_content; string[] insert_file_list; string[] images_list; ST_doc_digest doc_digest; } ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) { ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src); return _0_header_1_body_content_2_insert_filelist_struct; } struct RawMarkupContent { final sourceContent(in string fn_src) { auto raw = MarkupRawUnit(); string source_txt_str = raw.markupSourceReadIn(fn_src); return source_txt_str; } final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)( O _opt_action, in string source_txt_str, in string fn_src="" ) { auto raw = MarkupRawUnit(); string[] insert_file_list_get; string[] images_list_get; ST_header_content_inserts_images st = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str); char[] header_raw = st.header; char[][] sourcefile_body_content = st.src_txt; if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise auto ins = Inserts(); ST_contents_inserts_images _cii = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); sourcefile_body_content = _cii.contents; insert_file_list_get = _cii.insert_files.dup; images_list_get = _cii.images.dup; } else if (_opt_action.source_or_pod) { auto ins = Inserts(); ST_contents_inserts_images _cii = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src); images_list_get = _cii.images.dup; } // image_list, if path could take sha256 digests already here? string header_type = ""; ST_doc_digest dig; { dig.markup_doc = source_txt_str.sha256Of; dig.header = st.header.sha256Of; dig.text = sourcefile_body_content.sha256Of; } ST_doc_parts ret; { ret.header_raw = st.header; ret.sourcefile_body_content = sourcefile_body_content; ret.insert_file_list = insert_file_list_get; ret.images_list = images_list_get; ret.doc_digest = dig; } return ret; } } struct MarkupRawUnit { import std.digest.sha; import std.file; <> <> <> <> <> <> } struct Inserts { struct ST_contents_and_images { char[][] insert_contents; string[] images; } ST_contents_and_images scan_subdoc_source(O)( O _opt_action, char[][] markup_sourcefile_insert_content, string fn_src ) { <> foreach (line; markup_sourcefile_insert_content) { <> } // end src subdoc (inserts) loop <> } ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)( O _opt_action, char[][] sourcefile_body_content, string fn_src ) { import std.algorithm; <> foreach (line; sourcefile_body_content) { <> } // end src doc loop <> } } } #+END_SRC ** get markup source, read file :source:markup: *** read file, source string [#A] :string: #+NAME: meta_markup_source_raw_read_file_source_string #+BEGIN_SRC d final private string readInMarkupSource(in char[] fn_src) { enforce( exists(fn_src) != 0, "file not found: «" ~ fn_src ~ "»" ); string source_txt_str; try { if (exists(fn_src)) { if (fn_src.getLinkAttributes.attrIsFile) { source_txt_str = fn_src.readText; } else { } } } catch (ErrnoException ex) { } catch (UTFException ex) { // Handle validation errors } catch (FileException ex) { // Handle errors } std.utf.validate(source_txt_str); return source_txt_str; } #+END_SRC *** document header & content, array.length == 2 [#A] :array: here you split document header and body, an array.length == 2 split is on first match of level A~ (which is required) #+NAME: meta_markup_source_raw_doc_header_and_content_split #+BEGIN_SRC d @trusted final private char[][] header0Content1(in string src_text) { // cast(char[]) /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/ char[][] header_and_content; auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a); header_and_content ~= m.pre; header_and_content ~= m.hit ~ m.post; assert(header_and_content.length == 2, "document markup is broken, header body split == " ~ header_and_content.length.to!string ~ "; (header / body array split should == 2 (split is on level A~))" ); return header_and_content; } #+END_SRC *** source line array :array: #+NAME: meta_markup_source_raw_source_line_array #+BEGIN_SRC d @trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[]) char[][] source_line_arr = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding); return source_line_arr; } #+END_SRC *** source content raw line array :array: - used for regular .sst files; master .ssm files and; .ssi inserts - regex is passed for relevant enforce match **** read in file #+NAME: meta_markup_source_raw_read_in_file #+BEGIN_SRC d string markupSourceReadIn(in string fn_src) { static auto rgx_files = RgxFiles(); enforce( fn_src.match(rgx_files.src_pth_sst_or_ssm), "not a dr markup filename: «" ~ fn_src ~ "»" ); string source_txt_str = readInMarkupSource(fn_src); return source_txt_str; } #+END_SRC **** tuple (a) header, (b) body content, (c) file insert list & (d) image list? - header - body content - file insert list - [image list?] #+NAME: meta_markup_source_raw_struct_of_header_and_body #+BEGIN_SRC d ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) { string[] file_insert_list = []; string[] images_list = []; char[][] hc = header0Content1(source_txt_str); char[] header = hc[0]; char[] source_txt = hc[1]; char[][] source_line_arr = markupSourceLineArray(source_txt); ST_header_content_inserts_images ret; { ret.header = header; ret.src_txt = source_line_arr; ret.insert_files = file_insert_list; ret.images = images_list; } return ret; } #+END_SRC **** get insert source line array #+NAME: meta_markup_source_raw_get_insert_source_line_array #+BEGIN_SRC d final char[][] getInsertMarkupSourceContentRawLineArray( in char[] fn_src_insert, Regex!(char) rgx_file ) { enforce( fn_src_insert.match(rgx_file), "not a dr markup filename: «" ~ fn_src_insert ~ "»" ); string source_txt_str = readInMarkupSource(fn_src_insert); char[][] source_line_arr = markupSourceLineArray(source_txt_str); return source_line_arr; } #+END_SRC ** get markup source, master file & inserts :masterfile:inserts: - [[./doc-reform.org][doc-reform.org]] [[./][org/]] *** scan inserts (sub-document) source :scan_insert_src: **** scan subdoc source #+NAME: meta_inserts_scan #+BEGIN_SRC d char[][] contents_insert; int code_block_status = 0; enum codeBlock { off, curly, tic, quotemarks } auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); auto markup_src_file_path = fn_pth_full.captures[1]; #+END_SRC **** loop insert (sub-document) #+NAME: meta_inserts_scan_loop #+BEGIN_SRC d if (code_block_status == codeBlock.curly) { if (line.matchFirst(rgx.block_curly_code_close)) { code_block_status = codeBlock.off; } contents_insert ~= line; } else if (line.matchFirst(rgx.block_curly_code_open)) { code_block_status = codeBlock.curly; contents_insert ~= line; } else if (code_block_status == codeBlock.quotemarks) { if (line.matchFirst(rgx.block_quotemarks_close)) { code_block_status = codeBlock.off; } contents_insert ~= line; } else if (code_block_status == codeBlock.tic) { if (line.matchFirst(rgx.block_tic_close)) { code_block_status = codeBlock.off; } contents_insert ~= line; } else if (line.matchFirst(rgx.block_quotemarks_code_open)) { code_block_status = codeBlock.quotemarks; contents_insert ~= line; } else if (line.matchFirst(rgx.block_tic_code_open)) { code_block_status = codeBlock.tic; contents_insert ~= line; } else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { auto insert_fn = m.captures[2]; auto insert_sub_pth = m.captures[1]; auto fn_src_insert = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; auto raw = MarkupRawUnit(); auto markup_sourcesubfile_insert_content = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); debug(insert_file) { writeln(line); writeln(fn_src_insert); writeln( " length contents insert array: ", markup_sourcesubfile_insert_content.length ); } if (_opt_action.source_or_pod) { _images ~= _extract_images(markup_sourcesubfile_insert_content); } auto ins = Inserts(); /+ - 1. load file - 2. read lines - 3. scan lines - a. if filename insert, and insert filename - repeat 1 - b. else - add line to new array; - build image list, search for any image files to add to image list +/ } else { contents_insert ~= line; // images to extract for image list? if (_opt_action.source_or_pod) { string[] _image_linelist = _extract_images(line); if (_image_linelist.length > 0) { _images ~= _image_linelist; } } } #+END_SRC **** post loop #+NAME: meta_inserts_scan_post #+BEGIN_SRC d ST_contents_and_images ret; { ret.insert_contents = contents_insert; ret.images = _images; } return ret; #+END_SRC *** scan document source :scan_src: **** scan doc source #+NAME: meta_master_doc_scan_for_insert_filenames #+BEGIN_SRC d char[][] contents; int code_block_status = 0; enum codeBlock { off, curly, tic, quotemarks } auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm); auto markup_src_file_path = fn_pth_full.captures[1]; char[][] contents_insert; string[] _images =[]; string[] insert_file_list =[]; #+END_SRC **** include inserts: _loop master_ scan for inserts (insert documents) #+NAME: meta_master_doc_scan_for_insert_filenames_loop #+BEGIN_SRC d if (code_block_status == codeBlock.curly) { if (line.matchFirst(rgx.block_curly_code_close)) { code_block_status = codeBlock.off; } contents ~= line; } else if (line.matchFirst(rgx.block_curly_code_open)) { code_block_status = codeBlock.curly; contents ~= line; } else if (code_block_status == codeBlock.quotemarks) { if (line.matchFirst(rgx.block_quotemarks_close)) { code_block_status = codeBlock.off; } contents ~= line; } else if (code_block_status == codeBlock.tic) { if (line.matchFirst(rgx.block_tic_close)) { code_block_status = codeBlock.off; } contents ~= line; } else if (line.matchFirst(rgx.block_quotemarks_code_open)) { code_block_status = codeBlock.quotemarks; contents ~= line; } else if (line.matchFirst(rgx.block_tic_code_open)) { code_block_status = codeBlock.tic; contents ~= line; } else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) { auto insert_fn = m.captures[2]; auto insert_sub_pth = m.captures[1]; auto fn_src_insert = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array; insert_file_list ~= fn_src_insert.to!string; auto raw = MarkupRawUnit(); /+ TODO +/ auto markup_sourcefile_insert_content = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts); debug(insert_file) { writeln(line); writeln(fn_src_insert); writeln( " length contents insert array: ", markup_sourcefile_insert_content.length ); } auto ins = Inserts(); ST_contents_and_images contents_insert_st = ins.scan_subdoc_source( _opt_action, markup_sourcefile_insert_content, fn_src_insert.to!string ); contents ~= contents_insert_st.insert_contents; if (_opt_action.source_or_pod) { string[] _image_linelist = _extract_images(contents_insert_st.images); if (_image_linelist.length > 0) { _images ~= _image_linelist; } } /+ - 1. load file - 2. read lines - 3. scan lines - a. if filename insert, and insert filename - repeat 1 - b. else - add line to new array; - build image list, search for any image files to add to image list +/ } else { contents ~= line; if (_opt_action.source_or_pod) { string[] _image_linelist = _extract_images(line); if (_image_linelist.length > 0) { _images ~= _image_linelist; } } } #+END_SRC **** post loop #+NAME: meta_master_doc_scan_for_insert_filenames_post #+BEGIN_SRC d string[] images = []; foreach(i; uniq(_images.sort())) { images ~= i; } debug(insert_file) { writeln(__LINE__); writeln(contents.length); } ST_contents_inserts_images ret; { ret.contents = contents; ret.insert_files = insert_file_list; ret.images = images; } return ret; #+END_SRC * C. read a .ssp file back into the document abstraction ** _module template_ :module:ssp_files: #+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp_in.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> module sisudoc.ocda.abstraction.ssp_in; @safe: /+ ↓ read a .ssp file back into the document abstraction . the reverse of sisudoc.ocda.abstraction.ssp. what it returns is the same value the parser produces, ObjGenericComposite[][string], so anything that consumes the abstraction can be fed from a .ssp instead of from markup. . the check that it is faithful is a round trip: load a .ssp, emit each object again with sspObjectRecord (the writer's own definition of a record) and require the result to be byte identical to the file read. +/ template spineAbstractionRead() { import std.algorithm : startsWith; import std.array; import std.conv : to; import std.file; import std.json : JSONValue; import std.stdio; import std.string; import sisudoc.ocda.meta.metadoc_object_setter; import sisudoc.ocda.abstraction.ssp : sspEscape; mixin sspEscape; mixin ObjectSetter; /+ ↓ what a .ssp file holds: the three header blocks as key/value, and the object sections in the order the file gives them +/ struct SSPdocument { string format; // "% SiSU Document Abstraction v0.1" string source; // "% Source: ..." string[string] meta; string[] meta_order; string[string] make; string[] make_order; string[string] doc_has; string[] doc_has_order; string[] section_order; // as found in the file ObjGenericComposite[][string] abstraction; } private int _levOfMarkedUp(string lev) { switch (lev) { case "A": return 0; case "B": return 1; case "C": return 2; case "D": return 3; case "1": return 4; case "2": return 5; case "3": return 6; case "4": return 7; default: return 9; } } private int[] _ints(string val) { int[] _out; foreach (f; val.split(" ")) { if (f.length > 0) { _out ~= f.to!int; } } return _out; } /+ ↓ the eight slot arrays are int[] with a literal default, so every default constructed object shares one array; write into a fresh one +/ private int[8] _eight(string val) { int[8] _out; foreach (i, v; _ints(val)) { if (i < _out.length) { _out[i] = v; } } return _out; } private ubyte[32] _hexToBytes(string hex) { ubyte[32] _out; if (hex.length < 64) { return _out; } foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } return _out; } SSPdocument sspRead(string[] lines) { SSPdocument doc; string section; // the @section block currently open string block; // "meta", "make", "doc_has", or "" for objects bool in_object; string[] text_lines; ObjGenericComposite obj; void _closeObject() { if (!in_object) { return; } obj.text = text_lines.join("\n"); doc.abstraction[section] ~= obj; obj = ObjGenericComposite(); text_lines = []; in_object = false; } foreach (line; lines) { if (line.length == 0) { continue; } if (line.startsWith("% SiSU Document Abstraction")) { doc.format = line; continue; } if (line.startsWith("% Source: ")) { doc.source = line["% Source: ".length .. $]; continue; } if (line.startsWith("@") && line.endsWith(" {")) { _closeObject(); string _name = line[1 .. $-2]; switch (_name) { case "meta": case "make": case "doc_has": block = _name; section = ""; break; default: block = ""; section = _name; doc.section_order ~= _name; doc.abstraction[_name] = []; break; } continue; } if (line == "}") { _closeObject(); block = ""; section = ""; continue; } if (block.length > 0) { // a header block key: value line auto _kv = line.stripLeft.findSplit(": "); // leading only: a value may end in a space if (_kv[1].length > 0) { final switch (block) { case "meta": doc.meta[_kv[0].to!string] = _kv[2].to!string; doc.meta_order ~= _kv[0].to!string; break; case "make": doc.make[_kv[0].to!string] = _kv[2].to!string; doc.make_order ~= _kv[0].to!string; break; case "doc_has": doc.doc_has[_kv[0].to!string] = _kv[2].to!string; doc.doc_has_order ~= _kv[0].to!string; break; } } continue; } if (line.startsWith("[")) { // object declaration line _closeObject(); in_object = true; auto _close = line.indexOf("] "); obj.metainfo.ocn = line[1 .. _close].to!int; string _rest = line[_close + 2 .. $]; if (_rest.startsWith("heading :")) { obj.metainfo.is_a = "heading"; string _tail = _rest["heading :".length .. $]; auto _sp = _tail.indexOf(" "); if (_sp > 0) { obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail[0 .. _sp]); obj.metainfo.identifier = _tail[_sp + 1 .. $]; } else { obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail); /+ ↓ omitted means it equals the ocn, and for an object with no ocn it was empty: "a"~N identifiers are always written +/ obj.metainfo.identifier = (obj.metainfo.ocn != 0) ? obj.metainfo.ocn.to!string : ""; } } else { obj.metainfo.is_a = _rest; obj.metainfo.identifier = (obj.metainfo.ocn != 0) ? obj.metainfo.ocn.to!string : ""; } obj.metainfo.is_of_section = section; continue; } if (line.startsWith("| ")) { text_lines ~= line[2 .. $]; continue; } if (!line.startsWith(".")) { continue; } auto _p = line[1 .. $].findSplit(": "); string _key = (_p[1].length > 0) ? _p[0].to!string : line[1 .. $].to!string; string _val = (_p[1].length > 0) ? _p[2].to!string : ""; switch (_key) { case "identifier": obj.metainfo.identifier = _val; break; case "part": obj.metainfo.is_of_part = _val; break; case "section": obj.metainfo.is_of_section = _val; break; case "parent": obj.metainfo.parent_ocn = _val.to!int; break; case "last_descendant": obj.metainfo.last_descendant_ocn = _val.to!int; break; case "children": obj.metainfo.children_headings = _ints(_val); break; case "ancestors": obj.metainfo.markedup_ancestors = _eight(_val); break; case "ancestors_collapsed": obj.metainfo.collapsed_ancestors = _eight(_val); break; case "dom_status": obj.metainfo.dom_structure_markedup_tags_status = _eight(_val); break; case "dom_status_collapsed": obj.metainfo.dom_structure_collapsed_tags_status = _eight(_val); break; case "heading_lev_collapsed": obj.metainfo.heading_lev_collapsed = _val.to!int; break; case "parent_lev": obj.metainfo.parent_lev_markup = _val.to!int; break; case "dummy": obj.metainfo.dummy_heading = true; break; case "ocn_off": obj.metainfo.object_number_off = true; break; case "is_of_type": obj.metainfo.is_of_type = _val; break; case "attrib": obj.metainfo.attrib = _val; break; case "meta_lang": obj.metainfo.lang = _val; break; case "meta_syntax": obj.metainfo.syntax = _val; break; case "sha256": obj.metainfo.sha256.text = _hexToBytes(_val); break; case "image": { ST_file_name_hash_size_ _img; auto _fields = _val.split(" "); if (_fields.length > 0) { _img.fileName = _fields[0].to!string; } foreach (f; _fields[1 .. $]) { if (f == "missing:true") { _img.fileMissing = true; } else if (f.startsWith("sha256:")) { _img.fileHash_sha256 = _hexToBytes(f[7 .. $].to!string); } else if (f.startsWith("bytes:")) { _img.fileSize = f[6 .. $].to!ulong; } else if (f.startsWith("px:")) { auto _wh = f[3 .. $].findSplit("x"); if (_wh[1].length > 0) { _img.imageWidth = _wh[0].to!int; _img.imageHeight = _wh[2].to!int; } } } obj.metainfo.sha256.images ~= _img; } break; case "indent": { auto _i = _ints(_val); if (_i.length > 0) { obj.attrib.indent_base = _i[0]; } if (_i.length > 1) { obj.attrib.indent_hang = _i[1]; } } break; case "bullet": obj.attrib.bullet = true; break; case "lang": obj.attrib.language = _val; break; case "has": foreach (f; _val.split(" ")) { switch (f) { case "links": obj.has.inline_links = true; break; case "notes_reg": obj.has.inline_notes_reg = true; break; case "notes_star": obj.has.inline_notes_star = true; break; case "images": obj.has.images = true; break; case "images_no_dim": obj.has.image_without_dimensions = true; break; default: break; } } break; case "table_cols": obj.table.number_of_columns = _val.to!int; break; case "table_widths": foreach (w; _val.split(" ")) { obj.table.column_widths ~= w.to!double; } break; case "table_aligns": foreach (a; _val.split(" ")) { obj.table.column_aligns ~= a.to!string; } break; case "table_header": obj.table.heading = true; break; case "code_linenumbers": obj.code_block.linenumbers = true; break; case "stow_link": obj.stow.link ~= _val; break; case "segment": obj.tags.in_segment_html = _val; break; case "anchor": obj.tags.anchor_tag_html = _val; break; case "segment_prev": obj.tags.segname_prev = _val; break; case "segment_next": obj.tags.segname_next = _val; break; case "heading_lev_anchor": obj.tags.heading_lev_anchor_tag = _val; break; case "segment_epub": obj.tags.segment_anchor_tag_epub = _val; break; case "segment_html_is": obj.tags.html_segment_anchor_tag_is = _val; break; case "segment_epub_is": obj.tags.epub_segment_anchor_tag_is = _val; break; case "heading_ancestors_text": { string[8] _h; foreach (i, h; sspUnescapeSplit(_val)) { if (i < _h.length) { _h[i] = h; } } obj.tags.heading_ancestors_text = _h; } break; case "lev4_subtoc": obj.tags.lev4_subtoc ~= _val; break; case "anchor_tag": obj.tags.anchor_tags ~= _val; break; default: break; } } _closeObject(); /+ ↓ .anchor is written only when it differs from .segment, and .section only when it differs from the enclosing block; restore both +/ foreach (sect; doc.section_order) { foreach (ref o; doc.abstraction[sect]) { if (o.tags.anchor_tag_html.length == 0 && o.tags.in_segment_html.length > 0) { o.tags.anchor_tag_html = o.tags.in_segment_html; } } } return doc; } /+ ↓ the abstraction as it survives a trip through the .ssp text: emitted as lines by the writer, read straight back. what a consumer built from this cannot see, the .ssp does not carry, which is what makes a second artefact built from it unable to drift +/ auto sspRoundTripAbstraction(D)(D doc) { import sisudoc.ocda.abstraction.ssp; mixin spineAbstractionTxt; return sspRead(sspDocumentLines(doc)).abstraction; } SSPdocument sspReadFile(string file_path) { string[] _lines; try { _lines = (cast(char[]) file_path.read).to!string.split("\n"); } catch (Exception ex) { writeln("ERROR: could not read ", file_path, ": ", ex.msg); return SSPdocument(); } return sspRead(_lines); } } #+END_SRC * D. read a .db file into the document abstraction ** _module template_ :module:db_files: #+HEADER: :tangle "../src/sisudoc/ocda/abstraction/db_in.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> module sisudoc.ocda.abstraction.db_in; @safe: /+ ↓ read a .ocda.db back into the document abstraction the counterpart of sisudoc.outputs.io_out.create_abstraction_db, and the sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, so a consumer does not have to know which of the two artefacts it was handed. the check is the same one: read a database, emit it as .ssp with the writer's own record definition, and require the result to equal the .ssp written from the same document. +/ template spineAbstractionDbRead() { import std.conv : to; import std.file; import std.stdio; import std.string; import std.array; import std.json; import d2sqlite3; import sisudoc.ocda.abstraction.ssp_in : spineAbstractionRead; mixin spineAbstractionRead; // for SSPdocument, and the shared unescape private int[8] _jsonEight(string _s) { int[8] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (i, v; _j.array) { if (i < _out.length) { _out[i] = v.integer.to!int; } } } catch (Exception ex) {} return _out; } private int[] _jsonInts(string _s) { int[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= v.integer.to!int; } } catch (Exception ex) {} return _out; } private string[] _jsonStrs(string _s) { string[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= v.str; } } catch (Exception ex) {} return _out; } private double[] _jsonDoubles(string _s) { double[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= (v.type == JSONType.integer) ? v.integer.to!double : v.floating; } } catch (Exception ex) {} return _out; } private ubyte[32] _hex32(string hex) { ubyte[32] _out; if (hex.length < 64) { return _out; } foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } return _out; } private string _levMarkedUp(int lev) { switch (lev) { case 0: return "A"; case 1: return "B"; case 2: return "C"; case 3: return "D"; case 4: return "1"; case 5: return "2"; case 6: return "3"; case 7: return "4"; default: return ""; } } @trusted SSPdocument dbReadFile(string db_file) { SSPdocument doc; if (!db_file.exists) { writeln("ERROR: no such file: ", db_file); return doc; } auto db = Database(db_file, SQLITE_OPEN_READONLY); /+ ↓ the three header blocks, from the metadata table. the key prefix says which block a row belongs to, and insertion order is kept +/ foreach (row; db.execute("SELECT key, value FROM metadata ORDER BY rowid")) { string _k = row["key"].as!string; string _v = row["value"].as!string; if (_k.startsWith("make.")) { string _kk = _k["make.".length .. $]; doc.make[_kk] = _v; doc.make_order ~= _kk; } else if (_k.startsWith("doc_has.")) { string _kk = _k["doc_has.".length .. $]; doc.doc_has[_kk] = _v; doc.doc_has_order ~= _kk; } else if (_k.startsWith("schema.")) { if (_k == "schema.version") { doc.format = "% SiSU Document Abstraction v0.1"; } } else if (_k == "source.filename") { doc.source = _v; } else if (_k == "source.language") { // carried in the path, not in the .ssp header } else { doc.meta[_k] = _v; doc.meta_order ~= _k; } } /+ ↓ the objects, section by section, in the order they were written +/ string[] _sections; foreach (row; db.execute( "SELECT section FROM objects GROUP BY section ORDER BY MIN(id)") ) { _sections ~= row["section"].as!string; } foreach (section; _sections) { doc.section_order ~= section; doc.abstraction[section] = []; foreach (row; db.execute( "SELECT * FROM objects WHERE section = '" ~ section ~ "' ORDER BY seq") ) { ObjGenericComposite obj; long _id = row["id"].as!long; obj.metainfo.ocn = row["ocn"].as!int; obj.metainfo.is_a = row["is_a"].as!string; obj.metainfo.is_of_part = row["is_of_part"].as!string; obj.metainfo.is_of_section = (row["is_of_section"].as!string.length > 0) ? row["is_of_section"].as!string : section; obj.metainfo.is_of_type = row["is_of_type"].as!string; obj.metainfo.identifier = row["identifier"].as!string; obj.metainfo.heading_lev_markup = (obj.metainfo.is_a == "heading") ? row["heading_level"].as!int : 9; obj.metainfo.heading_lev_collapsed = (row["heading_lev_collapsed"].as!string.length > 0) ? row["heading_lev_collapsed"].as!int : 9; obj.metainfo.parent_ocn = row["parent_ocn"].as!int; obj.metainfo.parent_lev_markup = row["parent_lev"].as!int; obj.metainfo.last_descendant_ocn = row["last_descendant_ocn"].as!int; obj.metainfo.children_headings = _jsonInts(row["children"].as!string); obj.metainfo.markedup_ancestors = _jsonEight(row["ancestors"].as!string); obj.metainfo.collapsed_ancestors = _jsonEight(row["ancestors_collapsed"].as!string); obj.metainfo.dom_structure_markedup_tags_status = _jsonEight(row["dom_status"].as!string); obj.metainfo.dom_structure_collapsed_tags_status = _jsonEight(row["dom_status_collapsed"].as!string); obj.metainfo.dummy_heading = (row["dummy_heading"].as!int == 1); obj.metainfo.object_number_off = (row["object_number_off"].as!int == 1); obj.metainfo.attrib = row["attrib"].as!string; obj.metainfo.lang = row["meta_lang"].as!string; obj.metainfo.syntax = row["meta_syntax"].as!string; obj.metainfo.sha256.text = _hex32(row["sha256"].as!string); obj.attrib.indent_base = row["indent_base"].as!int; obj.attrib.indent_hang = row["indent_hang"].as!int; obj.attrib.bullet = (row["bullet"].as!int == 1); obj.attrib.language = row["lang"].as!string; obj.has.inline_links = (row["has_links"].as!int == 1); obj.has.inline_notes_reg = (row["has_notes_reg"].as!int == 1); obj.has.inline_notes_star = (row["has_notes_star"].as!int == 1); obj.has.images = (row["has_images"].as!int == 1); obj.has.image_without_dimensions = (row["has_images_no_dim"].as!int == 1); obj.tags.in_segment_html = row["segment"].as!string; obj.tags.segname_prev = row["segment_prev"].as!string; obj.tags.segname_next = row["segment_next"].as!string; obj.tags.segment_anchor_tag_epub = row["segment_epub"].as!string; obj.tags.html_segment_anchor_tag_is = row["segment_html_is"].as!string; obj.tags.epub_segment_anchor_tag_is = row["segment_epub_is"].as!string; obj.tags.anchor_tag_html = row["anchor"].as!string; obj.tags.heading_lev_anchor_tag = row["heading_lev_anchor"].as!string; { string[8] _h; foreach (i, h; _jsonStrs(row["heading_ancestors_text"].as!string)) { if (i < _h.length) { _h[i] = h; } } obj.tags.heading_ancestors_text = _h; } if (obj.metainfo.is_a == "table") { obj.table.number_of_columns = row["table_cols"].as!int; obj.table.column_widths = _jsonDoubles(row["table_widths"].as!string); obj.table.column_aligns = _jsonStrs(row["table_aligns"].as!string); obj.table.heading = (row["table_header"].as!int == 1); } obj.code_block.linenumbers = (row["code_linenumbers"].as!int == 1); obj.text = row["text"].as!string; /+ ↓ the per object lists +/ foreach (r; db.execute( "SELECT name, bytes, sha256, width, height, missing FROM object_images" ~ " WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq") ) { ST_file_name_hash_size_ _img; _img.fileName = r["name"].as!string; _img.fileSize = r["bytes"].as!ulong; _img.fileHash_sha256 = _hex32(r["sha256"].as!string); _img.imageWidth = r["width"].as!int; _img.imageHeight = r["height"].as!int; _img.fileMissing = (r["missing"].as!int == 1); obj.metainfo.sha256.images ~= _img; } foreach (r; db.execute( "SELECT url FROM object_links WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq") ) { obj.stow.link ~= r["url"].as!string; } foreach (r; db.execute( "SELECT anchor FROM object_anchors WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq") ) { obj.tags.anchor_tags ~= r["anchor"].as!string; } foreach (r; db.execute( "SELECT entry FROM object_subtoc WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq") ) { obj.tags.lev4_subtoc ~= r["entry"].as!string; } doc.abstraction[section] ~= obj; } } return doc; } } #+END_SRC * E. read a .db file into the document abstraction ** _module template_ :module:db_files: #+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d" #+HEADER: :noweb yes #+BEGIN_SRC d <> module sisudoc.ocda.abstraction.load; @safe: /+ ↓ one way in, whatever the document is being read from spine's pipeline is markup -> abstraction -> output. once the abstraction is serialised, there is more than one thing an abstraction can be read from, and a consumer should not have to know which it was handed: .sst / .ssm + images the markup source pod (dir) + images the same, bundled pod .zip the same, zipped .ssp + images the abstraction, as text .ocda.db the abstraction, sqlite, images inside This module names those sources, tells them apart, and loads the two that are self-describing artefacts, returning the value the parser produces. The three source forms are deliberately *not* loaded here. They are the parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs the environment, the options and the configuration that spine.d assembles, none of which belongs in a loader. What this module gives that case is the dispatch and a plain statement of where to go. +/ template spineAbstractionLoad() { import std.algorithm : endsWith; import std.conv : to; import std.file; import std.path; import std.stdio; import std.string; import sisudoc.ocda.abstraction.db_in; mixin spineAbstractionDbRead; // brings the .ssp reader with it enum AbstractionSource { unknown, markup, // .sst or .ssm, with its images beside it pod, // a directory holding pod.manifest pod_zip, // that directory, zipped ssp, // .ssp, the abstraction as text, images beside it ocda_db, // .ocda.db, the abstraction as sqlite, images inside it } string abstractionSourceName(AbstractionSource _s) { final switch (_s) { case AbstractionSource.unknown: return "unknown"; case AbstractionSource.markup: return "markup source (.sst/.ssm)"; case AbstractionSource.pod: return "pod directory"; case AbstractionSource.pod_zip: return "pod zip"; case AbstractionSource.ssp: return ".ssp (abstraction as text)"; case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)"; } } /+ ↓ what is this? by name, and for a directory by what it holds +/ AbstractionSource abstractionSourceOf(string _path) { if (_path.length == 0) { return AbstractionSource.unknown; } if (_path.isValidPath && _path.exists && _path.isDir) { return (_path.chainPath("pod.manifest").array.exists) ? AbstractionSource.pod : AbstractionSource.unknown; } if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } return AbstractionSource.unknown; } struct LoadedAbstraction { AbstractionSource source; string path; bool loaded; // is .doc filled string note; // why not, when it is not SSPdocument doc; } /+ ↓ load what can be loaded from the path alone. .ssp and .ocda.db come back filled. the three source forms come back with loaded = false and a note saying where they are handled, because reading them needs the manifest, environment and configuration that spine.d builds, not a file path. +/ LoadedAbstraction abstractionLoad(string _path) { LoadedAbstraction _out; _out.path = _path; _out.source = abstractionSourceOf(_path); if (!_path.exists) { _out.note = "no such file or directory"; return _out; } final switch (_out.source) { case AbstractionSource.ssp: _out.doc = sspReadFile(_path); _out.loaded = (_out.doc.section_order.length > 0); if (!_out.loaded) { _out.note = "no object sections found"; } break; case AbstractionSource.ocda_db: _out.doc = dbReadFile(_path); _out.loaded = (_out.doc.section_order.length > 0); if (!_out.loaded) { _out.note = "no object sections found"; } break; case AbstractionSource.markup: case AbstractionSource.pod: case AbstractionSource.pod_zip: _out.note = "a source form: read by the parser," ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the" ~ " manifest, environment and configuration spine assembles"; break; case AbstractionSource.unknown: _out.note = "not a document source spine knows"; break; } return _out; } /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a given artefact really does hold what it should +/ string[] abstractionLoadSummary(LoadedAbstraction _l) { string[] _out; _out ~= "source: " ~ abstractionSourceName(_l.source); _out ~= "path: " ~ _l.path; if (!_l.loaded) { _out ~= "loaded: no"; if (_l.note.length > 0) { _out ~= "note: " ~ _l.note; } return _out; } _out ~= "loaded: yes"; if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; } if ("title.main" in _l.doc.meta) { _out ~= "title: " ~ _l.doc.meta["title.main"]; } if ("creator.author" in _l.doc.meta) { _out ~= "author: " ~ _l.doc.meta["creator.author"]; } _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, " ~ _l.doc.make.length.to!string ~ " make, " ~ _l.doc.doc_has.length.to!string ~ " doc_has"; int _objs, _citable, _headings; string[] _sections; foreach (section; _l.doc.section_order) { int _n; foreach (obj; _l.doc.abstraction[section]) { ++_objs; ++_n; if (obj.metainfo.ocn > 0) { ++_citable; } if (obj.metainfo.is_a == "heading") { ++_headings; } } _sections ~= section ~ " " ~ _n.to!string; } _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string ~ " citable, " ~ _headings.to!string ~ " headings)"; _out ~= "sections: " ~ _sections.join(", "); return _out; } } #+END_SRC * org includes ** project version #+NAME: spine_version #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> #+END_SRC ** year #+NAME: year #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> #+END_SRC ** document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC * __END__