path: root/org/in_source_files.org

-*- mode: org -*-
#+TITLE:       sisudoc spine (doc_reform) markup source raw
#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
#+FILETAGS:    :spine:sourcefile:read:
#+AUTHOR:      Ralph Amissah
#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
#+COPYRIGHT:   Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
#+LANGUAGE:    en
#+STARTUP:     content hideblocks hidestars noindent entitiespretty
#+PROPERTY:    header-args+ :eval never-export :exports code
#+PROPERTY:    header-args+ :noweb yes :padline no
#+PROPERTY:    header-args+ :results silent :cache no
#+PROPERTY:    header-args+ :mkdirp yes
#+OPTIONS:     H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
- magic single double-quote → " ← FIX changes hilighting behavior (occuring
  after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX

- [[./doc-reform.org][doc-reform.org]]  [[./][org/]]

* A. get _config file_, read in (.readText) [#A]
** _module template_ :module:config_files:

#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_config_files.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/++
  read configuration files<BR>
  - read config files<BR>
  meta_config_files.d
+/
module sisudoc.ocda.io_in.read_config_files;
@safe:
import std.file;
import std.path;
import sisudoc.ocda.meta;
import sisudoc.ocda.io_in.paths_source;
import sisudoc.ocda.meta.rgx_files;
import sisudoc.ocda.meta.rgx;
<<meta_config_file_hub_read_site_config>>
<<meta_config_file_hub_read_document_config>>
<<meta_config_file_hub_read_site_yaml>>
#+END_SRC

*** read config files (config local site & dr document make) (yaml)
**** site configuration

SEE NOTES on configuration hierarchy in spine.org

#+NAME: meta_config_file_hub_read_site_config
#+BEGIN_SRC d
template readConfigSite() {
  @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) {
    mixin spineRgxIn;
    static auto rgx = RgxI();
    string conf_filename = "NONE";
    string config_file_str;
    string default_config_file_str = format(q"┃
flag:
  act0:                  "--html"
  act1:                  "--html --epub"
output:
  path:                  "%s"
default:
  language:              "en"
  papersize:             "a4"
  text_wrap:             "80"
  digest:                "sha256"
webserv:
  http:                  "%s"
  host:                  "%s"
  data_http:             "%s"
  data_host:             "%s"
  data_root_url:         "%s"
  data_root_path:        "%s"
  data_root_part:        ""
  images_root_part:      "image"
  cgi_search_form_title: "%s"
  cgi_http:              "%s"
  cgi_host:              "%s"
  cgi_bin_url:           "%s"
  cgi_bin_subpath:       "%s"
  cgi_bin_path:          "%s"
  cgi_search_script:     "%s"
  cgi_port:              ""
  cgi_user:              ""
  cgi_action:            "%s"
  db_sqlite_path:        "%s"
  db_sqlite_filename:    "%s"
  db_pg_table:           ""
  db_pg_user:            ""
┃",
  _cfg.processing_path_doc_root, // doc root
  _cfg.http_request_type,        // http
  _cfg.http_host,                // host / domain
  _cfg.http_request_type,        // data "http" or "https"
  _cfg.http_host,                // data domain "localhost"
  _cfg.www_url_doc_root,         // data root url "http://locahost" "https://sisudoc.org"
  _cfg.processing_path_doc_root, // data root path
  _cfg.cgi_search_form_title,    // cgi title // e.g. "≅ SiSU Spine search"
  _cfg.http_request_type,        // cgi http
  _cfg.http_host,                // cgi host
  _cfg.cgi_url_root,             // cgi bin url
  _cfg.cgi_bin_subpath,          // cgi bin path
  _cfg.cgi_bin_root,             // cgi bin path
  _cfg.cgi_filename,             // cgi filename
  _cfg.cgi_url_action,           // cgi action
  _cfg.db_sqlite_path,           // sqlite db path
  _cfg.db_sqlite_filename,       // sqlite db filename
);
    foreach(conf_fn; [_conf_file_details.config_filename_site]) {
      foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) {
        char[] conf_file;
        conf_filename = conf_fn;
        if (exists(pth)) {
          auto f_attrib = pth.getLinkAttributes;
          if (
            _conf_file_details.possible_config_path_locations.config_local_site.length == 1
            && f_attrib.attrIsFile
          ) {
            conf_file = pth.to!(char[]);
            conf_filename = pth.baseName;
          } else if (f_attrib.attrIsDir) {
            conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
            conf_filename = conf_fn;
          }
          try {
            if (exists(conf_file)) {
              if (conf_file.getLinkAttributes.attrIsFile) {
                if (_opt_action.vox_gt_2 || _opt_action.debug_do) {
                  writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)");
                }
                config_file_str = conf_file.readText;
                break;
              }
            }
          } catch (ErrnoException ex) {
          } catch (FileException ex) {
          }
        }
      }
      if (config_file_str.length > 0) { break; }
    }
    if (config_file_str.length > 0) {
      import dyaml;
      Node yaml_root;
      try {
        yaml_root = Loader.fromString(config_file_str).load();
      } catch (Throwable) {
        import std.stdio;
        writeln("ERROR failed to read config file content, not parsed as yaml, program default used");
        conf_filename = "VIRTUAL";
        config_file_str = default_config_file_str;
      }
    }
    if (config_file_str.length == 0) { /+ use dummy default config file +/
      // writeln("WARNING config file NOT found, default provided");
      conf_filename = "VIRTUAL";
      config_file_str = default_config_file_str;
    }
    struct _ConfContent {
      string filename() {
        return conf_filename;
      }
      string filetype() {
        string _ft = "";
        if (content.match(rgx.yaml_config)) {
          _ft = "yaml";
        }
        return _ft;
      }
      string content() {
        return config_file_str;
      }
    }
    return _ConfContent();
  }
}
#+END_SRC

**** document make/config

#+NAME: meta_config_file_hub_read_document_config
#+HEADER: :noweb yes
#+BEGIN_SRC d
static template readConfigDoc() {
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  import sisudoc.ocda.meta.rgx_files;
  import sisudoc.ocda.meta.rgx;
  @system final auto readConfigDoc(M,E)(M _manifested, E _env) {
    mixin spineRgxIn;
    static auto rgx = RgxI();
    mixin spineRgxFiles;
    static auto rgx_files = RgxFiles();
    string config_file_str;
    string conf_filename = "NONE";
    auto _conf_file_details = configFilePaths!()(_manifested, _env);
    string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make;
    foreach(conf_fn; [_conf_file_details.config_filename_document]) {
      foreach(pth; possible_config_path_locations) {
        char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
        conf_filename = conf_fn;
        if (config_file_str.length > 0) {
          break;
        }
        try {
          if (exists(conf_file)) {
            if (conf_file.getLinkAttributes.attrIsFile) {
              config_file_str = conf_file.readText;
              break;
            }
          }
        } catch (ErrnoException ex) {
        } catch (FileException ex) {
        }
      }
      if (config_file_str.length > 0) { break; }
    }
    struct _ConfContent {
      string filename() {
        return conf_filename;
      }
      string content() {
        return config_file_str;
      }
      string filetype() {
        string _ft = "";
        if (content.match(rgx.yaml_config)) {
          _ft = "yaml";
        }
        return _ft;
      }
    }
    return _ConfContent();
  }
}
#+END_SRC

*** YAML config (config local site & dr document make) :file:config:hub:

#+NAME: meta_config_file_hub_read_site_yaml
#+HEADER: :noweb yes
#+BEGIN_SRC d
static template configReadSiteYAML() {
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  import sisudoc.ocda.meta.rgx_files;
  import sisudoc.ocda.meta.rgx;
  final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) {
    string _configuration = configReadInSiteYAML!()(_manifested, _env);
    auto _conf_file_details = configFilePaths!()(_manifested, _env);
    string _conf_yaml_fn = _conf_file_details.config_filename_site;
    YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
    return _yaml_conf;
  }
}
static template configReadDocYAML() {
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) {
    string _configuration = configReadInDocYAML!()(_manifested, _env);
    auto _conf_file_details = configFilePaths!()(_manifested, _env);
    string _conf_yaml_fn = _conf_file_details.config_filename_document;
    YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
    return _yaml_conf;
  }
}
#+END_SRC

* B. get _markup source_, read file (.readText) [#A]:module:source_files:
** _module template_

#+HEADER: :tangle "../src/sisudoc/ocda/io_in/read_source_files.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
/++
  module source_read_source_files;<BR>
  - open markup files<BR>
  - if master file scan for addional files to import/insert
+/
module sisudoc.ocda.io_in.read_source_files;
@safe:
template spineRawMarkupContent() {
  import std.digest.sha;
  import std.file;
  import std.path;
  import sisudoc.ocda.meta;
  import sisudoc.ocda.io_in.paths_source;
  import sisudoc.ocda.meta.rgx_files;
  import sisudoc.ocda.meta.rgx;
  mixin spineRgxIn;
  static auto rgx = RgxI();
  mixin spineRgxFiles;
  static auto rgx_files = RgxFiles();
  string[] _images=[];
  string[] _extract_images(S)(S content_block) {
    string[] images_;
    string _content_block = content_block.to!string;
    if (auto m = _content_block.matchAll(rgx.image)) {
      images_ ~= m.captures[1].to!string;
    }
    return images_;
  }
  auto rawsrc = RawMarkupContent();
  struct ST_contents_inserts_images {
    char[][]   contents;
    string[]   insert_files;
    string[]   images;
  }
  struct ST_header_content_inserts_images {
    char[]     header;
    char[][]   src_txt;
    string[]   insert_files;
    string[]   images;
  }
  struct ST_doc_digest {
    ubyte[32]  markup_doc;
    ubyte[32]  header;
    ubyte[32]  text;
  }
  struct ST_doc_parts {
    char[]        header_raw;
    char[][]      sourcefile_body_content;
    string[]      insert_file_list;
    string[]      images_list;
    ST_doc_digest doc_digest;
  }
  ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) {
    ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct
      = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src);
    return _0_header_1_body_content_2_insert_filelist_struct;
  }
  struct RawMarkupContent {
    final sourceContent(in string fn_src) {
      auto raw = MarkupRawUnit();
      string source_txt_str
        = raw.markupSourceReadIn(fn_src);
      return source_txt_str;
    }
    final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)(
      O         _opt_action,
      in string source_txt_str,
      in string fn_src=""
    ) {
      auto raw = MarkupRawUnit();
      string[] insert_file_list_get;
      string[] images_list_get;
      ST_header_content_inserts_images st
        = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str);
      char[] header_raw = st.header;
      char[][] sourcefile_body_content = st.src_txt;
      if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise
        auto ins = Inserts();
        ST_contents_inserts_images _cii
          = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
        sourcefile_body_content = _cii.contents;
        insert_file_list_get = _cii.insert_files.dup;
        images_list_get = _cii.images.dup;
      } else if (_opt_action.source_or_pod) {
        auto ins = Inserts();
        ST_contents_inserts_images _cii
          = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
        images_list_get = _cii.images.dup;
      } // image_list, if path could take sha256 digests already here?
      string header_type = "";
      ST_doc_digest dig;
      {
        dig.markup_doc = source_txt_str.sha256Of;
        dig.header = st.header.sha256Of;
        dig.text = sourcefile_body_content.sha256Of;
      }
      ST_doc_parts ret;
      {
        ret.header_raw = st.header;
        ret.sourcefile_body_content = sourcefile_body_content;
        ret.insert_file_list = insert_file_list_get;
        ret.images_list = images_list_get;
        ret.doc_digest = dig;
      }
      return ret;
    }
  }
  struct MarkupRawUnit {
    import std.digest.sha;
    import std.file;
    <<meta_markup_source_raw_read_file_source_string>>
    <<meta_markup_source_raw_doc_header_and_content_split>>
    <<meta_markup_source_raw_source_line_array>>
    <<meta_markup_source_raw_read_in_file>>
    <<meta_markup_source_raw_struct_of_header_and_body>>
    <<meta_markup_source_raw_get_insert_source_line_array>>
  }
  struct Inserts {
    struct ST_contents_and_images {
      char[][]   insert_contents;
      string[]   images;
    }
    ST_contents_and_images scan_subdoc_source(O)(
      O        _opt_action,
      char[][] markup_sourcefile_insert_content,
      string   fn_src
    ) {
      <<meta_inserts_scan>>
      foreach (line; markup_sourcefile_insert_content) {
        <<meta_inserts_scan_loop>>
      } // end src subdoc (inserts) loop
      <<meta_inserts_scan_post>>
    }
    ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)(
      O        _opt_action,
      char[][] sourcefile_body_content,
      string   fn_src
    ) {
      import std.algorithm;
      <<meta_master_doc_scan_for_insert_filenames>>
      foreach (line; sourcefile_body_content) {
        <<meta_master_doc_scan_for_insert_filenames_loop>>
      } // end src doc loop
      <<meta_master_doc_scan_for_insert_filenames_post>>
    }
  }
}
#+END_SRC

** get markup source, read file :source:markup:
*** read file, source string [#A] :string:

#+NAME: meta_markup_source_raw_read_file_source_string
#+BEGIN_SRC d
final private string readInMarkupSource(in char[] fn_src) {
  enforce(
    exists(fn_src) != 0,
    "file not found: «" ~
    fn_src ~ "»"
  );
  string source_txt_str;
  try {
    if (exists(fn_src)) {
      if (fn_src.getLinkAttributes.attrIsFile) {
        source_txt_str = fn_src.readText;
      } else {
      }
    }
  } catch (ErrnoException ex) {
  } catch (UTFException ex) {
    // Handle validation errors
  } catch (FileException ex) {
    // Handle errors
  }
  std.utf.validate(source_txt_str);
  return source_txt_str;
}
#+END_SRC

*** document header & content, array.length == 2 [#A] :array:

here you split document header and body, an array.length == 2
split is on first match of level A~ (which is required)

#+NAME: meta_markup_source_raw_doc_header_and_content_split
#+BEGIN_SRC d
@trusted final private char[][] header0Content1(in string src_text) { // cast(char[])
  /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/
  char[][] header_and_content;
  auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a);
  header_and_content ~= m.pre;
  header_and_content ~= m.hit ~ m.post;
  assert(header_and_content.length == 2,
    "document markup is broken, header body split == "
    ~ header_and_content.length.to!string
    ~ "; (header / body array split should == 2 (split is on level A~))"
  );
  return header_and_content;
}
#+END_SRC

*** source line array :array:

#+NAME: meta_markup_source_raw_source_line_array
#+BEGIN_SRC d
@trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[])
  char[][] source_line_arr
    = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding);
  return source_line_arr;
}
#+END_SRC

*** source content raw line array :array:
- used for regular .sst files; master .ssm files and; .ssi inserts
- regex is passed for relevant enforce match

**** read in file

#+NAME: meta_markup_source_raw_read_in_file
#+BEGIN_SRC d
string markupSourceReadIn(in string fn_src) {
  static auto rgx_files = RgxFiles();
  enforce(
    fn_src.match(rgx_files.src_pth_sst_or_ssm),
    "not a dr markup filename: «" ~
    fn_src ~ "»"
  );
  string source_txt_str = readInMarkupSource(fn_src);
  return source_txt_str;
}
#+END_SRC

**** tuple (a) header, (b) body content, (c) file insert list & (d) image list?

- header
- body content
- file insert list
- [image list?]

#+NAME: meta_markup_source_raw_struct_of_header_and_body
#+BEGIN_SRC d
ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) {
  string[] file_insert_list = [];
  string[] images_list = [];
  char[][] hc = header0Content1(source_txt_str);
  char[] header = hc[0];
  char[] source_txt = hc[1];
  char[][] source_line_arr = markupSourceLineArray(source_txt);
  ST_header_content_inserts_images ret;
  {
    ret.header          = header;
    ret.src_txt         = source_line_arr;
    ret.insert_files    = file_insert_list;
    ret.images          = images_list;
  }
  return ret;
}
#+END_SRC

**** get insert source line array

#+NAME: meta_markup_source_raw_get_insert_source_line_array
#+BEGIN_SRC d
final char[][] getInsertMarkupSourceContentRawLineArray(
  in char[]    fn_src_insert,
  Regex!(char) rgx_file
) {
  enforce(
    fn_src_insert.match(rgx_file),
    "not a dr markup filename: «" ~
    fn_src_insert  ~ "»"
  );
  string source_txt_str = readInMarkupSource(fn_src_insert);
  char[][] source_line_arr = markupSourceLineArray(source_txt_str);
  return source_line_arr;
}
#+END_SRC

** get markup source, master file & inserts :masterfile:inserts:

- [[./doc-reform.org][doc-reform.org]]  [[./][org/]]

*** scan inserts (sub-document) source :scan_insert_src:
**** scan subdoc source

#+NAME: meta_inserts_scan
#+BEGIN_SRC d
char[][] contents_insert;
int code_block_status     = 0;
enum codeBlock { off, curly, tic, quotemarks }
auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
auto markup_src_file_path = fn_pth_full.captures[1];
#+END_SRC

**** loop insert (sub-document)

#+NAME: meta_inserts_scan_loop
#+BEGIN_SRC d
if (code_block_status == codeBlock.curly) {
  if (line.matchFirst(rgx.block_curly_code_close)) {
    code_block_status = codeBlock.off;
  }
  contents_insert ~= line;
} else if (line.matchFirst(rgx.block_curly_code_open)) {
  code_block_status   = codeBlock.curly;
  contents_insert ~= line;
} else if (code_block_status == codeBlock.quotemarks) {
  if (line.matchFirst(rgx.block_quotemarks_close)) {
    code_block_status = codeBlock.off;
  }
  contents_insert ~= line;
} else if (code_block_status == codeBlock.tic) {
  if (line.matchFirst(rgx.block_tic_close)) {
    code_block_status = codeBlock.off;
  }
  contents_insert ~= line;
} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
  code_block_status   = codeBlock.quotemarks;
  contents_insert ~= line;
} else if (line.matchFirst(rgx.block_tic_code_open)) {
  code_block_status   = codeBlock.tic;
  contents_insert ~= line;
} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
  auto insert_fn = m.captures[2];
  auto insert_sub_pth = m.captures[1];
  auto fn_src_insert
    = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
  auto raw = MarkupRawUnit();
  auto markup_sourcesubfile_insert_content
    = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
  debug(insert_file) {
    writeln(line);
    writeln(fn_src_insert);
    writeln(
      "  length contents insert array: ",
      markup_sourcesubfile_insert_content.length
    );
  }
  if (_opt_action.source_or_pod) {
    _images ~= _extract_images(markup_sourcesubfile_insert_content);
  }
  auto ins = Inserts();
  /+
    - 1. load file
    - 2. read lines
    - 3. scan lines
      - a. if filename insert, and insert filename
         - repeat 1
      - b. else
         - add line to new array;
         - build image list, search for any image files to add to image list
  +/
} else {
  contents_insert ~= line; // images to extract for image list?
  if (_opt_action.source_or_pod) {
    string[] _image_linelist = _extract_images(line);
    if (_image_linelist.length > 0) {
      _images ~= _image_linelist;
    }
  }
}
#+END_SRC

**** post loop

#+NAME: meta_inserts_scan_post
#+BEGIN_SRC d
ST_contents_and_images ret;
{
  ret.insert_contents = contents_insert;
  ret.images          = _images;
}
return ret;
#+END_SRC

*** scan document source :scan_src:
**** scan doc source

#+NAME: meta_master_doc_scan_for_insert_filenames
#+BEGIN_SRC d
char[][] contents;
int code_block_status     = 0;
enum codeBlock { off, curly, tic, quotemarks }
auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
auto markup_src_file_path = fn_pth_full.captures[1];
char[][] contents_insert;
string[] _images          =[];
string[] insert_file_list =[];
#+END_SRC

**** include inserts: _loop master_ scan for inserts (insert documents)

#+NAME: meta_master_doc_scan_for_insert_filenames_loop
#+BEGIN_SRC d
if (code_block_status == codeBlock.curly) {
  if (line.matchFirst(rgx.block_curly_code_close)) {
    code_block_status = codeBlock.off;
  }
  contents ~= line;
} else if (line.matchFirst(rgx.block_curly_code_open)) {
  code_block_status = codeBlock.curly;
  contents ~= line;
} else if (code_block_status == codeBlock.quotemarks) {
  if (line.matchFirst(rgx.block_quotemarks_close)) {
    code_block_status = codeBlock.off;
  }
  contents ~= line;
} else if (code_block_status == codeBlock.tic) {
  if (line.matchFirst(rgx.block_tic_close)) {
    code_block_status = codeBlock.off;
  }
  contents ~= line;
} else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
  code_block_status = codeBlock.quotemarks;
  contents ~= line;
} else if (line.matchFirst(rgx.block_tic_code_open)) {
  code_block_status = codeBlock.tic;
  contents ~= line;
} else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
  auto insert_fn      = m.captures[2];
  auto insert_sub_pth = m.captures[1];
  auto fn_src_insert
    = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
    insert_file_list ~= fn_src_insert.to!string;
  auto raw = MarkupRawUnit();
  /+ TODO +/
  auto markup_sourcefile_insert_content
    = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
  debug(insert_file) {
    writeln(line);
    writeln(fn_src_insert);
    writeln(
      "  length contents insert array: ",
      markup_sourcefile_insert_content.length
    );
  }
  auto ins = Inserts();
  ST_contents_and_images contents_insert_st = ins.scan_subdoc_source(
    _opt_action,
    markup_sourcefile_insert_content,
    fn_src_insert.to!string
  );
  contents ~= contents_insert_st.insert_contents;
  if (_opt_action.source_or_pod) {
    string[] _image_linelist = _extract_images(contents_insert_st.images);
    if (_image_linelist.length > 0) {
      _images ~= _image_linelist;
    }
  }
  /+
    - 1. load file
    - 2. read lines
    - 3. scan lines
      - a. if filename insert, and insert filename
         - repeat 1
      - b. else
         - add line to new array;
         - build image list, search for any image files to add to image list
  +/
} else {
  contents ~= line;
  if (_opt_action.source_or_pod) {
    string[] _image_linelist = _extract_images(line);
    if (_image_linelist.length > 0) {
      _images ~= _image_linelist;
    }
  }
}
#+END_SRC

**** post loop

#+NAME: meta_master_doc_scan_for_insert_filenames_post
#+BEGIN_SRC d
string[] images = [];
foreach(i; uniq(_images.sort())) {
  images ~= i;
}
debug(insert_file) {
  writeln(__LINE__);
  writeln(contents.length);
}
ST_contents_inserts_images ret;
{
  ret.contents = contents;
  ret.insert_files = insert_file_list;
  ret.images = images;
}
return ret;
#+END_SRC

* C. read a .ssp file back into the document abstraction
** _module template_ :module:ssp_files:

#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/ssp_in.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
module sisudoc.ocda.abstraction.ssp_in;
@safe:
/+ ↓ read a .ssp file back into the document abstraction
     .
     the reverse of sisudoc.ocda.abstraction.ssp. what it returns is the same
     value the parser produces, ObjGenericComposite[][string], so anything
     that consumes the abstraction can be fed from a .ssp instead of from
     markup.
     .
     the check that it is faithful is a round trip: load a .ssp, emit each
     object again with sspObjectRecord (the writer's own definition of a
     record) and require the result to be byte identical to the file read.
+/
template spineAbstractionRead() {
  import std.algorithm : startsWith;
  import std.array;
  import std.conv : to;
  import std.file;
  import std.json : JSONValue;
  import std.stdio;
  import std.string;
  import sisudoc.ocda.meta.metadoc_object_setter;
  import sisudoc.ocda.abstraction.ssp : sspEscape;
  mixin sspEscape;
  mixin ObjectSetter;
  /+ ↓ what a .ssp file holds: the three header blocks as key/value, and the
       object sections in the order the file gives them
  +/
  struct SSPdocument {
    string                         format;        // "% SiSU Document Abstraction v0.1"
    string                         source;        // "% Source: ..."
    string[string]                 meta;
    string[]                       meta_order;
    string[string]                 make;
    string[]                       make_order;
    string[string]                 doc_has;
    string[]                       doc_has_order;
    string[]                       section_order; // as found in the file
    ObjGenericComposite[][string]  abstraction;
  }
  private int _levOfMarkedUp(string lev) {
    switch (lev) {
    case "A": return 0;
    case "B": return 1;
    case "C": return 2;
    case "D": return 3;
    case "1": return 4;
    case "2": return 5;
    case "3": return 6;
    case "4": return 7;
    default:  return 9;
    }
  }
  private int[] _ints(string val) {
    int[] _out;
    foreach (f; val.split(" ")) {
      if (f.length > 0) { _out ~= f.to!int; }
    }
    return _out;
  }
  /+ ↓ the eight slot arrays are int[] with a literal default, so every
       default constructed object shares one array; write into a fresh one
  +/
  private int[8] _eight(string val) {
    int[8] _out;
    foreach (i, v; _ints(val)) {
      if (i < _out.length) { _out[i] = v; }
    }
    return _out;
  }
  private ubyte[32] _hexToBytes(string hex) {
    ubyte[32] _out;
    if (hex.length < 64) { return _out; }
    foreach (i; 0..32) {
      _out[i] = hex[i*2 .. i*2+2].to!ubyte(16);
    }
    return _out;
  }
  SSPdocument sspRead(string[] lines) {
    SSPdocument doc;
    string  section;          // the @section block currently open
    string  block;            // "meta", "make", "doc_has", or "" for objects
    bool    in_object;
    string[] text_lines;
    ObjGenericComposite obj;
    void _closeObject() {
      if (!in_object) { return; }
      obj.text = text_lines.join("\n");
      doc.abstraction[section] ~= obj;
      obj       = ObjGenericComposite();
      text_lines = [];
      in_object = false;
    }
    foreach (line; lines) {
      if (line.length == 0) { continue; }
      if (line.startsWith("% SiSU Document Abstraction")) {
        doc.format = line; continue;
      }
      if (line.startsWith("% Source: ")) {
        doc.source = line["% Source: ".length .. $]; continue;
      }
      if (line.startsWith("@") && line.endsWith(" {")) {
        _closeObject();
        string _name = line[1 .. $-2];
        switch (_name) {
        case "meta": case "make": case "doc_has":
          block = _name; section = ""; break;
        default:
          block = ""; section = _name;
          doc.section_order ~= _name;
          doc.abstraction[_name] = [];
          break;
        }
        continue;
      }
      if (line == "}") { _closeObject(); block = ""; section = ""; continue; }
      if (block.length > 0) {          // a header block key: value line
        auto _kv = line.stripLeft.findSplit(": ");  // leading only: a value may end in a space
        if (_kv[1].length > 0) {
          final switch (block) {
          case "meta":
            doc.meta[_kv[0].to!string]    = _kv[2].to!string;
            doc.meta_order    ~= _kv[0].to!string;
            break;
          case "make":
            doc.make[_kv[0].to!string]    = _kv[2].to!string;
            doc.make_order    ~= _kv[0].to!string;
            break;
          case "doc_has":
            doc.doc_has[_kv[0].to!string] = _kv[2].to!string;
            doc.doc_has_order ~= _kv[0].to!string;
            break;
          }
        }
        continue;
      }
      if (line.startsWith("[")) {      // object declaration line
        _closeObject();
        in_object = true;
        auto _close = line.indexOf("] ");
        obj.metainfo.ocn = line[1 .. _close].to!int;
        string _rest = line[_close + 2 .. $];
        if (_rest.startsWith("heading :")) {
          obj.metainfo.is_a = "heading";
          string _tail = _rest["heading :".length .. $];
          auto _sp = _tail.indexOf(" ");
          if (_sp > 0) {
            obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail[0 .. _sp]);
            obj.metainfo.identifier         = _tail[_sp + 1 .. $];
          } else {
            obj.metainfo.heading_lev_markup = _levOfMarkedUp(_tail);
            /+ ↓ omitted means it equals the ocn, and for an object with no ocn
                 it was empty: "a"~N identifiers are always written
            +/
            obj.metainfo.identifier         = (obj.metainfo.ocn != 0)
              ? obj.metainfo.ocn.to!string : "";
          }
        } else {
          obj.metainfo.is_a     = _rest;
          obj.metainfo.identifier = (obj.metainfo.ocn != 0)
            ? obj.metainfo.ocn.to!string : "";
        }
        obj.metainfo.is_of_section = section;
        continue;
      }
      if (line.startsWith("| ")) { text_lines ~= line[2 .. $]; continue; }
      if (!line.startsWith(".")) { continue; }
      auto _p = line[1 .. $].findSplit(": ");
      string _key = (_p[1].length > 0) ? _p[0].to!string : line[1 .. $].to!string;
      string _val = (_p[1].length > 0) ? _p[2].to!string : "";
      switch (_key) {
      case "identifier":            obj.metainfo.identifier            = _val; break;
      case "part":                  obj.metainfo.is_of_part            = _val; break;
      case "section":               obj.metainfo.is_of_section         = _val; break;
      case "parent":                obj.metainfo.parent_ocn            = _val.to!int; break;
      case "last_descendant":       obj.metainfo.last_descendant_ocn   = _val.to!int; break;
      case "children":              obj.metainfo.children_headings     = _ints(_val); break;
      case "ancestors":
        obj.metainfo.markedup_ancestors = _eight(_val);
        break;
      case "ancestors_collapsed":
        obj.metainfo.collapsed_ancestors = _eight(_val);
        break;
      case "dom_status":
        obj.metainfo.dom_structure_markedup_tags_status = _eight(_val);
        break;
      case "dom_status_collapsed":
        obj.metainfo.dom_structure_collapsed_tags_status = _eight(_val);
        break;
      case "heading_lev_collapsed":  obj.metainfo.heading_lev_collapsed = _val.to!int; break;
      case "parent_lev":             obj.metainfo.parent_lev_markup     = _val.to!int; break;
      case "dummy":                  obj.metainfo.dummy_heading         = true; break;
      case "ocn_off":                obj.metainfo.object_number_off     = true; break;
      case "is_of_type":             obj.metainfo.is_of_type            = _val; break;
      case "attrib":                 obj.metainfo.attrib                = _val; break;
      case "meta_lang":              obj.metainfo.lang                  = _val; break;
      case "meta_syntax":            obj.metainfo.syntax                = _val; break;
      case "sha256":                 obj.metainfo.sha256.text           = _hexToBytes(_val); break;
      case "image":
        {
          ST_file_name_hash_size_ _img;
          auto _fields = _val.split(" ");
          if (_fields.length > 0) { _img.fileName = _fields[0].to!string; }
          foreach (f; _fields[1 .. $]) {
            if (f == "missing:true")        { _img.fileMissing = true; }
            else if (f.startsWith("sha256:")) { _img.fileHash_sha256 = _hexToBytes(f[7 .. $].to!string); }
            else if (f.startsWith("bytes:"))  { _img.fileSize = f[6 .. $].to!ulong; }
            else if (f.startsWith("px:")) {
              auto _wh = f[3 .. $].findSplit("x");
              if (_wh[1].length > 0) {
                _img.imageWidth  = _wh[0].to!int;
                _img.imageHeight = _wh[2].to!int;
              }
            }
          }
          obj.metainfo.sha256.images ~= _img;
        }
        break;
      case "indent":
        {
          auto _i = _ints(_val);
          if (_i.length > 0) { obj.attrib.indent_base = _i[0]; }
          if (_i.length > 1) { obj.attrib.indent_hang = _i[1]; }
        }
        break;
      case "bullet":                 obj.attrib.bullet                  = true; break;
      case "lang":                   obj.attrib.language                = _val; break;
      case "has":
        foreach (f; _val.split(" ")) {
          switch (f) {
          case "links":         obj.has.inline_links             = true; break;
          case "notes_reg":     obj.has.inline_notes_reg         = true; break;
          case "notes_star":    obj.has.inline_notes_star        = true; break;
          case "images":        obj.has.images                   = true; break;
          case "images_no_dim": obj.has.image_without_dimensions = true; break;
          default: break;
          }
        }
        break;
      case "table_cols":             obj.table.number_of_columns        = _val.to!int; break;
      case "table_widths":
        foreach (w; _val.split(" ")) { obj.table.column_widths ~= w.to!double; }
        break;
      case "table_aligns":
        foreach (a; _val.split(" ")) { obj.table.column_aligns ~= a.to!string; }
        break;
      case "table_header":           obj.table.heading                  = true; break;
      case "code_linenumbers":       obj.code_block.linenumbers         = true; break;
      case "stow_link":              obj.stow.link                     ~= _val; break;
      case "segment":                obj.tags.in_segment_html           = _val; break;
      case "anchor":                 obj.tags.anchor_tag_html           = _val; break;
      case "segment_prev":           obj.tags.segname_prev              = _val; break;
      case "segment_next":           obj.tags.segname_next              = _val; break;
      case "heading_lev_anchor":     obj.tags.heading_lev_anchor_tag    = _val; break;
      case "segment_epub":           obj.tags.segment_anchor_tag_epub   = _val; break;
      case "segment_html_is":        obj.tags.html_segment_anchor_tag_is = _val; break;
      case "segment_epub_is":        obj.tags.epub_segment_anchor_tag_is = _val; break;
      case "heading_ancestors_text":
        {
          string[8] _h;
          foreach (i, h; sspUnescapeSplit(_val)) {
            if (i < _h.length) { _h[i] = h; }
          }
          obj.tags.heading_ancestors_text = _h;
        }
        break;
      case "lev4_subtoc":            obj.tags.lev4_subtoc              ~= _val; break;
      case "anchor_tag":             obj.tags.anchor_tags              ~= _val; break;
      default: break;
      }
    }
    _closeObject();
    /+ ↓ .anchor is written only when it differs from .segment, and .section
         only when it differs from the enclosing block; restore both
    +/
    foreach (sect; doc.section_order) {
      foreach (ref o; doc.abstraction[sect]) {
        if (o.tags.anchor_tag_html.length == 0
          && o.tags.in_segment_html.length > 0) {
          o.tags.anchor_tag_html = o.tags.in_segment_html;
        }
      }
    }
    return doc;
  }
  /+ ↓ the abstraction as it survives a trip through the .ssp text: emitted
       as lines by the writer, read straight back. what a consumer built
       from this cannot see, the .ssp does not carry, which is what makes a
       second artefact built from it unable to drift
  +/
  auto sspRoundTripAbstraction(D)(D doc) {
    import sisudoc.ocda.abstraction.ssp;
    mixin spineAbstractionTxt;
    return sspRead(sspDocumentLines(doc)).abstraction;
  }
  SSPdocument sspReadFile(string file_path) {
    string[] _lines;
    try {
      _lines = (cast(char[]) file_path.read).to!string.split("\n");
    } catch (Exception ex) {
      writeln("ERROR: could not read ", file_path, ": ", ex.msg);
      return SSPdocument();
    }
    return sspRead(_lines);
  }
}
#+END_SRC

* D. read a .db file into the document abstraction
** _module template_ :module:db_files:

#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/db_in.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
module sisudoc.ocda.abstraction.db_in;
@safe:
/+ ↓ read a <doc>.ocda.db back into the document abstraction

     the counterpart of sisudoc.outputs.io_out.create_ocda, and the
     sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value,
     so a consumer does not have to know which of the two artefacts it was
     handed.

     the check is the same one: read a database, emit it as .ssp with the
     writer's own record definition, and require the result to equal the .ssp
     written from the same document.
+/
template spineAbstractionDbRead() {
  import std.conv : to;
  import std.file;
  import std.stdio;
  import std.string;
  import std.array;
  import std.json;
  import d2sqlite3;
  import sisudoc.ocda.abstraction.ssp_in : spineAbstractionRead;
  mixin spineAbstractionRead;   // for SSPdocument, and the shared unescape
  private int[8] _jsonEight(string _s) {
    int[8] _out;
    if (_s.length == 0) { return _out; }
    try {
      auto _j = parseJSON(_s);
      foreach (i, v; _j.array) {
        if (i < _out.length) { _out[i] = v.integer.to!int; }
      }
    } catch (Exception ex) {}
    return _out;
  }
  private int[] _jsonInts(string _s) {
    int[] _out;
    if (_s.length == 0) { return _out; }
    try {
      auto _j = parseJSON(_s);
      foreach (v; _j.array) { _out ~= v.integer.to!int; }
    } catch (Exception ex) {}
    return _out;
  }
  private string[] _jsonStrs(string _s) {
    string[] _out;
    if (_s.length == 0) { return _out; }
    try {
      auto _j = parseJSON(_s);
      foreach (v; _j.array) { _out ~= v.str; }
    } catch (Exception ex) {}
    return _out;
  }
  private double[] _jsonDoubles(string _s) {
    double[] _out;
    if (_s.length == 0) { return _out; }
    try {
      auto _j = parseJSON(_s);
      foreach (v; _j.array) {
        _out ~= (v.type == JSONType.integer) ? v.integer.to!double : v.floating;
      }
    } catch (Exception ex) {}
    return _out;
  }
  private ubyte[32] _hex32(string hex) {
    ubyte[32] _out;
    if (hex.length < 64) { return _out; }
    foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); }
    return _out;
  }
  private string _levMarkedUp(int lev) {
    switch (lev) {
    case 0:  return "A";
    case 1:  return "B";
    case 2:  return "C";
    case 3:  return "D";
    case 4:  return "1";
    case 5:  return "2";
    case 6:  return "3";
    case 7:  return "4";
    default: return "";
    }
  }
  @trusted SSPdocument dbReadFile(string db_file) {
    SSPdocument doc;
    if (!db_file.exists) {
      writeln("ERROR: no such file: ", db_file);
      return doc;
    }
    auto db = Database(db_file, SQLITE_OPEN_READONLY);
    /+ ↓ the three header blocks, from the metadata table. the key prefix
         says which block a row belongs to, and insertion order is kept +/
    foreach (row; db.execute("SELECT key, value FROM metadata ORDER BY rowid")) {
      string _k = row["key"].as!string;
      string _v = row["value"].as!string;
      if (_k.startsWith("make.")) {
        string _kk = _k["make.".length .. $];
        doc.make[_kk] = _v;
        doc.make_order ~= _kk;
      } else if (_k.startsWith("doc_has.")) {
        string _kk = _k["doc_has.".length .. $];
        doc.doc_has[_kk] = _v;
        doc.doc_has_order ~= _kk;
      } else if (_k.startsWith("schema.")) {
        if (_k == "schema.version") { doc.format = "% SiSU Document Abstraction v0.1"; }
      } else if (_k == "source.filename") {
        doc.source = _v;
      } else if (_k == "source.language") {
        // carried in the path, not in the .ssp header
      } else {
        doc.meta[_k] = _v;
        doc.meta_order ~= _k;
      }
    }
    /+ ↓ the objects, section by section, in the order they were written +/
    string[] _sections;
    foreach (row; db.execute(
      "SELECT section FROM objects GROUP BY section ORDER BY MIN(id)")
    ) {
      _sections ~= row["section"].as!string;
    }
    foreach (section; _sections) {
      doc.section_order ~= section;
      doc.abstraction[section] = [];
      foreach (row; db.execute(
        "SELECT * FROM objects WHERE section = '" ~ section ~ "' ORDER BY seq")
      ) {
        ObjGenericComposite obj;
        long _id                          = row["id"].as!long;
        obj.metainfo.ocn                  = row["ocn"].as!int;
        obj.metainfo.is_a                 = row["is_a"].as!string;
        obj.metainfo.is_of_part           = row["is_of_part"].as!string;
        obj.metainfo.is_of_section        = (row["is_of_section"].as!string.length > 0)
          ? row["is_of_section"].as!string : section;
        obj.metainfo.is_of_type           = row["is_of_type"].as!string;
        obj.metainfo.identifier           = row["identifier"].as!string;
        obj.metainfo.heading_lev_markup   = (obj.metainfo.is_a == "heading")
          ? row["heading_level"].as!int : 9;
        obj.metainfo.heading_lev_collapsed = (row["heading_lev_collapsed"].as!string.length > 0)
          ? row["heading_lev_collapsed"].as!int : 9;
        obj.metainfo.parent_ocn           = row["parent_ocn"].as!int;
        obj.metainfo.parent_lev_markup    = row["parent_lev"].as!int;
        obj.metainfo.last_descendant_ocn  = row["last_descendant_ocn"].as!int;
        obj.metainfo.children_headings    = _jsonInts(row["children"].as!string);
        obj.metainfo.markedup_ancestors   = _jsonEight(row["ancestors"].as!string);
        obj.metainfo.collapsed_ancestors  = _jsonEight(row["ancestors_collapsed"].as!string);
        obj.metainfo.dom_structure_markedup_tags_status
          = _jsonEight(row["dom_status"].as!string);
        obj.metainfo.dom_structure_collapsed_tags_status
          = _jsonEight(row["dom_status_collapsed"].as!string);
        obj.metainfo.dummy_heading        = (row["dummy_heading"].as!int == 1);
        obj.metainfo.object_number_off    = (row["object_number_off"].as!int == 1);
        obj.metainfo.attrib               = row["attrib"].as!string;
        obj.metainfo.lang                 = row["meta_lang"].as!string;
        obj.metainfo.syntax               = row["meta_syntax"].as!string;
        obj.metainfo.sha256.text          = _hex32(row["sha256"].as!string);
        obj.attrib.indent_base            = row["indent_base"].as!int;
        obj.attrib.indent_hang            = row["indent_hang"].as!int;
        obj.attrib.bullet                 = (row["bullet"].as!int == 1);
        obj.attrib.language               = row["lang"].as!string;
        obj.has.inline_links              = (row["has_links"].as!int == 1);
        obj.has.inline_notes_reg          = (row["has_notes_reg"].as!int == 1);
        obj.has.inline_notes_star         = (row["has_notes_star"].as!int == 1);
        obj.has.images                    = (row["has_images"].as!int == 1);
        obj.has.image_without_dimensions  = (row["has_images_no_dim"].as!int == 1);
        obj.tags.in_segment_html          = row["segment"].as!string;
        obj.tags.segname_prev             = row["segment_prev"].as!string;
        obj.tags.segname_next             = row["segment_next"].as!string;
        obj.tags.segment_anchor_tag_epub  = row["segment_epub"].as!string;
        obj.tags.html_segment_anchor_tag_is = row["segment_html_is"].as!string;
        obj.tags.epub_segment_anchor_tag_is = row["segment_epub_is"].as!string;
        obj.tags.anchor_tag_html          = row["anchor"].as!string;
        obj.tags.heading_lev_anchor_tag   = row["heading_lev_anchor"].as!string;
        {
          string[8] _h;
          foreach (i, h; _jsonStrs(row["heading_ancestors_text"].as!string)) {
            if (i < _h.length) { _h[i] = h; }
          }
          obj.tags.heading_ancestors_text = _h;
        }
        if (obj.metainfo.is_a == "table") {
          obj.table.number_of_columns     = row["table_cols"].as!int;
          obj.table.column_widths         = _jsonDoubles(row["table_widths"].as!string);
          obj.table.column_aligns         = _jsonStrs(row["table_aligns"].as!string);
          obj.table.heading               = (row["table_header"].as!int == 1);
        }
        obj.code_block.linenumbers        = (row["code_linenumbers"].as!int == 1);
        obj.text                          = row["text"].as!string;
        /+ ↓ the per object lists +/
        foreach (r; db.execute(
          "SELECT name, bytes, sha256, width, height, missing FROM object_images"
          ~ " WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq")
        ) {
          ST_file_name_hash_size_ _img;
          _img.fileName        = r["name"].as!string;
          _img.fileSize        = r["bytes"].as!ulong;
          _img.fileHash_sha256 = _hex32(r["sha256"].as!string);
          _img.imageWidth      = r["width"].as!int;
          _img.imageHeight     = r["height"].as!int;
          _img.fileMissing     = (r["missing"].as!int == 1);
          obj.metainfo.sha256.images ~= _img;
        }
        foreach (r; db.execute(
          "SELECT url FROM object_links WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq")
        ) { obj.stow.link ~= r["url"].as!string; }
        foreach (r; db.execute(
          "SELECT anchor FROM object_anchors WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq")
        ) { obj.tags.anchor_tags ~= r["anchor"].as!string; }
        foreach (r; db.execute(
          "SELECT entry FROM object_subtoc WHERE object_id = " ~ _id.to!string ~ " ORDER BY seq")
        ) { obj.tags.lev4_subtoc ~= r["entry"].as!string; }
        doc.abstraction[section] ~= obj;
      }
    }
    return doc;
  }
}
#+END_SRC

* E. read a .db file into the document abstraction
** _module template_ :module:db_files:

#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/load.d"
#+HEADER: :noweb yes
#+BEGIN_SRC d
<<doc_header_including_copyright_and_license>>
module sisudoc.ocda.abstraction.load;
@safe:
/+ ↓ one way in, whatever the document is being read from

     spine's pipeline is markup -> abstraction -> output. once the abstraction
     is serialised, there is more than one thing an abstraction can be read
     from, and a consumer should not have to know which it was handed:

       .sst / .ssm  + images   the markup source
       pod (dir)    + images   the same, bundled
       pod .zip                the same, zipped
       .ssp         + images   the abstraction, as text
       .ocda.db                the abstraction, sqlite, images inside

     This module names those sources, tells them apart, and loads the two
     that are self-describing artefacts, returning the value the parser
     produces.

     The three source forms are deliberately *not* loaded here. They are the
     parser's job (sisudoc.ocda.meta.metadoc, spineAbstraction), and it needs
     the environment, the options and the configuration that spine.d
     assembles, none of which belongs in a loader. What this module gives
     that case is the dispatch and a plain statement of where to go.
+/
template spineAbstractionLoad() {
  import std.algorithm : endsWith;
  import std.conv : to;
  import std.file;
  import std.path;
  import std.stdio;
  import std.string;
  import sisudoc.ocda.abstraction.db_in;
  mixin spineAbstractionDbRead;  // brings the .ssp reader with it

  enum AbstractionSource {
    unknown,
    markup,     // .sst or .ssm, with its images beside it
    pod,        // a directory holding pod.manifest
    pod_zip,    // that directory, zipped
    ssp,        // .ssp, the abstraction as text, images beside it
    ocda_db,    // .ocda.db, the abstraction as sqlite, images inside it
  }

  string abstractionSourceName(AbstractionSource _s) {
    final switch (_s) {
    case AbstractionSource.unknown: return "unknown";
    case AbstractionSource.markup:  return "markup source (.sst/.ssm)";
    case AbstractionSource.pod:     return "pod directory";
    case AbstractionSource.pod_zip: return "pod zip";
    case AbstractionSource.ssp:     return ".ssp (abstraction as text)";
    case AbstractionSource.ocda_db: return ".ocda.db (abstraction as sqlite)";
    }
  }

  /+ ↓ what is this? by name, and for a directory by what it holds +/
  AbstractionSource abstractionSourceOf(string _path) {
    if (_path.length == 0) { return AbstractionSource.unknown; }
    if (_path.isValidPath && _path.exists && _path.isDir) {
      return (_path.chainPath("pod.manifest").array.exists)
        ? AbstractionSource.pod : AbstractionSource.unknown;
    }
    if (_path.endsWith(".ocda.db"))                    { return AbstractionSource.ocda_db; }
    if (_path.endsWith(".ssp"))                        { return AbstractionSource.ssp; }
    if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; }
    if (_path.endsWith(".zip"))                        { return AbstractionSource.pod_zip; }
    if (_path.endsWith(".db"))                         { return AbstractionSource.ocda_db; }
    return AbstractionSource.unknown;
  }

  struct LoadedAbstraction {
    AbstractionSource source;
    string            path;
    bool              loaded;   // is .doc filled
    string            note;     // why not, when it is not
    SSPdocument       doc;
  }

  /+ ↓ load what can be loaded from the path alone.

       .ssp and .ocda.db come back filled. the three source forms come back
       with loaded = false and a note saying where they are handled, because
       reading them needs the manifest, environment and configuration that
       spine.d builds, not a file path.
  +/
  LoadedAbstraction abstractionLoad(string _path) {
    LoadedAbstraction _out;
    _out.path   = _path;
    _out.source = abstractionSourceOf(_path);
    if (!_path.exists) {
      _out.note = "no such file or directory";
      return _out;
    }
    final switch (_out.source) {
    case AbstractionSource.ssp:
      _out.doc    = sspReadFile(_path);
      _out.loaded = (_out.doc.section_order.length > 0);
      if (!_out.loaded) { _out.note = "no object sections found"; }
      break;
    case AbstractionSource.ocda_db:
      _out.doc    = dbReadFile(_path);
      _out.loaded = (_out.doc.section_order.length > 0);
      if (!_out.loaded) { _out.note = "no object sections found"; }
      break;
    case AbstractionSource.markup:
    case AbstractionSource.pod:
    case AbstractionSource.pod_zip:
      _out.note = "a source form: read by the parser,"
        ~ " sisudoc.ocda.meta.metadoc spineAbstraction, which needs the"
        ~ " manifest, environment and configuration spine assembles";
      break;
    case AbstractionSource.unknown:
      _out.note = "not a document source spine knows";
      break;
    }
    return _out;
  }

  /+ ↓ what was loaded, in a few lines: for the eye, and for a check that a
       given artefact really does hold what it should +/
  string[] abstractionLoadSummary(LoadedAbstraction _l) {
    string[] _out;
    _out ~= "source: " ~ abstractionSourceName(_l.source);
    _out ~= "path:   " ~ _l.path;
    if (!_l.loaded) {
      _out ~= "loaded: no";
      if (_l.note.length > 0) { _out ~= "note:   " ~ _l.note; }
      return _out;
    }
    _out ~= "loaded: yes";
    if (_l.doc.source.length > 0) { _out ~= "document: " ~ _l.doc.source; }
    if ("title.main" in _l.doc.meta) {
      _out ~= "title:  " ~ _l.doc.meta["title.main"];
    }
    if ("creator.author" in _l.doc.meta) {
      _out ~= "author: " ~ _l.doc.meta["creator.author"];
    }
    _out ~= "header: " ~ _l.doc.meta.length.to!string ~ " meta, "
      ~ _l.doc.make.length.to!string ~ " make, "
      ~ _l.doc.doc_has.length.to!string ~ " doc_has";
    int _objs, _citable, _headings;
    string[] _sections;
    foreach (section; _l.doc.section_order) {
      int _n;
      foreach (obj; _l.doc.abstraction[section]) {
        ++_objs; ++_n;
        if (obj.metainfo.ocn > 0)              { ++_citable; }
        if (obj.metainfo.is_a == "heading")    { ++_headings; }
      }
      _sections ~= section ~ " " ~ _n.to!string;
    }
    _out ~= "objects: " ~ _objs.to!string ~ " (" ~ _citable.to!string
      ~ " citable, " ~ _headings.to!string ~ " headings)";
    _out ~= "sections: " ~ _sections.join(", ");
    return _out;
  }
}
#+END_SRC

* org includes
** project version

#+NAME: spine_version
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
#+END_SRC

** year

#+NAME: year
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
#+END_SRC

** document header including copyright & license

#+NAME: doc_header_including_copyright_and_license
#+HEADER: :noweb yes
#+BEGIN_SRC emacs-lisp
<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
#+END_SRC

* __END__