#+TITLE: sdp hub #+AUTHOR: Ralph Amissah #+EMAIL: ralph.amissah@gmail.com #+STARTUP: indent #+LANGUAGE: en #+OPTIONS: H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t #+OPTIONS: TeX:t LaTeX:t skip:nil d:nil todo:t pri:nil tags:not-in-toc #+OPTIONS: author:nil email:nil creator:nil timestamp:nil #+PROPERTY: header-args :padline no :exports code :noweb yes #+EXPORT_SELECT_TAGS: export #+EXPORT_EXCLUDE_TAGS: noexport #+FILETAGS: :sdp:rel:hub: #+TAGS: assert(a) class(c) debug(d) mixin(m) sdp(s) tangle(T) template(t) WEB(W) noexport(n) [[../maker.org][maker.org makefile]] [[./][org/]] * 0. version.txt (set version) :version: #+NAME: version_txt #+BEGIN_SRC d :tangle ../views/version.txt /+ obt - org generated file +/ struct Version { int major; int minor; int patch; } enum ver = Version(0, 13, 4); #+END_SRC * 1. sdp (sisu document parser) :sdp: - deal with imports - get options - get command line instructions - read config instructions - process files as instructed by options - read in file - process file - output ** 0. sdp src/sdp.d :template: #+BEGIN_SRC d :tangle ../src/sdp.d :shebang #!/usr/bin/env rdmd /+ sdp +/ import compile_time_info, abstraction; <<imports_sdp>> <<imports_sdlang>> <<imports_std>> <<sdp_output_selection>> <<mixin_sdp_version>> <<mixin_pre_main>> /++ A SiSU document parser writen in D. +/ void main(string[] args) { <<sdp_mixin>> <<sdp_args>> <<sdp_env>> <<sdp_conf_files>> foreach(fn_src; fns_src) { if (!empty(fn_src)) { <<sdp_each_file_do_scope>> <<sdp_abstraction>> <<sdp_each_file_do_debugs_checkdoc>> <<sdp_each_file_do_selected_output>> <<sdp_each_file_do_scope_exit>> } else { <<sdp_no_filename_provided>> } } } unittest { /++ name "sdp" description "A SiSU document parser writen in D." homepage "http://sisudoc.org" +/ } #+END_SRC ** 1. pre-loop init :init: *** init **** imports :import: ***** sdp :sdp: #+NAME: imports_sdp #+BEGIN_SRC d /+ sdp: sisu document parser, see http://sisudoc.org +/ import abstraction_summary, ao_abstract_doc_source, ao_conf_make_meta, ao_conf_make_meta_native, ao_conf_make_meta_sdlang, ao_defaults, ao_doc_debugs, ao_read_config_files, ao_read_source_files, ao_rgx, output_hub; #+END_SRC ****** notes ├── src │ ├── sdp.d │ └── sdp │ ├── ao_abstract_doc_source.d │ ├── ... │ └── compile_time_info.d └── views └── version.txt [[./ao_abstract_doc_source.org][ao_abstract_doc_source]] [[./ao_conf_make_meta.org][ao_conf_make_meta]] [[./ao_defaults.org][ao_defaults]] [[./ao_output_debugs.org][ao_output_debugs]] [[./ao_read_source_files.org][ao_read_source_files]] [[./compile_time_info.org][compile time info]] [[./output.org][output]] [[./sdp.org][sdp]] ***** sdlang :sdlang: keep up to date, configuration in ../maker.org check: - http://github.com/Abscissa/SDLang-D - https://github.com/abscissa/libInputVisitor #+NAME: imports_sdlang #+BEGIN_SRC d /+ sdlang http://sdlang.org +/ import sdlang; // sdlang.d #+END_SRC ****** notes sdlang.parser, sdlang.exceptions; ***** std :std: #+NAME: imports_std #+BEGIN_SRC d /+ std +/ private import std.array, std.exception, std.getopt, std.process, std.stdio, std.regex, std.string, std.traits, std.typecons, std.utf, std.conv : to; #+END_SRC ****** notes std.conv, std.variant, **** mixins :mixin: ***** version.txt :version: #+NAME: mixin_sdp_version #+BEGIN_SRC d mixin(import("version.txt")); #+END_SRC ***** pre main mixins #+NAME: mixin_pre_main #+BEGIN_SRC d mixin CompileTimeInfo; #+END_SRC ***** sdp "main" mixins :sdp: #+NAME: sdp_mixin #+BEGIN_SRC d mixin SiSUrgxInit; mixin SiSUregisters; mixin SiSUheaderExtractSDLang; mixin SiSUnode; mixin SiSUbiblio; mixin SiSUrgxInitFlags; mixin outputHub; #+END_SRC **** init :init: #+NAME: sdp_args #+BEGIN_SRC d string[] fns_src; string flag_action; string arg_unrecognized; enum dAM { abstraction, matters } auto rgx = Rgx(); #+END_SRC *** scope (run complete) :scope: #+NAME: sdp_args #+BEGIN_SRC d scope(success) { debug(checkdoc) { writefln( "~ run complete, ok ~ (sdp-%s.%s.%s, %s v%s, %s %s)", ver.major, ver.minor, ver.patch, __VENDOR__, __VERSION__, bits, os, ); } } scope(failure) { debug(checkdoc) { stderr.writefln( "run failure", ); } } #+END_SRC *** config files and command line arguements **** getopt args for loop :args:getopt: look into using getopt [[http://dlang.org/phobos/std_getopt.html][getopt]] [[http://dlang.org/library/std/getopt.html][getopt]] #+NAME: sdp_args #+BEGIN_SRC d bool[string] _opt_action_bool = [ "assertions" : false, "concordance" : false, "debug" : false, "digest" : false, "docbook" : false, "epub" : false, "html" : false, "html_seg" : false, "html_scroll" : false, "manifest" : false, "ocn" : true, "odt" : false, "pdf" : false, "postgresql" : false, "qrcode" : false, "sisupod" : false, "source" : false, "sqlite" : false, "text" : false, "verbose" : false, "xhtml" : false, "xml_dom" : false, "xml_sax" : false, "section_toc" : true, "section_body" : true, "section_endnotes" : true, "section_glossary" : true, "section_biblio" : true, "section_bookindex" : true, "section_blurb" : true, "backmatter" : true, "skip_output" : false, ]; auto helpInfo = getopt(args, std.getopt.config.passThrough, "assert", "--assert set optional assertions on", &_opt_action_bool["assertions"], "concordance", "--concordance file for document", &_opt_action_bool["concordance"], "debug", "--debug only relevant when debug options compiled in", &_opt_action_bool["debug"], "digest", "--digest hash digest for each object", &_opt_action_bool["digest"], "docbook", "--docbook process docbook output", &_opt_action_bool["docbook"], "epub", "--epub process epub output", &_opt_action_bool["epub"], "html", "--html process html output", &_opt_action_bool["html"], "html_seg", "--html-seg process html output", &_opt_action_bool["html_seg"], "html_scroll", "--html-seg process html output", &_opt_action_bool["html_scroll"], "manifest", "--manifest process manifest output", &_opt_action_bool["manifest"], "ocn", "--ocn object cite numbers (default)", &_opt_action_bool["ocn"], "odf", "--odf process odf:odt output", &_opt_action_bool["odt"], "odt", "--odt process odf:odt output", &_opt_action_bool["odt"], "pdf", "--pdf process pdf output", &_opt_action_bool["pdf"], "pg", "--pg process postgresql output", &_opt_action_bool["postgresql"], "postgresql", "--postgresql process postgresql output", &_opt_action_bool["postgresql"], "qrcode", "--qrcode with document metadata", &_opt_action_bool["qrcode"], "sisupod", "--sisupod sisupod source content bundled", &_opt_action_bool["sisupod"], "source", "--source markup source text content", &_opt_action_bool["source"], "sqlite", "--sqlite process sqlite output", &_opt_action_bool["sqlite"], "text", "--text process text output", &_opt_action_bool["text"], "txt", "--txt process text output", &_opt_action_bool["text"], "verbose|v", "--verbose output to terminal", &_opt_action_bool["verbose"], "xhtml", "--xhtml process xhtml output", &_opt_action_bool["xhtml"], "xml-dom", "--xml-dom process xml dom output", &_opt_action_bool["xml_dom"], "xml-sax", "--xml-sax process xml sax output", &_opt_action_bool["xml_sax"], "section-toc", "--section-toc process table of contents (default)", &_opt_action_bool["section_toc"], "section-body", "--section-body process document body (default)", &_opt_action_bool["section_body"], "section-endnotes", "--section-endnotes process document endnotes (default)", &_opt_action_bool["section_endnotes"], "section-glossary", "--section-glossary process document glossary (default)", &_opt_action_bool["section_glossary"], "section-biblio", "--section-biblio process document biblio (default)", &_opt_action_bool["section_biblio"], "section-bookindex", "--section-bookindex process document bookindex (default)", &_opt_action_bool["section_bookindex"], "section-blurb", "--section-blurb process document blurb (default)", &_opt_action_bool["section_blurb"], "backmatter", "--section-backmatter process document backmatter (default)", &_opt_action_bool["backmatter"], "skip_output", "--skip-output", &_opt_action_bool["skip_output"], ); if (helpInfo.helpWanted) { defaultGetoptPrinter("Some information about the program.", helpInfo.options); } foreach(arg; args) { if (match(arg, rgx.flag_action)) { flag_action ~= " " ~ arg; // flags not taken by getopt } else if (match(arg, rgx.src_pth)) { fns_src ~= arg; // gather input markup source file names for processing } else { // anything remaining, unused arg_unrecognized ~= " " ~ arg; } } #+END_SRC **** environment :environment: #+NAME: sdp_env #+BEGIN_SRC d auto env = [ "pwd" : environment["PWD"], "home" : environment["HOME"], ]; #+END_SRC **** TODO config files (load & read) (so far only SDLang) :config:files: #+NAME: sdp_conf_files #+BEGIN_SRC d auto sdl_root_configuration = ConfigHub!()("conf.sdl", env); auto sdl_root_doc_make = ConfigHub!()("sisu_document_make", env); auto confsdl = HeaderExtractSDL(); auto conf_settings_aa = confsdl.configSettingsSDLangToAAmake(sdl_root_configuration); auto conf_doc_make_aa = confsdl.documentMakeSDLangToAAmake(sdl_root_doc_make); #+END_SRC ** _2a. processing: loop each file_ [+2] :loop:files: *** scope (loop) :scope: #+NAME: sdp_each_file_do_scope #+BEGIN_SRC d scope(success) { debug(checkdoc) { writefln( "%s\n%s", "~ document complete, ok ~", "------------------------------------------------------------------", ); } } scope(failure) { debug(checkdoc) { stderr.writefln( "~ document run failure ~ (%s v%s)\n\t%s", __VENDOR__, __VERSION__, fn_src ); } } enforce( match(fn_src, rgx.src_pth), "not a sisu markup filename" ); #+END_SRC *** 1. _document abstraction_ [#A] - return tuple of: - doc_abstraction (the document) - doc_matters #+NAME: sdp_abstraction #+BEGIN_SRC d auto t = SiSUabstraction!()(fn_src, _opt_action_bool, env); static assert(!isTypeTuple!(t)); static assert(t.length==2); auto doc_abstraction = t[dAM.abstraction]; auto doc_matters = t[dAM.matters]; #+END_SRC *** 2. _output processing_ (post abstraction processing) **** 0. abstraction _print summary_ :abstraction:summary: #+NAME: sdp_each_file_do_debugs_checkdoc #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt_action_bool["verbose"]) { SiSUabstractionSummary!()(doc_abstraction, doc_matters); } #+END_SRC **** 1. _debug_ (document parts, checkdoc) :debug:checkdoc: - [[./ao_output_debugs.org][ao_output_debugs]] #+NAME: sdp_each_file_do_debugs_checkdoc #+BEGIN_SRC d /+ ↓ debugs +/ if ((doc_matters.opt_action_bool["debug"]) || (doc_matters.opt_action_bool["verbose"]) ) { SiSUdebugs!()(doc_abstraction, doc_matters); } #+END_SRC **** 2. _process outputs_ :outputs: - [[./output.org][output]] #+NAME: sdp_each_file_do_selected_output #+BEGIN_SRC d /+ ↓ output hub +/ if (!(_opt_action_bool["skip_output"])) { outputHub!()(doc_abstraction, doc_matters); } #+END_SRC *** scope (on loop exit) :scope:exit: #+NAME: sdp_each_file_do_scope_exit #+BEGIN_SRC d scope(exit) { debug(checkdoc) { writefln( "processed file: %s", fn_src ); } destroy(fn_src); } #+END_SRC ** 2b. no filename provided #+NAME: sdp_no_filename_provided #+BEGIN_SRC d /+ no recognized filename provided +/ writeln("no recognized filename"); break; // terminate, stop #+END_SRC * 2. _document abstraction functions_ ** 0. abstraction template [#A] :template: #+BEGIN_SRC d :tangle ../src/sdp/abstraction.d template SiSUabstraction() { <<imports_sdp>> <<imports_sdlang>> <<imports_std>> <<sdp_mixin>> enum headBody { header, body_content, insert_filelist } enum makeMeta { make, meta } enum docAbst { doc_abstraction, section_keys, segnames, images } auto rgx = Rgx(); auto SiSUabstraction(Fn,O,E)(Fn fn_src, O opts, E env){ <<sdp_conf_files>> <<sdp_each_file_do_read_and_split_sisu_markup_file_content_into_header_and_body>> <<sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta>> <<sdp_each_file_do_document_abstraction>> <<sdp_each_file_do_document_matters>> auto t = tuple(doc_abstraction, doc_matters); static assert(t.length==2); return t; } } #+END_SRC ** 1. (a) _read in raw file_ (b) split content into: _doc header & doc content_ - [[./ao_read_source_files.org][ao_read_source_files]] - read in the source marked up document and - split the document into: - document header - document body - if a master document make a list of insert files - return a tuple of: - header - body - insert file list #+NAME: sdp_each_file_do_read_and_split_sisu_markup_file_content_into_header_and_body #+BEGIN_SRC d /+ ↓ read file (filename with path) +/ /+ ↓ file tuple of header and content +/ auto _header_body_inserts = SiSUrawMarkupContent!()(fn_src); static assert(!isTypeTuple!(_header_body_inserts)); static assert(_header_body_inserts.length==3); debug(header_and_body) { writeln(header); writeln(_header_body_inserts.length); writeln(_header_body_inserts.length[headBody.body_content][0]); } #+END_SRC ** 2. _document metadata_ & _make instructions_ :doc:header:metadata:make: - [[./ao_conf_make_meta.org][ao_conf_make_meta]] - read document header, split into: - metadata - make instructions - read config files - consolidate make instructions - return tuple of: - document metadata - make instructions (from configuration files & document header make instructions) #+NAME: sdp_each_file_do_split_sisu_markup_file_header_into_make_and_meta #+BEGIN_SRC d /+ ↓ split header into make and meta +/ auto _make_and_meta = SiSUheaderExtractHub!()(_header_body_inserts[headBody.header], conf_doc_make_aa); static assert(!isTypeTuple!(_make_and_meta)); static assert(_make_and_meta.length==2); #+END_SRC ** 3. _document abstraction, tuple_ (pre-processing) :processing: - [[./ao_abstract_doc_source.org][ao_abstract_doc_source]] - prepare the document abstraction used in downstream processing - return tuple of: - document abstraction (the_document or doc_abstraction) - document abstraction keys - (head, toc, body, endnotes, glossary, bibliography, bookindex, blurb, tail) - (passed in doc_matters) - segnames for html epub (passed in doc_matters) - image list (passed in doc_matters) #+NAME: sdp_each_file_do_document_abstraction #+BEGIN_SRC d /+ ↓ document abstraction: process document, return abstraction as tuple +/ auto da = SiSUdocAbstraction!()( (_header_body_inserts[headBody.body_content]), (_make_and_meta[makeMeta.make]), (_make_and_meta[makeMeta.meta]), opts ); static assert(!isTypeTuple!(da)); static assert(da.length==4); auto doc_abstraction = da[docAbst.doc_abstraction]; // head ~ toc ~ body ~ endnotes_seg ~ glossary ~ bibliography ~ bookindex ~blurb; string[][string] _document_section_keys_sequenced = da[docAbst.section_keys]; string[] _doc_html_segnames = da[docAbst.segnames]; auto _images = da[docAbst.images]; #+END_SRC ** 4. _document matters_ (doc info gathered, various sources) - prepare document_matters, miscellany about processing and the document of use in downstream processing #+NAME: sdp_each_file_do_document_matters #+BEGIN_SRC d struct DocumentMatters { string[] keys_seq_seg() { string[] _k = _document_section_keys_sequenced["seg"]; return _k; } string[] keys_seq_scroll() { string[] _k = _document_section_keys_sequenced["scroll"]; return _k; } string[] segnames() { string[] _k = _doc_html_segnames; return _k; } auto dochead_make() { string[string][string] _k = _make_and_meta[makeMeta.make]; return _k; } auto dochead_meta() { string[string][string] _k = _make_and_meta[makeMeta.meta]; return _k; } auto source_filename() { string _k = fn_src; return _k; } auto language() { string _k; if (auto m = match(fn_src, rgx.language_code_and_filename)) { _k = m.captures[1]; } else { _k = "en"; } return _k; } auto file_insert_list() { string[] _k = _header_body_inserts[headBody.insert_filelist]; return _k; } auto image_list() { auto _k = _images; return _k; } auto opt_action_bool() { bool[string] _k = opts; return _k; } auto environment() { auto _k = env; return _k; } } auto doc_matters = DocumentMatters(); #+END_SRC * 3. document abstraction _summary_ :summary:doc: ** 0. template: :template: #+BEGIN_SRC d :tangle ../src/sdp/abstraction_summary.d template SiSUabstractionSummary() { auto SiSUabstractionSummary(S,T)( auto return ref const S doc_abstraction, auto return ref T doc_matters, ) { <<abstraction_summary_imports>> mixin InternalMarkup; <<abstraction_summary_initialize>> if (doc_matters.opt_action_bool["verbose"]) { <<ao_abstraction_summary>> } } } #+END_SRC ** init *** imports #+name: abstraction_summary_imports #+BEGIN_SRC d import ao_defaults, ao_rgx; import std.array, std.exception, std.stdio, std.regex, std.string, std.traits, std.typecons, std.uni, std.utf, std.conv : to; #+END_SRC *** initialize :report: #+name: abstraction_summary_initialize #+BEGIN_SRC d auto markup = InlineMarkup(); #+END_SRC ** (last ocn) #+name: ao_abstraction_summary #+BEGIN_SRC d string[string] check = [ "last_obj_cite_number" : "NA [debug \"checkdoc\" not run]", ]; foreach (k; doc_matters.keys_seq_seg) { foreach (obj; doc_abstraction[k]) { if (obj.use != "empty") { if (!empty(obj.obj_cite_number)) { check["last_obj_cite_number"] = obj.obj_cite_number; } } } } #+END_SRC ** summary #+name: ao_abstraction_summary #+BEGIN_SRC d // auto char_repeat_number = doc_matters.source_filename.length; // char_repeat_number = (char_repeat_number > 33) auto min_repeat_number = 66; auto char_repeat_number = (doc_matters.dochead_meta["title"]["full"].length + doc_matters.dochead_meta["creator"]["author"].length + 4); char_repeat_number = (char_repeat_number > min_repeat_number) ? char_repeat_number : min_repeat_number; // writeln(char_repeat_number); writefln( "%s\n\"%s\", %s\n%s\n%s\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n%s%10d\n(%s: %s)\n%s", markup.repeat_character_by_number_provided("-", char_repeat_number), doc_matters.dochead_meta["title"]["full"], doc_matters.dochead_meta["creator"]["author"], doc_matters.source_filename, markup.repeat_character_by_number_provided("-", char_repeat_number), "length toc arr: ", to!int(doc_abstraction["toc_seg"].length), "length doc_abstraction arr: ", to!int(doc_abstraction["body"].length), "last obj_cite_number: ", to!int(check["last_obj_cite_number"]), "length endnotes: ", (doc_abstraction["endnotes"].length > 1) ? (to!int(doc_abstraction["endnotes"].length)) : 0, "length glossary: ", (doc_abstraction["glossary"].length > 1) ? (to!int(doc_abstraction["glossary"].length)) : 0, "length biblio: ", (doc_abstraction["bibliography"].length > 1) ? (to!int(doc_abstraction["bibliography"].length)) : 0, "length bookindex: ", (doc_abstraction["bookindex_seg"].length > 1) ? (to!int(doc_abstraction["bookindex_seg"].length)) : 0, "length blurb: ", (doc_abstraction["blurb"].length > 1) ? (to!int(doc_abstraction["blurb"].length)) : 0, __FILE__, __LINE__, markup.repeat_character_by_number_provided("-", min_repeat_number), ); #+END_SRC markup.repeat_character_by_number_provided("-", 10) markup.repeat_character_by_number_provided("-", (doc_matters.dochead_meta["title"]["full"].length)) markup.repeat_character_by_number_provided("-", (doc_matters.source_filename.length))