/+
- Name: SisuDoc Spine, Doc Reform [a part of]
- Description: documents, structuring, processing, publishing, search
- static content generator
- Author: Ralph Amissah
[ralph.amissah@gmail.com]
- Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
- License: AGPL 3 or later:
Spine (SiSU), a framework for document structuring, publishing and
search
Copyright (C) Ralph Amissah
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU AFERO General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
You should have received a copy of the GNU General Public License along with
this program. If not, see [https://www.gnu.org/licenses/].
If you have Internet connection, the latest version of the AGPL should be
available at these locations:
[https://www.fsf.org/licensing/licenses/agpl.html]
[https://www.gnu.org/licenses/agpl.html]
- Spine (by Doc Reform, related to SiSU) uses standard:
- docReform markup syntax
- standard SiSU markup syntax with modified headers and minor modifications
- docReform object numbering
- standard SiSU object citation numbering & system
- Homepages:
[https://www.sisudoc.org]
[https://www.doc-reform.org]
- Git
[https://git.sisudoc.org/]
+/
module sisudoc.outputs.io_out.sqlite_collection_db;
import sisudoc.outputs.io_out;
import sisudoc.outputs.io_out.rgx;
import sisudoc.outputs.io_out.rgx_xhtml;
import std.file;
import std.uri;
// import std.digest.sha;
import std.conv : to;
import std.typecons : Nullable;
import d2sqlite3;
mixin spineRgxOut;
mixin spineRgxXHTML;
mixin InternalMarkup;
static auto rgx = RgxO();
static auto rgx_xhtml = RgxXHTML();
static auto mkup = InlineMarkup();
long _metadata_tid_lastrowid;
/+ ↓ sqlite db schema version, stamped into the db as PRAGMA user_version
bump whenever the table or index definitions in this module change, and
bump the matching expectation in sisudoc-spine-search-cgi
+/
enum sqlite_db_schema_version = 1;
/+ ↓ tally of failed sqlite statements, shared: sqlite output may run in parallel +/
private shared int _sqlite_failure_tally = 0;
/+ ↓ schema version mismatch is reported once per run, not once per document +/
private shared int _sqlite_schema_mismatch_reported = 0;
/+ ↓ record a failed sqlite statement, reported on stderr, tallied for exit status +/
void sqliteFailureRecord(string note, string msg) {
import core.atomic : atomicOp;
atomicOp!"+="(_sqlite_failure_tally, 1);
stderr.writeln("ERROR SQLite (", note, "): ", msg);
}
/+ ↓ count of failed sqlite statements this run, zero when all went well +/
int sqliteFailureTally() {
import core.atomic : atomicLoad;
return atomicLoad(_sqlite_failure_tally);
}
/+ ↓ compare the schema version stamped in an existing db with this spine's +/
void sqliteSchemaVersionCheck(Db)(Db db, string sqlite_file) {
import core.atomic : atomicOp;
int _found;
try {
_found = db.execute("PRAGMA user_version;").oneValue!int;
} catch (Exception) {
return;
}
if (_found == sqlite_db_schema_version
|| _found == 0) { /+ ↓ zero: db predates the stamp, or has no tables yet +/
return;
}
if (atomicOp!"+="(_sqlite_schema_mismatch_reported, 1) == 1) {
stderr.writeln("ERROR SQLite : db schema version mismatch: ", sqlite_file);
stderr.writeln(" db holds schema version ", _found,
", this spine writes schema version ", sqlite_db_schema_version);
stderr.writeln(" recreate the db, e.g.: spine --sqlite-db-recreate",
" --sqlite-db-path=... --sqlite-db-filename=...");
sqliteFailureRecord("schema version",
"db schema version " ~ _found.to!string
~ " != spine schema version " ~ sqlite_db_schema_version.to!string);
}
}
template SQLiteHubBuildTablesAndPopulate() {
void SQLiteHubBuildTablesAndPopulate(D)(D doc) {
auto pth_sqlite = spinePathsSQLite!()(doc.matters.sqlite.filename, doc.matters.sqlite.path);
if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) {
} else {
try {
pth_sqlite.base.mkdirRecurse;
} catch (FileException ex) {
stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base);
stderr.writeln(" (", ex.msg, ")");
import core.runtime;
core.runtime.Runtime.terminate();
}
}
template SQLiteDbStatementComposite() {
void SQLiteDbStatementComposite(Db,D)(
Db db,
D doc
) {
string _db_statement;
if ((doc.matters.opt.action.sqlite_db_create)) {
auto pth_sqlite = spinePathsSQLite!()(doc.matters.sqlite.filename, doc.matters.sqlite.path);
if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) {
} else {
try {
pth_sqlite.base.mkdirRecurse;
} catch (FileException ex) {
stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base);
stderr.writeln(" (", ex.msg, ")");
import core.runtime;
core.runtime.Runtime.terminate();
}
}
_db_statement ~= SQLiteTablesReCreate!()();
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "TABLE RE-CREATE");
_db_statement = [];
}
if (doc.matters.opt.action.sqlite_delete) {
_db_statement ~= SQLiteDeleteDocument!()(doc.matters);
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "DELETE Document");
_db_statement = [];
}
if (doc.matters.opt.action.sqlite_update) {
_db_statement ~= SQLiteDeleteDocument!()(doc.matters);
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "DELETE Document");
_db_statement = [];
_db_statement ~= SQLiteInsertMetadata!()(doc.matters);
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT MetaData");
_db_statement = [];
/+ get tid (lastrowid or max) for use in doc.objects table +/
_db_statement ~= doc.SQLiteInsertDocObjectsLoop!();
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT DocObjects");
_db_statement = [];
_db_statement ~= SQLiteInsertMetadataTopics!()(doc.matters);
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "INSERT MetaDataTopics");
_db_statement = [];
}
db.close;
if (doc.matters.opt.action.vox_gt_1) {
writeln(" ", pth_sqlite.sqlite_file);
}
}
}
try {
auto db = Database(pth_sqlite.sqlite_file);
sqliteSchemaVersionCheck(db, pth_sqlite.sqlite_file);
SQLiteDbStatementComposite!()(db, doc);
}
catch (FileException e) {
writeln("Failed (FileException): ", e.msg, " ", pth_sqlite.sqlite_file);
writeln(e.file, " line: ", e.line);
import core.runtime;
core.runtime.Runtime.terminate();
}
catch (ErrnoException e) {
writeln("Failed (ErrnoException): ", e.msg, " ", pth_sqlite.sqlite_file);
writeln(e.file, " line: ", e.line);
import core.runtime;
core.runtime.Runtime.terminate();
}
catch (Exception e) {
writeln("Failed (Exception): ", e.msg, " ", pth_sqlite.sqlite_file);
writeln(e.file, " line: ", e.line);
import core.runtime;
core.runtime.Runtime.terminate();
}
catch (Throwable) {
writeln("Failed (Trowable): ", pth_sqlite.sqlite_file);
import core.runtime;
core.runtime.Runtime.terminate();
}
}
}
template SQLiteHubDiscreteBuildTablesAndPopulate() {
void SQLiteHubDiscreteBuildTablesAndPopulate(D)(D doc) {
auto url_html = spineUrlsHTML!()(doc.matters.conf_make_meta.conf.w_srv_data_root_url_html, doc.matters.src.language);
auto pth_sqlite = spinePathsSQLiteDiscrete!()(doc.matters.output_path, doc.matters.src.language); // doc.matters.db_path
if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) {
} else {
try {
pth_sqlite.base.mkdirRecurse;
} catch (FileException ex) {
stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base);
stderr.writeln(" (", ex.msg, ")");
import core.runtime;
core.runtime.Runtime.terminate();
}
}
auto db = Database(pth_sqlite.sqlite_file(doc.matters.src.filename));
sqliteSchemaVersionCheck(db, pth_sqlite.sqlite_file(doc.matters.src.filename));
template SQLiteDiscreteDbStatementComposite() {
void SQLiteDiscreteDbStatementComposite(Db,D)(
Db db,
D doc
) {
try {
{
string _db_statement;
_db_statement ~= SQLiteTablesReCreate!()();
_db_statement ~= SQLiteInsertMetadata!()(doc.matters);
_db_statement ~= SQLiteInsertMetadataTopics!()(doc.matters);
_db_statement ~= doc.SQLiteInsertDocObjectsLoop!();
SQLiteDbRun!()(db, _db_statement, doc.matters.opt.action, "table CREATE Tables, INSERT DocObjects");
}
db.close;
}
catch (FileException e) {
writeln("Failed (FileException): ", e.msg);
writeln(e.file, " line: ", e.line);
import core.runtime;
core.runtime.Runtime.terminate();
}
catch (ErrnoException e) {
writeln("Failed (ErrnoException): ", e.msg);
writeln(e.file, " line: ", e.line);
import core.runtime;
core.runtime.Runtime.terminate();
}
catch (Exception e) {
writeln("Failed (Exception): ", e.msg);
writeln(e.file, " line: ", e.line);
import core.runtime;
core.runtime.Runtime.terminate();
}
catch (Throwable) {
import core.runtime;
core.runtime.Runtime.terminate();
}
if (doc.matters.opt.action.vox_gt_1) {
writeln(" ", pth_sqlite.sqlite_file(doc.matters.src.filename));
}
}
}
SQLiteDiscreteDbStatementComposite!()(db, doc);
}
}
template SQLiteDbRun() {
void SQLiteDbRun(Db,St,O)(
Db db,
St db_statement,
O opt_action,
string note,
) {
debug(sql_statement) {
writeln(db_statement);
}
try {
db.run(
"\nBEGIN TRANSACTION;\n" ~
db_statement ~
"\nCOMMIT TRANSACTION;\n"
);
} catch (ErrnoException ex) {
sqliteFailureRecord(note, ex.msg);
} catch (Exception ex) {
sqliteFailureRecord(note, ex.msg);
}
{ /+ debug +/
if (opt_action.debug_do_sqlite) {
writeln(note);
if (opt_action.vox_gt_3) {
writeln(db_statement);
}
}
}
}
}
template SQLinsertDelimiter() {
string SQLinsertDelimiter(string _txt) {
_txt = _txt
.replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0");
return _txt;
}
}
template SQLiteFormatAndLoadObject() {
auto SQLiteFormatAndLoadObject(M)(M doc_matters) {
mixin spineRgxOut;
mixin spineRgxXHTML;
struct sqlite_format_and_load_objects {
string generic_munge_sanitize_text_for_search(
string _txt,
) {
string _notes;
string _urls;
if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
_notes ~= "\n" ~ m["text"];
}
_txt = _txt.replaceAll(rgx.inline_notes_al_gen, "");
}
if (_txt.matchFirst(rgx.inline_link)) {
foreach (m; _txt.matchAll(rgx.inline_link)) {
if (m["link"].match(rgx.url)) {
_urls ~= "\n" ~ m["link"];
}
}
_txt = _txt.replaceAll(rgx.inline_link_clean, "");
}
if (_notes.length > 0) {
_txt ~= _notes;
}
if (_urls.length > 0) {
_txt ~= _urls;
}
{ /+ debug +/
if (doc_matters.opt.action.debug_do_sqlite) {
writeln(_txt, "\n");
}
}
debug(sql_text_clean) {
writeln(_txt);
}
return _txt;
}
string munge_html(M,O)(
M doc_matters,
const O obj,
) {
string _html_special_characters(string _txt){
_txt = _txt
.replaceAll(rgx_xhtml.ampersand, "&")
.replaceAll(rgx_xhtml.quotation, """)
.replaceAll(rgx_xhtml.less_than, "<")
.replaceAll(rgx_xhtml.greater_than, ">")
.replaceAll(rgx.nbsp_char, " ")
.replaceAll(rgx.br_line_inline, "
")
.replaceAll(rgx.br_line, "
")
.replaceAll(rgx.br_line_spaced, "
")
.replaceAll(rgx_xhtml.line_break, "
");
return _txt;
}
string _html_font_face(string _txt){
_txt = _txt
.replaceAll(rgx.inline_emphasis, "$1")
.replaceAll(rgx.inline_bold, "$1")
.replaceAll(rgx.inline_underscore, "$1")
.replaceAll(rgx.inline_italics, "$1")
.replaceAll(rgx.inline_superscript, "$1")
.replaceAll(rgx.inline_subscript, "$1")
.replaceAll(rgx.inline_strike, "$1")
.replaceAll(rgx.inline_insert, "$1")
.replaceAll(rgx.inline_mono, "$1")
.replaceAll(rgx.inline_cite, "$1");
return _txt;
}
string _notes;
string _urls;
string _txt = _html_font_face(_html_special_characters(obj.text));
{ /+ debug +/
if (doc_matters.opt.action.debug_do_sqlite) {
writeln(_txt, "\n");
}
}
return _txt;
}
string html_special_characters(string _txt){
_txt = _txt
.replaceAll(rgx_xhtml.ampersand, "&")
.replaceAll(rgx_xhtml.quotation, """)
.replaceAll(rgx_xhtml.less_than, "<")
.replaceAll(rgx_xhtml.greater_than, ">")
.replaceAll(rgx.nbsp_char, " ")
.replaceAll(rgx.br_line_inline, "
")
.replaceAll(rgx.br_line, "
")
.replaceAll(rgx.br_line_spaced, "
")
.replaceAll(rgx_xhtml.line_break, "
");
return _txt;
}
string html_special_characters_code(string _txt){
_txt = _txt
.replaceAll(rgx_xhtml.ampersand, "&")
.replaceAll(rgx_xhtml.quotation, """)
.replaceAll(rgx_xhtml.less_than, "<")
.replaceAll(rgx_xhtml.greater_than, ">")
.replaceAll(rgx.nbsp_char, " ");
return _txt;
}
string html_font_face(string _txt){
_txt = _txt
.replaceAll(rgx.inline_emphasis, "$1")
.replaceAll(rgx.inline_bold, "$1")
.replaceAll(rgx.inline_underscore, "$1")
.replaceAll(rgx.inline_italics, "$1")
.replaceAll(rgx.inline_superscript, "$1")
.replaceAll(rgx.inline_subscript, "$1")
.replaceAll(rgx.inline_strike, "$1")
.replaceAll(rgx.inline_insert, "$1")
.replaceAll(rgx.inline_mono, "$1")
.replaceAll(rgx.inline_cite, "$1");
return _txt;
}
string inline_grouped_text_bullets_indents(M,O)(
M doc_matters,
const O obj,
string _txt,
string _suffix = ".html",
string _xml_type = "seg",
) {
static auto rgx = RgxO();
static auto rgx_xhtml = RgxXHTML();
if (obj.metainfo.is_a == "group") {
_txt = (_txt)
.replaceAll(rgx.grouped_para_indent_1,
" ")
.replaceAll(rgx.grouped_para_indent_2,
" ")
.replaceAll(rgx.grouped_para_indent_3,
" ")
.replaceAll(rgx.grouped_para_indent_4,
" ")
.replaceAll(rgx.grouped_para_indent_5,
" ")
.replaceAll(rgx.grouped_para_indent_6,
" ")
.replaceAll(rgx.grouped_para_indent_7,
" ")
.replaceAll(rgx.grouped_para_indent_8,
" ")
.replaceAll(rgx.grouped_para_indent_9,
" ")
.replaceAll(rgx.grouped_para_indent_hang, " ")
.replaceAll(rgx.grouped_para_bullet, "● ")
.replaceAll(rgx.grouped_para_bullet_indent_1,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_2,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_3,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_4,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_5,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_6,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_7,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_8,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_9,
" ● ");
}
return _txt;
}
string inline_images(M,O)(
M doc_matters,
const O obj,
string _txt,
string _suffix = ".html",
string _xml_type = "seg",
) {
string _img_pth;
if (_xml_type == "epub") {
_img_pth = "image/";
} else if (_xml_type == "scroll") {
_img_pth = "../../image/";
} else if (_xml_type == "seg") {
_img_pth = "../../../image/";
}
if (_txt.match(rgx.inline_image)) {
_txt = replaceAll!(m =>
m["pre"].to!string
~ _img_element(_img_pth ~ m["img"].to!string, m["width"].to!string,
m["height"].to!string, _image_alt(m["post"].to!string))
~ " " ~ _image_rest(m["post"].to!string)
)(_txt, rgx.inline_image);
}
return _txt;
}
/+ ↓ the same treatment of an image as the html and epub writers give
it, because this is the html the cgi search renders: the markup's
description is the alt text, and dimensions that are not there are
not written as zeroes.
+/
string _image_alt(string _post) {
if (auto m = _post.matchFirst(rgx.inline_image_alt)) {
return m["alt"].to!string;
}
return "";
}
string _image_rest(string _post) {
if (auto m = _post.matchFirst(rgx.inline_image_alt)) {
return m.post.to!string;
}
return _post;
}
string _img_element(string _src, string _w, string _h, string _alt) {
string _dim = (_w == "0" && _h == "0")
? ""
: " width=\"" ~ _w ~ "\" height=\"" ~ _h ~ "\"";
return "";
}
string inline_links(M,O)(
M doc_matters,
const O obj,
string _txt,
string _xml_type = "seg",
) {
if (obj.has.inline_links) {
if (obj.metainfo.is_a != "code") {
_txt = replaceAll!(m =>
m["linked_text"] ~ "┤" ~ to!string((obj.stow.link[m["num"].to!ulong])).encode ~ "├"
)(_txt, rgx.inline_link_number_only);
}
if ((_txt.match(rgx.mark_internal_site_lnk))
&& (_xml_type == "scroll")) { // conditions reversed to avoid: gdc compiled program run segfault
_txt = _txt.replaceAll(
rgx.inline_seg_link,
"$1");
}
auto pth_html = spinePathsHTML!()(doc_matters.output_path, doc_matters.src.language);
if (_xml_type == "seg") {
foreach (m; _txt.matchAll(rgx.inline_link_seg_and_hash)) {
if (m.captures["hash"] in doc_matters.has.tag_associations) {
if (m.captures["hash"] == doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"]) {
_txt = _txt.replaceFirst(
rgx.inline_link_seg_and_hash,
"┥$1┝┤"
~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html
~ "/"
~ pth_html.tail_fn_seg(doc_matters.src.filename, "$2.html")
~ "├"
);
} else {
_txt = _txt.replaceFirst(
rgx.inline_link_seg_and_hash,
"┥$1┝┤"
~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html
~ "/"
~ doc_matters.src.filename_base
~ "/"
~ doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"]
~ ".html"
~ "#" ~ m.captures["hash"]
~ "├"
);
}
} else {
if (doc_matters.opt.action.vox_gt_1) {
writeln(
"WARNING on internal document links, anchor to link <<"
~ m.captures["hash"]
~ ">> not found in document, "
~ "anchor: " ~ m.captures["hash"]
~ " document: " ~ doc_matters.src.filename
);
}
}
}
} else {
if (auto m = _txt.match(rgx.inline_link_seg_and_hash)) {
_txt = _txt.replaceFirst(
rgx.inline_link_seg_and_hash,
"┥$1┝┤"
~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html
~ "/"
~ pth_html.tail_fn_scroll(doc_matters.src.filename)
~ "#" ~ m.captures["hash"]
~ "├"
);
}
}
_txt = _txt
.replaceAll(
rgx.inline_link_fn_suffix,
("$1.html"))
.replaceAll(
rgx.inline_link,
("$1"))
.replaceAll(
rgx.mark_internal_site_lnk,
"");
}
debug(markup_links) {
if (_txt.match(rgx.inline_link)) {
writeln(__LINE__,
" (missed) markup link identified (",
obj.has.inline_links,
"): ", obj.metainfo.is_a, ": ",
obj.text
);
}
// if (obj.metainfo.is_a == "bookindex") { // DEBUG LINE
// if (_txt.match(regex(r""
~ "" ~ m.captures["num"] ~ "."
~ m.captures["note"]
~ "
%s
┃", obj.metainfo.is_a, _txt, ); return o; } string html_para(M,O)( M doc_matters, const O obj, ) { assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter"); assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb"); assert(obj.metainfo.is_of_type == "para"); assert(obj.metainfo.is_a == "para" || "toc" || "endnote" || "glossary" || "bibliography" || "bookindex" || "blurb"); string _txt = munge_html(doc_matters, obj); _txt = (obj.attrib.bullet) ? ("● " ~ _txt) : _txt; _txt = inline_markup(doc_matters, obj, _txt); string o = format(q"┃%s
┃", obj.metainfo.is_a, obj.attrib.indent_hang, obj.attrib.indent_base, _txt ); return o; } string html_quote(M,O)( M doc_matters, const O obj, ) { assert(obj.metainfo.is_of_part == "body"); assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); assert(obj.metainfo.is_of_type == "block"); assert(obj.metainfo.is_a == "quote"); string _txt = munge_html(doc_matters, obj); _txt = inline_markup(doc_matters, obj, _txt); string o = format(q"┃%s
┃", obj.metainfo.is_a, _txt ); return o; } string html_group(M,O)( M doc_matters, const O obj, ) { assert(obj.metainfo.is_of_part == "body"); assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); assert(obj.metainfo.is_of_type == "block"); assert(obj.metainfo.is_a == "group"); string _txt = munge_html(doc_matters, obj); _txt = inline_markup(doc_matters, obj, _txt); string o = format(q"┃%s
┃", obj.metainfo.is_a, _txt ); return o; } string html_block(M,O)( M doc_matters, const O obj, ) { assert(obj.metainfo.is_of_part == "body"); assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); assert(obj.metainfo.is_of_type == "block"); assert(obj.metainfo.is_a == "block"); string _txt = munge_html(doc_matters, obj); _txt = inline_markup(doc_matters, obj, _txt); string o = format(q"┃%s
┃", obj.metainfo.is_a, _txt.stripRight ); return o; } string html_verse(M,O)( M doc_matters, const O obj, ) { assert(obj.metainfo.is_of_part == "body"); assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb"); assert(obj.metainfo.is_of_type == "block"); assert(obj.metainfo.is_a == "verse"); string _txt = munge_html(doc_matters, obj); _txt = inline_markup(doc_matters, obj, _txt); string o = format(q"┃%s
┃", obj.metainfo.is_a, _txt ); return o; } string html_code(O)( const O obj, ) { assert(obj.metainfo.is_of_part == "body"); assert(obj.metainfo.is_of_section == "body"); assert(obj.metainfo.is_of_type == "block"); assert(obj.metainfo.is_a == "code"); string _txt = html_special_characters_code(obj.text); string o = format(q"┃%s
┃", obj.metainfo.is_a, _txt ); return o; } string html_table(M,O)( M doc_matters, const O obj, ) { assert(obj.metainfo.is_of_part == "body"); assert(obj.metainfo.is_of_section == "body"); assert(obj.metainfo.is_of_type == "block"); assert(obj.metainfo.is_a == "table"); Tuple!(string, string) _tablarize(O)( const O obj, string _txt, ) { string[] _table_rows = _txt.split(rgx.table_delimiter_row); string[] _table_cols; string _table; string _tablenote; bool _in_head = false; bool _in_body = false; foreach(row_idx, row; _table_rows) { _table_cols = row.split(rgx.table_delimiter_col); if ((_table_cols.length == 1) && (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx) _tablenote ~= _table_cols[0]; continue; } /+ ↓ a heading row goes in , the rest in +/ if (row_idx == 0 && obj.table.heading) { _table ~= ""; _in_head = true; } else if (!_in_body) { if (_in_head) { _table ~= ""; _in_head = false; } _table ~= ""; _in_body = true; } _table ~= "" ~ _note.strip ~ "
") ); return o; } string sqlite_load_string(M,O)( M doc_matters, const O obj, ) { string o; return o; } string sqlite_statement(O)( const O obj, string _txt, string _html, ) { void _sql_exe(O)( string _sql, ) { writeln(_html); writeln(_sql); } string _sql; return _sql; } string[string] heading(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_heading(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] para(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_para(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] quote(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_quote(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] group(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_group(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] block(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_block(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] verse(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_verse(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] code(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_code(obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } string[string] table(M,O)( M doc_matters, const O obj, ) { string[string] obj_txt = [ "text": generic_munge_sanitize_text_for_search(obj.text), "html": html_table(doc_matters, obj) ]; { /+ debug +/ if (doc_matters.opt.action.debug_do_sqlite) { debug(sql_txt) { writeln(obj_txt["text"]); } debug(sql_html) { writeln(obj_txt["html"]); } } else { // load sql } } return obj_txt; } } return sqlite_format_and_load_objects(); } } template SQLiteTablesReCreate() { string SQLiteTablesReCreate()() { string _sql_instruct; _sql_instruct = format(q"┃ DROP INDEX IF EXISTS idx_ocn; DROP INDEX IF EXISTS idx_uid; DROP INDEX IF EXISTS idx_digest_clean; DROP INDEX IF EXISTS idx_digest_all; DROP INDEX IF EXISTS idx_clean; DROP INDEX IF EXISTS idx_title; DROP INDEX IF EXISTS idx_author; DROP INDEX IF EXISTS src_filename_base; DROP INDEX IF EXISTS idx_language_document_char; DROP INDEX IF EXISTS idx_classify_topic_register; DROP INDEX IF EXISTS idx_topic_list; DROP TABLE IF EXISTS metadata_and_text; DROP TABLE IF EXISTS topic_register; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE IF NOT EXISTS metadata_and_text ( uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, title_short VARCHAR(400) NULL, title_edition VARCHAR(10) NULL, title_language VARCHAR(100) NULL, title_language_char VARCHAR(6) NULL, creator_author VARCHAR(600) NOT NULL, creator_author_last_first VARCHAR(600) NOT NULL, creator_author_email VARCHAR(100) NULL, creator_author_hon VARCHAR(100) NULL, creator_author_nationality VARCHAR(100) NULL, creator_editor VARCHAR(600) NULL, creator_contributor VARCHAR(600) NULL, creator_illustrator VARCHAR(600) NULL, creator_photographer VARCHAR(600) NULL, creator_translator VARCHAR(600) NULL, creator_prepared_by VARCHAR(600) NULL, creator_digitized_by VARCHAR(600) NULL, creator_audio VARCHAR(600) NULL, creator_video VARCHAR(600) NULL, language_document VARCHAR(100) NULL, language_document_char VARCHAR(6) NOT NULL, language_original VARCHAR(100) NULL, language_original_char VARCHAR(6) NULL, date_added_to_site VARCHAR(10) NULL, date_available VARCHAR(10) NULL, date_created VARCHAR(10) NULL, date_issued VARCHAR(10) NULL, date_modified VARCHAR(10) NULL, date_published VARCHAR(10) NULL, date_valid VARCHAR(10) NULL, date_translated VARCHAR(10) NULL, date_original_publication VARCHAR(10) NULL, date_generated VARCHAR(10) NULL, original_title VARCHAR(800) NULL, original_publisher VARCHAR(600) NULL, original_language VARCHAR(100) NULL, original_language_char VARCHAR(6) NULL, original_source VARCHAR(600) NULL, original_institution VARCHAR(600) NULL, original_nationality VARCHAR(100) NULL, rights_copyright VARCHAR(2500) NULL, rights_copyright_audio VARCHAR(2500) NULL, rights_copyright_cover VARCHAR(2500) NULL, rights_copyright_illustrations VARCHAR(2500) NULL, rights_copyright_photographs VARCHAR(2500) NULL, rights_copyright_text VARCHAR(2500) NULL, rights_copyright_translation VARCHAR(2500) NULL, rights_copyright_video VARCHAR(2500) NULL, rights_license VARCHAR(2500) NULL, identifier_oclc VARCHAR(30) NULL, identifier_isbn VARCHAR(16) NULL, classify_topic_register VARCHAR(2500) NULL, classify_subject VARCHAR(600) NULL, classify_loc VARCHAR(30) NULL, classify_dewey VARCHAR(30) NULL, classify_keywords VARCHAR(600) NULL, notes_abstract TEXT NULL, notes_description TEXT NULL, notes_comment TEXT NULL, notes_coverage VARCHAR(200) NULL, notes_relation VARCHAR(200) NULL, notes_history VARCHAR(600) NULL, notes_type VARCHAR(600) NULL, notes_format VARCHAR(600) NULL, notes_prefix TEXT NULL, notes_prefix_a TEXT NULL, notes_prefix_b TEXT NULL, notes_suffix TEXT NULL, publisher VARCHAR(600) NULL, src_filename_base VARCHAR(256) NOT NULL, src_filename_suffix VARCHAR(6) NOT NULL, src_fingerprint VARCHAR(256) NULL, src_filesize VARCHAR(10) NULL, src_wordcount VARCHAR(10) NULL, pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here site_url_doc_root VARCHAR(256) NULL, -- url path to doc root site_url_html_toc VARCHAR(256) NULL, site_url_html_scroll VARCHAR(256) NULL, site_url_epub VARCHAR(256) NULL, links TEXT NULL ); CREATE TABLE IF NOT EXISTS topic_register ( -- tid BIGINT PRIMARY KEY, uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename topic_register VARCHAR(250) NOT NULL, site_url_doc_root VARCHAR(256) NULL, -- url path to doc root site_url_html_toc VARCHAR(256) NULL, site_url_html_scroll VARCHAR(256) NULL ); CREATE TABLE IF NOT EXISTS doc_objects ( lid BIGINT PRIMARY KEY, uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, ocn SMALLINT, obj_id VARCHAR(6) NULL, clean TEXT NULL, body TEXT NULL, seg VARCHAR(256) NULL, lev_an VARCHAR(1), is_of_type VARCHAR(16), is_a VARCHAR(16), lev SMALLINT NULL, node VARCHAR(16) NULL, -- not written to parent VARCHAR(16) NULL, last_descendant VARCHAR(16) NULL, -- headings only digest_clean CHAR(256), digest_all CHAR(256), seg_name CHAR(256), types CHAR(1) NULL ); CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn); CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean); CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all); CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean); CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title); CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first); CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid); CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base); CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char); CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register); CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register); PRAGMA user_version = %s; ┃", sqlite_db_schema_version, ); return _sql_instruct; } } template SQLiteDeleteDocument() { string SQLiteDeleteDocument(M)(M doc_matters) { string _uid = doc_matters.src.doc_uid; string _delete_uid = format(q"┃ DELETE FROM metadata_and_text WHERE uid = '%s'; DELETE FROM doc_objects WHERE uid_metadata_and_text = '%s'; ┃", _uid, _uid, ); return _delete_uid; } } template SQLiteInsertMetadata() { string SQLiteInsertMetadata(M)(M doc_matters) { string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); string _insert_metadata = format(q"┃ INSERT INTO metadata_and_text ( uid, src_filename_base, src_filename_suffix, src_composite_id_per_txt, src_composite_id_per_pod, title, title_main, title_sub, title_short, title_edition, title_language, creator_author, creator_author_last_first, creator_author_email, creator_illustrator, creator_translator, language_document, language_document_char, date_added_to_site, date_available, date_created, date_issued, date_modified, date_published, date_valid, rights_copyright, rights_copyright_audio, rights_copyright_cover, rights_copyright_illustrations, rights_copyright_photographs, rights_copyright_text, rights_copyright_translation, rights_copyright_video, rights_license, identifier_oclc, identifier_isbn, classify_dewey, classify_keywords, classify_loc, classify_subject, classify_topic_register, original_title, original_publisher, original_language, original_language_char, original_source, notes_abstract, notes_description, publisher, site_url_doc_root ) VALUES ( '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' ); ┃", _uid, SQLinsertDelimiter!()(doc_matters.src.filename_base), SQLinsertDelimiter!()(doc_matters.src.filename_extension), SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc), SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_surname_fn), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source), SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher), SQLinsertDelimiter!()(doc_matters.conf_make_meta.conf.w_srv_data_root_url_html) ); return _insert_metadata; } } template SQLiteInsertMetadataTopics() { string SQLiteInsertMetadataTopics(M)(M doc_matters) { string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid); string[] _insert_topics; foreach (topic_line; doc_matters.conf_make_meta.meta.classify_topic_register_expanded_arr) { // writeln(topic_line); _insert_topics ~= format(q"┃ INSERT INTO topic_register ( uid_metadata_and_text, topic_register ) VALUES ( '%s', '%s' ); ┃", _uid, SQLinsertDelimiter!()(topic_line) ); } return _insert_topics.join.to!(char[]).toUTF8; } } template SQLiteInsertDocObjectsLoop() { string SQLiteInsertDocObjectsLoop(D)(D doc) { string _uid = SQLinsertDelimiter!()(doc.matters.src.doc_uid); auto url_html = spineUrlsHTML!()(doc.matters.conf_make_meta.conf.w_srv_data_root_url_html, doc.matters.src.language); string insertDocObjectsRow(O)(O obj) { string _insert_doc_objects_row = format(q"┃ INSERT INTO doc_objects ( uid_metadata_and_text, ocn, obj_id, clean, body, lev, is_of_type, is_a, seg_name ) VALUES ( '%s', %s, '%s', '%s', '%s', %s, '%s', '%s', '%s' ); ┃", _uid, obj.metainfo.ocn, obj.metainfo.identifier, SQLinsertDelimiter!()(obj_txt["text"]), SQLinsertDelimiter!()(obj_txt["html"]), obj.metainfo.heading_lev_markup, obj.metainfo.is_of_type, obj.metainfo.is_a, obj.tags.html_segment_anchor_tag_is ); return _insert_doc_objects_row; } auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc.matters); string[string] obj_txt; string doc_text; string[] _insert_doc_objects; foreach (part; doc.matters.has.keys_seq.sql) { foreach (obj; doc.abstraction[part]) { switch (obj.metainfo.is_of_part) { case "frontmatter": assert(part == "head", part); switch (obj.metainfo.is_of_type) { case "para": switch (obj.metainfo.is_a) { case "heading": obj_txt = format_and_sqlite_load.heading(doc.matters, obj); break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); } } break; } break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); } } break; } break; case "body": // assert(part == "body", part); switch (obj.metainfo.is_of_type) { case "para": switch (obj.metainfo.is_a) { case "heading": debug (asserts) { if (part != "body") { writeln(__LINE__, ": ", obj.text); } } obj_txt = format_and_sqlite_load.heading(doc.matters, obj); break; case "para": obj_txt = format_and_sqlite_load.para(doc.matters, obj); break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); } } break; } break; case "block": switch (obj.metainfo.is_a) { case "quote": obj_txt = format_and_sqlite_load.quote(doc.matters, obj); break; case "group": obj_txt = format_and_sqlite_load.group(doc.matters, obj); break; case "block": obj_txt = format_and_sqlite_load.block(doc.matters, obj); break; case "poem": // double check on keeping both poem & verse break; case "verse": obj_txt = format_and_sqlite_load.verse(doc.matters, obj); break; case "code": obj_txt = format_and_sqlite_load.code(doc.matters, obj); break; case "table": obj_txt = format_and_sqlite_load.table(doc.matters, obj); break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); } } break; } break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); } } break; } break; case "backmatter": assert(part == "glossary" || "bibliography" || "bookindex" || "blurb" || "tail", part); switch (obj.metainfo.is_of_type) { case "para": switch (obj.metainfo.is_a) { case "heading": obj_txt = format_and_sqlite_load.heading(doc.matters, obj); break; case "glossary": assert(part == "glossary", part); obj_txt = format_and_sqlite_load.para(doc.matters, obj); break; case "bibliography": assert(part == "bibliography", part); obj_txt = format_and_sqlite_load.para(doc.matters, obj); break; case "bookindex": assert(part == "bookindex", part); obj_txt = format_and_sqlite_load.para(doc.matters, obj); break; case "blurb": assert(part == "blurb", part); obj_txt = format_and_sqlite_load.para(doc.matters, obj); break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); } } break; } break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type); } } break; } break; case "comment": break; default: { /+ debug +/ if (doc.matters.opt.action.debug_do_sqlite) { writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_part); // check where empty value could come from writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a); writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from } } break; } if (obj.metainfo.is_a == "heading") { if (doc.matters.opt.action.show_sqlite) { if (obj.metainfo.heading_lev_markup == 0) { writeln(doc.matters.src.filename); } writeln( "markup: ", obj.metainfo.heading_lev_markup, "> ", obj.metainfo.dom_structure_markedup_tags_status, "; collapsed: ", obj.metainfo.heading_lev_collapsed, "> ", obj.metainfo.dom_structure_collapsed_tags_status, "; ocn: ", obj.metainfo.ocn, "; parent: ", obj.metainfo.parent_lev_markup, "; ocn: ", obj.metainfo.parent_ocn, "; ", ); } } if (!(obj.metainfo.is_a == "comment")) { _insert_doc_objects ~= insertDocObjectsRow(obj); } } // loop closes } return _insert_doc_objects.join.to!(char[]).toUTF8; } } template SQLiteTablesCreate() { void SQLiteTablesCreate(E,O,C)(E env, O opt_action, C config) { import d2sqlite3; template SQLiteTablesReCreate() { string SQLiteTablesReCreate()() { string _sql_instruct; _sql_instruct = format(q"┃ DROP INDEX IF EXISTS idx_ocn; DROP INDEX IF EXISTS idx_uid; DROP INDEX IF EXISTS idx_digest_clean; DROP INDEX IF EXISTS idx_digest_all; DROP INDEX IF EXISTS idx_clean; DROP INDEX IF EXISTS idx_title; DROP INDEX IF EXISTS idx_author; DROP INDEX IF EXISTS src_filename_base; DROP INDEX IF EXISTS idx_language_document_char; DROP INDEX IF EXISTS idx_classify_topic_register; DROP INDEX IF EXISTS idx_topic_list; DROP TABLE IF EXISTS metadata_and_text; DROP TABLE IF EXISTS topic_register; DROP TABLE IF EXISTS doc_objects; DROP TABLE IF EXISTS urls; CREATE TABLE IF NOT EXISTS metadata_and_text ( uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/]) src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename title VARCHAR(800) NOT NULL, title_main VARCHAR(400) NOT NULL, title_sub VARCHAR(400) NULL, title_short VARCHAR(400) NULL, title_edition VARCHAR(10) NULL, title_language VARCHAR(100) NULL, title_language_char VARCHAR(6) NULL, creator_author VARCHAR(600) NOT NULL, creator_author_last_first VARCHAR(600) NOT NULL, creator_author_email VARCHAR(100) NULL, creator_author_hon VARCHAR(100) NULL, creator_author_nationality VARCHAR(100) NULL, creator_editor VARCHAR(600) NULL, creator_contributor VARCHAR(600) NULL, creator_illustrator VARCHAR(600) NULL, creator_photographer VARCHAR(600) NULL, creator_translator VARCHAR(600) NULL, creator_prepared_by VARCHAR(600) NULL, creator_digitized_by VARCHAR(600) NULL, creator_audio VARCHAR(600) NULL, creator_video VARCHAR(600) NULL, language_document VARCHAR(100) NULL, language_document_char VARCHAR(6) NOT NULL, language_original VARCHAR(100) NULL, language_original_char VARCHAR(6) NULL, date_added_to_site VARCHAR(10) NULL, date_available VARCHAR(10) NULL, date_created VARCHAR(10) NULL, date_issued VARCHAR(10) NULL, date_modified VARCHAR(10) NULL, date_published VARCHAR(10) NULL, date_valid VARCHAR(10) NULL, date_translated VARCHAR(10) NULL, date_original_publication VARCHAR(10) NULL, date_generated VARCHAR(10) NULL, original_title VARCHAR(800) NULL, original_publisher VARCHAR(600) NULL, original_language VARCHAR(100) NULL, original_language_char VARCHAR(6) NULL, original_source VARCHAR(600) NULL, original_institution VARCHAR(600) NULL, original_nationality VARCHAR(100) NULL, rights_copyright VARCHAR(2500) NULL, rights_copyright_audio VARCHAR(2500) NULL, rights_copyright_cover VARCHAR(2500) NULL, rights_copyright_illustrations VARCHAR(2500) NULL, rights_copyright_photographs VARCHAR(2500) NULL, rights_copyright_text VARCHAR(2500) NULL, rights_copyright_translation VARCHAR(2500) NULL, rights_copyright_video VARCHAR(2500) NULL, rights_license VARCHAR(2500) NULL, identifier_oclc VARCHAR(30) NULL, identifier_isbn VARCHAR(16) NULL, classify_topic_register VARCHAR(2500) NULL, classify_subject VARCHAR(600) NULL, classify_loc VARCHAR(30) NULL, classify_dewey VARCHAR(30) NULL, classify_keywords VARCHAR(600) NULL, notes_abstract TEXT NULL, notes_description TEXT NULL, notes_comment TEXT NULL, notes_coverage VARCHAR(200) NULL, notes_relation VARCHAR(200) NULL, notes_history VARCHAR(600) NULL, notes_type VARCHAR(600) NULL, notes_format VARCHAR(600) NULL, notes_prefix TEXT NULL, notes_prefix_a TEXT NULL, notes_prefix_b TEXT NULL, notes_suffix TEXT NULL, publisher VARCHAR(600) NULL, src_filename_base VARCHAR(256) NOT NULL, src_filename_suffix VARCHAR(6) NOT NULL, src_fingerprint VARCHAR(256) NULL, src_filesize VARCHAR(10) NULL, src_wordcount VARCHAR(10) NULL, pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here site_url_doc_root VARCHAR(256) NULL, -- url path to doc root site_url_html_toc VARCHAR(256) NULL, site_url_html_scroll VARCHAR(256) NULL, site_url_epub VARCHAR(256) NULL, links TEXT NULL ); CREATE TABLE IF NOT EXISTS topic_register ( -- tid BIGINT PRIMARY KEY, uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, -- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code -- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename topic_register VARCHAR(250) NOT NULL, site_url_doc_root VARCHAR(256) NULL, -- url path to doc root site_url_html_toc VARCHAR(256) NULL, site_url_html_scroll VARCHAR(256) NULL ); CREATE TABLE IF NOT EXISTS doc_objects ( lid BIGINT PRIMARY KEY, uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE, ocn SMALLINT, obj_id VARCHAR(6) NULL, clean TEXT NULL, body TEXT NULL, seg VARCHAR(256) NULL, lev_an VARCHAR(1), is_of_type VARCHAR(16), is_a VARCHAR(16), lev SMALLINT NULL, node VARCHAR(16) NULL, -- not written to parent VARCHAR(16) NULL, last_descendant VARCHAR(16) NULL, -- headings only digest_clean CHAR(256), digest_all CHAR(256), seg_name CHAR(256), types CHAR(1) NULL ); CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn); CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean); CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all); CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean); CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title); CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first); CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid); CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base); CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char); CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register); CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register); PRAGMA user_version = %s; ┃", sqlite_db_schema_version, ); return _sql_instruct; } } try { if (opt_action.sqlite_db_create) { string _db_statement; string db_filename = (opt_action.sqliteDB_filename.length > 0) ? opt_action.sqliteDB_filename : (config.conf.w_srv_db_sqlite_filename.length > 0) ? config.conf.w_srv_db_sqlite_filename : ""; string db_path = (opt_action.sqliteDB_path.length > 0) ? opt_action.sqliteDB_path : (config.conf.w_srv_db_sqlite_path.length > 0) ? config.conf.w_srv_db_sqlite_path : ""; if (db_filename.length > 0 && db_path.length > 0) { if (opt_action.vox_gt_3) { writeln("db name: ", db_filename); writeln("db path: ", db_path); writeln("db name & path: ", db_path, "/", db_filename); } if (opt_action.vox_gt_2) { writeln("attempting to create db: ", db_path, "/", db_filename); } auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path); if ((isValidPath(pth_sqlite.base) && exists(pth_sqlite.base) != 0 && pth_sqlite.base.isDir)) { } else { try { pth_sqlite.base.mkdirRecurse; } catch (FileException ex) { stderr.writeln("FATAL: cannot create --sqlite-db-path directory: ", pth_sqlite.base); stderr.writeln(" (", ex.msg, ")"); import core.runtime; core.runtime.Runtime.terminate(); } } auto db = Database(pth_sqlite.sqlite_file); { _db_statement ~= SQLiteTablesReCreate!()(); } SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE"); } else { writeln("must provide db name & output root path either on the command line or in configuration file"); writeln("db name: ", db_filename); writeln("db path: ", db_path); } } } catch (FileException e) { writeln("Failed (FileException): ", e.msg); writeln(e.file, " line: ", e.line); import core.runtime; core.runtime.Runtime.terminate(); } catch (ErrnoException e) { writeln("Failed (ErrnoException): ", e.msg); writeln(e.file, " line: ", e.line); import core.runtime; core.runtime.Runtime.terminate(); } catch (Exception e) { writeln("Failed (Exception): ", e.msg); writeln(e.file, " line: ", e.line); import core.runtime; core.runtime.Runtime.terminate(); } catch (Throwable) { import core.runtime; core.runtime.Runtime.terminate(); } } } template SQLiteDbDrop() { void SQLiteDbDrop(O,C)(O opt_action, C config) { writeln("db drop"); if ((opt_action.sqlite_db_drop)) { string db_filename = (opt_action.sqliteDB_filename.length > 0) ? opt_action.sqliteDB_filename : (config.conf.w_srv_db_sqlite_filename.length > 0) ? config.conf.w_srv_db_sqlite_filename : ""; string db_path = (opt_action.sqliteDB_path.length > 0) // ? opt_action.sqliteDB_path : (config.conf.w_srv_db_sqlite_path.length > 0) ? config.conf.w_srv_db_sqlite_path : ""; if (db_filename.length > 0 && db_path.length > 0) { auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path); writeln("remove(", pth_sqlite.sqlite_file, ")"); try { remove(pth_sqlite.sqlite_file); } catch (FileException ex) { // handle error } } else { writeln("must provide db name & output root path either on the command line or in configuration file"); writeln("db name: ", db_filename); writeln("db path: ", db_path); } } } }