/+ - Name: SisuDoc Spine, Doc Reform [a part of] - Description: documents, structuring, processing, publishing, search - static content generator - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later: Spine (SiSU), a framework for document structuring, publishing and search Copyright (C) Ralph Amissah This program is free software: you can redistribute it and/or modify it under the terms of the GNU AFERO General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at these locations: [https://www.fsf.org/licensing/licenses/agpl.html] [https://www.gnu.org/licenses/agpl.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.sisudoc.org] [https://www.doc-reform.org] - Git [https://git.sisudoc.org/] +/ module sisudoc.ocda.abstraction.db_in; @safe: /+ ↓ read a .ocda.db back into the document abstraction the counterpart of sisudoc.outputs.io_out.create_ocda, and the sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, so a consumer does not have to know which of the two artefacts it was handed. the check is the same one: read a database, emit it as .ssp with the writer's own record definition, and require the result to equal the .ssp written from the same document. +/ template spineAbstractionDbRead() { import std.conv : to; import std.file; import std.stdio; import std.string; import std.array; import std.json; import d2sqlite3; import sisudoc.ocda.abstraction.ssp_in : spineAbstractionRead; mixin spineAbstractionRead; // for SSPdocument, and the shared unescape private int[8] _jsonEight(string _s) { int[8] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (i, v; _j.array) { if (i < _out.length) { _out[i] = v.integer.to!int; } } } catch (Exception ex) {} return _out; } private int[] _jsonInts(string _s) { int[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= v.integer.to!int; } } catch (Exception ex) {} return _out; } private string[] _jsonStrs(string _s) { string[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= v.str; } } catch (Exception ex) {} return _out; } private double[] _jsonDoubles(string _s) { double[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= (v.type == JSONType.integer) ? v.integer.to!double : v.floating; } } catch (Exception ex) {} return _out; } private ubyte[32] _hex32(string hex) { ubyte[32] _out; if (hex.length < 64) { return _out; } foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } return _out; } private string _levMarkedUp(int lev) { switch (lev) { case 0: return "A"; case 1: return "B"; case 2: return "C"; case 3: return "D"; case 4: return "1"; case 5: return "2"; case 6: return "3"; case 7: return "4"; default: return ""; } } @trusted SSPdocument dbReadFile(string db_file) { SSPdocument doc; if (!db_file.exists) { writeln("ERROR: no such file: ", db_file); return doc; } auto db = Database(db_file, SQLITE_OPEN_READONLY); /+ ↓ the four header blocks, from the metadata table. the key prefix says which block a row belongs to, and insertion order is kept +/ foreach (row; db.execute("SELECT key, value FROM metadata ORDER BY rowid")) { string _k = row["key"].as!string; string _v = row["value"].as!string; if (_k.startsWith("make.")) { string _kk = _k["make.".length .. $]; doc.make[_kk] = _v; doc.make_order ~= _kk; } else if (_k.startsWith("doc_has.")) { string _kk = _k["doc_has.".length .. $]; doc.doc_has[_kk] = _v; doc.doc_has_order ~= _kk; } else if (_k.startsWith("schema.")) { if (_k == "schema.version") { doc.format = "% SiSU Document Abstraction v" ~ _v; } } else if (_k == "source.filename") { doc.source = _v; } else if (_k == "source.ssp_digest") { /+ ↓ this database's own note of the .ssp it was built from. it is not part of the .ssp, so it does not go back into the block +/ doc.ssp_digest = _v; } else if (_k.startsWith("source.")) { string _kk = _k["source.".length .. $]; doc.source_info[_kk] = _v; doc.source_info_order ~= _kk; } else { doc.meta[_k] = _v; doc.meta_order ~= _k; } } if (!sspFormatVersionOk(doc.format, db_file)) { return SSPdocument(); } /+ ↓ the per object lists, gathered first, one sweep of each table. these four tables hold what a single object row cannot: the images an object carries, the links stowed off it, its anchor tags and its lev4 subtoc entries. They are read here in four ordered passes and kept by object id, rather than queried per object as the objects are built. The cost is then what the tables hold, not what the document holds: war and peace has 12,135 objects and 138 rows across all four, which as four statements per object was 48,540 statement preparations and made reading a .ocda.db slower than parsing the markup it came from. +/ ST_file_name_hash_size_[][long] _images_by_id; string[][long] _links_by_id; string[][long] _anchors_by_id; string[][long] _subtoc_by_id; foreach (r; db.execute( "SELECT object_id, name, bytes, sha256, width, height, missing" ~ " FROM object_images ORDER BY object_id, seq") ) { ST_file_name_hash_size_ _img; _img.fileName = r["name"].as!string; _img.fileSize = r["bytes"].as!ulong; _img.fileHash_sha256 = _hex32(r["sha256"].as!string); _img.imageWidth = r["width"].as!int; _img.imageHeight = r["height"].as!int; _img.fileMissing = (r["missing"].as!int == 1); _images_by_id[r["object_id"].as!long] ~= _img; } foreach (r; db.execute( "SELECT object_id, url FROM object_links ORDER BY object_id, seq") ) { _links_by_id[r["object_id"].as!long] ~= r["url"].as!string; } foreach (r; db.execute( "SELECT object_id, anchor FROM object_anchors ORDER BY object_id, seq") ) { _anchors_by_id[r["object_id"].as!long] ~= r["anchor"].as!string; } foreach (r; db.execute( "SELECT object_id, entry FROM object_subtoc ORDER BY object_id, seq") ) { _subtoc_by_id[r["object_id"].as!long] ~= r["entry"].as!string; } /+ ↓ the objects, section by section, in the order they were written +/ string[] _sections; foreach (row; db.execute( "SELECT section FROM objects GROUP BY section ORDER BY MIN(id)") ) { _sections ~= row["section"].as!string; } foreach (section; _sections) { doc.section_order ~= section; doc.abstraction[section] = []; /+ ↓ column name to index, resolved once for the statement. d2sqlite3's row["name"] is indexForName, a linear scan over the columns that calls sqlite3_column_name and allocates a D string for each one it passes. With some fifty named reads per object over forty columns that is around a thousand of those per object, and it was the larger half of what made reading a database slow. Resolved once here, the reads below are an integer index. +/ int[string] _col; foreach (row; db.execute( "SELECT * FROM objects WHERE section = '" ~ section ~ "' ORDER BY seq") ) { if (_col.length == 0) { foreach (_i; 0 .. row.length) { _col[row.columnName(_i)] = _i.to!int; } } ObjGenericComposite obj; long _id = row[_col["id"]].as!long; obj.metainfo.ocn = row[_col["ocn"]].as!int; obj.metainfo.is_a = row[_col["is_a"]].as!string; obj.metainfo.is_of_part = row[_col["is_of_part"]].as!string; obj.metainfo.is_of_section = (row[_col["is_of_section"]].as!string.length > 0) ? row[_col["is_of_section"]].as!string : section; obj.metainfo.is_of_type = row[_col["is_of_type"]].as!string; obj.metainfo.identifier = row[_col["identifier"]].as!string; obj.metainfo.heading_lev_markup = (obj.metainfo.is_a == "heading") ? row[_col["heading_level"]].as!int : 9; obj.metainfo.heading_lev_collapsed = (row[_col["heading_lev_collapsed"]].as!string.length > 0) ? row[_col["heading_lev_collapsed"]].as!int : 9; obj.metainfo.parent_ocn = row[_col["parent_ocn"]].as!int; obj.metainfo.parent_lev_markup = row[_col["parent_lev"]].as!int; obj.metainfo.last_descendant_ocn = row[_col["last_descendant_ocn"]].as!int; obj.metainfo.children_headings = _jsonInts(row[_col["children"]].as!string); obj.metainfo.markedup_ancestors = _jsonEight(row[_col["ancestors"]].as!string); obj.metainfo.collapsed_ancestors = _jsonEight(row[_col["ancestors_collapsed"]].as!string); obj.metainfo.dom_structure_markedup_tags_status = _jsonEight(row[_col["dom_status"]].as!string); obj.metainfo.dom_structure_collapsed_tags_status = _jsonEight(row[_col["dom_status_collapsed"]].as!string); obj.metainfo.dummy_heading = (row[_col["dummy_heading"]].as!int == 1); obj.metainfo.object_number_off = (row[_col["object_number_off"]].as!int == 1); obj.metainfo.attrib = row[_col["attrib"]].as!string; obj.metainfo.lang = row[_col["meta_lang"]].as!string; obj.metainfo.syntax = row[_col["meta_syntax"]].as!string; obj.metainfo.sha256.text = _hex32(row[_col["sha256"]].as!string); obj.attrib.indent_base = row[_col["indent_base"]].as!int; obj.attrib.indent_hang = row[_col["indent_hang"]].as!int; obj.attrib.bullet = (row[_col["bullet"]].as!int == 1); obj.attrib.language = row[_col["lang"]].as!string; obj.has.inline_links = (row[_col["has_links"]].as!int == 1); obj.has.inline_notes_reg = (row[_col["has_notes_reg"]].as!int == 1); obj.has.inline_notes_star = (row[_col["has_notes_star"]].as!int == 1); obj.has.images = (row[_col["has_images"]].as!int == 1); obj.has.image_without_dimensions = (row[_col["has_images_no_dim"]].as!int == 1); obj.tags.in_segment_html = row[_col["segment"]].as!string; obj.tags.segname_prev = row[_col["segment_prev"]].as!string; obj.tags.segname_next = row[_col["segment_next"]].as!string; obj.tags.segment_anchor_tag_epub = row[_col["segment_epub"]].as!string; obj.tags.html_segment_anchor_tag_is = row[_col["segment_html_is"]].as!string; obj.tags.epub_segment_anchor_tag_is = row[_col["segment_epub_is"]].as!string; obj.tags.anchor_tag_html = row[_col["anchor"]].as!string; obj.tags.heading_lev_anchor_tag = row[_col["heading_lev_anchor"]].as!string; { string[8] _h; foreach (i, h; _jsonStrs(row[_col["heading_ancestors_text"]].as!string)) { if (i < _h.length) { _h[i] = h; } } obj.tags.heading_ancestors_text = _h; } if (obj.metainfo.is_a == "table") { obj.table.number_of_columns = row[_col["table_cols"]].as!int; obj.table.column_widths = _jsonDoubles(row[_col["table_widths"]].as!string); obj.table.column_aligns = _jsonStrs(row[_col["table_aligns"]].as!string); obj.table.heading = (row[_col["table_header"]].as!int == 1); } obj.code_block.linenumbers = (row[_col["code_linenumbers"]].as!int == 1); obj.text = row[_col["text"]].as!string; /+ ↓ the per object lists, from the sweeps above +/ if (auto _v = _id in _images_by_id) { obj.metainfo.sha256.images = *_v; } if (auto _v = _id in _links_by_id) { obj.stow.link = *_v; } if (auto _v = _id in _anchors_by_id) { obj.tags.anchor_tags = *_v; } if (auto _v = _id in _subtoc_by_id) { obj.tags.lev4_subtoc = *_v; } doc.abstraction[section] ~= obj; } } return doc; } }