aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/sisudoc/spine.d
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-21 14:15:34 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-22 17:02:01 -0400
commitbf1294cf89265cfab7e4886a505e0338c9013203 (patch)
treec60d5f0cb527ea359f3789c053d63d602f5fe00f /src/sisudoc/spine.d
parentdispatch: pod materialised if source required (diff)
verify: this spine still produces this abstraction?
does this spine still produce this abstraction? A database states an abstraction and carries the markup it was built from, so re-parsing the one and comparing against the other says whether the two still agree. Checked as follows: - automatically and unskippably when a document is built from a database, at the moment it is parsed and before anything has been written from it. - on --ocda-verify=<file>, an action of its own that materialises, parses every language, compares, reports and writes nothing, the exit status being the answer. - on --no-verify as the escape, warning per document, for a newer spine reading an older artefact where the abstraction is expected to differ and the output is wanted anyway. A build ends at the first failure rather than skipping the language. A document rebuilt from a database now has its whole abstraction built. (assisted by Claude-Code)
Diffstat (limited to 'src/sisudoc/spine.d')
-rw-r--r--src/sisudoc/spine.d270
1 files changed, 268 insertions, 2 deletions
diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d
index fa6ef03..95e48c7 100644
--- a/src/sisudoc/spine.d
+++ b/src/sisudoc/spine.d
@@ -219,6 +219,7 @@ string program_name = "spine";
"section_bookindex" : true,
"section_blurb" : true,
"backmatter" : true,
+ "no-verify" : false,
"skip-output" : false,
"strict" : false,
"theme-dark" : false,
@@ -230,6 +231,7 @@ string program_name = "spine";
"pod-compression" : "",
"ssp-round-trip" : "",
"db-round-trip" : "",
+ "ocda-verify" : "",
"abstraction-source" : "",
"www-http" : "",
"www-host" : "",
@@ -295,6 +297,8 @@ string program_name = "spine";
"output", "=/path/to/output/dir specify where to place output", &settings["output"],
"ssp-round-trip", "=/path/to/file.ssp read a .ssp back and re-emit it on stdout", &settings["ssp-round-trip"],
"db-round-trip", "=/path/to/file.ocda.db read it back and emit .ssp on stdout", &settings["db-round-trip"],
+ "ocda-verify", "=/path/to/file.ocda.db does this spine still make this abstraction from the markup it carries", &settings["ocda-verify"],
+ "no-verify", "do not check carried markup against the abstraction it was built with", &opts["no-verify"],
"abstraction-source", "=/path/to/(.sst|pod|.ssp|.ocda.db) identify it, and load it if it is an abstraction", &settings["abstraction-source"],
"parallel", "parallelise document processing (opt-in; slower than serial)", &opts["parallel"],
"parallel-subprocesses", "nested parallelisation", &opts["parallel-subprocesses"],
@@ -721,7 +725,33 @@ string program_name = "spine";
asked for in the same run still happens from the abstraction.
+/
@trusted bool needs_markup_source() {
- return (source_or_pod || show_abstraction || ocda_db) ? true : false;
+ return (source_or_pod || show_abstraction || ocda_db || ocda_verify)
+ ? true : false;
+ }
+ /+ ↓ verify: does this spine still make this abstraction from the markup the
+ artefact carries?
+ .
+ A .ssp or a database states an abstraction and, since a .ocda.db carries
+ the markup it was built from; re-parsing the one and comparing against
+ the other is the only check that the two still agree, and it is the check
+ that says whether a document travels through time as well as through a
+ file.
+ .
+ --ocda-verify names a file and asks only that. It is an action in its own
+ right: no output is written, and the run's exit status is the answer.
+ +/
+ @trusted bool ocda_verify() {
+ return (settings["ocda-verify"].length > 0);
+ }
+ @trusted string ocda_verify_file() {
+ return settings["ocda-verify"];
+ }
+ /+ ↓ the escape, for the case it is actually wanted: a newer spine reading an
+ older artefact, where the abstraction is expected to differ and the
+ output is wanted anyway. It warns.
+ +/
+ @trusted bool no_verify() {
+ return opts["no-verify"];
}
@trusted bool sqlite_discrete() {
return opts["sqlite-discrete"];
@@ -817,7 +847,11 @@ string program_name = "spine";
return opts["backmatter"];
}
@trusted bool skip_output() {
- return opts["skip-output"];
+ /+ ↓ --ocda-verify answers a question and writes nothing: the run's
+ exit status is the answer, and a check that left output behind
+ could not be run against a published tree
+ +/
+ return (opts["skip-output"] || ocda_verify) ? true : false;
}
@trusted bool strict() {
return opts["strict"];
@@ -961,6 +995,7 @@ string program_name = "spine";
opts["abstraction"]
|| show_abstraction
|| ocda_db
+ || ocda_verify
|| concordance
|| source_or_pod
|| curate
@@ -989,6 +1024,7 @@ string program_name = "spine";
|| manifest
|| show_abstraction
|| ocda_db
+ || ocda_verify
|| show_make
|| show_metadata
|| show_summary
@@ -1016,11 +1052,28 @@ string program_name = "spine";
|| skel
) ? true :false;
}
+ /+ ↓ the dom structure tags, which the .ssp records and which are only
+ computed when something needs them.
+ .
+ --ocda-verify belongs on this list and the omission cost an hour:
+ verify re-parses the carried markup and compares the .ssp it would
+ write against the one the artefact holds, and every artefact is
+ written by a run that has this on (show_abstraction and ocda_db
+ both turn it on). Without it, verify built an abstraction missing
+ the dom tags, and reported ten sound languages as ten failures.
+ .
+ Worth knowing more generally: the abstraction is not one thing
+ regardless of what was asked for. --text alone builds one without
+ these tags. That is invisible in output, since only html, epub, odt
+ and the databases use them, but anything comparing abstractions has
+ to compare like with like.
+ +/
@trusted bool meta_processing_xml_dom() {
return (
opts["abstraction"]
|| show_abstraction
|| ocda_db
+ || ocda_verify
|| html
|| epub
|| odt
@@ -1135,7 +1188,17 @@ string program_name = "spine";
A database without source markup still refuses, as it has to: no artefact
written by an older spine carries any.
+/
+ /+ ↓ --ocda-verify names its file rather than taking it as an argument, so
+ that the question is unmistakable in the command line. It joins the
+ arguments here and is materialised and parsed like any other database;
+ what makes it a verification rather than a build is that nothing is
+ written and the comparison below decides the exit status.
+ +/
+ if (_opt_action.ocda_verify) {
+ _resolved_args ~= _opt_action.ocda_verify_file;
+ }
string[] _pod_materialisations;
+ string[string] _pod_came_from_db; // pod directory -> the database
if (_opt_action.needs_markup_source) {
import sisudoc.ocda.abstraction.pod_from_db;
import std.process : thisProcessID;
@@ -1153,7 +1216,18 @@ string program_name = "spine";
auto _mat = podFromDb!()(arg, _root, _opt_action);
if (_mat.ok) {
_pod_materialisations ~= _root;
+ _pod_came_from_db[_mat.pod_dir] = arg;
_args_after ~= _mat.pod_dir;
+ /+ ↓ a document rebuilt from a database gets the whole abstraction
+ built, dom structure tags and all.
+ .
+ Because that is what it will be compared against: the artefact
+ recorded its abstraction from a run that had these on, every
+ run that writes one having them on. Parsed without them, the
+ same markup makes a different .ssp, and verify would report ten
+ sound languages as ten failures. It did, for an hour.
+ +/
+ opts["abstraction"] = true;
if (_opt_action.vox_gt_1) {
writeln("pod from database: ", arg.baseName, " -> ", _mat.pod_dir);
}
@@ -1546,6 +1620,10 @@ string program_name = "spine";
+/
mixin spineOcnAlign _ocna;
_ocna.ST_OcnProfile[] _ocn_profiles;
+ /+ ↓ what verify found, gathered as the documents go by +/
+ size_t _verify_checked = 0;
+ size_t _verify_failed = 0;
+ string[] _verify_notes;
if (_manifests.length > 1 // _manifests[0] initialized dummy element
&& _opt_action.abstraction) {
/+ ↓ output hub +/
@@ -1585,6 +1663,84 @@ string program_name = "spine";
) {
writeln("- stepX complete for [", manifest.src.filename, "]");
}
+ /+ ↓ verify: the markup the database carried, re-parsed, against
+ the abstraction the database holds.
+ .
+ Here and not later because this is the last moment before
+ anything is written from it. The document has been parsed and
+ nothing has been produced yet, so a mismatch can still stop
+ the writing rather than be discovered in it.
+ .
+ Only for a document that came from a database: a pod given on
+ the command line is the source, and there is nothing to hold
+ it against.
+ +/
+ bool _verified = true;
+ if (_pod_came_from_db.length > 0) {
+ import sisudoc.ocda.abstraction.ssp_in;
+ import sisudoc.ocda.abstraction.db_in : spineDbSspDigests;
+ mixin spineAbstractionRead _vfy;
+ string _pod_key
+ = (doc.matters.pod.manifest_path.asNormalizedPath).array.to!string;
+ if (auto _db = _pod_key in _pod_came_from_db) {
+ string _made = _vfy.sspRoundTripDocument(doc).ssp_digest;
+ auto _recorded = spineDbSspDigests!()(*_db);
+ string _lang = doc.matters.src.language;
+ string _was = (_lang in _recorded) ? _recorded[_lang] : "";
+ synchronized {
+ _verify_checked += 1;
+ if (_was.length == 0) {
+ _verify_notes ~= (*_db).baseName ~ " [" ~ _lang
+ ~ "] records no abstraction digest to check against";
+ _verify_failed += 1;
+ _verified = false;
+ } else if (_was != _made) {
+ _verify_notes ~= (*_db).baseName ~ " [" ~ _lang
+ ~ "] the markup it carries no longer makes the"
+ ~ " abstraction it holds";
+ _verify_notes ~= " recorded " ~ _was;
+ _verify_notes ~= " made now " ~ _made;
+ _verify_failed += 1;
+ _verified = false;
+ }
+ }
+ }
+ }
+ if (!_verified) {
+ if (_opt_action.no_verify) {
+ stderr.writeln("WARNING: --no-verify, so this is built from",
+ " markup that no longer makes the abstraction recorded with",
+ " it: ", manifest.src.filename, " [", manifest.src.language, "]");
+ } else if (_opt_action.ocda_verify) {
+ /+ ↓ verify writes nothing, so the rest of the languages are
+ worth checking: the answer wanted is how much of the file
+ is sound, not which language failed first
+ +/
+ continue;
+ } else {
+ /+ ↓ a build stops here, and does not skip the language and
+ carry on.
+ .
+ Skipping one language of a pod leaves the pod writer
+ carrying markup for a language whose directory was never
+ made, which threw a FileException out of the run: a half
+ written pod, and an exception instead of an explanation.
+ A document whose carried markup no longer makes its
+ recorded abstraction is not one to write a pod from at
+ all, so the run ends here and says so.
+ +/
+ foreach (_n; _verify_notes) { stderr.writeln(" ", _n); }
+ stderr.writeln("~ run FAILED ~ verify: ", manifest.src.filename,
+ " [", manifest.src.language, "] is built from markup that no",
+ " longer makes the abstraction recorded with it.");
+ stderr.writeln(" Anything already written is incomplete.",
+ " --ocda-verify=<file> reports on every language, and",
+ " --no-verify builds anyway.");
+ import core.stdc.stdlib : exit;
+ stdout.flush;
+ exit(1);
+ }
+ }
/+ ↓ debugs +/
if (doc.matters.opt.action.show_summary) {
import sisudoc.ocda.meta.metadoc_show_summary;
@@ -1716,6 +1872,84 @@ string program_name = "spine";
) {
writeln("- stepX complete for [", manifest.src.filename, "]");
}
+ /+ ↓ verify: the markup the database carried, re-parsed, against
+ the abstraction the database holds.
+ .
+ Here and not later because this is the last moment before
+ anything is written from it. The document has been parsed and
+ nothing has been produced yet, so a mismatch can still stop
+ the writing rather than be discovered in it.
+ .
+ Only for a document that came from a database: a pod given on
+ the command line is the source, and there is nothing to hold
+ it against.
+ +/
+ bool _verified = true;
+ if (_pod_came_from_db.length > 0) {
+ import sisudoc.ocda.abstraction.ssp_in;
+ import sisudoc.ocda.abstraction.db_in : spineDbSspDigests;
+ mixin spineAbstractionRead _vfy;
+ string _pod_key
+ = (doc.matters.pod.manifest_path.asNormalizedPath).array.to!string;
+ if (auto _db = _pod_key in _pod_came_from_db) {
+ string _made = _vfy.sspRoundTripDocument(doc).ssp_digest;
+ auto _recorded = spineDbSspDigests!()(*_db);
+ string _lang = doc.matters.src.language;
+ string _was = (_lang in _recorded) ? _recorded[_lang] : "";
+ synchronized {
+ _verify_checked += 1;
+ if (_was.length == 0) {
+ _verify_notes ~= (*_db).baseName ~ " [" ~ _lang
+ ~ "] records no abstraction digest to check against";
+ _verify_failed += 1;
+ _verified = false;
+ } else if (_was != _made) {
+ _verify_notes ~= (*_db).baseName ~ " [" ~ _lang
+ ~ "] the markup it carries no longer makes the"
+ ~ " abstraction it holds";
+ _verify_notes ~= " recorded " ~ _was;
+ _verify_notes ~= " made now " ~ _made;
+ _verify_failed += 1;
+ _verified = false;
+ }
+ }
+ }
+ }
+ if (!_verified) {
+ if (_opt_action.no_verify) {
+ stderr.writeln("WARNING: --no-verify, so this is built from",
+ " markup that no longer makes the abstraction recorded with",
+ " it: ", manifest.src.filename, " [", manifest.src.language, "]");
+ } else if (_opt_action.ocda_verify) {
+ /+ ↓ verify writes nothing, so the rest of the languages are
+ worth checking: the answer wanted is how much of the file
+ is sound, not which language failed first
+ +/
+ continue;
+ } else {
+ /+ ↓ a build stops here, and does not skip the language and
+ carry on.
+ .
+ Skipping one language of a pod leaves the pod writer
+ carrying markup for a language whose directory was never
+ made, which threw a FileException out of the run: a half
+ written pod, and an exception instead of an explanation.
+ A document whose carried markup no longer makes its
+ recorded abstraction is not one to write a pod from at
+ all, so the run ends here and says so.
+ +/
+ foreach (_n; _verify_notes) { stderr.writeln(" ", _n); }
+ stderr.writeln("~ run FAILED ~ verify: ", manifest.src.filename,
+ " [", manifest.src.language, "] is built from markup that no",
+ " longer makes the abstraction recorded with it.");
+ stderr.writeln(" Anything already written is incomplete.",
+ " --ocda-verify=<file> reports on every language, and",
+ " --no-verify builds anyway.");
+ import core.stdc.stdlib : exit;
+ stdout.flush;
+ exit(1);
+ }
+ }
/+ ↓ debugs +/
if (doc.matters.opt.action.show_summary) {
import sisudoc.ocda.meta.metadoc_show_summary;
@@ -1816,6 +2050,24 @@ string program_name = "spine";
}
}
}
+ /+ ↓ what verify found, said once at the end.
+ A run that was only asked to verify says so plainly and its exit status
+ is the answer. A run that was building says it too, because a document
+ whose markup no longer makes its abstraction was skipped and the user has
+ to know why their output is short.
+ +/
+ if (_verify_checked > 0) {
+ foreach (_n; _verify_notes) { stderr.writeln(" ", _n); }
+ if (_opt_action.ocda_verify || _opt_action.vox_gt_1 || _verify_failed > 0) {
+ writeln("verify: ", _verify_checked - _verify_failed, " of ",
+ _verify_checked, " re-parse to the abstraction recorded with them");
+ }
+ } else if (_opt_action.ocda_verify) {
+ stderr.writeln("verify: nothing was checked. ",
+ _opt_action.ocda_verify_file.baseName,
+ " carries no markup, or none that could be read");
+ _verify_failed += 1;
+ }
/+ ↓ the languages of a document, held against each other.
.
Here because it is the first point at which it can be done: a language
@@ -1980,6 +2232,20 @@ string program_name = "spine";
import core.stdc.stdlib : exit;
exit(1);
}
+ /+ ↓ a verification that failed is a failed run, whatever else went well.
+ --no-verify has already said its piece per document and does not reach
+ here: it turns the failure into a warning and the output is produced
+ deliberately.
+ +/
+ if (_verify_failed > 0 && !(_opt_action.no_verify)) {
+ stderr.writefln(
+ "~ run FAILED ~ verify: %s of %s did not re-parse to the abstraction"
+ ~ " recorded with them",
+ _verify_failed, _verify_checked,
+ );
+ import core.stdc.stdlib : exit;
+ exit(1);
+ }
/+ ↓ sqlite statements that failed are fatal to the run, report and exit non-zero +/
{
import sisudoc.outputs.io_out.sqlite_collection_db : sqliteFailureTally;