1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
|
/+
- Name: SisuDoc Spine
- Description: part of SiSU spine (Doc Reform):
for document structuring, processing, publishing and search
- Author: Ralph Amissah
[ralph.amissah@gmail.com]
- Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah,
All Rights Reserved.
- License: AGPL 3 or later, for everything in this repository unless a file or
a directory specifies otherwise.
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License (AGPLv3) as
published by the Free Software Foundation (FSF), either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see [https://www.gnu.org/licenses/].
If you have Internet connection, the latest version of the AGPL should be
available at: [https://www.gnu.org/licenses/agpl-3.0.html]
- Spine (by Doc Reform, related to SiSU) uses standard:
- docReform markup syntax
- standard SiSU markup syntax with modified headers and minor modifications
- docReform object numbering
- standard SiSU object citation numbering & system
- Homepages:
[https://www.sisudoc.org]
[https://www.doc-reform.org]
- Git
[https://git.sisudoc.org/]
+/
module sisudoc.ocda.abstraction.doc_from_artefact;
@safe:
/+ ↓ a document, from an artefact rather than from markup
spineAbstraction() parses markup and hands the output writers a doc:
an abstraction and a doc_matters. This does the same from a .ssp or a
.ocda.db, filling the same docMattersMake() arguments, so that what
comes out is the value the writers already take and they never learn
which of the two made it.
.
Three of the seven arguments come from the run and are the same either
way: program_info, opt_action, and the conf half of conf_make_meta,
which is site and run scoped (urls, papersize, the search database) and
has no business in a document's own artefact. The other four come from
the artefact:
.
conf_make_meta meta and make from the @meta and @make blocks, over
the site config the run assembled
doc_has docHasFromAbstraction, over the objects
doc_digest the markup digest from @source
manifest synthesised from the artefact's path and the
document's own name and language
.
What is not here, deliberately: insert_file_list, which is read only by
source_pod.d for --source and --pod2. No artefact carries the markup,
so a pod cannot be rebuilt from one, and the empty list is the truthful
answer rather than a gap.
+/
/+ ↓ which languages of an artefact there are to build, and in what order.
a .ssp is one language and answers [""], meaning "the one it holds".
a .ocda.db holds every language of its document, so each one named in it
is a document to build, filtered by --lang where that was given. --lang
naming a language the database does not hold leaves an empty list, and
the caller says so rather than silently building nothing.
.
Its own template rather than a function inside spineDocFromArtefact:
that one is eponymous, so nothing beside the function of the same name
can be reached from outside it. The listing itself is db_in's
spineDbLanguages, which is separate from the reader for the same reason:
asking what languages a file holds should not instantiate the reader.
+/
template spineArtefactLanguages() {
string[] spineArtefactLanguages(string _artefact, string[] _langs_wanted) {
import std.algorithm : endsWith;
import sisudoc.ocda.abstraction.db_in : spineDbLanguages;
if (!_artefact.endsWith(".db")) { return [""]; }
string[] _held = spineDbLanguages!()(_artefact);
if (_held.length == 0) { return [""]; } // let the read report why
if (_langs_wanted.length == 0 || _langs_wanted[0] == "all") { return _held; }
string[] _out;
foreach (_l; _held) {
foreach (_w; _langs_wanted) { if (_l == _w) { _out ~= _l; break; } }
}
return _out;
}
}
template spineDocFromArtefact() {
import std.algorithm : endsWith;
import std.array;
import std.conv : to;
import std.path;
import std.process : thisProcessID;
import std.regex;
import std.stdio;
import std.string;
import sisudoc.ocda.meta.conf_make_meta_structs;
import sisudoc.ocda.meta.doc_matters;
import sisudoc.ocda.io_in.paths_source;
import sisudoc.ocda.io_in.carried_names;
import sisudoc.ocda.abstraction.doc_has;
import sisudoc.ocda.abstraction.load;
import sisudoc.ocda.meta.topic_register;
mixin spineDocMatters;
mixin spineDocHasFromAbstraction;
mixin spineAbstractionLoad;
mixin spineTopicRegister;
mixin spineCarriedNames;
/+ ↓ a .ssp header key from a field name: the first underscore becomes the
dot that separates the group, so title_main is title.main and
rights_copyright_text is rights.copyright_text. The writer's keys are
formed the same way, which is what lets this be read off the struct
rather than kept as a second list to fall out of step with it.
+/
private string _sspKeyOf(string _member) {
foreach (i, c; _member) {
if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; }
}
return _member;
}
/+ ↓ the document's own header, over the site config the run assembled.
every string field of MetaComposite is looked for under its own key,
so a field added there and emitted by the writer arrives here without
anything being added. The arr fields are split from the string they
come from, as agreed: that is parsing a value, not deriving a
structure.
+/
ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) {
ConfComposite _out = _site_conf; // conf: site and run scoped, kept
static foreach (_m; __traits(allMembers, MetaComposite)) {
static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{
enum string _key = _sspKeyOf(_m);
if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; }
}}
}
/+ ↓ title_sub is a copy of title_subtitle, made where the header is
read, and is what epub3 puts in dc:title id="subtitle". It is not
emitted, being the same string twice; it is remade here the way the
yaml reader makes it +/
_out.meta.title_sub = _out.meta.title_subtitle;
/+ ↓ creator.author is the author list joined with ", ", so splitting on
it gives the list back +/
if (_out.meta.creator_author.length > 0) {
foreach (_a; _out.meta.creator_author.split(", ")) {
if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; }
}
}
/+ ↓ the topic register arrays, by the one rule the yaml reader uses:
the split is not a plain one and deriving it twice would be two
answers to the same question +/
{
auto _tr = topicRegisterArrays(_out.meta.classify_topic_register);
_out.meta.classify_topic_register_arr = _tr.arr;
_out.meta.classify_topic_register_expanded_arr = _tr.expanded;
}
/+ ↓ make: the keys are the field names, with no group to separate +/
if (auto _v = "doc_type" in _ssp.make) {
_out.make.doc_type = *_v; _out.make_str.doc_type = *_v;
}
if (auto _v = "auto_num_top_at_level" in _ssp.make) {
_out.make.auto_num_top_at_level = *_v;
}
if (auto _v = "auto_num_top_lv" in _ssp.make) {
try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {}
}
if (auto _v = "auto_num_depth" in _ssp.make) {
try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {}
}
if (auto _v = "breaks" in _ssp.make) {
_out.make.breaks = *_v; _out.make_str.breaks = *_v;
}
if (auto _v = "home_button_text" in _ssp.make) {
_out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v;
}
/+ ↓ footer is a list, joined with the separator the format escapes +/
if (auto _v = "footer" in _ssp.make) {
_out.make.footer = sspUnescapeSplit(*_v);
_out.make_str.footer = _out.make.footer;
}
return _out;
}
/+ ↓ where a document loaded from an artefact says it is.
an artefact names its document (% Source:) and its language (@source
language) but not where the pod holding its images sits, so that is
taken from where the artefact itself was found:
.
pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/
pod/<uid>.ocda.db -> pod/<doc>/
.
which is where --pod2 puts them both. The images beside a .ssp are
then found where html.d already looks; a .ocda.db carries its own and
does not need the directory to exist.
+/
struct ST_ArtefactPaths {
string pod_dir;
string src_file_with_path;
}
ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) {
ST_ArtefactPaths _out;
string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en";
string _dir = _artefact.dirName;
if (_artefact.endsWith(".ssp")) {
/+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/
_out.pod_dir = (_dir.baseName == "abstraction")
? _dir.dirName.dirName : _dir;
} else {
/+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/
string _uid = _artefact.baseName;
foreach (_sfx; [".ocda.db", ".db"]) {
if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; }
}
if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; }
_out.pod_dir = (_dir.chainPath(_uid).array).to!string;
}
_out.src_file_with_path = (_out.pod_dir
.chainPath("media").chainPath("text").chainPath(_lang)
.chainPath(_ssp.source).array).to!string;
return _out;
}
/+ ↓ a database's images, written where the writers look for them.
returns the pod directory to use in place of the one beside the
artefact, or "" when the database carries no images and the pod
beside it should stand. Each blob is checked against the sha256 the
database recorded with it: the two were written together, so a
mismatch means the file is damaged, and saying so is the point of
having recorded the digest at all.
+/
@trusted private string _imagesExtract(O)(string _artefact, O _opt_action) {
import std.digest : toHexString;
import std.digest.sha : sha256Of;
import std.file : mkdirRecurse, write, tempDir;
import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead;
import sisudoc.ocda.abstraction.pod_from_db : spinePodFromDb;
mixin spineAbstractionDbRead _dbr;
mixin spinePodFromDb _pfd;
auto _files = _dbr.dbReadFiles(_artefact, "image");
auto _manifest = _dbr.dbReadFiles(_artefact, "manifest");
if (_files.length == 0 && _manifest.length == 0) { return ""; }
/+ ↓ every name checked before anything is created or written.
the name comes out of the database and a database can be
downloaded, so it is attacker-controlled: it can climb with "..",
and chainPath drops everything before an absolute segment, so "/x"
would not land under the image directory at all. One bad name means
the artefact cannot be trusted for the rest of them, so this
refuses the lot rather than skipping one, which is what the zip
reader does with a zip.
+/
foreach (_f; _files) {
string _bad = validateCarriedFileName(_f.name);
if (_bad.length > 0) {
stderr.writeln("WARNING: ", _artefact.baseName,
" carries an image spine will not write: ", _bad,
"; no image is taken from this artefact");
return "";
}
}
/+ ↓ the directory this makes is a pod, named for the document, and not
a bare place to put images.
.
Because a document's output files are named from its pod: no pod,
and doc_uid_out takes its other branch and prefixes an underscore,
so the same database rendered here wrote _live-manual.en.txt where
the pod it came from writes live-manual.en.txt. It mixed environment:
a database sitting beside a pod tree borrowed that pod's name and got
it right, and the same database moved elsewhere did not.
.
A database carries its own pod.manifest, so it can say what it is
called without being told. Written here beside the images, which
makes this a pod as far as PathMatters is concerned, and the naming
is then the document's own wherever the file happens to be.
+/
string _tmp = (tempDir.chainPath("spine-ocda-"
~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string;
string _root = (_tmp.chainPath(_pfd.podNameFromDbPath(_artefact))
.array).to!string;
string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string;
try {
_img_dir.mkdirRecurse;
} catch (Exception ex) {
stderr.writeln("WARNING: could not make an image directory for ", _artefact,
": ", ex.msg);
return "";
}
foreach (_m; _manifest) {
if (validateCarriedPath(_m.name).length > 0) { continue; }
string _m_path = (_root.chainPath(_m.name).array).to!string;
if (!(carriedPathIsWithin(_root, _m_path))) { continue; }
try {
_m_path.write(_m.data);
} catch (Exception ex) {
stderr.writeln("WARNING: could not write ", _m.name, ": ", ex.msg);
}
}
foreach (_f; _files) {
string _got = _f.data.sha256Of.toHexString.to!string;
if (_f.sha256.length > 0 && _got != _f.sha256) {
stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName,
" does not match the digest recorded with it (", _f.sha256, " expected, ",
_got, " found); it is written out as it stands");
}
string _out_path = (_img_dir.chainPath(_f.name).array).to!string;
/+ ↓ the check on the check: the name rules above already forbid a
directory part, so this can only fire if they were loosened
+/
if (!(carriedPathIsWithin(_img_dir, _out_path))) {
stderr.writeln("WARNING: ", _f.name, " in ", _artefact.baseName,
" resolves outside the image directory; no image is taken from",
" this artefact");
return "";
}
try {
_out_path.write(_f.data);
} catch (Exception ex) {
stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg);
}
}
if (_opt_action.vox_gt_1) {
writeln(" ", _files.length, " image(s) from the database, in ", _img_dir);
}
return _root;
}
/+ ↓ the images a .ssp describes, against the ones beside it.
a .ssp carries no image bytes, so what it can do is say whether the
files in the pod it sits in are the ones the abstraction was built
from. Nothing checked that before: a changed or wrong image in the
directory paired silently.
+/
@trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) {
import std.digest : toHexString;
import std.digest.sha : sha256Of;
import std.file : exists, read;
string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string;
int _checked, _missing, _wrong;
foreach (_section; _ssp.section_order) {
foreach (obj; _ssp.abstraction[_section]) {
foreach (_img; obj.metainfo.sha256.images) {
if (_img.fileName.length == 0) { continue; }
string _pth = (_img_dir.chainPath(_img.fileName).array).to!string;
if (!_pth.exists) { ++_missing; continue; }
++_checked;
string _want = _img.fileHash_sha256.toHexString.to!string;
string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string;
if (_want != _got && _want != "00000000000000000000000000000000"
~ "00000000000000000000000000000000") {
++_wrong;
stderr.writeln("WARNING: image ", _img.fileName, " beside ",
_ssp.source, " is not the one the abstraction was built from");
}
}
}
}
if (_missing > 0) {
stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are",
" not in ", _img_dir);
}
if (_opt_action.vox_gt_1 && _checked > 0) {
writeln(" ", _checked, " image(s) checked against the abstraction's digests",
(_wrong == 0) ? ", all matching" : "");
}
}
/+ ↓ the doc the output writers take, from an artefact.
the reading is done here rather than by the caller, and that is not
only tidiness: main() is @system, so a template mixed in there infers
@system for everything it declares, and the output writers are @safe
and will not take objects of such a type. Doing it inside this module,
which is @safe:, is what makes the loaded objects the same kind of
thing the parser's are. Called as spineDocFromArtefact!()(...), the
way spineAbstraction!() is, rather than mixed in.
+/
auto spineDocFromArtefact(P,O,E)(
string _artefact,
P program_info,
O _opt_action,
E _env,
ConfComposite _site_conf,
string _lang = "",
) {
auto _loaded = abstractionLoad(_artefact, _lang);
auto _ssp = _loaded.doc;
auto _pths = artefactPaths(_artefact, _ssp);
/+ ↓ the images.
a .ocda.db carries its own, which is what lets it travel on its
own, so they are taken from it and written where the output writers
look for images: a directory of this run's making, not the pod
beside the artefact, which may be somebody's published tree. A .ssp
only describes its images, so those are the ones in the pod it sits
in, and are checked against the digests it recorded.
+/
string _images_tmp;
if (_loaded.loaded) {
if (_artefact.endsWith(".ssp")) {
_imagesVerify(_pths.pod_dir, _ssp, _opt_action);
} else {
string _extracted = _imagesExtract(_artefact, _opt_action);
if (_extracted.length > 0) {
/+ ↓ the source path has to move with it: image_dir_path is reached
from the document's own file, not from the pod. What is removed
afterwards is the pod's *parent*, the directory this run made:
the pod inside it is named for the document, and removing only
that would leave the parent.
+/
_images_tmp = _extracted.dirName;
_pths.pod_dir = _extracted;
_pths.src_file_with_path = (_extracted
.chainPath("media").chainPath("text")
.chainPath(("language" in _ssp.source_info)
? _ssp.source_info["language"] : "en")
.chainPath(_ssp.source).array).to!string;
}
}
}
auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path);
auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf);
auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action);
/+ ↓ the digests: the markup digest is what @source carries, and it is
the one that says which source this abstraction was made from. The
header and text digests are of parts an artefact does not keep
separately, and are left unset rather than guessed at
+/
ST_ArtefactDigest _dig;
if (auto _v = "digest" in _ssp.source_info) {
_dig.markup_doc = _hexToDigest(*_v);
}
auto _abstraction = _ssp.abstraction;
auto _matters = docMattersMake(
program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init,
);
/+ ↓ the same shape spineAbstraction() returns, built the same way: a
struct of accessors over the locals, with whether the artefact
could be read at all, which the caller has to know +/
auto theDOC() {
struct ST_DOC {
const auto abstraction() { return _abstraction; }
auto matters() { return _matters; }
bool loaded() { return _loaded.loaded; }
string images_tmp() { return _images_tmp; }
string note() { return _loaded.note; }
string source_name() { return abstractionSourceName(_loaded.source); }
}
return ST_DOC();
}
return theDOC();
}
/+ ↓ the digests doc_matters carries. Only markup_doc can be filled from
an artefact, @source having recorded it; the header and text digests
are of parts no artefact keeps separately and are left unset rather
than guessed at. Shaped to match ST_doc_digest, which is declared
inside spineRawMarkupContent and reachable only by parsing a file.
+/
struct ST_ArtefactDigest {
ubyte[32] markup_doc;
ubyte[32] header;
ubyte[32] text;
}
private ubyte[32] _hexToDigest(string _hex) {
ubyte[32] _out;
if (_hex.length < 64) { return _out; }
foreach (i; 0..32) {
try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {}
}
return _out;
}
}
|