diff options
author | Ralph Amissah <ralph.amissah@gmail.com> | 2024-07-10 08:51:11 -0400 |
---|---|---|
committer | Ralph Amissah <ralph.amissah@gmail.com> | 2024-07-10 12:20:03 -0400 |
commit | 18bdad0fd7ced5fecb39e9e73d7c4bd9a3956c6f (patch) | |
tree | e3399ab289ecc7cf13f09822bf103cc15b55ed2f /src/sisudoc | |
parent | nix dlang overlay ldc-1.39.0 (diff) |
- serial processing (need to be built serially)
- multilingual pods, copy all languages before zip
Diffstat (limited to 'src/sisudoc')
-rw-r--r-- | src/sisudoc/io_out/hub.d | 16 | ||||
-rw-r--r-- | src/sisudoc/io_out/metadata.d | 11 | ||||
-rw-r--r-- | src/sisudoc/io_out/source_pod.d | 193 | ||||
-rwxr-xr-x | src/sisudoc/spine.d | 31 |
4 files changed, 120 insertions, 131 deletions
diff --git a/src/sisudoc/io_out/hub.d b/src/sisudoc/io_out/hub.d index b68eb0d..0b7fe8c 100644 --- a/src/sisudoc/io_out/hub.d +++ b/src/sisudoc/io_out/hub.d @@ -70,17 +70,11 @@ template outputHub() { auto msg = Msg!()(doc_matters); if (sched == outTask.source_or_pod) { msg.v("spine (doc reform) source processing... "); - if (doc_matters.opt.action.pod) { - msg.v("spine (doc reform) source pod processing... "); - } + if (doc_matters.opt.action.pod) { msg.v("spine (doc reform) source pod processing... "); } import sisudoc.io_out.source_pod; spinePod!()(doc_matters); - if (doc_matters.opt.action.source) { - msg.vv("spine (doc reform) source done"); - } - if (doc_matters.opt.action.pod) { - msg.vv("spine (doc reform) source pod done"); - } + if (doc_matters.opt.action.source) { msg.vv("spine (doc reform) source done"); } + if (doc_matters.opt.action.pod) { msg.vv("spine (doc reform) source pod done"); } } if (sched == outTask.epub) { msg.v("epub3 processing... "); @@ -140,9 +134,7 @@ template outputHub() { msg.vv("sqlite done"); } } - if (doc_matters.opt.action.vox_gt0) { - writeln(" ", doc_matters.src.filename_base); - } + if (doc_matters.opt.action.vox_gt0) { writeln(doc_matters.src.filename_base); } if (!(doc_matters.opt.action.parallelise_subprocesses)) { foreach(schedule; doc_matters.opt.action.output_task_scheduler) { Scheduled!()(schedule, doc_abstraction, doc_matters); diff --git a/src/sisudoc/io_out/metadata.d b/src/sisudoc/io_out/metadata.d index 95fc465..0917969 100644 --- a/src/sisudoc/io_out/metadata.d +++ b/src/sisudoc/io_out/metadata.d @@ -50,7 +50,7 @@ module sisudoc.io_out.metadata; // @safe: template outputMetadata() { - void outputMetadata(T)(T doc_matters) { + void outputMetadata(T)(T doc_matters) { string inline_search_form(M)( M doc_matters, ) { @@ -393,9 +393,7 @@ string theme_light_1 = format(q"┃ } catch (ErrnoException ex) { // Handle error } - if (doc_matters.opt.action.vox_gt0) { - writeln(" ", pth_html.fn_scroll("metadata." ~ doc_matters.src.filename)); - } + if (doc_matters.opt.action.vox_gt0) { writeln(" ", pth_html.fn_scroll("metadata." ~ doc_matters.src.filename)); } } static auto mkup = InlineMarkup(); import sisudoc.io_out.html_snippet; @@ -524,11 +522,6 @@ string theme_light_1 = format(q"┃ } } } - metadata_ ~= "<p class=\"lev2\"><tt>" - ~ doc_matters.doc_digest.markup_doc.toHexString - ~ "</tt> - " - ~ doc_matters.src.filename - ~ "</p>"; if (doc_matters.conf_make_meta.meta.classify_topic_register_arr.length > 0) { metadata_ ~= "<hr /><p class=\"lev0\">Topics:</p>"; string[] _top = ["", "", "", "", ""]; diff --git a/src/sisudoc/io_out/source_pod.d b/src/sisudoc/io_out/source_pod.d index 6b880d9..2b9953f 100644 --- a/src/sisudoc/io_out/source_pod.d +++ b/src/sisudoc/io_out/source_pod.d @@ -73,33 +73,38 @@ template spinePod() { auto lang = Lang(); static auto rgx_files = RgxFiles(); assert (doc_matters.src.filename.match(rgx_files.src_fn)); - try { - { - pod_archive_directory_tree(doc_matters, pths_pod); - } - auto t = pod_zip_make_ready(doc_matters, pths_pod); - static assert(t.length==3); - auto zip = t[0]; - auto fn_pod = t[1]; - auto _digests = t[2]; - { - zipArchive(doc_matters, fn_pod, zip); - } { - zipArchiveDigest(doc_matters, fn_pod, _digests); + if (doc_matters.opt.action.pod) { + try { + { + podArchive_directory_tree(doc_matters, pths_pod); + } + { + struct STsrcDigests { + std.zip.ZipArchive zip; + string fn_pod; + string[string][string] digests; + } + STsrcDigests _st; + _st = pod_zipMakeReady(doc_matters, pths_pod, _st); + { + zipArchive(doc_matters, _st.fn_pod, _st.zip); + if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { + zipArchiveDigest(doc_matters, _st.fn_pod, _st.digests); + } + } + } + } catch (ErrnoException ex) { + // Handle error } - } catch (ErrnoException ex) { - // Handle error } } - void pod_archive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure + void podArchive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure if (!exists(pths_pod.pod_dir_())) { // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) pths_pod.pod_dir_().mkdirRecurse; } if (doc_matters.opt.action.source_or_pod) { - if (doc_matters.opt.action.vox_gt0) { - writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); - } + // if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); } if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) { pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse; } @@ -132,10 +137,9 @@ template spinePod() { )); } } - auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) { + auto pod_zipMakeReady(M,P,S)(M doc_matters, P pths_pod, S _st) { auto pth_dr_doc_src = doc_matters.src_path_info; - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln(__LINE__, ": ", doc_matters.src.filename, " -> ", pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod @@ -170,11 +174,10 @@ template spinePod() { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); + zip = podArchive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (image): ", fn_src_in); } } @@ -195,11 +198,10 @@ template spinePod() { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); } } @@ -223,8 +225,7 @@ template spinePod() { = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); Node _pmy; string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { try { _pmy = Loader.fromString(_pm).load(); } catch (ErrnoException ex) { @@ -242,7 +243,7 @@ template spinePod() { pod_filelist_yaml_string.writeln(_pm); } if (doc_matters.opt.action.pod) { - zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip); + zip = podArchive("string", _pm, fn_src_out_pod_zip_base, zip); } } } @@ -256,47 +257,55 @@ template spinePod() { = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: string[] filelist_src_out_pod_arr; string[] filelist_src_zpod_arr; - if (exists(fn_src_in)) { // what of language? - debug(io) { writeln("(io debug) src in found: ", fn_src_in); } - { // take DIGEST write to pod file digests.txt - auto data = (cast(byte[]) (fn_src_in).read); - _digests["en"]["sst"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename; // FIX language issue - // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename); - } - filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; - filelist_src_zpod_arr ~= fn_src_out_inside_pod; - string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename; - if (doc_matters.opt.action.source_or_pod) { - fn_src_in.copy(fn_src_out_filesystem); - } - if (doc_matters.opt.action.pod) { - auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); - if (auto _x = fn_src_in.match(_rgx)){ - if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { - string _path_to_pod = _x.captures["path_to_pod"]; - string _podname = _x.captures["podname"]; - string _root_to_lang = _x.captures["from_root"]; - string _language = _x.captures["language"]; - string _filename = _x.captures["filename"]; - foreach (_lang; doc_matters.pod.manifest_list_of_languages) { - string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename; - zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssm parsed + foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions + string fn_src_out_filesystem_lng + = pths_pod.fn_doc(doc_matters.src.filename, _lang).filesystem_open_zpod.to!string; + string _sstm = (doc_matters.pod.manifest_path ~ "/media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename); + // writeln(_sstm); + if (exists(_sstm)) { // what of language? + debug(io) { writeln("(io debug) src in found: ", _sstm); } + { // take DIGEST write to pod file digests.txt + auto data = (cast(byte[]) (_sstm).read); + _digests[_lang]["sstm"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename ~ " - [" ~ _lang ~ "]"; + // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename); + } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; + string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename; + if (doc_matters.opt.action.source_or_pod) { + _sstm.copy(fn_src_out_filesystem_lng); + } + if (doc_matters.opt.action.pod) { + auto _rgx_sstm = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])"); + if (auto _x = _sstm.match(_rgx_sstm)){ + if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { // again wait until all language versions of .ssm parsed + string _path_to_pod = _x.captures["path_to_pod"]; + string _podname = _x.captures["podname"]; + string _root_to_lang = _x.captures["from_root"]; + string _language = _x.captures["language"]; // .ssi inserts expected to have same name across languages + string _filename = _x.captures["filename"]; + foreach (_lang1; doc_matters.pod.manifest_list_of_languages) { // do for all language versions + string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename; + string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename; + // writeln("\nin: ", _pth_mkup_src_in, "\nout: ", _pth_mkup_src_out); // DEBUG, REMOVE + zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + } + } + } else { + zip = podArchive("file_path_text", _sstm, fn_src_out_pod_zip_base, zip); } } } else { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { + writeln("WARNING (io) src in NOT found (markup source): ", _sstm); + } } } - } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { - writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); - } } } { // bundle insert files (.ssi) - get digest if (doc_matters.srcs.file_insert_list.length > 0) { - auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); + auto _rgx_ssi = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])"); foreach (insert_file; doc_matters.srcs.file_insert_list) { debug(pod) { writeln( @@ -308,7 +317,7 @@ template spinePod() { ).zpod ); } - if (auto _x = insert_file.match(_rgx)){ + if (auto _x = insert_file.match(_rgx_ssi)){ if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { string _path_to_pod = _x.captures["path_to_pod"]; string _podname = _x.captures["podname"]; @@ -334,11 +343,10 @@ template spinePod() { _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere } if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); + zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip); } } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in); } } @@ -369,11 +377,10 @@ template spinePod() { fn_src_in.copy(fn_src_out_filesystem); } if (doc_matters.opt.action.pod) { - zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); + zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip); } } else { - if (doc_matters.opt.action.debug_do_pod - && doc_matters.opt.action.vox_gt1) { + if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) { writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); } } @@ -381,10 +388,14 @@ template spinePod() { } } } - auto t = tuple(zip, fn_pod, _digests); - return t; + { + _st.zip = zip; + _st.fn_pod = fn_pod; + _st.digests = _digests; + } + return _st; } - @system auto pod_archive(Z)( + @system auto podArchive(Z)( string _source_type, string _data_in, string _pth_out, @@ -425,6 +436,7 @@ template spinePod() { void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) { import sisudoc.io_out.paths_output; auto pths_pod = spinePathsPods!()(doc_matters); + char[] _zip_digest; try { if (!exists(pths_pod.pod_dir_())) { // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs) @@ -434,23 +446,20 @@ template spinePod() { // Handle error } try { - writeln(pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); + // if (doc_matters.opt.action.vox_gt1) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); } string _digest_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt"; - writeln(_digest_fn); + // if (doc_matters.opt.action.vox_gt1) { writeln(_digest_fn); } auto f = File(_digest_fn, "w"); if (exists(fn_pod)) { try { - if (doc_matters.opt.action.vox_gt0 - && doc_matters.opt.action.pod) { - auto data = (cast(byte[]) (fn_pod).read); - if (doc_matters.opt.action.vox_gt1) { - writeln(doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); - } - if (doc_matters.opt.action.pod) { - auto _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"); - writeln(_zip_digest); - f.writeln(_zip_digest); - } + auto data = (cast(byte[]) (fn_pod).read); + // if (doc_matters.opt.action.vox_gt1) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); } + if (doc_matters.opt.action.pod) { + _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"); + if (doc_matters.opt.action.vox_gt0) { writeln(" ", _zip_digest); } + if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, "/"); } + if (doc_matters.opt.action.vox_gt0) { writeln(" ", _digest_fn); } + f.writeln(_zip_digest); } } catch (ErrnoException ex) { // Handle errors @@ -458,19 +467,19 @@ template spinePod() { } foreach (_lang; doc_matters.pod.manifest_list_of_languages) { if (_lang in _digests) { - if (("sst" in _digests[_lang]) && (_digests[_lang]["sst"].length > 0)) { - writeln(_digests[_lang]["sst"]); - f.writeln(_digests[_lang]["sst"]); + if (("sstm" in _digests[_lang]) && (_digests[_lang]["sstm"].length > 0)) { + // if (doc_matters.opt.action.vox_gt1) { writeln(_digests[_lang]["sstm"]); } + f.writeln(_digests[_lang]["sstm"]); } if (("ssi" in _digests[_lang]) && (_digests[_lang]["ssi"].length > 0)) { - writeln(_digests[_lang]["ssi"]); + // if (doc_matters.opt.action.vox_gt1) { writeln(_digests[_lang]["ssi"]); } f.writeln(_digests[_lang]["ssi"]); } } } if ("shared" in _digests) { if (("images" in _digests["shared"]) && (_digests["shared"]["images"].length > 0)) { - writeln(_digests["shared"]["images"]); + // if (doc_matters.opt.action.vox_gt1) { writeln(_digests["shared"]["images"]); } f.writeln(_digests["shared"]["images"]); } } diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d index b79bc54..39563fb 100755 --- a/src/sisudoc/spine.d +++ b/src/sisudoc/spine.d @@ -656,11 +656,19 @@ string program_name = "spine"; bool _is; if (opts["serial"] == true) { _is = false; - } else if (sqlite_shared_db_action) { + } else if ( + sqlite_shared_db_action + || source_or_pod + ) { _is = false; } else if (opts["parallel"] == true) { _is = true; - if (sqlite_shared_db_action) { _is = false; } + if ( + sqlite_shared_db_action + || source_or_pod + ) { + _is = false; + } } else if ( opts["abstraction"] || concordance @@ -670,7 +678,6 @@ string program_name = "spine"; || odt || latex || manifest - || source_or_pod || sqlite_discrete ) { _is = true; @@ -1030,12 +1037,7 @@ string program_name = "spine"; foreach(manifest; parallel(_manifests[1..$])) { if (!empty(manifest.src.filename)) { scope(success) { - if (_opt_action.vox_gt0) { - writefln( - "%s", - "-- ~ document complete, ok ~ ------------------------------------", - ); - } + if (_opt_action.vox_gt0) { writeln("-- ~ document complete, ok ~ ------------------------------------"); } } scope(failure) { debug(checkdoc) { @@ -1138,17 +1140,10 @@ string program_name = "spine"; } } else { // note cannot parallelise sqlite shared db foreach(manifest; _manifests[1..$]) { - if (_opt_action.vox_gt2) { - writeln("parallelisation off: actions include sqlite shared db"); - } + if (_opt_action.vox_gt2) { writeln("parallelisation off: actions include sqlite shared db"); } if (!empty(manifest.src.filename)) { scope(success) { - if (_opt_action.vox_gt0) { - writefln( - "%s", - "-- ~ document complete, ok ~ ------------------------------------", - ); - } + if (_opt_action.vox_gt0) { writeln("-- ~ document complete, ok ~ ------------------------------------"); } } scope(failure) { debug(checkdoc) { |