aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/sisudoc/io_in
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-05-25 10:21:59 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-05-25 12:20:54 -0400
commitdb3dd3c3a5f4c5cde9a736cb688f05af9f79d718 (patch)
tree1ce0c71cbeeae23566001904b9de3609ced76ebe /src/sisudoc/io_in
parent0.22.0 (diff)
separate abstraction lib from output processing
create new directories under ./src/sisudoc ocda & outputs in order to separate the document abstraction library from downstream output processing (stuff broken till paths & modules fixed)
Diffstat (limited to 'src/sisudoc/io_in')
-rw-r--r--src/sisudoc/io_in/paths_source.d902
-rw-r--r--src/sisudoc/io_in/read_config_files.d271
-rw-r--r--src/sisudoc/io_in/read_source_files.d435
-rw-r--r--src/sisudoc/io_in/read_zip_pod.d401
4 files changed, 0 insertions, 2009 deletions
diff --git a/src/sisudoc/io_in/paths_source.d b/src/sisudoc/io_in/paths_source.d
deleted file mode 100644
index 41353ed..0000000
--- a/src/sisudoc/io_in/paths_source.d
+++ /dev/null
@@ -1,902 +0,0 @@
-/+
-- Name: SisuDoc Spine, Doc Reform [a part of]
- - Description: documents, structuring, processing, publishing, search
- - static content generator
-
- - Author: Ralph Amissah
- [ralph.amissah@gmail.com]
-
- - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
-
- - License: AGPL 3 or later:
-
- Spine (SiSU), a framework for document structuring, publishing and
- search
-
- Copyright (C) Ralph Amissah
-
- This program is free software: you can redistribute it and/or modify it
- under the terms of the GNU AFERO General Public License as published by the
- Free Software Foundation, either version 3 of the License, or (at your
- option) any later version.
-
- This program is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- You should have received a copy of the GNU General Public License along with
- this program. If not, see [https://www.gnu.org/licenses/].
-
- If you have Internet connection, the latest version of the AGPL should be
- available at these locations:
- [https://www.fsf.org/licensing/licenses/agpl.html]
- [https://www.gnu.org/licenses/agpl.html]
-
- - Spine (by Doc Reform, related to SiSU) uses standard:
- - docReform markup syntax
- - standard SiSU markup syntax with modified headers and minor modifications
- - docReform object numbering
- - standard SiSU object citation numbering & system
-
- - Homepages:
- [https://www.sisudoc.org]
- [https://www.doc-reform.org]
-
- - Git
- [https://git.sisudoc.org/]
-
-+/
-/++
- read configuration files<BR>
- - read config files<BR>
- meta_config_files.d
-+/
-module sisudoc.io_in.paths_source;
-@safe:
-import std.array;
-import std.file;
-import std.path;
-import std.regex;
-import std.stdio;
-import std.conv : to;
-import sisudoc.meta.defaults;
-import sisudoc.meta.rgx_files;
-template PodManifest() {
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- auto PodManifest(O)(
- O _opt_action,
- string _pth=""
- ) {
- struct ManifestFile_ {
- string pod_manifest_filename() {
- return "pod.manifest";
- }
- string pod_manifest_path() {
- string _manifest_path;
- if ((isValidPath(_pth) && exists(_pth) != 0 && _pth.isDir)
- && (exists(_pth.chainPath(pod_manifest_filename).array) != 0
- && (_pth.chainPath(pod_manifest_filename).array).isFile)) {
- _manifest_path = _pth;
- } else if (_pth.match(rgx_files.src_pth_contents)
- && exists(_pth) != 0 && _pth.isFile) {
- _manifest_path = _pth.dirName;
- } else if (_pth.match(rgx_files.src_pth_pod_sst_or_ssm)
- && exists(_pth) != 0 && (_pth.isFile)) {
- if (auto m = _pth.match(rgx_files.src_pth_pod_sst_or_ssm)) {
- _manifest_path = m.captures["podpath"];
- }
- } else {
- if (_opt_action.vox_gt_3 || _opt_action.debug_do) {
- if (exists(_pth) == 0 && (_pth.isFile)) {
- writeln("WARNING, src file is not .sst or .ssm (nor is it a pod directory): ", _pth); // remove? unless can distinguish pod
- }
- }
- _manifest_path = "";
- }
- return _manifest_path;
- }
- string pods_collection_root_path() {
- return (pod_manifest_path.length > 0) ? ((chainPath(pod_manifest_path, "..")).asNormalizedPath).array.to!string : "";
- }
- string pod_manifest_file_with_path() {
- string _k;
- if (exists(pod_manifest_path.chainPath(pod_manifest_filename).array)!=0) {
- _k = pod_manifest_path.chainPath(pod_manifest_filename).array;
- } else if (exists(pod_manifest_path)!=0) {
- _k = pod_manifest_path;
- }
- if (exists(_k)==0) {
- // writeln("ERROR >> Processing Skipped! Manifest not found: ", _k);
- _k = null;
- }
- return _k;
- }
- }
- return ManifestFile_();
- }
-}
-template PathMatters() {
- mixin InternalMarkup;
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- static auto mkup = InlineMarkup();
- auto PathMatters(O,E)(
- O _opt_action,
- E _env,
- string _pth,
- string _fns = "",
- char[][] _manifest_fn_list = [[]],
- ) {
- auto _manifested = PodManifest!()(_opt_action, _pth);
- struct ManifestMatters_ {
- auto env() {
- auto _env = _env;
- struct Env_ {
- auto pwd() {
- return _env["pwd"];
- }
- auto home() {
- return _env["home"];
- }
- }
- return Env_();
- }
- auto opt() {
- struct Opt_ {
- auto action() {
- return _opt_action;
- }
- }
- return Opt_();
- }
- bool src_is_pod() {
- return (_manifested.pod_manifest_path.length > 0) ? true : false;
- }
- auto pod() {
- struct Pod_ {
- bool src_is_pod() {
- return (_manifested.pod_manifest_path.length > 0) ? true : false;
- }
- string collection_root() {
- return _manifested.pods_collection_root_path;
- }
- string manifest_filename() {
- return _manifested.pod_manifest_filename;
- }
- string manifest_path() {
- return _manifested.pod_manifest_path;
- }
- string pod_name_with_path() {
- return _manifested.pod_manifest_path.baseName;
- }
- string manifest_file_with_path() {
- return _manifested.pod_manifest_file_with_path;
- }
- string[] config_dr_document_make_dirs() {
- string[] _config_dirs;
- return _config_dirs;
- }
- string[] config_local_site_dirs() {
- string[] _config_dirs;
- return _config_dirs;
- }
- string[] image_dirs() {
- string[] _image_dirs;
- return _image_dirs;
- }
- auto manifest_list_of_filenames() {
- return _manifest_fn_list;
- }
- string[] manifest_list_of_languages() {
- string[] _lngs;
- foreach (filename_; manifest_list_of_filenames) {
- string _k = "en";
- if (auto m = (filename_).match(rgx_files.language_code_and_filename)) {
- _k = m.captures[1].to!string;
- }
- _lngs ~= _k; // all the languages from the manifest list of filenames with paths
- }
- return _lngs;
- }
- }
- return Pod_();
- }
- auto src() {
- string _fns = _fns; // required here by dmd & not by ldc (for D:2078)
- auto _opt_action = _opt_action;
- auto _env = _env;
- struct SRC_ {
- bool is_pod() {
- return (_manifested.pod_manifest_path.length > 0) ? true : false;
- }
- string path_and_fn() {
- return _fns;
- }
- string pod_name_with_path() {
- return (is_pod) ? _manifested.pod_manifest_path : "";
- }
- string pods_collection_root_path() {
- return (is_pod) ? _manifested.pods_collection_root_path : "";
- }
- string pod_name() {
- return pod_name_with_path.baseName;
- }
- string filename() {
- return path_and_fn.baseName;
- }
- string filename_base() {
- return filename.stripExtension;
- }
- string filename_extension() {
- return filename.match(rgx_files.src_pth_sst_or_ssm).captures["extension"];
- }
- string lng() {
- string _k;
- if (auto m = path_and_fn.match(rgx_files.language_code_and_filename)) {
- _k = m.captures[1];
- } else {_k = "en"; }
- return _k;
- }
- string doc_uid() {
- string _uid;
- if (is_pod && !(pod_name_with_path.empty)) {
- if (pod_name_with_path.baseName == filename_base) {
- _uid = filename_base ~ "." ~ filename_extension ~ mkup.uid_sep ~ lng;
- } else {
- _uid = pod_name_with_path.baseName ~ mkup.uid_sep ~ filename_base ~ "." ~ filename_extension ~ mkup.uid_sep ~ lng;
- }
- } else {
- _uid = mkup.uid_sep ~ filename_base ~ "." ~ filename_extension ~ mkup.uid_sep ~ lng;
- }
- return _uid;
- }
- string doc_uid_out() {
- string _uid;
- if (is_pod && !(pod_name_with_path.empty)) {
- if (pod_name_with_path.baseName == filename_base) {
- _uid = filename_base ~ "." ~ lng;
- } else {
- _uid = pod_name_with_path.baseName ~ mkup.uid_sep ~ filename_base ~ "." ~ lng;
- }
- } else {
- _uid = "_" ~ filename_base ~ "." ~ lng;
- }
- return _uid;
- }
- string docname_composite_unique_per_src_doc() {
- string _fn;
- if (pod_name_with_path.baseName == filename_base) {
- _fn = filename_base ~ mkup.uid_sep ~ filename_extension ~ mkup.uid_sep ~ lng;
- } else if (!(pod_name_with_path.empty)) {
- _fn = pod_name_with_path.baseName ~ mkup.uid_sep ~ filename_base ~ mkup.uid_sep ~ filename_extension ~ mkup.uid_sep ~ lng;
- } else {
- _fn = "_" ~ mkup.uid_sep ~ filename_base ~ mkup.uid_sep ~ filename_extension ~ mkup.uid_sep ~ lng;
- }
- return _fn;
- }
- string docname_composite_unique_per_src_pod() {
- /+
- z pod name if any + src filename (without lng code)
- filename ~ mkup.uid_sep ~ lng
- * unique per src pod
- used by
- - pod (multilingual collection)
- - sqlite discrete index (multilingual collection)
- +/
- string _fn;
- if (pod_name_with_path.baseName == filename_base) {
- _fn = filename_base ~ mkup.uid_sep ~ filename_extension;
- } else if (!(pod_name_with_path.empty)) {
- _fn = pod_name_with_path.baseName ~ mkup.uid_sep ~ filename_base ~ mkup.uid_sep ~ filename_extension;
- } else {
- _fn = "_" ~ mkup.uid_sep ~ filename_base ~ mkup.uid_sep ~ filename_extension;
- }
- return _fn;
- }
- string language() {
- return lng();
- }
- string file_with_absolute_path() {
- return _env["pwd"].chainPath(path_and_fn).array;
- }
- string absolute_path_to_src() {
- return (_env["pwd"].chainPath(path_and_fn)).dirName.array;
- }
- string path_to_doc_root_path_to_lang_and_filename() {
- return _env["pwd"].chainPath(path_and_fn).array;
- }
- string base_dir() {
- string _dir;
- if (
- auto m = (absolute_path_to_src)
- .match(regex(r"[/](?P<dir>(?:[a-zA-Z0-9._-]+))/pod/" ~ filename.stripExtension))
- ) {
- _dir = ((path_and_fn.chainPath("../../")).asNormalizedPath).array;
- assert(_dir == m.captures["dir"]);
- } else {
- _dir = ((path_and_fn.chainPath("../../../")).asNormalizedPath).array;
- assert(_dir == absolute_path_to_src
- .match(rgx_files.src_base_parent_dir_name).captures["dir"]);
- }
- if (_opt_action.debug_do) {
- writeln("--> (base_dir) ", _dir);
- }
- return _dir;
- }
- string base_parent_dir_path() {
- string _dir;
- if (
- auto m = (absolute_path_to_src)
- .match(regex(r"[/](?P<dir>(?:[a-zA-Z0-9._-]+))/pod/" ~ filename.stripExtension))
- ) {
- _dir = ((path_and_fn.chainPath("../../")).asNormalizedPath).array;
- } else {
- _dir = ((path_and_fn.chainPath("../../../")).asNormalizedPath).array;
- }
- return _dir;
- }
- string base_dir_path() {
- string _dir;
- if (
- auto m = (absolute_path_to_src)
- .match(rgx_files.src_formalised_file_path_parts)
- ) {
- _dir = ((m.captures["pth"]).asNormalizedPath).array;
- } else if (
- auto m = (absolute_path_to_src)
- .match(regex(r"[/](?P<dir>(?:[a-zA-Z0-9._-]+))/pod/" ~ filename.stripExtension))
- ) {
- _dir = ((path_and_fn.chainPath("../")).asNormalizedPath).array;
- } else {
- _dir = ((path_and_fn.chainPath("../../")).asNormalizedPath).array;
- }
- if (_opt_action.debug_do) {
- writeln("--> (base_dir_path) ", _dir);
- }
- return _dir;
- }
- string media_dir_path() {
- string _dir = ((base_dir_path.chainPath("media")).asNormalizedPath).array;
- return _dir;
- }
- string image_dir_path() {
- string _paths;
- string[] _possible_img_pths = [ "./image", "../image", "../../image" ];
- string _img_pth_found = "";
- if (is_pod) {
- _img_pth_found = ((file_with_absolute_path.dirName ~ "/../../image").asNormalizedPath).array;
- } else {
- string _img_pth(string _possible_img_pth) {
- return ((file_with_absolute_path.dirName ~ "/" ~ _possible_img_pth).asNormalizedPath).array;
- }
- foreach(_possible_img_pth; _possible_img_pths) {
- if (exists(_img_pth(_possible_img_pth))) {
- _img_pth_found = _img_pth(_possible_img_pth);
- break;
- } else {
- _paths ~= " " ~ _img_pth(_possible_img_pth);
- }
- }
- }
- if (_img_pth_found.empty) {
- writeln("WARNING not image path found, searched: ", _paths);
- }
- return _img_pth_found;
- }
- auto conf_dir_path() {
- return ((base_dir_path.chainPath("conf")).asNormalizedPath).array;
- }
- auto base_parent_dir() {
- string _dir;
- if (
- auto m = (absolute_path_to_src)
- .match(regex(r"[/](?P<dir>(?:[a-zA-Z0-9._-]+))/pod/" ~ filename.stripExtension))
- ) {
- _dir = m.captures["dir"];
- } else {
- _dir = (absolute_path_to_src).match(rgx_files.src_base_parent_dir_name).captures["dir"];
- }
- if (_opt_action.debug_do) {
- writeln("--> (base_parent_dir) ", _dir);
- }
- return _dir;
- }
- string[] config_dirs() {
- string[] _config_dirs;
- if (is_pod) {
- } else {}
- return _config_dirs;
- }
- string[] image_dirs() {
- string[] _image_dirs;
- if (is_pod) {
- } else {}
- return _image_dirs;
- }
- }
- return SRC_();
- }
- auto output() {
- /+
- - command line if output path set
- - config file if found and set set
- - search for and if exists read config
- - default paths to config related to:
- - source markup path;
- - current dir;
- - home dir
- - get output path if set
- - (program) default within current directory?
- +/
- auto _env = _env;
- struct Out_ {
- auto path() {
- auto _output_path = _env["pwd"];
- if ((_opt_action.output_dir_set.length > 0)
- && isValidPath(_opt_action.output_dir_set)
- ) {
- _output_path = ((_opt_action.output_dir_set).asNormalizedPath).array;
- if (!exists(_output_path)) {
- try {
- _output_path.mkdirRecurse;
- // } catch (ErrnoException ex) {
- } catch (Exception ex) {
- // Handle error
- }
- }
- assert(_output_path.isDir,
- "not a directory: " ~ _output_path);
- // TODO always test that is a directory and it is writable
- }
- return _output_path;
- }
- }
- return Out_();
- }
- }
- return ManifestMatters_();
- }
-}
-template configFilePaths() {
- auto configFilePaths(M,E)(
- M _manifested,
- E _env,
- string _cli_config_path_set = ""
- ) {
- struct ConfFilePaths {
- string config_filename_document() {
- return "dr_document_make";
- }
- string config_filename_site() {
- return "config_local_site";
- }
- auto possible_config_path_locations() {
- struct _ConfFilePaths {
- string[] dr_document_make() {
- /+ FIX clean up conf paths ↓ +/
- /+ config local site (file system only, not in pod) +/
- /+ return paths +/
- string[] _possible_config_path_locations;
- if (_cli_config_path_set.empty) {
- if (_manifested.src.is_pod) {
- /+ config document in pod +/
- string _dr_doc_conf_pod;
- string _dr_doc_conf_pod_text;
- _dr_doc_conf_pod = asNormalizedPath(chainPath(
- to!string(_env["pwd"]),
- _manifested.pod.manifest_path ~ "/conf"
- )).array;
- _dr_doc_conf_pod_text = asNormalizedPath(chainPath(
- to!string(_env["pwd"]),
- _manifested.pod.manifest_path ~ "/media/text/" ~ _manifested.src.lng ~ "/conf"
- )).array;
- /+ return paths +/
- _possible_config_path_locations = [
- _dr_doc_conf_pod_text,
- _dr_doc_conf_pod,
- ];
- } else {
- /+ config document (& or local site) on filesystem +/
- string _dr_doc_conf_pwd = ((chainPath(to!string(_env["pwd"]), "dr_doc/conf")).asNormalizedPath).array; // think about
- string _dr_doc_conf_pwd_a = ((chainPath(to!string(_env["pwd"]), "conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_b = ((chainPath(to!string(_env["pwd"]), "../conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_c = ((chainPath(to!string(_env["pwd"]), "../../conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_d = ((chainPath(to!string(_env["pwd"]), "../../../conf")).asNormalizedPath).array;
- /+ return paths +/
- _possible_config_path_locations = [
- _dr_doc_conf_pwd,
- _dr_doc_conf_pwd_a,
- _dr_doc_conf_pwd_b,
- _dr_doc_conf_pwd_c,
- _dr_doc_conf_pwd_d,
- ];
- }
- } else if (_cli_config_path_set.isDir) {
- _possible_config_path_locations = [_cli_config_path_set ];
- // } else if (_cli_config_path_set.isFile) { // use file, taken care of elsewhere
- }
- /+ FIX clean up conf paths ↑
- (compare pwd to doc path location, and build config path)
- +/
- return _possible_config_path_locations;
- }
- string[] config_local_site() {
- /+ FIX clean up conf paths ↓ +/
- /+ config local site (file system only, not in pod) +/
- string[] _possible_config_path_locations;
- if (_cli_config_path_set.empty) {
- string _dot_pwd = ((chainPath(to!string(_env["pwd"]), ".dr")).asNormalizedPath).array;
- string _underscore_pwd = ((chainPath(to!string(_env["pwd"]), "_dr")).asNormalizedPath).array;
- string _dot_home = ((chainPath(to!string(_env["home"]), ".dr")).asNormalizedPath).array;
- /+ return paths +/
- if (_manifested.src.is_pod) {
- string _collection_root_a = ((chainPath(to!string(_manifested.pod.collection_root.to!string), ".dr")).asNormalizedPath).array;
- string _collection_root_b = ((chainPath(to!string(_manifested.pod.collection_root.to!string), "_dr")).asNormalizedPath).array;
- _possible_config_path_locations = [
- _dot_pwd,
- _underscore_pwd,
- _collection_root_a,
- _collection_root_b,
- _dot_home,
- "/etc/dr",
- ];
- } else {
- /+ config document (& or local site) on filesystem +/
- string _dr_doc_conf_pwd = ((chainPath(to!string(_env["pwd"]), "dr_doc/conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_a = ((chainPath(to!string(_env["pwd"]), "conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_b = ((chainPath(to!string(_env["pwd"]), "../conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_c = ((chainPath(to!string(_env["pwd"]), "../../conf")).asNormalizedPath).array;
- string _dr_doc_conf_pwd_d = ((chainPath(to!string(_env["pwd"]), "../../../conf")).asNormalizedPath).array;
- _possible_config_path_locations = [
- _dr_doc_conf_pwd,
- _dr_doc_conf_pwd_a,
- _dr_doc_conf_pwd_b,
- _dr_doc_conf_pwd_c,
- _dr_doc_conf_pwd_d,
- _dot_pwd,
- _underscore_pwd,
- _dot_home,
- "/etc/dr"
- ];
- }
- } else {
- _possible_config_path_locations = [
- _cli_config_path_set
- ];
- }
- /+ FIX clean up conf paths ↑
- (compare pwd to doc path location, and build config path)
- +/
- return _possible_config_path_locations;
- }
- }
- return _ConfFilePaths();
- }
- }
- return ConfFilePaths();
- }
-}
-template spinePathsSRC() {
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- auto spinePathsSRC(D,Fn)(
- D _pwd,
- Fn _fn_src_and_path,
- ) {
- struct drSrcPaths {
- auto pwd() {
- return _pwd;
- }
- string language() {
- // use command line info as well?
- string _k;
- if (auto m = _fn_src_and_path.match(rgx_files.language_code_and_filename)) {
- _k = m.captures[1];
- } else { /+ unknown until doc_meta read, (could provide & use command line info?) +/
- _k = "xx"; // original default was "en" but is not known
- }
- return _k;
- }
- string doc_root() {
- return "dr_doc";
- }
- auto media_root() {
- return ((doc_root.chainPath("media")).asNormalizedPath).array;
- }
- auto conf_root() {
- return ((doc_root.chainPath("conf")).asNormalizedPath).array;
- }
- auto text_root() {
- return ((media_root.chainPath("text")).asNormalizedPath).array;
- }
- auto image_root() {
- return ((media_root.chainPath("image")).asNormalizedPath).array;
- }
- auto doc_src_fn_with_path_for_text_root_and_lng() {
- return ((text_root.chainPath(language)).asNormalizedPath).array;
- }
- auto doc_src_fn() {
- return ((_fn_src_and_path.baseName).asNormalizedPath).array;
- }
- auto doc_src_with_path() {
- return ((pwd.chainPath(_fn_src_and_path)).asNormalizedPath).array;
- }
- }
- return drSrcPaths();
- }
-}
-
-
-template spinePathsPods() {
- string _suffix = ".zip";
- auto spinePathsPods(M)(M doc_matters) {
- string _base_dir_pod = (doc_matters.output_path.length > 0)
- ? doc_matters.output_path ~ "/pod"
- : "/pod";
- string _base_dir_doc = "dr_doc";
- struct _PodPaths {
- string base_filename_(string fn_src) {
- return fn_src.baseName.stripExtension;
- }
- string internal_base() {
- return "pod";
- }
- string pod_dir_() {
- return _base_dir_pod;
- }
- string dr_doc_dir_() {
- return _base_dir_doc;
- }
- string pod_filename_(string fn_src) {
- return _base_dir_pod.chainPath(base_filename_(fn_src) ~ _suffix).array;
- }
- string base_filesystem_(string fn_src) {
- string pth = _base_dir_pod.chainPath(base_filename_(fn_src)).array;
- assert(pth == _base_dir_pod ~ "/" ~ base_filename_(fn_src),
- pth ~ " == " ~ _base_dir_pod ~ "/" ~ base_filename_(fn_src) ~ "?");
- return pth;
- }
- string output_pod_manifest_file(string fn_src) {
- return base_filesystem_(fn_src).chainPath("pod.manifest").array;
- }
- string base_pod_(string fn_src) {
- return _base_dir_pod.chainPath(base_filename_(fn_src)).array; // change this
- }
- auto base_filename(string fn_src) {
- auto pth_1_ = base_filename_(fn_src);
- auto pth_2_ = base_filename_(fn_src);
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- return pth_2_;
- }
- }
- return _pods();
- }
- auto pod_filename(string fn_src) {
- auto pth_1_ = pod_filename_(fn_src);
- auto pth_2_ = pod_filename_(fn_src);
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- return pth_2_;
- }
- }
- return _pods();
- }
- auto base(string fn_src) {
- auto pth_1_ = "";
- auto pth_2_ = base_filesystem_(fn_src);
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- return pth_2_;
- }
- }
- return _pods();
- }
- auto pod_root(string fn_src) {
- auto pth_1_ = "pod";
- auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array; // "dr_doc"
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- return pth_2_;
- }
- }
- return _pods();
- }
- auto conf_root(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = "conf";
- auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("conf")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto css(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((conf_root(fn_src).zpod.chainPath("css")).asNormalizedPath).array;
- auto pth_2_ = ((conf_root(fn_src).filesystem_open_zpod.chainPath("css")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto pod_manifest(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("pod.manifest")).asNormalizedPath).array;
- auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("pod.manifest")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto media_root(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("media")).asNormalizedPath).array;
- auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("media")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto text_root(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((media_root(fn_src).zpod.chainPath("text")).asNormalizedPath).array;
- auto pth_2_ = ((media_root(fn_src).filesystem_open_zpod.chainPath("text")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto doc(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = text_root(fn_src).zpod;
- auto pth_2_ = text_root(fn_src).filesystem_open_zpod;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto doc_lng(string fn_src, string lng) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((text_root(fn_src).zpod.chainPath(lng)).asNormalizedPath).array;
- auto pth_2_ = ((text_root(fn_src).filesystem_open_zpod.chainPath(lng)).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto abstraction_root(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((media_root(fn_src).zpod.chainPath("abstraction")).asNormalizedPath).array;
- auto pth_2_ = ((media_root(fn_src).filesystem_open_zpod.chainPath("abstraction")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- return pth_2_;
- }
- }
- return _pods();
- }
- auto image_root(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((media_root(fn_src).zpod.chainPath("image")).asNormalizedPath).array;
- auto pth_2_ = ((media_root(fn_src).filesystem_open_zpod.chainPath("image")).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto fn_pod_filelist(string fn_src) {
- auto pod_root_ = pod_root(fn_src);
- auto _manifested = PodManifest!()(doc_matters.opt.action, fn_src).pod_manifest_filename;
- auto pth_1_ = _manifested;
- auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath(_manifested)).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto fn_doc(string fn_src, string lng) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((doc_lng(fn_src, lng).zpod.chainPath(fn_src.baseName)).asNormalizedPath).array;
- auto pth_2_ = ((doc_lng(fn_src, lng).filesystem_open_zpod.chainPath(fn_src.baseName)).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- auto fn_doc_insert(string fn_src, string fn_insert, string lng) {
- auto pod_root_ = pod_root(fn_src);
- auto pth_1_ = ((doc_lng(fn_src, lng).zpod.chainPath(fn_insert.baseName)).asNormalizedPath).array;
- auto pth_2_ = ((doc_lng(fn_src, lng).filesystem_open_zpod.chainPath(fn_insert.baseName)).asNormalizedPath).array;
- struct _pods {
- auto zpod() {
- return pth_1_;
- }
- auto filesystem_open_zpod() {
- // assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
- return pth_2_;
- }
- }
- return _pods();
- }
- }
- return _PodPaths();
- }
-}
diff --git a/src/sisudoc/io_in/read_config_files.d b/src/sisudoc/io_in/read_config_files.d
deleted file mode 100644
index d3a3f45..0000000
--- a/src/sisudoc/io_in/read_config_files.d
+++ /dev/null
@@ -1,271 +0,0 @@
-/+
-- Name: SisuDoc Spine, Doc Reform [a part of]
- - Description: documents, structuring, processing, publishing, search
- - static content generator
-
- - Author: Ralph Amissah
- [ralph.amissah@gmail.com]
-
- - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
-
- - License: AGPL 3 or later:
-
- Spine (SiSU), a framework for document structuring, publishing and
- search
-
- Copyright (C) Ralph Amissah
-
- This program is free software: you can redistribute it and/or modify it
- under the terms of the GNU AFERO General Public License as published by the
- Free Software Foundation, either version 3 of the License, or (at your
- option) any later version.
-
- This program is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- You should have received a copy of the GNU General Public License along with
- this program. If not, see [https://www.gnu.org/licenses/].
-
- If you have Internet connection, the latest version of the AGPL should be
- available at these locations:
- [https://www.fsf.org/licensing/licenses/agpl.html]
- [https://www.gnu.org/licenses/agpl.html]
-
- - Spine (by Doc Reform, related to SiSU) uses standard:
- - docReform markup syntax
- - standard SiSU markup syntax with modified headers and minor modifications
- - docReform object numbering
- - standard SiSU object citation numbering & system
-
- - Homepages:
- [https://www.sisudoc.org]
- [https://www.doc-reform.org]
-
- - Git
- [https://git.sisudoc.org/]
-
-+/
-/++
- read configuration files<BR>
- - read config files<BR>
- meta_config_files.d
-+/
-module sisudoc.io_in.read_config_files;
-@safe:
-import std.file;
-import std.path;
-import sisudoc.meta;
-import sisudoc.io_in.paths_source;
-import sisudoc.meta.rgx_files;
-import sisudoc.meta.rgx;
-template readConfigSite() {
- @system final auto readConfigSite(Cf,O,Cfg)(Cf _conf_file_details, O _opt_action, Cfg _cfg) {
- mixin spineRgxIn;
- static auto rgx = RgxI();
- string conf_filename = "NONE";
- string config_file_str;
- string default_config_file_str = format(q"┃
-flag:
- act0: "--html"
- act1: "--html --epub"
-output:
- path: "%s"
-default:
- language: "en"
- papersize: "a4"
- text_wrap: "80"
- digest: "sha256"
-webserv:
- http: "%s"
- host: "%s"
- data_http: "%s"
- data_host: "%s"
- data_root_url: "%s"
- data_root_path: "%s"
- data_root_part: ""
- images_root_part: "image"
- cgi_search_form_title: "%s"
- cgi_http: "%s"
- cgi_host: "%s"
- cgi_bin_url: "%s"
- cgi_bin_subpath: "%s"
- cgi_bin_path: "%s"
- cgi_search_script: "%s"
- cgi_port: ""
- cgi_user: ""
- cgi_action: "%s"
- db_sqlite_path: "%s"
- db_sqlite_filename: "%s"
- db_pg_table: ""
- db_pg_user: ""
-┃",
- _cfg.processing_path_doc_root, // doc root
- _cfg.http_request_type, // http
- _cfg.http_host, // host / domain
- _cfg.http_request_type, // data "http" or "https"
- _cfg.http_host, // data domain "localhost"
- _cfg.www_url_doc_root, // data root url "http://locahost" "https://sisudoc.org"
- _cfg.processing_path_doc_root, // data root path
- _cfg.cgi_search_form_title, // cgi title // e.g. "≅ SiSU Spine search"
- _cfg.http_request_type, // cgi http
- _cfg.http_host, // cgi host
- _cfg.cgi_url_root, // cgi bin url
- _cfg.cgi_bin_subpath, // cgi bin path
- _cfg.cgi_bin_root, // cgi bin path
- _cfg.cgi_filename, // cgi filename
- _cfg.cgi_url_action, // cgi action
- _cfg.db_sqlite_path, // sqlite db path
- _cfg.db_sqlite_filename, // sqlite db filename
-);
- foreach(conf_fn; [_conf_file_details.config_filename_site]) {
- foreach(pth; _conf_file_details.possible_config_path_locations.config_local_site) {
- char[] conf_file;
- conf_filename = conf_fn;
- if (exists(pth)) {
- auto f_attrib = pth.getLinkAttributes;
- if (
- _conf_file_details.possible_config_path_locations.config_local_site.length == 1
- && f_attrib.attrIsFile
- ) {
- conf_file = pth.to!(char[]);
- conf_filename = pth.baseName;
- } else if (f_attrib.attrIsDir) {
- conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
- conf_filename = conf_fn;
- }
- try {
- if (exists(conf_file)) {
- if (conf_file.getLinkAttributes.attrIsFile) {
- if (_opt_action.vox_gt_2 || _opt_action.debug_do) {
- writeln("config file used: \"", conf_file, "\" (cli flag settings override config file's individual settings)");
- }
- config_file_str = conf_file.readText;
- break;
- }
- }
- } catch (ErrnoException ex) {
- } catch (FileException ex) {
- }
- }
- }
- if (config_file_str.length > 0) { break; }
- }
- if (config_file_str.length > 0) {
- import dyaml;
- Node yaml_root;
- try {
- yaml_root = Loader.fromString(config_file_str).load();
- } catch (Throwable) {
- import std.stdio;
- writeln("ERROR failed to read config file content, not parsed as yaml, program default used");
- conf_filename = "VIRTUAL";
- config_file_str = default_config_file_str;
- }
- }
- if (config_file_str.length == 0) { /+ use dummy default config file +/
- // writeln("WARNING config file NOT found, default provided");
- conf_filename = "VIRTUAL";
- config_file_str = default_config_file_str;
- }
- struct _ConfContent {
- string filename() {
- return conf_filename;
- }
- string filetype() {
- string _ft = "";
- if (content.match(rgx.yaml_config)) {
- _ft = "yaml";
- }
- return _ft;
- }
- string content() {
- return config_file_str;
- }
- }
- return _ConfContent();
- }
-}
-static template readConfigDoc() {
- import std.file;
- import std.path;
- import sisudoc.meta;
- import sisudoc.io_in.paths_source;
- import sisudoc.meta.rgx_files;
- import sisudoc.meta.rgx;
- @system final auto readConfigDoc(M,E)(M _manifested, E _env) {
- mixin spineRgxIn;
- static auto rgx = RgxI();
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- string config_file_str;
- string conf_filename = "NONE";
- auto _conf_file_details = configFilePaths!()(_manifested, _env);
- string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make;
- foreach(conf_fn; [_conf_file_details.config_filename_document]) {
- foreach(pth; possible_config_path_locations) {
- char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array;
- conf_filename = conf_fn;
- if (config_file_str.length > 0) {
- break;
- }
- try {
- if (exists(conf_file)) {
- if (conf_file.getLinkAttributes.attrIsFile) {
- config_file_str = conf_file.readText;
- break;
- }
- }
- } catch (ErrnoException ex) {
- } catch (FileException ex) {
- }
- }
- if (config_file_str.length > 0) { break; }
- }
- struct _ConfContent {
- string filename() {
- return conf_filename;
- }
- string content() {
- return config_file_str;
- }
- string filetype() {
- string _ft = "";
- if (content.match(rgx.yaml_config)) {
- _ft = "yaml";
- }
- return _ft;
- }
- }
- return _ConfContent();
- }
-}
-static template configReadSiteYAML() {
- import std.file;
- import std.path;
- import sisudoc.meta;
- import sisudoc.io_in.paths_source;
- import sisudoc.meta.rgx_files;
- import sisudoc.meta.rgx;
- final YAMLDocument configReadSiteYAML(M,E)(M _manifested, E _env) {
- string _configuration = configReadInSiteYAML!()(_manifested, _env);
- auto _conf_file_details = configFilePaths!()(_manifested, _env);
- string _conf_yaml_fn = _conf_file_details.config_filename_site;
- YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
- return _yaml_conf;
- }
-}
-static template configReadDocYAML() {
- import std.file;
- import std.path;
- import sisudoc.meta;
- import sisudoc.io_in.paths_source;
- final YAMLDocument configReadDocYAML(M,E)(M _manifested, E _env) {
- string _configuration = configReadInDocYAML!()(_manifested, _env);
- auto _conf_file_details = configFilePaths!()(_manifested, _env);
- string _conf_yaml_fn = _conf_file_details.config_filename_document;
- YAMLDocument _yaml_conf = configYAML!()(_configuration, _conf_yaml_fn);
- return _yaml_conf;
- }
-}
diff --git a/src/sisudoc/io_in/read_source_files.d b/src/sisudoc/io_in/read_source_files.d
deleted file mode 100644
index 31cbd37..0000000
--- a/src/sisudoc/io_in/read_source_files.d
+++ /dev/null
@@ -1,435 +0,0 @@
-/+
-- Name: SisuDoc Spine, Doc Reform [a part of]
- - Description: documents, structuring, processing, publishing, search
- - static content generator
-
- - Author: Ralph Amissah
- [ralph.amissah@gmail.com]
-
- - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
-
- - License: AGPL 3 or later:
-
- Spine (SiSU), a framework for document structuring, publishing and
- search
-
- Copyright (C) Ralph Amissah
-
- This program is free software: you can redistribute it and/or modify it
- under the terms of the GNU AFERO General Public License as published by the
- Free Software Foundation, either version 3 of the License, or (at your
- option) any later version.
-
- This program is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- You should have received a copy of the GNU General Public License along with
- this program. If not, see [https://www.gnu.org/licenses/].
-
- If you have Internet connection, the latest version of the AGPL should be
- available at these locations:
- [https://www.fsf.org/licensing/licenses/agpl.html]
- [https://www.gnu.org/licenses/agpl.html]
-
- - Spine (by Doc Reform, related to SiSU) uses standard:
- - docReform markup syntax
- - standard SiSU markup syntax with modified headers and minor modifications
- - docReform object numbering
- - standard SiSU object citation numbering & system
-
- - Homepages:
- [https://www.sisudoc.org]
- [https://www.doc-reform.org]
-
- - Git
- [https://git.sisudoc.org/]
-
-+/
-/++
- module source_read_source_files;<BR>
- - open markup files<BR>
- - if master file scan for addional files to import/insert
-+/
-module sisudoc.io_in.read_source_files;
-@safe:
-template spineRawMarkupContent() {
- import std.digest.sha;
- import std.file;
- import std.path;
- import sisudoc.meta;
- import sisudoc.io_in.paths_source;
- import sisudoc.meta.rgx_files;
- import sisudoc.meta.rgx;
- mixin spineRgxIn;
- static auto rgx = RgxI();
- mixin spineRgxFiles;
- static auto rgx_files = RgxFiles();
- string[] _images=[];
- string[] _extract_images(S)(S content_block) {
- string[] images_;
- string _content_block = content_block.to!string;
- if (auto m = _content_block.matchAll(rgx.image)) {
- images_ ~= m.captures[1].to!string;
- }
- return images_;
- }
- auto rawsrc = RawMarkupContent();
- struct ST_contents_inserts_images {
- char[][] contents;
- string[] insert_files;
- string[] images;
- }
- struct ST_header_content_inserts_images {
- char[] header;
- char[][] src_txt;
- string[] insert_files;
- string[] images;
- }
- struct ST_doc_digest {
- ubyte[32] markup_doc;
- ubyte[32] header;
- ubyte[32] text;
- }
- struct ST_doc_parts {
- char[] header_raw;
- char[][] sourcefile_body_content;
- string[] insert_file_list;
- string[] images_list;
- ST_doc_digest doc_digest;
- }
- ST_doc_parts spineRawMarkupContent(O,Fn)(O _opt_action, Fn fn_src) {
- ST_doc_parts _0_header_1_body_content_2_insert_filelist_struct
- = rawsrc.sourceContentSplitIntoHeaderAndBody(_opt_action, rawsrc.sourceContent(fn_src), fn_src);
- return _0_header_1_body_content_2_insert_filelist_struct;
- }
- struct RawMarkupContent {
- final sourceContent(in string fn_src) {
- auto raw = MarkupRawUnit();
- string source_txt_str
- = raw.markupSourceReadIn(fn_src);
- return source_txt_str;
- }
- final ST_doc_parts sourceContentSplitIntoHeaderAndBody(O)(
- O _opt_action,
- in string source_txt_str,
- in string fn_src=""
- ) {
- auto raw = MarkupRawUnit();
- string[] insert_file_list_get;
- string[] images_list_get;
- ST_header_content_inserts_images st
- = raw.markupSourceHeaderContentRawLineStructArray(source_txt_str);
- char[] header_raw = st.header;
- char[][] sourcefile_body_content = st.src_txt;
- if (fn_src.match(rgx_files.src_fn_master)) { // filename with path needed if master file (.ssm) not otherwise
- auto ins = Inserts();
- ST_contents_inserts_images _cii
- = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
- sourcefile_body_content = _cii.contents;
- insert_file_list_get = _cii.insert_files.dup;
- images_list_get = _cii.images.dup;
- } else if (_opt_action.source_or_pod) {
- auto ins = Inserts();
- ST_contents_inserts_images _cii
- = ins.scan_master_src_for_insert_files_and_import_content(_opt_action, sourcefile_body_content, fn_src);
- images_list_get = _cii.images.dup;
- } // image_list, if path could take sha256 digests already here?
- string header_type = "";
- ST_doc_digest dig;
- {
- dig.markup_doc = source_txt_str.sha256Of;
- dig.header = st.header.sha256Of;
- dig.text = sourcefile_body_content.sha256Of;
- }
- ST_doc_parts ret;
- {
- ret.header_raw = st.header;
- ret.sourcefile_body_content = sourcefile_body_content;
- ret.insert_file_list = insert_file_list_get;
- ret.images_list = images_list_get;
- ret.doc_digest = dig;
- }
- return ret;
- }
- }
- struct MarkupRawUnit {
- import std.digest.sha;
- import std.file;
- final private string readInMarkupSource(in char[] fn_src) {
- enforce(
- exists(fn_src) != 0,
- "file not found: «" ~
- fn_src ~ "»"
- );
- string source_txt_str;
- try {
- if (exists(fn_src)) {
- if (fn_src.getLinkAttributes.attrIsFile) {
- source_txt_str = fn_src.readText;
- } else {
- }
- }
- } catch (ErrnoException ex) {
- } catch (UTFException ex) {
- // Handle validation errors
- } catch (FileException ex) {
- // Handle errors
- }
- std.utf.validate(source_txt_str);
- return source_txt_str;
- }
- @trusted final private char[][] header0Content1(in string src_text) { // cast(char[])
- /+ split string on _first_ match of "^:?A~\s" into [header, content] array/tuple +/
- char[][] header_and_content;
- auto m = (cast(char[]) src_text).matchFirst(rgx.heading_a);
- header_and_content ~= m.pre;
- header_and_content ~= m.hit ~ m.post;
- assert(header_and_content.length == 2,
- "document markup is broken, header body split == "
- ~ header_and_content.length.to!string
- ~ "; (header / body array split should == 2 (split is on level A~))"
- );
- return header_and_content;
- }
- @trusted final private char[][] markupSourceLineArray(in char[] src_text) { // cast(char[])
- char[][] source_line_arr
- = (cast(char[]) src_text).split(rgx.newline_eol_strip_preceding);
- return source_line_arr;
- }
- string markupSourceReadIn(in string fn_src) {
- static auto rgx_files = RgxFiles();
- enforce(
- fn_src.match(rgx_files.src_pth_sst_or_ssm),
- "not a dr markup filename: «" ~
- fn_src ~ "»"
- );
- string source_txt_str = readInMarkupSource(fn_src);
- return source_txt_str;
- }
- ST_header_content_inserts_images markupSourceHeaderContentRawLineStructArray(in string source_txt_str) {
- string[] file_insert_list = [];
- string[] images_list = [];
- char[][] hc = header0Content1(source_txt_str);
- char[] header = hc[0];
- char[] source_txt = hc[1];
- char[][] source_line_arr = markupSourceLineArray(source_txt);
- ST_header_content_inserts_images ret;
- {
- ret.header = header;
- ret.src_txt = source_line_arr;
- ret.insert_files = file_insert_list;
- ret.images = images_list;
- }
- return ret;
- }
- final char[][] getInsertMarkupSourceContentRawLineArray(
- in char[] fn_src_insert,
- Regex!(char) rgx_file
- ) {
- enforce(
- fn_src_insert.match(rgx_file),
- "not a dr markup filename: «" ~
- fn_src_insert ~ "»"
- );
- string source_txt_str = readInMarkupSource(fn_src_insert);
- char[][] source_line_arr = markupSourceLineArray(source_txt_str);
- return source_line_arr;
- }
- }
- struct Inserts {
- struct ST_contents_and_images {
- char[][] insert_contents;
- string[] images;
- }
- ST_contents_and_images scan_subdoc_source(O)(
- O _opt_action,
- char[][] markup_sourcefile_insert_content,
- string fn_src
- ) {
- char[][] contents_insert;
- int code_block_status = 0;
- enum codeBlock { off, curly, tic, quotemarks }
- auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
- auto markup_src_file_path = fn_pth_full.captures[1];
- foreach (line; markup_sourcefile_insert_content) {
- if (code_block_status == codeBlock.curly) {
- if (line.matchFirst(rgx.block_curly_code_close)) {
- code_block_status = codeBlock.off;
- }
- contents_insert ~= line;
- } else if (line.matchFirst(rgx.block_curly_code_open)) {
- code_block_status = codeBlock.curly;
- contents_insert ~= line;
- } else if (code_block_status == codeBlock.quotemarks) {
- if (line.matchFirst(rgx.block_quotemarks_close)) {
- code_block_status = codeBlock.off;
- }
- contents_insert ~= line;
- } else if (code_block_status == codeBlock.tic) {
- if (line.matchFirst(rgx.block_tic_close)) {
- code_block_status = codeBlock.off;
- }
- contents_insert ~= line;
- } else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
- code_block_status = codeBlock.quotemarks;
- contents_insert ~= line;
- } else if (line.matchFirst(rgx.block_tic_code_open)) {
- code_block_status = codeBlock.tic;
- contents_insert ~= line;
- } else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
- auto insert_fn = m.captures[2];
- auto insert_sub_pth = m.captures[1];
- auto fn_src_insert
- = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
- auto raw = MarkupRawUnit();
- auto markup_sourcesubfile_insert_content
- = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
- debug(insert_file) {
- writeln(line);
- writeln(fn_src_insert);
- writeln(
- " length contents insert array: ",
- markup_sourcesubfile_insert_content.length
- );
- }
- if (_opt_action.source_or_pod) {
- _images ~= _extract_images(markup_sourcesubfile_insert_content);
- }
- auto ins = Inserts();
- /+
- - 1. load file
- - 2. read lines
- - 3. scan lines
- - a. if filename insert, and insert filename
- - repeat 1
- - b. else
- - add line to new array;
- - build image list, search for any image files to add to image list
- +/
- } else {
- contents_insert ~= line; // images to extract for image list?
- if (_opt_action.source_or_pod) {
- string[] _image_linelist = _extract_images(line);
- if (_image_linelist.length > 0) {
- _images ~= _image_linelist;
- }
- }
- }
- } // end src subdoc (inserts) loop
- ST_contents_and_images ret;
- {
- ret.insert_contents = contents_insert;
- ret.images = _images;
- }
- return ret;
- }
- ST_contents_inserts_images scan_master_src_for_insert_files_and_import_content(O)(
- O _opt_action,
- char[][] sourcefile_body_content,
- string fn_src
- ) {
- import std.algorithm;
- char[][] contents;
- int code_block_status = 0;
- enum codeBlock { off, curly, tic, quotemarks }
- auto fn_pth_full = fn_src.match(rgx_files.src_pth_sst_or_ssm);
- auto markup_src_file_path = fn_pth_full.captures[1];
- char[][] contents_insert;
- string[] _images =[];
- string[] insert_file_list =[];
- foreach (line; sourcefile_body_content) {
- if (code_block_status == codeBlock.curly) {
- if (line.matchFirst(rgx.block_curly_code_close)) {
- code_block_status = codeBlock.off;
- }
- contents ~= line;
- } else if (line.matchFirst(rgx.block_curly_code_open)) {
- code_block_status = codeBlock.curly;
- contents ~= line;
- } else if (code_block_status == codeBlock.quotemarks) {
- if (line.matchFirst(rgx.block_quotemarks_close)) {
- code_block_status = codeBlock.off;
- }
- contents ~= line;
- } else if (code_block_status == codeBlock.tic) {
- if (line.matchFirst(rgx.block_tic_close)) {
- code_block_status = codeBlock.off;
- }
- contents ~= line;
- } else if (line.matchFirst(rgx.block_quotemarks_code_open)) {
- code_block_status = codeBlock.quotemarks;
- contents ~= line;
- } else if (line.matchFirst(rgx.block_tic_code_open)) {
- code_block_status = codeBlock.tic;
- contents ~= line;
- } else if (auto m = line.match(rgx_files.insert_src_fn_ssi_or_sst)) {
- auto insert_fn = m.captures[2];
- auto insert_sub_pth = m.captures[1];
- auto fn_src_insert
- = chainPath(markup_src_file_path, insert_sub_pth ~ insert_fn).array;
- insert_file_list ~= fn_src_insert.to!string;
- auto raw = MarkupRawUnit();
- /+ TODO +/
- auto markup_sourcefile_insert_content
- = raw.getInsertMarkupSourceContentRawLineArray(fn_src_insert, rgx_files.src_fn_find_inserts);
- debug(insert_file) {
- writeln(line);
- writeln(fn_src_insert);
- writeln(
- " length contents insert array: ",
- markup_sourcefile_insert_content.length
- );
- }
- auto ins = Inserts();
- ST_contents_and_images contents_insert_st = ins.scan_subdoc_source(
- _opt_action,
- markup_sourcefile_insert_content,
- fn_src_insert.to!string
- );
- contents ~= contents_insert_st.insert_contents;
- if (_opt_action.source_or_pod) {
- string[] _image_linelist = _extract_images(contents_insert_st.images);
- if (_image_linelist.length > 0) {
- _images ~= _image_linelist;
- }
- }
- /+
- - 1. load file
- - 2. read lines
- - 3. scan lines
- - a. if filename insert, and insert filename
- - repeat 1
- - b. else
- - add line to new array;
- - build image list, search for any image files to add to image list
- +/
- } else {
- contents ~= line;
- if (_opt_action.source_or_pod) {
- string[] _image_linelist = _extract_images(line);
- if (_image_linelist.length > 0) {
- _images ~= _image_linelist;
- }
- }
- }
- } // end src doc loop
- string[] images = [];
- foreach(i; uniq(_images.sort())) {
- images ~= i;
- }
- debug(insert_file) {
- writeln(__LINE__);
- writeln(contents.length);
- }
- ST_contents_inserts_images ret;
- {
- ret.contents = contents;
- ret.insert_files = insert_file_list;
- ret.images = images;
- }
- return ret;
- }
- }
-}
diff --git a/src/sisudoc/io_in/read_zip_pod.d b/src/sisudoc/io_in/read_zip_pod.d
deleted file mode 100644
index d228f4e..0000000
--- a/src/sisudoc/io_in/read_zip_pod.d
+++ /dev/null
@@ -1,401 +0,0 @@
-/+
-- Name: SisuDoc Spine, Doc Reform [a part of]
- - Description: documents, structuring, processing, publishing, search
- - static content generator
-
- - Author: Ralph Amissah
- [ralph.amissah@gmail.com]
-
- - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
-
- - License: AGPL 3 or later:
-
- Spine (SiSU), a framework for document structuring, publishing and
- search
-
- Copyright (C) Ralph Amissah
-
- This program is free software: you can redistribute it and/or modify it
- under the terms of the GNU AFERO General Public License as published by the
- Free Software Foundation, either version 3 of the License, or (at your
- option) any later version.
-
- This program is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
-
- You should have received a copy of the GNU General Public License along with
- this program. If not, see [https://www.gnu.org/licenses/].
-
- If you have Internet connection, the latest version of the AGPL should be
- available at these locations:
- [https://www.fsf.org/licensing/licenses/agpl.html]
- [https://www.gnu.org/licenses/agpl.html]
-
- - Spine (by Doc Reform, related to SiSU) uses standard:
- - docReform markup syntax
- - standard SiSU markup syntax with modified headers and minor modifications
- - docReform object numbering
- - standard SiSU object citation numbering & system
-
- - Homepages:
- [https://www.sisudoc.org]
- [https://www.doc-reform.org]
-
- - Git
- [https://git.sisudoc.org/]
-
-+/
-/++
- module read_zip_pod;<BR>
- - extract pod zip archives to temp directory for processing<BR>
- - validate zip entries for security (path traversal, size limits)
-+/
-module sisudoc.io_in.read_zip_pod;
-@safe:
-template spineExtractZipPod() {
- import std.algorithm : canFind;
- import std.array : array;
- import std.conv : to;
- import std.file;
- import std.path;
- import std.regex;
- import std.stdio;
- import std.string : indexOf;
-
- /+ security limits for zip extraction +/
- enum size_t MAX_ENTRY_SIZE = 50 * 1024 * 1024; /+ 50 MB per entry +/
- enum size_t MAX_TOTAL_SIZE = 500 * 1024 * 1024; /+ 500 MB total +/
- enum size_t MAX_ENTRY_COUNT = 500; /+ max entries in archive +/
- enum size_t MAX_PATH_DEPTH = 10; /+ max path components +/
-
- /+ allowed entry name pattern: alphanumeric, dots, dashes, underscores, forward slashes +/
- static auto rgx_safe_entry_name = ctRegex!(`^[a-zA-Z0-9._/ -]+$`);
-
- struct ZipPodResult {
- string tmp_dir; /+ temp directory containing extracted pod +/
- string pod_dir; /+ path to pod directory within tmp_dir +/
- bool ok; /+ extraction succeeded +/
- string error_msg; /+ error description if !ok +/
- }
-
- /+ ↓ validate a single zip entry name for security +/
- string validateEntryName(string name) {
- /+ reject empty names +/
- if (name.length == 0)
- return "empty entry name";
- /+ reject absolute paths +/
- if (name[0] == '/')
- return "absolute path in zip entry: " ~ name;
- /+ reject path traversal +/
- if (name.canFind(".."))
- return "path traversal in zip entry: " ~ name;
- /+ reject null bytes +/
- if (name.indexOf('\0') >= 0)
- return "null byte in zip entry name: " ~ name;
- /+ reject backslashes (windows path separator tricks) +/
- if (name.canFind("\\"))
- return "backslash in zip entry: " ~ name;
- /+ check path depth +/
- size_t depth = 0;
- foreach (c; name) {
- if (c == '/') depth++;
- }
- if (depth > MAX_PATH_DEPTH)
- return "path too deep in zip entry: " ~ name;
- /+ check allowed characters +/
- if (!(name.matchFirst(rgx_safe_entry_name)))
- return "disallowed characters in zip entry: " ~ name;
- return ""; /+ empty string means valid +/
- }
-
- /+ ↓ extract zip pod to temp directory, returns ZipPodResult +/
- @trusted ZipPodResult extractZipPod(string zip_path) {
- import std.zip;
- ZipPodResult result;
- result.ok = false;
- /+ ↓ verify zip file exists +/
- if (!exists(zip_path) || !zip_path.isFile) {
- result.error_msg = "zip file not found: " ~ zip_path;
- return result;
- }
- /+ ↓ derive pod name from zip filename +/
- string zip_basename = zip_path.baseName.stripExtension;
- /+ ↓ read and parse zip archive +/
- ZipArchive zip;
- try {
- zip = new ZipArchive(read(zip_path));
- } catch (ZipException ex) {
- result.error_msg = "failed to read zip archive: " ~ zip_path ~ " - " ~ ex.msg;
- return result;
- } catch (Exception ex) {
- result.error_msg = "error reading zip file: " ~ zip_path ~ " - " ~ ex.msg;
- return result;
- }
- /+ ↓ validate entry count +/
- if (zip.directory.length > MAX_ENTRY_COUNT) {
- result.error_msg = "zip archive has too many entries ("
- ~ zip.directory.length.to!string ~ " > " ~ MAX_ENTRY_COUNT.to!string ~ "): " ~ zip_path;
- return result;
- }
- /+ ↓ validate all entries before extracting any +/
- size_t total_size = 0;
- foreach (name, member; zip.directory) {
- /+ validate entry name +/
- string name_err = validateEntryName(name);
- if (name_err.length > 0) {
- result.error_msg = name_err;
- return result;
- }
- /+ check per-entry size +/
- if (member.expandedSize > MAX_ENTRY_SIZE) {
- result.error_msg = "zip entry too large ("
- ~ member.expandedSize.to!string ~ " bytes): " ~ name;
- return result;
- }
- /+ check total size +/
- total_size += member.expandedSize;
- if (total_size > MAX_TOTAL_SIZE) {
- result.error_msg = "zip archive total size exceeds limit ("
- ~ MAX_TOTAL_SIZE.to!string ~ " bytes): " ~ zip_path;
- return result;
- }
- }
- /+ ↓ create temp directory +/
- string tmp_base = tempDir.buildPath("spine-zip-pod");
- try {
- if (!exists(tmp_base))
- mkdirRecurse(tmp_base);
- } catch (FileException ex) {
- result.error_msg = "failed to create temp base directory: " ~ ex.msg;
- return result;
- }
- /+ pod directory inside temp: tmp_base/pod_name/ +/
- string pod_dir = tmp_base.buildPath(zip_basename);
- try {
- if (exists(pod_dir))
- rmdirRecurse(pod_dir);
- mkdirRecurse(pod_dir);
- } catch (FileException ex) {
- result.error_msg = "failed to create temp pod directory: " ~ ex.msg;
- return result;
- }
- /+ ↓ extract entries +/
- /+ zip internal structure uses paths like:
- pod.manifest, conf/dr_document_make,
- pod/media/text/en/filename.sst, image/filename.png
- but the extracted pod directory needs to look like a normal pod:
- pod.manifest, conf/dr_document_make,
- media/text/en/filename.sst, image/filename.png
- The "pod/" prefix in zip entries for text files maps to the pod root.
- +/
- /+ ↓ pre-compute canonical pod path for containment checks +/
- auto canonical_pod = (pod_dir.asNormalizedPath).array.to!string ~ "/";
- foreach (name, member; zip.directory) {
- /+ skip directory entries +/
- if (name.length > 0 && name[$-1] == '/')
- continue;
- /+ ↓ map zip internal path to filesystem path +/
- /+ entries with "pod/" prefix: strip it so media/text/en/file.sst ends up at pod_dir/media/text/en/file.sst +/
- string entry_path = name;
- if (entry_path.length > 4 && entry_path[0..4] == "pod/") {
- entry_path = entry_path[4..$];
- }
- string out_path = pod_dir.buildPath(entry_path);
- /+ ↓ verify resolved path is within pod_dir (defense in depth) +/
- auto canonical_out = (out_path.asNormalizedPath).array.to!string;
- if (canonical_out.length < canonical_pod.length
- || canonical_out[0..canonical_pod.length] != canonical_pod)
- {
- result.error_msg = "zip entry escapes extraction directory: " ~ name;
- try { rmdirRecurse(pod_dir); } catch (FileException) {}
- return result;
- }
- /+ ↓ create parent directories +/
- string parent = out_path.dirName;
- try {
- if (!exists(parent))
- mkdirRecurse(parent);
- } catch (FileException ex) {
- result.error_msg = "failed to create directory for: " ~ name ~ " - " ~ ex.msg;
- try { rmdirRecurse(pod_dir); } catch (FileException) {}
- return result;
- }
- /+ ↓ decompress and write file +/
- try {
- auto data = zip.expand(member);
- std.file.write(out_path, data);
- } catch (Exception ex) {
- result.error_msg = "failed to extract: " ~ name ~ " - " ~ ex.msg;
- try { rmdirRecurse(pod_dir); } catch (FileException) {}
- return result;
- }
- }
- /+ ↓ verify no symlinks were created (defense in depth) +/
- string symlink_err = checkForSymlinks(pod_dir);
- if (symlink_err.length > 0) {
- result.error_msg = symlink_err;
- try { rmdirRecurse(pod_dir); } catch (FileException) {}
- return result;
- }
- /+ ↓ verify pod.manifest exists in extracted content +/
- if (!exists(pod_dir.buildPath("pod.manifest"))) {
- result.error_msg = "zip archive does not contain pod.manifest: " ~ zip_path;
- try { rmdirRecurse(pod_dir); } catch (FileException) {}
- return result;
- }
- result.tmp_dir = tmp_base;
- result.pod_dir = pod_dir;
- result.ok = true;
- return result;
- }
-
- /+ ↓ recursively check for symlinks in extracted directory +/
- @trusted string checkForSymlinks(string dir_path) {
- try {
- foreach (entry; dirEntries(dir_path, SpanMode.depth)) {
- if (entry.isSymlink) {
- return "symlink found in zip extraction: " ~ entry.name;
- }
- }
- } catch (FileException ex) {
- return "error checking for symlinks: " ~ ex.msg;
- }
- return "";
- }
-
- /+ ↓ download a zip pod from a URL to a temp file +/
- enum size_t MAX_DOWNLOAD_SIZE = 200 * 1024 * 1024; /+ 200 MB download limit +/
- enum int DOWNLOAD_TIMEOUT = 120; /+ seconds +/
-
- static auto rgx_url_zip = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+[.]zip$`);
-
- struct DownloadResult {
- string local_path; /+ path to downloaded temp file +/
- bool ok;
- string error_msg;
- }
-
- bool isUrl(string arg) {
- return arg.length > 8
- && (arg[0..8] == "https://" || arg[0..7] == "http://");
- }
-
- @trusted DownloadResult downloadZipUrl(string url) {
- import std.process : execute, environment;
- DownloadResult result;
- result.ok = false;
- /+ ↓ validate URL scheme +/
- if (url.length < 8 || (url[0..8] != "https://" && url[0..7] != "http://")) {
- result.error_msg = "only http/https URLs are supported: " ~ url;
- return result;
- }
- if (url[0..7] == "http://" && url[0..8] != "https://") {
- stderr.writeln("WARNING: downloading over insecure http: ", url);
- }
- /+ ↓ validate URL format +/
- if (!(url.matchFirst(rgx_url_zip))) {
- result.error_msg = "URL does not match expected zip URL pattern: " ~ url;
- return result;
- }
- /+ ↓ reject URLs that could target internal services +/
- {
- import std.uni : toLower;
- string url_lower = url.toLower;
- /+ strip scheme to get host portion +/
- string after_scheme = (url_lower[0..8] == "https://")
- ? url_lower[8..$]
- : url_lower[7..$];
- /+ extract host (up to first / or :) +/
- string host;
- foreach (i, c; after_scheme) {
- if (c == '/' || c == ':') {
- host = after_scheme[0..i];
- break;
- }
- }
- if (host.length == 0) host = after_scheme;
- if (host == "localhost"
- || host == "127.0.0.1"
- || host == "::1"
- || host == "[::1]"
- || host == "0.0.0.0"
- || host.canFind("169.254.")
- || host.canFind("10.")
- || host.canFind("192.168.")
- ) {
- result.error_msg = "URL targets a local/private address: " ~ url;
- return result;
- }
- }
- /+ ↓ derive filename from URL +/
- string url_basename = url.baseName;
- if (url_basename.length == 0 || url_basename.indexOf('.') < 0) {
- result.error_msg = "cannot determine filename from URL: " ~ url;
- return result;
- }
- /+ ↓ create temp directory for download +/
- string tmp_base = tempDir.buildPath("spine-zip-pod");
- try {
- if (!exists(tmp_base))
- mkdirRecurse(tmp_base);
- } catch (FileException ex) {
- result.error_msg = "failed to create temp directory: " ~ ex.msg;
- return result;
- }
- string tmp_file = tmp_base.buildPath(url_basename);
- /+ ↓ download using curl +/
- auto curl_result = execute([
- "curl",
- "--silent", "--show-error",
- "--fail", /+ fail on HTTP errors +/
- "--location", /+ follow redirects +/
- "--max-redirs", "5", /+ limit redirects +/
- "--max-time", DOWNLOAD_TIMEOUT.to!string,
- "--max-filesize", MAX_DOWNLOAD_SIZE.to!string,
- "--proto", "=https,http", /+ restrict protocols +/
- "--output", tmp_file,
- url
- ]);
- if (curl_result.status != 0) {
- result.error_msg = "download failed: " ~ url;
- if (curl_result.output.length > 0)
- result.error_msg ~= " - " ~ curl_result.output;
- /+ clean up partial download +/
- try { if (exists(tmp_file)) remove(tmp_file); } catch (FileException) {}
- return result;
- }
- if (!exists(tmp_file) || !tmp_file.isFile) {
- result.error_msg = "download produced no file: " ~ url;
- return result;
- }
- result.local_path = tmp_file;
- result.ok = true;
- return result;
- }
-
- /+ ↓ clean up a downloaded temp file +/
- void cleanupDownload(ref DownloadResult dlr) {
- if (dlr.local_path.length > 0 && exists(dlr.local_path)) {
- try {
- remove(dlr.local_path);
- } catch (FileException ex) {
- stderr.writeln("WARNING: failed to clean up downloaded file: ", dlr.local_path);
- }
- }
- dlr.ok = false;
- }
-
- /+ ↓ clean up extracted temp directory +/
- void cleanupZipPod(ref ZipPodResult zpr) {
- if (zpr.pod_dir.length > 0 && exists(zpr.pod_dir)) {
- try {
- rmdirRecurse(zpr.pod_dir);
- } catch (FileException ex) {
- stderr.writeln("WARNING: failed to clean up temp zip extraction: ", zpr.pod_dir);
- }
- }
- zpr.ok = false;
- }
-}