pod zip fixesHEADmain

- serial processing (need to be built serially)
- multilingual pods, copy all languages before zip

1 files changed, 103 insertions, 92 deletions

diff --git a/org/out_src_pod.org b/org/out_src_pod.org
index da2b050..c16280e 100644
--- a/org/out_src_pod.org
+++ b/org/out_src_pod.org
@@ -31,22 +31,29 @@ template spinePod() {
   <<output_imports>>
   void spinePod(T)(T doc_matters) {
     <<source_pod_init>>
-    try {
-      {
-        pod_archive_directory_tree(doc_matters, pths_pod);
-      }
-      auto t = pod_zip_make_ready(doc_matters, pths_pod);
-      static assert(t.length==3);
-      auto zip = t[0];
-      auto fn_pod = t[1];
-      auto _digests = t[2];
-      {
-        zipArchive(doc_matters, fn_pod, zip);
-      } {
-        zipArchiveDigest(doc_matters, fn_pod, _digests);
+    if (doc_matters.opt.action.pod) {
+      try {
+        {
+          podArchive_directory_tree(doc_matters, pths_pod);
+        }
+        {
+          struct STsrcDigests {
+            std.zip.ZipArchive     zip;
+            string                 fn_pod;
+            string[string][string] digests;
+          }
+          STsrcDigests _st;
+          _st = pod_zipMakeReady(doc_matters, pths_pod, _st);
+          {
+            zipArchive(doc_matters, _st.fn_pod, _st.zip);
+            if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) {
+              zipArchiveDigest(doc_matters, _st.fn_pod, _st.digests);
+            }
+          }
+        }
+      } catch (ErrnoException ex) {
+        // Handle error
       }
-    } catch (ErrnoException ex) {
-      // Handle error
     }
   }
   <<source_pod_mkdirs>>
@@ -96,10 +103,9 @@ assert (doc_matters.src.filename.match(rgx_files.src_fn));
 
 #+NAME: source_pod_archive_prepare_for_zip_and_get_digests
 #+BEGIN_SRC d
-auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
+auto pod_zipMakeReady(M,P,S)(M doc_matters, P pths_pod, S _st) {
   auto pth_dr_doc_src = doc_matters.src_path_info;
-  if (doc_matters.opt.action.debug_do_pod
-  && doc_matters.opt.action.vox_gt1) {
+  if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
     writeln(__LINE__, ": ",
       doc_matters.src.filename, " -> ",
       pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod
@@ -134,11 +140,10 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
           fn_src_in.copy(fn_src_out_filesystem);
         }
         if (doc_matters.opt.action.pod) {
-          zip = pod_archive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip);
+          zip = podArchive("file_path_bin", fn_src_in, fn_src_out_pod_zip_base, zip);
         }
       } else {
-        if (doc_matters.opt.action.debug_do_pod
-        && doc_matters.opt.action.vox_gt1) {
+        if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
           writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
         }
       }
@@ -159,11 +164,10 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
         fn_src_in.copy(fn_src_out_filesystem);
       }
       if (doc_matters.opt.action.pod) {
-        zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
+        zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
       }
     } else {
-      if (doc_matters.opt.action.debug_do_pod
-      && doc_matters.opt.action.vox_gt1) {
+      if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
         writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
       }
     }
@@ -187,8 +191,7 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
           = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
         Node _pmy;
         string _pm = "doc:\n  filename: " ~ doc_matters.src.filename ~ "\n  language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n";
-        if (doc_matters.opt.action.debug_do_pod
-        && doc_matters.opt.action.vox_gt1) {
+        if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
           try {
             _pmy = Loader.fromString(_pm).load();
           } catch (ErrnoException ex) {
@@ -206,7 +209,7 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
           pod_filelist_yaml_string.writeln(_pm);
         }
         if (doc_matters.opt.action.pod) {
-          zip = pod_archive("string", _pm, fn_src_out_pod_zip_base, zip);
+          zip = podArchive("string", _pm, fn_src_out_pod_zip_base, zip);
         }
       }
     }
@@ -220,47 +223,55 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
       = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path:
     string[] filelist_src_out_pod_arr;
     string[] filelist_src_zpod_arr;
-    if (exists(fn_src_in)) { // what of language?
-      debug(io) { writeln("(io debug) src in found: ", fn_src_in); }
-      { // take DIGEST write to pod file digests.txt
-        auto data = (cast(byte[]) (fn_src_in).read);
-        _digests["en"]["sst"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename; // FIX language issue
-        // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename);
-      }
-      filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
-      filelist_src_zpod_arr ~= fn_src_out_inside_pod;
-      string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ doc_matters.src.language ~ "/" ~ doc_matters.src.filename;
-      if (doc_matters.opt.action.source_or_pod) {
-        fn_src_in.copy(fn_src_out_filesystem);
-      }
-      if (doc_matters.opt.action.pod) {
-        auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])");
-        if (auto _x = fn_src_in.match(_rgx)){
-          if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
-            string _path_to_pod = _x.captures["path_to_pod"];
-            string _podname = _x.captures["podname"];
-            string _root_to_lang = _x.captures["from_root"];
-            string _language = _x.captures["language"];
-            string _filename = _x.captures["filename"];
-            foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
-              string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
-              string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang ~ "/" ~ _filename;
-              zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
+    if (doc_matters.src.language == doc_matters.pod.manifest_list_of_languages[$-1]) { // wait until all language versions of .ssm parsed
+      foreach (_lang; doc_matters.pod.manifest_list_of_languages) { // do for all language versions
+        string fn_src_out_filesystem_lng
+          = pths_pod.fn_doc(doc_matters.src.filename, _lang).filesystem_open_zpod.to!string;
+        string _sstm = (doc_matters.pod.manifest_path ~ "/media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename);
+        // writeln(_sstm);
+        if (exists(_sstm)) { // what of language?
+          debug(io) { writeln("(io debug) src in found: ", _sstm); }
+          { // take DIGEST write to pod file digests.txt
+            auto data = (cast(byte[]) (_sstm).read);
+            _digests[_lang]["sstm"] ~= data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename ~ " - [" ~ _lang ~ "]";
+            // writeln(data.sha256Of.toHexString, "::", data.length, " - ", doc_matters.src.filename);
+          }
+          filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
+          filelist_src_zpod_arr ~= fn_src_out_inside_pod;
+          string _pod_to_markup_file = doc_matters.src.pod_name ~ "/" ~ "media/text/" ~ _lang ~ "/" ~ doc_matters.src.filename;
+          if (doc_matters.opt.action.source_or_pod) {
+            _sstm.copy(fn_src_out_filesystem_lng);
+          }
+          if (doc_matters.opt.action.pod) {
+            auto _rgx_sstm = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[mt])");
+            if (auto _x = _sstm.match(_rgx_sstm)){
+              if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) { // again wait until all language versions of .ssm parsed
+                string _path_to_pod = _x.captures["path_to_pod"];
+                string _podname = _x.captures["podname"];
+                string _root_to_lang = _x.captures["from_root"];
+                string _language = _x.captures["language"]; // .ssi inserts expected to have same name across languages
+                string _filename = _x.captures["filename"];
+                foreach (_lang1; doc_matters.pod.manifest_list_of_languages) { // do for all language versions
+                  string _pth_mkup_src_in = _path_to_pod ~ _podname ~ "/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename;
+                  string _pth_mkup_src_out = "pod/" ~ _root_to_lang ~ _lang1 ~ "/" ~ _filename;
+                  // writeln("\nin:  ", _pth_mkup_src_in, "\nout: ", _pth_mkup_src_out); // DEBUG, REMOVE
+                  zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
+                }
+              }
+            } else {
+              zip = podArchive("file_path_text", _sstm, fn_src_out_pod_zip_base, zip);
             }
           }
         } else {
-          zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
+          if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
+            writeln("WARNING (io) src in NOT found (markup source): ", _sstm);
+          }
         }
       }
-    } else {
-      if (doc_matters.opt.action.debug_do_pod
-      && doc_matters.opt.action.vox_gt1) {
-        writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
-      }
     }
   } { // bundle insert files (.ssi) - get digest
     if (doc_matters.srcs.file_insert_list.length > 0) {
-      auto _rgx = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])");
+      auto _rgx_ssi = regex(r"(?P<path_to_pod>\S+?)(?P<podname>[a-z_-]+)/(?P<from_root>media/text/)(?P<language>\S+?)/(?P<filename>\S+?\.ss[i])");
       foreach (insert_file; doc_matters.srcs.file_insert_list) {
         debug(pod) {
           writeln(
@@ -272,7 +283,7 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
             ).zpod
           );
         }
-        if (auto _x = insert_file.match(_rgx)){
+        if (auto _x = insert_file.match(_rgx_ssi)){
           if (doc_matters.src.lng == doc_matters.pod.manifest_list_of_languages[$-1]) {
             string _path_to_pod = _x.captures["path_to_pod"];
             string _podname = _x.captures["podname"];
@@ -298,11 +309,10 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
                   _pth_mkup_src_in.copy(fn_src_out_filesystem); // check why here, thought dealt with elsewhere
                 }
                 if (doc_matters.opt.action.pod) {
-                   zip = pod_archive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
+                   zip = podArchive("file_path_text", _pth_mkup_src_in, _pth_mkup_src_out, zip);
                 }
               } else {
-                if (doc_matters.opt.action.debug_do_pod
-                && doc_matters.opt.action.vox_gt1) {
+                if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
                   writeln("WARNING (io) src out NOT found (insert file): ", _pth_mkup_src_in);
                 }
               }
@@ -333,11 +343,10 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
               fn_src_in.copy(fn_src_out_filesystem);
             }
             if (doc_matters.opt.action.pod) {
-              zip = pod_archive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
+              zip = podArchive("file_path_text", fn_src_in, fn_src_out_pod_zip_base, zip);
             }
           } else {
-            if (doc_matters.opt.action.debug_do_pod
-            && doc_matters.opt.action.vox_gt1) {
+            if (doc_matters.opt.action.debug_do_pod && doc_matters.opt.action.vox_gt1) {
               writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
             }
           }
@@ -345,24 +354,28 @@ auto pod_zip_make_ready(M,P)(M doc_matters, P pths_pod) {
       }
     }
   }
-  auto t = tuple(zip, fn_pod, _digests);
-  return t;
+  {
+    _st.zip     = zip;
+    _st.fn_pod  = fn_pod;
+    _st.digests = _digests;
+  }
+  return _st;
 }
 #+END_SRC
 
 ** mkdir :mkdir:
 
+- create directory structure
+
 #+NAME: source_pod_mkdirs
 #+BEGIN_SRC d
-void pod_archive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure
+void podArchive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create directory structure
   if (!exists(pths_pod.pod_dir_())) {
     // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
     pths_pod.pod_dir_().mkdirRecurse;
   }
   if (doc_matters.opt.action.source_or_pod) {
-    if (doc_matters.opt.action.vox_gt0) {
-      writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod);
-    }
+    // if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod); }
     if (!exists(pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod)) {
       pths_pod.text_root(doc_matters.src.filename).filesystem_open_zpod.mkdirRecurse;
     }
@@ -401,7 +414,7 @@ void pod_archive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create dire
 
 #+NAME: source_pod_archive_zip
 #+BEGIN_SRC d
-@system auto pod_archive(Z)(
+@system auto podArchive(Z)(
   string _source_type,
   string _data_in,
   string _pth_out,
@@ -454,6 +467,7 @@ void zipArchive(M,F,Z)(M doc_matters, F fn_pod, Z zip) {
 void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) {
   import sisudoc.io_out.paths_output;
   auto pths_pod = spinePathsPods!()(doc_matters);
+  char[] _zip_digest;
   try {
     if (!exists(pths_pod.pod_dir_())) {
       // used both by pod zipped (& pod filesystem (unzipped) which makes its own recursive dirs)
@@ -463,23 +477,20 @@ void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) {
     // Handle error
   }
   try {
-    writeln(pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt");
+    // if (doc_matters.opt.action.vox_gt1) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, ".digests.txt"); }
     string _digest_fn = pths_pod.pod_dir_() ~ "/" ~ doc_matters.src.filename_base ~ ".digests.txt";
-    writeln(_digest_fn);
+    // if (doc_matters.opt.action.vox_gt1) { writeln(_digest_fn); }
     auto f = File(_digest_fn, "w");
     if (exists(fn_pod)) {
       try {
-        if (doc_matters.opt.action.vox_gt0
-        && doc_matters.opt.action.pod) {
-          auto data = (cast(byte[]) (fn_pod).read);
-          if (doc_matters.opt.action.vox_gt1) {
-            writeln(doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip");
-          }
-          if (doc_matters.opt.action.pod) {
-            auto _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip");
-            writeln(_zip_digest);
-            f.writeln(_zip_digest);
-          }
+        auto data = (cast(byte[]) (fn_pod).read);
+        // if (doc_matters.opt.action.vox_gt1) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); }
+        if (doc_matters.opt.action.pod) {
+          _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip");
+          if (doc_matters.opt.action.vox_gt0) { writeln(" ", _zip_digest); }
+          if (doc_matters.opt.action.vox_gt0) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, "/"); }
+          if (doc_matters.opt.action.vox_gt0) { writeln(" ", _digest_fn); }
+          f.writeln(_zip_digest);
         }
       } catch (ErrnoException ex) {
         // Handle errors
@@ -487,19 +498,19 @@ void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) {
     }
     foreach (_lang; doc_matters.pod.manifest_list_of_languages) {
       if (_lang in _digests) {
-        if (("sst" in _digests[_lang]) && (_digests[_lang]["sst"].length > 0)) {
-          writeln(_digests[_lang]["sst"]);
-          f.writeln(_digests[_lang]["sst"]);
+        if (("sstm" in _digests[_lang]) && (_digests[_lang]["sstm"].length > 0)) {
+          // if (doc_matters.opt.action.vox_gt1) { writeln(_digests[_lang]["sstm"]); }
+          f.writeln(_digests[_lang]["sstm"]);
         }
         if (("ssi" in _digests[_lang]) && (_digests[_lang]["ssi"].length > 0)) {
-          writeln(_digests[_lang]["ssi"]);
+          // if (doc_matters.opt.action.vox_gt1) { writeln(_digests[_lang]["ssi"]); }
           f.writeln(_digests[_lang]["ssi"]);
         }
       }
     }
     if ("shared" in _digests) {
       if (("images" in _digests["shared"]) && (_digests["shared"]["images"].length > 0)) {
-        writeln(_digests["shared"]["images"]);
+        // if (doc_matters.opt.action.vox_gt1) { writeln(_digests["shared"]["images"]); }
         f.writeln(_digests["shared"]["images"]);
       }
     }