aboutsummaryrefslogtreecommitdiffhomepage
path: root/org/in_zip_pod.org
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-21 08:58:06 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-22 13:46:31 -0400
commitf89f0152a913cec14c6ee688a4f1b2ac7c4758ff (patch)
tree9701dd52ec89b5a8e5431eb1dbd29d2c6c349d2c /org/in_zip_pod.org
parentodt: office:meta from the document, not the run (diff)
pod: read when zstd-wrapped
sisupod using zstd compression recognised by first bytes. <doc>.sisupod: spine reads one before it writes one, so when the name changes (from .zip to .ssiupod) every pod already published stays readable. A .sisupod is one zstd frame wrapping the archive spine already builds. The reader unwraps the bytes and hands the same archive to the same parser, so every guard downstream is untouched: entry names, per-entry and total size, path depth, escape and symlinks. Which container it is comes from the first four bytes (rather than the name). A pod published as a plain .zip reads as it always did, a .sisupod reads, and either one renamed reads too. The suffix is still recognised, both spellings, for the argument and for a url. libzstd is declared rather than bound: provides the whole surface of fifteen extern C prototypes (there is nothing to generate and no upstream tree to track, which is the arrangement sqlite3 already has). dub links it with "libs": [ "zstd" ]; nix needs zstd.out rather than zstd, whose default output is the binaries and carries no library at all. A frame declares its uncompressed size in its own header, and for a pod fetched over https that number is attacker controlled. The declared size is checked against a ceiling before a buffer is asked for, a frame that will not declare one is refused, and what comes out is checked against what was promised. The ceiling is the extraction limit the archive reader already applies, so the two bounds agree. Measured: output built from a .sisupod is byte identical to output built from the same pod's .zip, 359 files over text, html, epub, odt and .ssp for three documents, live-manual's ten languages included. A truncated frame is refused and the document skipped. free_culture 2863504 -> 1099736 2.60x the_wealth_of_networks 4294022 -> 1172649 3.66x live-manual 6972418 -> 675970 10.31x (assisted by Claude-Code)
Diffstat (limited to 'org/in_zip_pod.org')
-rw-r--r--org/in_zip_pod.org26
1 files changed, 23 insertions, 3 deletions
diff --git a/org/in_zip_pod.org b/org/in_zip_pod.org
index 2922ecf..c542916 100644
--- a/org/in_zip_pod.org
+++ b/org/in_zip_pod.org
@@ -39,6 +39,7 @@ template spineExtractZipPod() {
import std.regex;
import std.stdio;
import std.string : indexOf;
+ import sisudoc.ocda.zstd;
/+ security limits for zip extraction +/
enum size_t MAX_ENTRY_SIZE = 50 * 1024 * 1024; /+ 50 MB per entry +/
@@ -98,10 +99,29 @@ template spineExtractZipPod() {
}
/+ ↓ derive pod name from zip filename +/
string zip_basename = zip_path.baseName.stripExtension;
- /+ ↓ read and parse zip archive +/
+ /+ ↓ read and parse the pod archive.
+ a .sisupod is one zstd frame wrapping the archive spine writes, so
+ the bytes are unwrapped before the archive is parsed. Determined from
+ the first four bytes and not from the filename: a pod published as a
+ plain .zip, or one a reader has renamed, is read either way. Everything
+ after this point sees the same archive it always did, guards included.
+ +/
+ ubyte[] _archive_bytes;
+ try {
+ _archive_bytes = cast(ubyte[]) read(zip_path);
+ if (zstdIsFrame(_archive_bytes)) {
+ _archive_bytes = zstdDecompress(_archive_bytes, MAX_TOTAL_SIZE);
+ }
+ } catch (ZstdException ex) {
+ result.error_msg = "failed to decompress pod archive: " ~ zip_path ~ " - " ~ ex.msg;
+ return result;
+ } catch (Exception ex) {
+ result.error_msg = "error reading pod archive: " ~ zip_path ~ " - " ~ ex.msg;
+ return result;
+ }
ZipArchive zip;
try {
- zip = new ZipArchive(read(zip_path));
+ zip = new ZipArchive(_archive_bytes);
} catch (ZipException ex) {
result.error_msg = "failed to read zip archive: " ~ zip_path ~ " - " ~ ex.msg;
return result;
@@ -250,7 +270,7 @@ template spineExtractZipPod() {
its own, carrying its images. A .ssp is not here: it describes its
images without carrying them, so it would arrive without them.
+/
- static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]zip|[.]ocda[.]db)$`);
+ static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]sisupod|[.]zip|[.]ocda[.]db)$`);
struct DownloadResult {
string local_path; /+ path to downloaded temp file +/