diff options
Diffstat (limited to 'org')
| -rw-r--r-- | org/config_nix.org | 389 | ||||
| -rw-r--r-- | org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org | 9 |
2 files changed, 397 insertions, 1 deletions
diff --git a/org/config_nix.org b/org/config_nix.org index cea53f2..146c9ea 100644 --- a/org/config_nix.org +++ b/org/config_nix.org @@ -16,12 +16,390 @@ * nix :nix: ** flake.nix :flake: +flake devshell, issues with diffoscope and with datasette + +diffoscope sorted by pinning for the time being to stable branch + +nix devshell: fix datasette with patched asgi-csrf + +datasette is useful for viewing sisudoc spine .ocda.db files without the need to +write any sql. + +datasette broken in nixpkgs (both in stable nixos-26.05 and unstable) nix +refuses to evaluate it, fix with patched asgi-csrf (used nixos-26.05) + +python-multipart 0.0.26 removed FormParser's FileClass argument, which asgi-csrf +passes (simonw/asgi-csrf#38, open), and 26.05 carries 0.0.29. + +A nixpkgs-stable input (nixos-26.05, by its channel url, locked), and from it, +for datasette alone, a python package set in which: + +- asgi-csrf hands its own ErrorOnWrite(...).write to on_file, which + python-multipart still calls as each file part ends, in place of FileClass: a + file arriving before the csrf token is still refused, and asgi-csrf's 48 tests + pass, the multipart ones included; +- datasette skips test_max_csv_mb, which streams a large csv under the query + time limit and is interrupted on a loaded builder. + +datasette is made available in dsh-production and dsh-production-post. datasette +0.65.2 serves an .ocda.db, its tables, views, json api, sql and facets. + #+HEADER: :tangle ../flake.nix #+HEADER: :noweb yes #+BEGIN_SRC nix { description = "a sisu like parser & document generator"; inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; + inputs.nixpkgs-stable.url = "<<nixpkgs_stable>>"; + outputs = { + self, + nixpkgs, + ... + }@inputs: let + pname = "spine"; + version = "<<spine_version>>"; + supportedSystems = ["x86_64-linux"]; # [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + nixpkgsFor = forAllSystems (system: import nixpkgs {inherit system;}); # nixpkgs instantiated for supported system types + localOverlay = (final: prev: { + ldc = prev.callPackage ./nix-overlays/ldc { }; # -> ok <<ldc_version>> + # dmd overlay: revert to GCC14; (current) GCC 15 introduced nullptr in its headers, and DMD's ImportC parser needs update to handle it, monitor and update: + dmd = prev.callPackage ./nix-overlays/dmd { stdenv = prev.gcc14Stdenv; }; # -> ok <<dmd_version>> + dub = prev.callPackage ./nix-overlays/dub { }; # -> ? <<dub_version>> + dtools = prev.callPackage ./nix-overlays/dtools { }; # -> ok <<dtools_version>> + #gdc = prev.callPackage ./nix-overlays/gdc { }; + }); + pkgsForSystem = system: import nixpkgs { + overlays = [ + localOverlay + ]; + inherit system; + }; + mkSpine = { pkgs, compilerPkg, compilerBin, buildType }: pkgs.stdenv.mkDerivation { + inherit pname version; + meta.mainProgram = "spine"; + executable = true; # not used + src = self; + buildInputs = [ pkgs.sqlite pkgs.zstd.out ]; + nativeBuildInputs = [ pkgs.dub compilerPkg pkgs.gnumake ]; + preBuild = '' + export HOME=$TMPDIR + ''; + buildPhase = '' + runHook preBuild + buildCMD="dub run --cache=local --compiler=$(type -P ${compilerBin}) --build=${buildType} --combined --skip-registry=all" + echo $buildCMD + $buildCMD + echo $buildCMD + runHook postBuild + ''; + checkPhase = '' + runHook preCheck + dub test --combined --skip-registry=all + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ./bin/spine $out/bin/spine + runHook postInstall + ''; + postInstall = '' + echo "❯❯ ./result/bin/spine -v --source --pod --text --epub --html --html-link-pdf --html-link-curate --html-link-markup --curate --output=./OUTPUT_TEST ../sisu-spine-samples/markup/pod-samples/pod/*"; + echo `ls -la $out/bin/spine` + echo "❯❯ spine-v${version} (rev: ${self.shortRev or "unknown"})" + $out/bin/spine -v + ''; + }; + shellHook = '' + #export Date=`date "+%Y%m%d"` + ## set local values in .envrc-local (or here if you must) + echo '❯❯ nix build'; + ## ImPure Nix environment + SpineGitVer=`git describe | sed "s/^[a-z_-]\+\([0-9.]\+\)/\1/" | sed "s/\([^-]*-g\)/r\1/" | sed "s/-/./g"` && \ + SpineGitBranch=`git branch --show-current` && \ + echo "❯❯ spine-v${version} - ($SpineGitBranch: $SpineGitVer)" + ## Pure Nix environment - version info from flake inputs + #echo "❯❯ spine-v${version} (rev: ${self.shortRev or "unknown"})" + echo '❯❯ $SpineBIN -v --source --pod --text --epub --html --html-link-pdf --html-link-curate --html-link-markup --curate --output=$SpineOUT $SpinePOD/*'; + echo "❯❯ $SpineBIN -v --source --pod --text --epub --html --html-link-pdf --html-link-curate --html-link-markup --curate --output=$SpineOUT $SpinePOD/*"; + echo '❯❯ nix flake update && nix flake check && nix flake show'; + ''; + mkDevShell = { pkgs, name, compiler, extraPackages ? [] }: pkgs.mkShell { + inherit name shellHook; + packages = [ + pkgs.direnv + pkgs.nix-direnv + compiler + pkgs.dub + pkgs.dtools + pkgs.gnumake + pkgs.sqlite + pkgs.zstd + pkgs.gnugrep + pkgs.gnused + pkgs.ripgrep + ] ++ extraPackages; + }; + in { + packages = forAllSystems (system: let + pkgs-ovl = pkgsForSystem system; + pkgs-nix = nixpkgsFor.${system}; + in { + default = self.packages.${system}.spine-nixpkgs-ldc; + spine-nixpkgs-dmd = mkSpine { + pkgs = pkgs-nix; + # nixpkgs dmd: revert to GCC14 (same nullptr/ImportC fix as overlay dmd) + compilerPkg = pkgs-nix.dmd.override { stdenv = pkgs-nix.gcc14Stdenv; }; + #compilerPkg = pkgs-nix.dmd; + compilerBin = "dmd"; + buildType = "dmd"; + }; + spine-nixpkgs-ldc = mkSpine { + pkgs = pkgs-nix; + compilerPkg = pkgs-nix.ldc; + compilerBin = "ldmd2"; + buildType = "ldmd2"; + }; + spine-overlay-dmd = mkSpine { + pkgs = pkgs-ovl; + compilerPkg = pkgs-ovl.dmd; + compilerBin = "dmd"; + buildType = "dmd"; + }; + spine-overlay-ldc = mkSpine { + pkgs = pkgs-ovl; + compilerPkg = pkgs-ovl.ldc; + compilerBin = "ldmd2"; + buildType = "ldmd2"; + }; + #spine-overlay-gdc = mkSpine { + # pkgs = pkgs-ovl; + # compilerPkg = pkgs-ovl.gdc; + # compilerBin = "gdc"; + # buildType = "gdc"; + #}; + }); + apps = forAllSystems (system: { + default = { + type = "app"; + program = "${self.packages.${system}.default}/bin/spine"; + }; + }); + devShells = forAllSystems (system: let + pkgs-ovl = pkgsForSystem system; + pkgs-nix = nixpkgsFor.${system}; + # datasette, from the stable channel. It is held back both in stable + # and in unstable by asgi-csrf 0.11, marked broken because + # python-multipart 0.0.26 dropped FormParser's FileClass argument + # (simonw/asgi-csrf#38, open). asgi-csrf used it only to refuse a file + # arriving before the csrf token, by a file class whose write raises; + # given instead as on_file, which python-multipart still calls as each + # file ends, that write refuses it the same way, and asgi-csrf's own + # tests then pass. + pkgs-stable = import inputs.nixpkgs-stable { + inherit system; + overlays = [ (final: prev: { + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ (pyfinal: pyprev: { + asgi-csrf = pyprev.asgi-csrf.overridePythonAttrs (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace asgi_csrf.py \ + --replace-fail "lambda: None," "ErrorOnWrite(None, None, None).write," \ + --replace-fail "FileClass=ErrorOnWrite," "" + ''; + meta = old.meta // { broken = false; }; + }); + # unbuilt by hydra while asgi-csrf was broken; this one test streams + # a large csv under the query time limit, and is interrupted on a + # loaded builder (sqlite "interrupted", a 500): timing, not datasette + datasette = pyprev.datasette.overridePythonAttrs (old: { + disabledTests = (old.disabledTests or [ ]) ++ [ "test_max_csv_mb" ]; + }); + }) ]; + }) ]; + }; + in { + default = self.devShells.${system}.dsh-overlay; + #default = import ./shell.nix {pkgs = pkgs-nix; flake = self;}; + dsh-overlay = mkDevShell { + pkgs = pkgs-ovl; + # "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>> - dtools-<<dtools_version>>"; + name = "spine-${version} dev shell (overlay ldc)"; + compiler = pkgs-ovl.ldc; + }; + dsh-nixpkgs-dmd = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell (nixpkgs dmd)"; + # nixpkgs dmd: revert to GCC14 (same nullptr/ImportC fix as overlay dmd) + compiler = pkgs-nix.dmd.override { stdenv = pkgs-nix.gcc14Stdenv; }; + #compiler = pkgs-nix.dmd; + }; + dsh-nixpkgs-ldc = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell (nixpkgs ldc)"; + compiler = pkgs-nix.ldc; + }; + dsh-overlay-dmd = mkDevShell { + pkgs = pkgs-ovl; + # "spine-<<spine_version>> base dev shell, dmd-<<dmd_version>>, dub-<<dub_version>> - dtools-<<dtools_version>>"; + name = "spine-${version} dev shell (overlay dmd)"; + compiler = pkgs-ovl.dmd; + }; + dsh-overlay-ldc = mkDevShell { + pkgs = pkgs-ovl; + # "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>> - dtools-<<dtools_version>>"; + name = "spine-${version} dev shell (overlay ldc)"; + compiler = pkgs-ovl.ldc; + }; + dsh-epub = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for epub output"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + libxml2 + html-tidy + xmlstarlet + epubcheck + ebook_tools + epr + sigil + calibre #(suite includes: ebook-viewer) + koreader + foliate + ]; + }; + dsh-html = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for html output"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + # ❯❯ text-mode web browsers + elinks + #links2 + lynx + w3m + # ❯❯ light graphical + #dillo + ]; + }; + dsh-odf = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for epub output"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + jing + ]; + }; + dsh-typst-pdf = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for latex & pdf output"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + typst + #typstyle + #typst-live + #typstwriter + ]; + }; + dsh-latex-pdf = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for latex & pdf output"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + source-sans-pro + source-serif-pro + source-code-pro + texliveFull + ]; + }; + dsh-sqlite = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for sqlite3 output"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + #sqlite + sqlite-interactive + litecli + visidata + sqlitebrowser + letos + dbeaver-bin + ]; + }; + dsh-translations = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell for internationalization, po4a"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + po4a + gettext + ]; + }; + dsh-production = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell: production run and its checks"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + # the suite shells out to these two; without them run-tests.sh + # re-execs itself into a nix shell that has them + sqlite-interactive # readline cli; base sqlite has none + pkgs-stable.datasette # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration + epubcheck + unzip # containers: every one spine writes is a zip + # xml and html by eye and by check + libxml2 # xmllint + xmlstarlet # xml sel -t -v + html-tidy # the scroll html, which epubcheck never sees + # pdf + poppler-utils # pdfinfo, pdftotext + typst + #texliveFull # NOT texlive.combined.scheme-full: deprecated, + file # what a file is + ]; + }; + dsh-production-post = mkDevShell { + pkgs = pkgs-nix; + name = "spine-${version} dev shell: production run and its checks"; + compiler = pkgs-nix.ldc; + extraPackages = with pkgs-nix; [ + # the suite shells out to these two; without them run-tests.sh + # re-execs itself into a nix shell that has them + sqlite-interactive # readline cli; base sqlite has none + pkgs-stable.datasette # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration + letos + epubcheck + pkgs-stable.diffoscope # diffs two epubs, two odts or two pdfs by content rather (than as bytes) (useful when output moves and cmp only says "differ") + unzip # containers: every one spine writes is a zip + # xml and html by eye and by check + libxml2 # xmllint + xmlstarlet # xml sel -t -v + html-tidy # the scroll html, which epubcheck never sees + xmlstarlet + ebook_tools + #jing + # pdf + poppler-utils # pdfinfo, pdftotext + typst + #texliveFull # NOT texlive.combined.scheme-full: deprecated, + file # what a file is + coreutils # sha256sum and numfmt guaranteed, for checking a digests.txt line by hand + po4a + gettext + ]; + }; + }); + }; +} +#+END_SRC + +*** reinstate this flake.nix once datasette builds again without intervention + +#+HEADER: :noweb yes +#+BEGIN_SRC nix +{ + description = "a sisu like parser & document generator"; + inputs.nixpkgs.url = "<<nixpkgs_unstable>>"; + #inputs.nixpkgs-stable.url = "<<nixpkgs_stable>>"; outputs = { self, nixpkgs, @@ -306,7 +684,7 @@ # the suite shells out to these two; without them run-tests.sh # re-execs itself into a nix shell that has them sqlite-interactive # readline cli; base sqlite has none - #datasette # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration + # datasette # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration letos epubcheck diffoscope # diffs two epubs, two odts or two pdfs by content rather (than as bytes) (useful when output moves and cmp only says "differ") @@ -1073,6 +1451,7 @@ postInstall: #+END_SRC * nixpkgs source +** unstable #+NAME: nixpkgs_unstable #+HEADER: :noweb yes @@ -1080,4 +1459,12 @@ postInstall: <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>> #+END_SRC +** stable + +#+NAME: nixpkgs_stable +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_stable()>> +#+END_SRC + * __END__ diff --git a/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org b/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org index 59936d2..347779d 100644 --- a/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org +++ b/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org @@ -978,6 +978,8 @@ git+file:///home/ralph/grotto/repo/git.repo/projects/doc-reform/packages/nix-fla - github:NixOS/nixpkgs/nixpkgs-unstable - https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst +** nipkgs unstable + #+NAME: nixpkgs_unstable #+BEGIN_SRC org https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst @@ -986,3 +988,10 @@ https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst #+BEGIN_SRC org github:NixOS/nixpkgs/nixpkgs-unstable #+END_SRC + +** nipkgs stable + +#+NAME: nixpkgs_stable +#+BEGIN_SRC org +https://channels.nixos.org/nixos-26.05/nixexprs.tar.zst +#+END_SRC |