open-webui: more consistent example

Open WebUI 0.9.6 still initializes its embedding function during startup, even when OFFLINE_MODE=true and BYPASS_EMBEDDING_AND_RETRIEVAL=true. With OFFLINE_MODE=true and no cached SentenceTransformer model, startup failed with:

ValueError: No embedding model is loaded. Set RAG_EMBEDDING_MODEL to a valid SentenceTransformer model name, or configure an external RAG_EMBEDDING_ENGINE (ollama, openai, azure_openai).

Configure the tests to use a dummy Ollama embedding endpoint so startup does not try to load or download a local embedding model. The endpoint is not meant to be contacted by these tests; it only keeps startup on the external embedding configuration path.

.github/workflows/auto-merge.yaml

@@ -23,7 +23,7 @@ on:
         required: false
 
 jobs:
-  automerge:
+  automerge-rebase:
     runs-on: ubuntu-latest
     steps:
       - uses: reitermarkus/automerge@v2
@@ -35,3 +35,16 @@ jobs:
           pull-request: ${{ github.event.inputs.pull-request }}
           review: ${{ github.event.inputs.review }}
           dry-run: false
+
+  automerge-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: reitermarkus/automerge@v2
+        with:
+          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          merge-method: merge
+          do-not-merge-labels: never-merge
+          required-labels: automerge-merge
+          pull-request: ${{ github.event.inputs.pull-request }}
+          review: ${{ github.event.inputs.review }}
+          dry-run: false

.github/workflows/lock-update.yaml

@@ -1,9 +1,16 @@
+# See shb.skarabox.com/misc.html
 name: Update Flake Lock
 
 on:
-  workflow_dispatch:
   schedule:
-    - cron: '0 0 * * *' # runs daily at 00:00
+    - cron: '0 */3 * * *' # runs every 3 hours at :00
+
+  workflow_dispatch:
+    inputs:
+      bisect_future:
+        description: "Bisect in the future instead of the past"
+        required: false
+        default: "false"
 
 jobs:
   lockfile:
@@ -15,9 +22,22 @@ jobs:
         uses: cachix/install-nix-action@v31
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Update flake.lock
+      - name: Cache nixpkgs mirror
-        uses: DeterminateSystems/update-flake-lock@main
+        uses: actions/cache@v5
         with:
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          path: .cache/nixpkgs.git
-          pr-labels: |
+          key: nixpkgs-mirror-v1
-            automerge
+      - name: Update flake.lock
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          BISECT_FUTURE: ${{ inputs.bisect_future }}
+        run: |
+          git config user.email "update-flake-lock-pr@selfhostblocks.com"
+          git config user.name "Update Flake Lock PR"
+          if [ "$BISECT_FUTURE" = false ]; then
+            BISECT_FUTURE=
+          elif [ -n "$BISECT_FUTURE" ]; then
+            BISECT_FUTURE=future
+          fi
+          nix run .#update-flake-lock-pr $BISECT_FUTURE
+

.github/workflows/pages.yml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/configure-pages@v6
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v4
+        uses: actions/upload-pages-artifact@v5
         with:
           path: ./public
 

.github/workflows/update-flake-lock-pr.nix

@@ -0,0 +1,223 @@
+{
+  gh,
+  writeShellApplication,
+}:
+writeShellApplication {
+  name = "update-flake-lock-pr";
+  runtimeInputs = [
+    gh
+  ];
+  text = ''
+    branch=ci/nixpkgs-update
+    if [ "''${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
+      branch=ci/nixpkgs-update-auto
+    fi
+
+    goto_future=no
+    if [ -n "''${1:-}" ]; then
+      goto_future=yes
+    fi
+
+    main () {
+      PR=$(get_pr)
+
+      if [ -z "$PR" ]; then
+        create_pr
+        exit 0
+      fi
+
+      case "$(get_pr_check_status "$PR")" in
+        pending)
+          echo "Checks are running on PR $PR, nothing to do yet."
+          exit 0
+          ;;
+
+        succeeded)
+          echo "Checks succeeded, nothing to do."
+          exit 0
+          ;;
+
+        failing)
+          echo "Checks failed, updating PR to midpoint commit."
+
+          local CURRENT_COMMIT
+          local LAST_WORKING_COMMIT
+          local FAILING_COMMIT
+          local NEW_COMMIT
+
+          CURRENT_COMMIT="$(get_main_branch_commit)"
+          if [ "$goto_future" = "no" ]; then
+            LAST_WORKING_COMMIT="$CURRENT_COMMIT"
+            FAILING_COMMIT="$(get_pr_commit "$PR")"
+          else
+            LAST_WORKING_COMMIT="$(get_pr_commit "$PR")"
+            FAILING_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+          NEW_COMMIT="$(midpoint_commit "$LAST_WORKING_COMMIT" "$FAILING_COMMIT")"
+
+          if [ "$NEW_COMMIT" = "$LAST_WORKING_COMMIT" ] || [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            NEW_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+
+          if [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            echo "Latest nixpkgs commit is still same failing commit, will retry later."
+            exit 0
+          fi
+
+          update_pr "$PR" "$CURRENT_COMMIT" "$NEW_COMMIT"
+          exit 0
+          ;;
+      esac
+    }
+
+    get_main_branch_commit() {
+      git fetch origin main
+
+      git show origin/main:flake.lock \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    midpoint_commit() {
+      local good="$1"
+      local bad="$2"
+
+      mkdir -p .cache
+      if [ ! -d .cache/nixpkgs.git ]; then
+        git clone --mirror https://github.com/NixOS/nixpkgs.git .cache/nixpkgs.git
+      else
+        git --git-dir=.cache/nixpkgs.git fetch origin
+      fi
+
+      local commits=()
+
+      mapfile -t commits < <(
+        git --git-dir=.cache/nixpkgs.git \
+          rev-list \
+          --reverse \
+          --ancestry-path \
+          "''${good}..''${bad}"
+      )
+
+      local count="''${#commits[@]}"
+
+      if [ "$count" -eq 0 ]; then
+        echo "$bad"
+        return
+      fi
+
+      local midpoint_index=$((count / 2))
+
+      echo "''${commits[$midpoint_index]}"
+    }
+
+    get_latest_nixpkgs_commit () {
+      git ls-remote https://github.com/NixOS/nixpkgs nixos-unstable | cut -f1
+    }
+
+    get_pr () {
+      gh pr list \
+        --head "$branch" \
+        --state open \
+        --json number \
+        --jq '.[0].number // empty'
+    }
+
+    create_pr() {
+      local test_commit
+
+      test_commit="$(get_latest_nixpkgs_commit)"
+      echo "Creating PR on nixpkgs' latest commit $test_commit"
+
+      git checkout -B "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push -u origin "$branch" --force
+
+      current_commit="$(get_main_branch_commit)"
+      gh pr create \
+        --title "update nixpkgs to $test_commit" \
+        --body "$(printf "%s\n\n%s" "Automated nixpkgs update. Latest tries:" " - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit")" \
+        --label automerge-merge
+    }
+
+    append_pr_body() {
+      local pr="$1"
+      local text="$2"
+
+      local current_body
+
+      current_body="$(gh pr view "$pr" --json body --jq .body)"
+
+      gh pr edit "$pr" \
+        --body "$(printf '%s%b' "$current_body" "$text")"
+    }
+
+    update_pr() {
+      local pr="$1"
+      local current_commit="$2"
+      local test_commit="$3"
+
+      echo "Updating PR to nixpkgs' commit $test_commit"
+
+      git checkout "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push --force-with-lease origin "$branch"
+
+      local future_text
+      if [ "$goto_future" = "yes" ]; then
+        future_text="bisected in the future"
+      else
+        future_text="bisected in the past"
+      fi
+
+      gh pr edit "$pr" \
+        --title "update nixpkgs to $test_commit"
+
+      append_pr_body "$pr" \
+        " ❌\n - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit ($future_text)"
+    }
+
+    get_pr_check_status() {
+      local pr="$1"
+
+      local status
+      status="$(gh pr checks "$pr" --json state --jq '.[].state' || true)"
+
+      if echo "$status" | grep -qE 'PENDING|QUEUED|IN_PROGRESS'; then
+        echo "pending"
+        return
+      fi
+
+      if echo "$status" | grep -qE 'FAILURE|TIMED_OUT|CANCELLED'; then
+        echo "failing"
+        return
+      fi
+
+      echo "success"
+    }
+
+    get_pr_commit() {
+      local pr="$1"
+
+      git fetch origin "$branch"
+
+      git show "origin/$branch:flake.lock" \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    main
+  '';
+}

CHANGELOG.md

@@ -16,6 +16,45 @@ Template:
 
 # Upcoming Release
 
+# v0.9.0
+
+Commits: https://github.com/ibizaman/selfhostblocks/compare/v0.8.0...v0.9.0
+
+## Breaking Changes
+
+- Updated simple-nixos-mailserver. Update all options following this pattern:
+
+  ```diff
+  - mailserver.mailboxes.<name>.specialUse = "value"
+  + mailserver.mailboxes.<name>.special_use = "\\value"
+  ```
+
+  Note that simple-nixos-mailserver wanted to upgrade the location of the storage path
+  to include the ldap UID instead of the email but I kept the email address.
+  This means there is no migration to do.
+- Update nixpkgs [from 6201e2 to abd1ea](https://github.com/ibizaman/selfhostblocks/compare/6201e203d09599479a3b3450ed24fa81537ebc4e...abd1ea17fdbedd48cbca770847b39e3a7a09ab5b).
+
+## New Features
+
+- Add `shb.zfs.snapshotBeforeActivation` option to take a ZFS snapshot of given datasets before activation.
+  See [the manual](https://shb.skarabox.com/blocks-zfs.html) for more details.
+- Add [sanoid block](https://shb.skarabox.com/blocks-sanoid.html)
+- Add option to take snapshot before activation with ZFS block.
+
+## Fixes
+
+- Fix oidc_login build for Nextcloud
+- Fix mailserver build
+
+## Other Changes
+
+- Harden lldap configuration
+- Convert ZFS block to system and [add documentation](https://shb.skarabox.com/blocks-zfs.html)
+- Switch monitoring log fetching from deprecated promtail to fluent-bit
+- Add test to catch drift for Let's Encrypt DNS provider
+
+# [BROKEN] v0.8.0
+
 ## Breaking Changes
 
 - Bump of Nextcloud version to 32 and 33 because of nixpkgs bump. All provided apps are verified compatible with Nextcloud 33 thanks to new tests.

VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

demo/minimal/flake.nix

@@ -24,7 +24,10 @@
           # This module makes the assertions happy and the build succeed.
           # This is of course wrong and will not work on any real system.
           filesystemModule = {
-            fileSystems."/".device = "/dev/null";
+            fileSystems."/" = {
+              device = "/dev/null";
+              fsType = "none";
+            };
             boot.loader.grub.devices = [ "/dev/null" ];
           };
         in

docs/blocks.md

@@ -31,6 +31,10 @@ modules/blocks/borgbackup/docs/default.md
 modules/blocks/restic/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//blocks-sanoid.html
+modules/blocks/sanoid/docs/default.md
+```
+
 ## Database {#blocks-category-database}
 
 ```{=include=} chapters html:into-file=//blocks-postgresql.html
@@ -43,6 +47,12 @@ modules/blocks/postgresql/docs/default.md
 modules/blocks/sops/docs/default.md
 ```
 
+## Filesystem {#blocks-category-filesystem}
+
+```{=include=} chapters html:into-file=//blocks-zfs.html
+modules/blocks/zfs/docs/default.md
+```
+
 ## Network {#blocks-category-network}
 
 ```{=include=} chapters html:into-file=//blocks-ssl.html

docs/contracts.md

@@ -39,7 +39,9 @@ Provided contracts are:
 - [Backup contract][] to backup directories.
   Two providers are implemented: [BorgBackup][] and [Restic][].
 - [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
-  One provider is implemented: [BorgBackup][] and [Restic][].
+  Two providers are implemented: [BorgBackup][] and [Restic][].
+- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets.
+  One provider is implemented: [Sanoid][]
 - [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
   One provider is implemented: [SOPS][].
 - [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard.
@@ -49,6 +51,7 @@ Provided contracts are:
 [borgbackup]: blocks-borgbackup.html
 [homepage]: services-homepage.html
 [restic]: blocks-restic.html
+[sanoid]: blocks-sanoid.html
 [sops]: blocks-sops.html
 
 ```{=include=} chapters html:into-file=//contracts-ssl.html
@@ -63,6 +66,10 @@ modules/contracts/backup/docs/default.md
 modules/contracts/databasebackup/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//contracts-datasetbackup.html
+modules/contracts/datasetbackup/docs/default.md
+```
+
 ```{=include=} chapters html:into-file=//contracts-secret.html
 modules/contracts/secret/docs/default.md
 ```

docs/contributing.md

@@ -61,6 +61,25 @@ If the test includes playwright tests, you can see the playwright trace with:
 $ nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
 ```
 
+If the test fails, you won't have the output directory available.
+Instead, run the test with `--keep-failed` and at the end you will see a line saying
+
+```
+Keeping build directory '/nix/var/nix/builds/nix-31776-4270051001/build'
+```
+
+Copy the path and run:
+
+```bash
+sudo cp -r /nix/var/nix/builds/nix-31776-4270051001/build/shared-xchg/trace trace && sudo chown -R $USER: trace
+```
+
+Now, open that file with playwright:
+
+```bash
+nix run .#playwright -- show-trace trace/0.zip
+```
+
 ### Debug Tests {#contributing-debug-tests}
 
 Run the test in driver interactive mode:

docs/manual.md

@@ -28,6 +28,10 @@ blocks.md
 recipes.md
 ```
 
+```{=include=} chapters html:into-file=//misc.html
+misc.md
+```
+
 ```{=include=} chapters html:into-file=//demos.html
 demos.md
 ```

docs/misc.md

@@ -0,0 +1,39 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Miscellaneous {#misc}
+
+Here goes meta-discussions around the repository and other topics not related directly to the usage of SHB.
+
+## Lock file update {#misc-lock-file-update}
+
+*SHB has an unusual strategy to keep up with its flake inputs. This section explains why.*
+
+SHB only depends on `nixpkgs` as far as flake inputs goes.
+There are others but they only matter for tasks around taking care of the repository itself.
+So only the `nixpkgs` input is important for downstream consumers of SHB.
+
+To keep up to date with nixpkgs unstable,
+initially a job was updating the nixpkgs input by simply updating the lock file and creating a PR.
+Now, this job failed most of the time because the tip of unstable often breaks modules and packages, resulting in failing SHB tests.
+To fix this, we usually needed to choose a commit around 2 weeks prior to the latest unstable commit.
+This usually did not lead to successful tests but the reason was then not
+because of failing packages but because nixpkgs modules evolved and SHB needs to adapt to that.
+The net effect was SHB was lagging too far behind unstable and updating it was becoming annoying.
+
+The new strategy now is as follows. On every job run:
+
+- If there is no PR, update nixpkgs input to latest unstable commit and create a PR which auto-merges when tests are successful.
+- If there is a PR:
+  - If the checks succeeded, do nothing because the PR will auto-merge.
+  - If the checks are pending, do nothing yet.
+  - If the checks failed, bisect and update nixpkgs input to between the commit on main and the commit in the PR.
+
+The effect here is as long as the tests are failing, we'll try a commit further in the past from nixpkgs unstable
+and closer to the tip of main branch.
+
+This could be made smarter by distinguishing between types of failures.
+We will see if this is needed.
+
+One can also run the bisect manually with `nix run .#update-flake-lock-pr`.
+This will create a PR on a different branch than the one used for the automated workflow.
+It accepts also one argument `future` which instead of trying a commit in between the tip of SHB main and the PR's failing one,
+it tries a commit between the PR's failing one and the nixpkgs unstable latest commit.

docs/redirects.json

@@ -347,6 +347,12 @@
   "blocks-borgbackup-maintenance": [
     "blocks-borgbackup.html#blocks-borgbackup-maintenance"
   ],
+  "blocks-borgbackup-maintenance-manuql": [
+    "blocks-borgbackup.html#blocks-borgbackup-maintenance-manuql"
+  ],
+  "blocks-borgbackup-maintenance-restore": [
+    "blocks-borgbackup.html#blocks-borgbackup-maintenance-restore"
+  ],
   "blocks-borgbackup-maintenance-troubleshooting": [
     "blocks-borgbackup.html#blocks-borgbackup-maintenance-troubleshooting"
   ],
@@ -581,6 +587,9 @@
   "blocks-category-database": [
     "blocks.html#blocks-category-database"
   ],
+  "blocks-category-filesystem": [
+    "blocks.html#blocks-category-filesystem"
+  ],
   "blocks-category-introspection": [
     "blocks.html#blocks-category-introspection"
   ],
@@ -917,6 +926,9 @@
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName"
   ],
+  "blocks-mitmdump-options-shb.mitmdump.instances._name_.timeout": [
+    "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.timeout"
+  ],
   "blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [
     "blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost"
   ],
@@ -959,6 +971,9 @@
   "blocks-monitoring-error-dashboard": [
     "blocks-monitoring.html#blocks-monitoring-error-dashboard"
   ],
+  "blocks-monitoring-impermanence": [
+    "blocks-monitoring.html#blocks-monitoring-impermanence"
+  ],
   "blocks-monitoring-nextcloud-dashboard": [
     "blocks-monitoring.html#blocks-monitoring-nextcloud-dashboard"
   ],
@@ -1022,6 +1037,9 @@
   "blocks-monitoring-options-shb.monitoring.grafanaPort": [
     "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.grafanaPort"
   ],
+  "blocks-monitoring-options-shb.monitoring.impermanence": [
+    "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.impermanence"
+  ],
   "blocks-monitoring-options-shb.monitoring.ldap": [
     "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ldap"
   ],
@@ -1373,6 +1391,12 @@
   "blocks-restic-maintenance": [
     "blocks-restic.html#blocks-restic-maintenance"
   ],
+  "blocks-restic-maintenance-manuql": [
+    "blocks-restic.html#blocks-restic-maintenance-manuql"
+  ],
+  "blocks-restic-maintenance-restore": [
+    "blocks-restic.html#blocks-restic-maintenance-restore"
+  ],
   "blocks-restic-maintenance-troubleshooting": [
     "blocks-restic.html#blocks-restic-maintenance-troubleshooting"
   ],
@@ -1589,6 +1613,63 @@
   "blocks-restic-usage-provider-remote": [
     "blocks-restic.html#blocks-restic-usage-provider-remote"
   ],
+  "blocks-sanoid": [
+    "blocks-sanoid.html#blocks-sanoid"
+  ],
+  "blocks-sanoid-contract-provider": [
+    "blocks-sanoid.html#blocks-sanoid-contract-provider"
+  ],
+  "blocks-sanoid-options": [
+    "blocks-sanoid.html#blocks-sanoid-options"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request.excludePatterns": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.excludePatterns"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.afterBackup": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.afterBackup"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.beforeBackup": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.beforeBackup"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request.sourceDirectories": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.sourceDirectories"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.request.user": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.user"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.result": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.result"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.result.backupService": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.result.backupService"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.result.restoreScript": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.result.restoreScript"
+  ],
+  "blocks-sanoid-options-shb.sanoid.backup._name_.settings": [
+    "blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.settings"
+  ],
+  "blocks-sanoid-usage": [
+    "blocks-sanoid.html#blocks-sanoid-usage"
+  ],
+  "blocks-sanoid-usage-custom-template": [
+    "blocks-sanoid.html#blocks-sanoid-usage-custom-template"
+  ],
+  "blocks-sanoid-usage-default-template": [
+    "blocks-sanoid.html#blocks-sanoid-usage-default-template"
+  ],
+  "blocks-sanoid-usage-without-contract": [
+    "blocks-sanoid.html#blocks-sanoid-usage-without-contract"
+  ],
   "blocks-sops": [
     "blocks-sops.html#blocks-sops"
   ],
@@ -1760,6 +1841,117 @@
   "blocks-ssl-options-shb.certs.systemdService": [
     "blocks-ssl.html#blocks-ssl-options-shb.certs.systemdService"
   ],
+  "blocks-zfs": [
+    "blocks-zfs.html#blocks-zfs"
+  ],
+  "blocks-zfs-features": [
+    "blocks-zfs.html#blocks-zfs-features"
+  ],
+  "blocks-zfs-options": [
+    "blocks-zfs.html#blocks-zfs-options"
+  ],
+  "blocks-zfs-options-shb.zfs.pools": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.after": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.after"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.excludePatterns": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.excludePatterns"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.afterBackup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.afterBackup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.beforeBackup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.beforeBackup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.sourceDirectories": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.sourceDirectories"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.user": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.user"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.backupService": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.backupService"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.enable": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.enable"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.mode": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.mode"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path"
+  ],
+  "blocks-zfs-options-shb.zfs.snapshotBeforeActivation.datasets": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.datasets"
+  ],
+  "blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable"
+  ],
+  "blocks-zfs-options-shb.zfs.snapshotBeforeActivation.recursive": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.recursive"
+  ],
+  "blocks-zfs-options-shb.zfs.snapshotBeforeActivation.template": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.template"
+  ],
+  "blocks-zfs-usage": [
+    "blocks-zfs.html#blocks-zfs-usage"
+  ],
+  "blocks-zfs-usage-backup-dataset": [
+    "blocks-zfs.html#blocks-zfs-usage-backup-dataset"
+  ],
+  "blocks-zfs-usage-backup-files": [
+    "blocks-zfs.html#blocks-zfs-usage-backup-files"
+  ],
+  "blocks-zfs-usage-snapshot-before-activation": [
+    "blocks-zfs.html#blocks-zfs-usage-snapshot-before-activation"
+  ],
   "build-time-validation": [
     "service-implementation-guide.html#build-time-validation"
   ],
@@ -1820,6 +2012,21 @@
   "contract-databasebackup-usage": [
     "contracts-databasebackup.html#contract-databasebackup-usage"
   ],
+  "contract-dataset-backup": [
+    "contracts-datasetbackup.html#contract-dataset-backup"
+  ],
+  "contract-datasetbackup-providers": [
+    "contracts-datasetbackup.html#contract-datasetbackup-providers"
+  ],
+  "contract-datasetbackup-requesters": [
+    "contracts-datasetbackup.html#contract-datasetbackup-requesters"
+  ],
+  "contract-datasetbackup-usage": [
+    "contracts-datasetbackup.html#contract-datasetbackup-usage"
+  ],
+  "contract-datatsetbackup-options": [
+    "contracts-datasetbackup.html#contract-datatsetbackup-options"
+  ],
   "contract-secret": [
     "contracts-secret.html#contract-secret"
   ],
@@ -1949,6 +2156,27 @@
   "contracts-databasebackup-options-shb.contracts.databasebackup.settings": [
     "contracts-databasebackup.html#contracts-databasebackup-options-shb.contracts.databasebackup.settings"
   ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup"
+  ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup.request": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request"
+  ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset"
+  ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup.result": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result"
+  ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService"
+  ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript"
+  ],
+  "contracts-datasetbackup-options-shb.contracts.datasetbackup.settings": [
+    "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.settings"
+  ],
   "contracts-nixpkgs": [
     "contracts.html#contracts-nixpkgs"
   ],
@@ -2183,6 +2411,12 @@
   "local-testing": [
     "service-implementation-guide.html#local-testing"
   ],
+  "misc": [
+    "misc.html#misc"
+  ],
+  "misc-lock-file-update": [
+    "misc.html#misc-lock-file-update"
+  ],
   "monitoring-failures": [
     "service-implementation-guide.html#monitoring-failures"
   ],
@@ -4790,6 +5024,9 @@
   "services-mailserver-options-shb.mailserver.ssl.systemdService": [
     "services-mailserver.html#services-mailserver-options-shb.mailserver.ssl.systemdService"
   ],
+  "services-mailserver-options-shb.mailserver.stateVersion": [
+    "services-mailserver.html#services-mailserver-options-shb.mailserver.stateVersion"
+  ],
   "services-mailserver-options-shb.mailserver.subdomain": [
     "services-mailserver.html#services-mailserver-options-shb.mailserver.subdomain"
   ],

flake.lock

@@ -20,11 +20,11 @@
     },
     "nix-flake-tests": {
       "locked": {
-        "lastModified": 1677844186,
+        "lastModified": 1775571237,
-        "narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
+        "narHash": "sha256-1f5Uvgcy3gx/eyRp4rqfDRBjcZc9uiHoIlfcFIL7GvI=",
         "owner": "antifuchs",
         "repo": "nix-flake-tests",
-        "rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
+        "rev": "86b789cff8aecbd7c1bed7cd421dd837c3f62201",
         "type": "github"
       },
       "original": {
@@ -35,11 +35,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1775036866,
+        "lastModified": 1781074563,
-        "narHash": "sha256-ZojAnPuCdy657PbTq5V0Y+AHKhZAIwSIT2cb8UgAz/U=",
+        "narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "6201e203d09599479a3b3450ed24fa81537ebc4e",
+        "rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca",
         "type": "github"
       },
       "original": {

flake.nix

@@ -78,6 +78,7 @@
           modules/contracts/backup/dummyModule.nix
           modules/contracts/dashboard/dummyModule.nix
           modules/contracts/databasebackup/dummyModule.nix
+          modules/contracts/datasetbackup/dummyModule.nix
           modules/contracts/secret/dummyModule.nix
           modules/contracts/ssl/dummyModule.nix
         ];
@@ -103,6 +104,13 @@
             "blocks/authelia" = ./modules/blocks/authelia.nix;
             "blocks/borgbackup" = ./modules/blocks/borgbackup.nix;
             "blocks/lldap" = ./modules/blocks/lldap.nix;
+            "blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
+            "blocks/monitoring" = ./modules/blocks/monitoring.nix;
+            "blocks/nginx" = ./modules/blocks/nginx.nix;
+            "blocks/postgresql" = ./modules/blocks/postgresql.nix;
+            "blocks/restic" = ./modules/blocks/restic.nix;
+            "blocks/sanoid" = ./modules/blocks/sanoid.nix;
+            "blocks/sops" = ./modules/blocks/sops.nix;
             "blocks/ssl" = {
               module = ./modules/blocks/ssl.nix;
               optionRoot = [
@@ -110,12 +118,8 @@
                 "certs"
               ];
             };
-            "blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
+            "blocks/zfs" = ./modules/blocks/zfs.nix;
-            "blocks/monitoring" = ./modules/blocks/monitoring.nix;
+
-            "blocks/nginx" = ./modules/blocks/nginx.nix;
-            "blocks/postgresql" = ./modules/blocks/postgresql.nix;
-            "blocks/restic" = ./modules/blocks/restic.nix;
-            "blocks/sops" = ./modules/blocks/sops.nix;
             "services/arr" = ./modules/services/arr.nix;
             "services/firefly-iii" = ./modules/services/firefly-iii.nix;
             "services/forgejo" = [
@@ -137,6 +141,7 @@
             "services/open-webui" = ./modules/services/open-webui.nix;
             "services/pinchflat" = ./modules/services/pinchflat.nix;
             "services/vaultwarden" = ./modules/services/vaultwarden.nix;
+
             "contracts/backup" = {
               module = ./modules/contracts/backup/dummyModule.nix;
               optionRoot = [
@@ -161,6 +166,14 @@
                 "databasebackup"
               ];
             };
+            "contracts/datasetbackup" = {
+              module = ./modules/contracts/datasetbackup/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "datasetbackup"
+              ];
+            };
             "contracts/secret" = {
               module = ./modules/contracts/secret/dummyModule.nix;
               optionRoot = [
@@ -244,6 +257,8 @@
           '';
         };
 
+        packages.update-flake-lock-pr = pkgs.callPackage ./.github/workflows/update-flake-lock-pr.nix { };
+
         lib = (pkgs.callPackage ./lib { }) // {
           test = pkgs.callPackage ./test/common.nix { };
           contracts = pkgs.callPackage ./modules/contracts {
@@ -388,6 +403,7 @@
             // (vm_test "homepage" ./test/services/homepage.nix)
             // (vm_test "jellyfin" ./test/services/jellyfin.nix)
             // (vm_test "karakeep" ./test/services/karakeep.nix)
+            // (vm_test "mailserver" ./test/services/mailserver.nix)
             // (vm_test "nextcloud" ./test/services/nextcloud.nix)
             // (vm_test "open-webui" ./test/services/open-webui.nix)
             // (vm_test "paperless" ./test/services/paperless.nix)
@@ -396,16 +412,18 @@
 
             // (vm_test "authelia" ./test/blocks/authelia.nix)
             // (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
-            // (vm_test "lldap" ./test/blocks/lldap.nix)
             // (vm_test "lib" ./test/blocks/lib.nix)
+            // (vm_test "lldap" ./test/blocks/lldap.nix)
             // (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
             // (vm_test "monitoring" ./test/blocks/monitoring.nix)
             // (vm_test "postgresql" ./test/blocks/postgresql.nix)
             // (vm_test "restic" ./test/blocks/restic.nix)
             // (vm_test "ssl" ./test/blocks/ssl.nix)
+            // (vm_test "zfs" ./test/blocks/zfs.nix)
 
             // (vm_test "contracts-backup" ./test/contracts/backup.nix)
             // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
+            // (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix)
             // (vm_test "contracts-secret" ./test/contracts/secret.nix)
           );
 
@@ -427,6 +445,7 @@
           self.nixosModules.nginx
           self.nixosModules.postgresql
           self.nixosModules.restic
+          self.nixosModules.sanoid
           self.nixosModules.ssl
           self.nixosModules.tinyproxy
           self.nixosModules.vpn
@@ -466,8 +485,9 @@
       nixosModules.nginx = modules/blocks/nginx.nix;
       nixosModules.postgresql = modules/blocks/postgresql.nix;
       nixosModules.restic = modules/blocks/restic.nix;
-      nixosModules.ssl = modules/blocks/ssl.nix;
+      nixosModules.sanoid = modules/blocks/sanoid.nix;
       nixosModules.sops = modules/blocks/sops.nix;
+      nixosModules.ssl = modules/blocks/ssl.nix;
       nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
       nixosModules.vpn = modules/blocks/vpn.nix;
       nixosModules.zfs = modules/blocks/zfs.nix;

lib/default.nix

@@ -1,7 +1,12 @@
 { pkgs, lib }:
 let
   inherit (builtins) isAttrs hasAttr;
-  inherit (lib) any concatMapStringsSep concatStringsSep;
+  inherit (lib)
+    any
+    concatMapStringsSep
+    concatStringsSep
+    escapeShellArg
+    ;
   shb = rec {
     # Replace secrets in a file.
     # - userConfig is an attrset that will produce a config file.
@@ -76,11 +81,41 @@ let
           pattern: "cat ${pattern.source} > /dev/null"
         ) replacements;
 
-        sedPatterns = concatMapStringsSep " " (pattern: "-e \"s|${pattern.name}|${pattern.value}|\"") (
+        generatedReplacements = map genReplacement replacements;
-          map genReplacement replacements
-        );
 
-        sedCmd = if replacements == [ ] then "cat" else "${pkgs.gnused}/bin/sed ${sedPatterns}";
+        replaceScript = pkgs.writers.writePython3 "replace-secret" { } ''
+          import argparse
+          import pathlib
+
+
+          def parse_args() -> argparse.Namespace:
+              parser = argparse.ArgumentParser()
+              parser.add_argument("--file", required=True, type=pathlib.Path)
+              parser.add_argument("--marker", required=True)
+              parser.add_argument("--replacement", required=True)
+              return parser.parse_args()
+
+
+          args = parse_args()
+          content = args.file.read_text()
+          args.file.write_text(content.replace(args.marker, args.replacement))
+        '';
+
+        replaceCommands = concatMapStringsSep "\n" (pattern: ''
+          ${replaceScript} \
+            --file ${escapeShellArg resultPath} \
+            --marker ${escapeShellArg pattern.name} \
+            --replacement "${pattern.value}"
+        '') generatedReplacements;
+
+        replaceCmd =
+          if replacements == [ ] then
+            "cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}"
+          else
+            ''
+              cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}
+              ${replaceCommands}
+            '';
       in
       ''
         set -euo pipefail
@@ -96,7 +131,7 @@ let
         chown ${user} ${resultPath}
       '')
       + ''
-        ${sedCmd} ${templatePath} > ${resultPath}
+        ${replaceCmd}
         chmod ${permissions} ${resultPath}
       '';
 
@@ -317,12 +352,16 @@ let
                   "expected"
                   "result"
                 ];
+
+                nativeBuildInputs = [
+                  (pkgs.python3.withPackages (ps: [ ps.deepdiff ] ++ ps.deepdiff.optional-dependencies.cli))
+                ];
               }
               ''
                 echo "${name} failed (- expected, + result)" > $out
                 cp ''${expectedPath} ''${expectedPath}.json
                 cp ''${resultPath} ''${resultPath}.json
-                ${pkgs.deepdiff}/bin/deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
+                deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
               ''
           );
 

modules/blocks/authelia.nix

@@ -645,6 +645,7 @@ in
     shb.mitmdump.instances."authelia-${fqdn}" = lib.mkIf cfg.debug {
       listenPort = 9091;
       upstreamPort = 9090;
+      timeout = 30;
       after = [ "authelia-${fqdn}.service" ];
       enabledAddons = [ config.shb.mitmdump.addons.logger ];
       extraArgs = [

modules/blocks/borgbackup.nix

@@ -386,8 +386,13 @@ in
                 ${serviceName} = mkMerge [
                   {
                     serviceConfig = {
-                      # Makes the systemd service wait for the backup to be done before changing state to inactive.
+                      # Purposely not a oneshot systemd service otherwise
-                      Type = "oneshot";
+                      # the service waits on the completion the backup before deactivating.
+                      # This seems like a nice property at first but there is one annoying
+                      # edge case when deploying. If a backup job somehow is started when
+                      # the deploy happens, the deploy will wait on the service to finish
+                      # before considering the deploy done. And worse, it will consider the
+                      # deploy as failed if the backup fails, which is not what you want.
                       Nice = lib.mkForce cfg.performance.niceness;
                       IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass;
                       IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority;
@@ -414,6 +419,7 @@ in
                   in
                   {
                     script = script.preStart;
+                    # Makes the systemd service wait for the backup to be done before changing state to inactive.
                     serviceConfig.Type = "oneshot";
                     serviceConfig.LoadCredential = script.loadCredentials;
                   }
@@ -451,39 +457,16 @@ in
           let
             mkBorgBackupBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "BORG_REPO"
-                  }
+                  "BORG_PASSCOMMAND"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
-                    exit 1
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
-                  echo "Will restore archive $archive"
-
-                  (cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive")
-                '';
               };
           in
           flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
@@ -493,39 +476,16 @@ in
           let
             mkBorgBackupBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "BORG_REPO"
-                  }
+                  "BORG_PASSCOMMAND"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
-                    exit 1
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
-                  echo "Will restore archive $archive"
-
-                  sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}"
-                '';
               };
           in
           flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);

modules/blocks/borgbackup/docs/default.md

@@ -3,6 +3,7 @@
 Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix).
 
 This block sets up a backup job using [BorgBackup][].
+It is heavily based on the nixpkgs BorgBackup module.
 
 [borgbackup]: https://www.borgbackup.org/
 
@@ -229,6 +230,36 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
 
 ## Maintenance {#blocks-borgbackup-maintenance}
 
+### Manual Backup {#blocks-borgbackup-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'borgbackup*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start borgbackup<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-borgbackup-maintenance-restore}
+
 One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
 
 The restore script has all the secrets needed to access the repo,

modules/blocks/davfs.nix

@@ -75,6 +75,30 @@ in
   config = {
     services.davfs2.enable = builtins.length cfg.mounts > 0;
 
+    systemd.services = lib.optionalAttrs (builtins.length cfg.mounts > 0) {
+      davfs2-password-generation = {
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig.Type = "oneshot";
+        script = ''
+          mkdir -p /etc/davfs2
+          [ -f /etc/davfs2/secrets ] && mv /etc/davfs2/secrets /etc/davfs2/secrets.prev
+          touch /etc/davfs2/secrets
+          chown root: /etc/davfs2
+          chown root: /etc/davfs2/secrets
+          chmod 700 /etc/davfs2
+          chmod 600 /etc/davfs2/secrets
+        ''
+        + (
+          let
+            mkPasswordCmd = cfg': ''
+              printf "%s %s %s\n" "${cfg'.mountPoint}" "${cfg'.username}" "$(cat ${cfg'.passwordFile})" >> /etc/davfs2/secrets
+            '';
+          in
+          lib.concatStringsSep "\n" (map mkPasswordCmd cfg.mounts)
+        );
+      };
+    };
+
     systemd.mounts =
       let
         mkMountCfg = c: {
@@ -82,6 +106,7 @@ in
           description = "Webdav mount point";
           after = [ "network-online.target" ];
           wants = [ "network-online.target" ];
+          wantedBy = [ "multi-user.target" ];
 
           what = c.remoteUrl;
           where = c.mountPoint;

modules/blocks/lldap.nix

@@ -418,6 +418,41 @@ in
       ) cfg.ensureUsers;
     };
 
+    # Harden lldap following https://github.com/NixOS/nixpkgs/pull/487933
+    systemd.services.lldap.serviceConfig = {
+      RemoveIPC = true;
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      RestrictAddressFamilies = [
+        "AF_UNIX"
+        "AF_INET"
+        "AF_INET6"
+      ];
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+        "~@resources"
+      ];
+      SystemCallArchitectures = "native";
+      CapabilityBoundingSet = "";
+      LockPersonality = true;
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      PrivateDevices = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectSystem = "strict";
+      ProtectProc = "invisible";
+      ProcSubset = "pid";
+      MemoryDenyWriteExecute = true;
+    };
+
     shb.mitmdump.instances."lldap-web" = lib.mkIf cfg.debug {
       listenPort = config.shb.lldap.webUIListenPort;
       upstreamPort = config.shb.lldap.webUIListenPort + 1;

modules/blocks/mitmdump.nix

@@ -48,7 +48,7 @@ let
         logging.basicConfig(level=logging.INFO, format='%(message)s')
 
 
-        def wait_for_port(host, port, timeout=10):
+        def wait_for_port(host, port, timeout):
             deadline = time.time() + timeout
             while time.time() < deadline:
                 try:
@@ -68,6 +68,7 @@ let
         parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
         parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
         parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
+        parser.add_argument("--timeout", required=False, type=int, default=10, help="Timeout to wait for port availability")
         args, rest = parser.parse_known_args()
 
         MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
@@ -75,7 +76,7 @@ let
             raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
 
         logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
-        wait_for_port(args.upstream_host, args.upstream_port, timeout=10)
+        wait_for_port(args.upstream_host, args.upstream_port, timeout=args.timeout)
         logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
 
         proc = subprocess.Popen(
@@ -90,10 +91,11 @@ let
         )
 
         logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
-        if wait_for_port("127.0.0.1", args.listen_port, timeout=10):
+        if wait_for_port("127.0.0.1", args.listen_port, timeout=args.timeout):
             logging.info(f"Mitmdump is started on port {args.listen_port}.")
             notify("READY=1")
         else:
+            print(f"Mitmdump instance did not start before the timeout of {args.timeout} seconds, consider increasing the timeout")
             proc.terminate()
             exit(1)
 
@@ -222,6 +224,14 @@ in
                 '';
               };
 
+              timeout = mkOption {
+                type = types.int;
+                default = 30;
+                description = ''
+                  Time to wait for upstream to start and mitmdump to start.
+                '';
+              };
+
               after = mkOption {
                 type = listOf str;
                 default = [ ];
@@ -282,7 +292,7 @@ in
               addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
               extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
             in
-            "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}";
+            "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} --timeout ${toString cfg'.timeout} ${addons} ${extraArgs}";
         };
         requires = cfg'.after;
         after = cfg'.after;

modules/blocks/monitoring.nix

@@ -293,6 +293,16 @@ in
         };
       };
     };
+
+    impermanence = lib.mkOption {
+      description = ''
+        Paths to save when using impermanence setup.
+      '';
+      type = lib.types.attrsOf lib.types.str;
+      default = {
+        fluent-bit = "/var/fluent-bit";
+      };
+    };
   };
 
   config = lib.mkMerge [
@@ -575,41 +585,62 @@ in
         };
       };
 
-      services.promtail = {
+      # I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
+      services.fluent-bit = {
         enable = true;
-        configuration = {
+        settings = {
-          server = {
+          service = {
-            http_listen_port = 9080;
+            flush = 1;
-            grpc_listen_port = 0;
+            log_level = "info";
+            http_server = "true";
+            http_listen = "127.0.0.1";
+            http_port = 9080;
+            grace = 30;
           };
 
-          positions.filename = "/tmp/positions.yaml";
+          pipeline = {
+            inputs = [
+              {
+                name = "systemd";
 
-          client.url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/api/prom/push";
+                # The asterisk appends the _SYSTEMD_UNIT to the prefix.
+                tag = "systemd.*";
 
-          scrape_configs = [
+                # Read logs from this systemd journal directory.
-            {
-              job_name = "systemd";
-              journal = {
-                json = false;
-                max_age = "12h";
                 path = "/var/log/journal";
-                # matches = "_TRANSPORT=kernel";
+
-                labels = {
+                # Database file to keep track of the journald cursor.
+                db = "/var/fluent-bit/systemd.db";
+
+                # Start reading new entries. Skip entries already stored in journald.
+                read_from_tail = true;
+
+                # Max entries to lookback on start.
+                max_entries = 10000;
+              }
+            ];
+
+            outputs = [
+              {
+                name = "loki";
+
+                match = "systemd.*";
+
+                host = "localhost";
+                port = config.services.loki.configuration.server.http_listen_port;
+
+                labels = lib.concatMapAttrsStringSep ", " (n: v: "${n}=${v}") {
+                  job = "systemd-journal";
                   domain = cfg.domain;
                   hostname = config.networking.hostName;
-                  job = "systemd-journal";
                 };
-              };
+
-              relabel_configs = [
+                label_keys = "$unit";
-                {
+              }
-                  source_labels = [ "__journal__systemd_unit" ];
+            ];
-                  target_label = "unit";
+          };
-                }
-              ];
-            }
-          ];
         };
+        graceLimit = "1m";
       };
 
       services.nginx = {
@@ -895,11 +926,11 @@ in
           src = pkgs.fetchFromGitHub {
             owner = "ibizaman";
             repo = "scrutiny";
-            rev = "7ff9a0530d3e54dd1323c2de34f32be330bfb48c";
+            rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
-            hash = "sha256-dE4HuZzaGZKBEkzXwBLQL3h+D55tJMm/EOTpr3wqGAI=";
+            hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
           };
 
-          vendorHash = "sha256-j3aGTeHNTr/FoVfFLwASkS96Ks0B/Ka9hPuLAKGZECs=";
+          vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
         });
 
         settings = {

modules/blocks/monitoring/docs/default.md

@@ -284,6 +284,16 @@ Graphs:
 
 ![Late SSL Jobs Alert Firing](./assets/alert_rules_LateSSL_1.png)
 
+## Impermanence {#blocks-monitoring-impermanence}
+
+To save the fluent-bit folder in an impermanence setup, add:
+
+```nix
+{
+  shb.zfs.datasets."safe/monitoring-fluent-bit".path = config.shb.jellyfin.impermanence.fluent-bit;
+}
+```
+
 ## Options Reference {#blocks-monitoring-options}
 
 ```{=include=} options

modules/blocks/postgresql.nix

@@ -73,7 +73,7 @@ in
           backupName = "postgres.sql";
 
           backupCmd = ''
-            ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
+            ${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
           '';
 
           restoreCmd = ''

modules/blocks/restic.nix

@@ -413,7 +413,13 @@ in
               nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
                 enable = true;
                 wantedBy = [ "multi-user.target" ];
-                serviceConfig.Type = "oneshot";
+                # Purposely not a oneshot systemd service otherwise
+                # the service waits on the completion the backup before deactivating.
+                # This seems like a nice property at first but there is one annoying
+                # edge case when deploying. If a backup job somehow is started when
+                # the deploy happens, the deploy will wait on the service to finish
+                # before considering the deploy done. And worse, it will consider the
+                # deploy as failed if the backup fails, which is not what you want.
                 script = (
                   shb.replaceSecrets {
                     userConfig = instance.settings.repository.secrets // {
@@ -434,38 +440,16 @@ in
           let
             mkResticBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "RESTIC_REPOSITORY"
-                  }
+                  "RESTIC_PASSWORD_FILE"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
-                    exit 1
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  echo "Will restore archive 'latest'"
-
-                  sudocmd ${pkgs.restic}/bin/restic restore latest --target /
-                '';
               };
           in
           flatten (mapAttrsToList mkResticBinary cfg.instances);
@@ -475,38 +459,16 @@ in
           let
             mkResticBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "RESTIC_REPOSITORY"
-                  }
+                  "RESTIC_PASSWORD_FILE"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
-                    exit 1
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  echo "Will restore archive 'latest'"
-
-                  sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}"
-                '';
               };
           in
           flatten (mapAttrsToList mkResticBinary cfg.databases);

modules/blocks/restic/docs/default.md

@@ -3,12 +3,12 @@
 Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
 
 This block sets up a backup job using [Restic][].
+It is heavily based on the nixpkgs Restic module.
 
 [restic]: https://restic.net/
 
 ## Provider Contracts {#blocks-restic-contract-provider}
 
-
 This block provides the following contracts:
 
 - [backup contract](contracts-backup.html) under the [`shb.restic.instances`][instances] option.
@@ -26,7 +26,6 @@ a backup Systemd service and a [restore script](#blocks-restic-maintenance) are
 
 ## Usage {#blocks-restic-usage}
 
-
 The following examples assume usage of the [sops block][] to provide secrets
 although any blocks providing the [secrets contract][] works too.
 
@@ -35,7 +34,6 @@ although any blocks providing the [secrets contract][] works too.
 
 ### One folder backed up manually {#blocks-restic-usage-provider-manual}
 
-
 The following snippet shows how to configure
 the backup of 1 folder to 1 repository.
 We assume that the folder `/var/lib/myfolder` of the service `myservice` must be backed up.
@@ -232,11 +230,44 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
 
 ## Maintenance {#blocks-restic-maintenance}
 
-One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
+### Manual Backup {#blocks-restic-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'restic*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start restic<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-restic-maintenance-restore}
+
+One command-line helper is provided per backup instance and repository pair which allows to:
+
+- list snapshots: `<script> snapshots`
+- to restore a snapshot: `<script> restore <snapshot>`
 
 The restore script has all the secrets needed to access the repo,
 it will run `sudo` automatically
-and the user running it needs to have correct permissions for privilege escalation
+and the user running it needs to have correct permissions for privilege escalation.
 
 In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
 

modules/blocks/sanoid.nix

@@ -0,0 +1,164 @@
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  utils,
+  ...
+}:
+let
+  cfg = config.shb.sanoid;
+
+  restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore";
+  backupScriptBase = "sanoid";
+
+  restoreScript =
+    name:
+    pkgs.writers.writePython3Bin (restoreScriptName name)
+      {
+        flakeIgnore = [ "E501" ];
+      }
+      ''
+        import argparse
+        import subprocess
+        import sys
+
+
+        dataset = "${name}"
+
+
+        class ZFSError(Exception):
+            pass
+
+
+        def run_command(cmd: list[str]) -> str:
+            try:
+                result = subprocess.run(
+                    cmd,
+                    check=True,
+                    text=True,
+                    capture_output=True,
+                )
+                return result.stdout.strip()
+            except subprocess.CalledProcessError as e:
+                raise ZFSError(
+                    f"Command failed: {' '.join(cmd)}\n"
+                    f"Exit code: {e.returncode}\n"
+                    f"stderr: {e.stderr.strip()}"
+                ) from e
+
+
+        def list_snapshots() -> None:
+            """List all ZFS snapshots."""
+            output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset])
+            if not output:
+                return []
+            return output.splitlines()
+
+
+        def restore_snapshot(snapshot: str) -> None:
+            """Rollback to a given snapshot."""
+            if not snapshot:
+                raise ValueError("Snapshot name must not be empty")
+
+            print(f"Rolling back to snapshot: {snapshot}")
+            run_command(["zfs", "rollback", "-r", snapshot])
+            print("Rollback completed successfully.")
+
+
+        def build_parser() -> argparse.ArgumentParser:
+            parser = argparse.ArgumentParser(description=f"Restore script for {dataset}")
+
+            subparsers = parser.add_subparsers(dest="command", required=True)
+
+            subparsers.add_parser(
+                "snapshots",
+                help="List all ZFS snapshots",
+            )
+
+            restore_parser = subparsers.add_parser(
+                "restore",
+                help="Rollback to a specific snapshot",
+            )
+            restore_parser.add_argument(
+                "snapshot",
+                help="Snapshot name (e.g. pool/dataset@snapname)",
+            )
+
+            return parser
+
+
+        def main():
+            parser = build_parser()
+            args = parser.parse_args()
+
+            try:
+                if args.command == "snapshots":
+                    snapshots = list_snapshots()
+                    for s in snapshots:
+                        print(s)
+                elif args.command == "restore":
+                    restore_snapshot(args.snapshot)
+                else:
+                    parser.print_help()
+                    sys.exit(1)
+
+            except ZFSError as e:
+                print(f"ERROR: {e}", file=sys.stderr)
+                sys.exit(1)
+            except Exception as e:
+                print(f"Unexpected error: {e}", file=sys.stderr)
+                sys.exit(1)
+
+
+        if __name__ == "__main__":
+            main()
+      '';
+in
+{
+  imports = [
+    ../../lib/module.nix
+  ];
+
+  options.shb.sanoid.backup = lib.mkOption {
+    description = "Sanoid prodiver for file backup contract";
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, ... }:
+        {
+          options = shb.contracts.backup.mkProvider {
+            resultCfg = {
+              restoreScript = restoreScriptName name;
+              restoreScriptText = restoreScriptName "<name>";
+
+              backupService = "${backupScriptBase}.service";
+              backupServiceText = "${backupScriptBase}.service";
+            };
+
+            settings = lib.mkOption {
+              description = "Options passed to the `services.sanoid.datasets.<name>` option.";
+              default = { };
+              type = lib.types.attrsOf lib.types.anything;
+            };
+          };
+        }
+      )
+    );
+  };
+
+  config = lib.mkIf (cfg.backup != { }) {
+    services.sanoid.enable = true;
+
+    services.sanoid.datasets =
+      let
+        mkDataset = name: cfg': {
+          inherit name;
+          value = cfg'.settings;
+        };
+      in
+      lib.mapAttrs' mkDataset cfg.backup;
+
+    environment.systemPackages = map restoreScript (lib.attrNames cfg.backup);
+  };
+}

modules/blocks/sanoid/docs/default.md

@@ -0,0 +1,97 @@
+# Sanoid Block {#blocks-sanoid}
+
+Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix):
+
+```nix
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.sanoid
+  ];
+}
+```
+
+## Provider Contracts {#blocks-sanoid-contract-provider}
+
+This block provides the following contracts:
+
+- [dataset backup contract](contracts-datasetbackup.html) under the [`shb.sanoid.backup`][backup] option.
+  It is tested with the [generic contract tests][backup contract tests].
+
+[backup]: #blocks-sanoid-options-shb.sanoid.backup
+[backup contract tests]: @REPO@/test/contracts/backup.nix
+
+## Usage {#blocks-sanoid-usage}
+
+Sanoid uses templates to know when snapshots should be kept or pruned.
+
+### Default Template {#blocks-sanoid-usage-default-template}
+
+Backup a dataset using the default Sanoid template:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+  };
+}
+```
+
+This uses the dataset backup contract which is exposed through the ZFS module's [`shb.zfs.pools.<name>.datasets.<name>.datasetBackup`](blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup) option.
+
+### Custom Template {#blocks-sanoid-usage-custom-template}
+
+Create a custom template and use it:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  services.sanoid.templates."yearly" = {
+    hourly = 10;
+    daily = 3;
+    monthly = 3;
+    yearly = 2;
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+    template = "yearly";
+  };
+}
+```
+
+Note we use the upstream `services.sanoid.templates` option to define the templates.
+
+### Without contract {#blocks-sanoid-usage-without-contract}
+
+To backup a dataset that does not provide the dataset backup contract,
+we can just set the request manually:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request.dataset = "root/home";
+  };
+}
+```
+
+Note the attr name under the `shb.sanoid.backup` option does not
+set the dataset name.
+
+## Options Reference {#blocks-sanoid-options}
+
+```{=include=} options
+id-prefix: blocks-sanoid-options-
+list-id: selfhostblocks-blocks-sanoid-options
+source: @OPTIONS_JSON@
+```

modules/blocks/ssl.nix

@@ -551,7 +551,7 @@ in
                     // lib.optionalAttrs (certCfg.dnsProvider != null) {
                       inherit (certCfg) dnsProvider dnsResolver;
                       inherit (certCfg) group reloadServices;
-                      credentialsFile = certCfg.credentialsFile;
+                      environmentFile = certCfg.credentialsFile;
                     };
                   }
                 ]

modules/blocks/zfs.nix

@@ -2,128 +2,258 @@
   config,
   pkgs,
   lib,
+  shb,
+  utils,
   ...
 }:
 
 let
   cfg = config.shb.zfs;
+
+  datasets =
+    if cfg.snapshotBeforeActivation.datasets != null then
+      cfg.snapshotBeforeActivation.datasets
+    else
+      config.boot.zfs.extraPools;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.zfs = {
-    defaultPoolName = lib.mkOption {
+    pools = lib.mkOption {
-      type = lib.types.nullOr lib.types.str;
-      default = null;
-      description = "ZFS pool name datasets should be created on if no pool name is given in the dataset.";
-    };
-
-    datasets = lib.mkOption {
       description = ''
-        ZFS Datasets.
+        Attrset of ZFS pools under which datasets will be created.
 
-        Each entry in the attrset will be created and mounted in the given path.
+        The ZFS pools are not managed by this module, they should already exist.
-        The attrset name is the dataset name.
 
-        This block implements the following contracts:
+        Each pool named here will be added to the [`boot.zfs.extraPools`](https://search.nixos.org/options?channel=unstable&include_modular_service_options=0&include_nixos_options=1&query=boot.zfs.extrapools&show=option:boot.zfs.extraPools) option.
-          - mount
       '';
       default = { };
-      example = lib.literalExpression ''
-        shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
-      '';
       type = lib.types.attrsOf (
         lib.types.submodule {
           options = {
-            enable = lib.mkEnableOption "shb.zfs.datasets";
+            datasets = lib.mkOption {
-
-            poolName = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              default = null;
-              description = "ZFS pool name this dataset should be created on. Overrides the defaultPoolName.";
-            };
-
-            path = lib.mkOption {
-              type = lib.types.str;
-              description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
-            };
-
-            mode = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              description = "If non null, unix mode to apply to the dataset root folder.";
-              default = null;
-              example = "ug=rwx,g+s";
-            };
-
-            owner = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              description = "If non null, unix user to apply to the dataset root folder.";
-              default = null;
-              example = "syncthing";
-            };
-
-            group = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              description = "If non null, unix group to apply to the dataset root folder.";
-              default = null;
-              example = "syncthing";
-            };
-
-            defaultACLs = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
               description = ''
-                If non null, default ACL to set on the dataset root folder.
+                ZFS Datasets.
 
-                Executes "setfacl -d -m $acl $path"
+                Each entry in the attrset will be created and mounted in the given path.
+                The attrset name is the dataset name.
+
+                This block implements the following contracts:
+                  - mount
               '';
-              default = null;
+              default = { };
-              example = "g:syncthing:rwX";
+              example = lib.literalExpression ''
+                shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
+              '';
+              type = lib.types.attrsOf (
+                lib.types.submodule (
+                  { config, name, ... }:
+                  {
+                    options = {
+                      enable = lib.mkEnableOption "shb.zfs.datasets";
+
+                      path = lib.mkOption {
+                        type = lib.types.str;
+                        description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
+                      };
+
+                      mode = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix mode to apply to the dataset root folder.";
+                        default = null;
+                        example = "ug=rwx,g+s";
+                      };
+
+                      owner = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix user to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      group = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix group to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      defaultACLs = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = ''
+                          If non null, default ACL to set on the dataset root folder.
+
+                          Executes "setfacl -d -m $acl $path"
+                        '';
+                        default = null;
+                        example = "g:syncthing:rwX";
+                      };
+
+                      after = lib.mkOption {
+                        type = lib.types.listOf lib.types.str;
+                        description = ''
+                          Order creating this dataset after the mentioned ones.
+                          This only works with datasets managed by this module.
+
+                          Use the name of the dataset without the pool name.
+                        '';
+                        default = [ ];
+                        example = lib.literalExpression ''
+                          [
+                            "backup"
+                          ]
+                        '';
+                      };
+
+                      backup = lib.mkOption {
+                        description = ''
+                          Backup contract consumer configuration.
+
+                          This contract will backup the files inside the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.backup.mkRequester {
+                            user = if config.owner == null then "root" else config.owner;
+                            sourceDirectories = [
+                              config.path
+                            ];
+                            sourceDirectoriesText = ''
+                              [
+                                shb.zfs.pools.<name>.datasets.<name>.path
+                              ]
+                            '';
+                          };
+                        };
+                      };
+
+                      datasetbackup = lib.mkOption {
+                        description = ''
+                          ZFS dataset backup contract configuration.
+
+                          This contract will take snaphots of the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.datasetbackup.mkRequester {
+                            dataset = name;
+                          };
+                        };
+                      };
+                    };
+                  }
+                )
+              );
             };
           };
         }
+
       );
     };
+
+    snapshotBeforeActivation = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        description = "Take a snapshot of all datasets before activation";
+        default = false;
+      };
+
+      template = lib.mkOption {
+        type = lib.types.str;
+        description = "Bash command to generate the snapshot name. $1 is the path to the new generation.";
+        default = ''pre-$(date --utc '+%y%m%dT%H%M%S')-$(basename "$1")'';
+      };
+
+      datasets = lib.mkOption {
+        type = lib.types.nullOr (lib.types.listOf lib.types.str);
+        description = ''
+          Defines all datasets to take a snapshot of.
+
+          If set to `null`, the default, take the list of datasets from `config.boot.zfs.extraPools`.
+
+          The snapshots are not recursive unless specificed in the `recursive` option.
+        '';
+      };
+
+      recursive = lib.mkOption {
+        type = lib.types.bool;
+        description = "If true, take snapshots recurisevly on the given datasets.";
+        default = false;
+      };
+    };
   };
 
+  # The implementation is greatly inspired by https://discourse.nixos.org/t/configure-zfs-filesystems-after-install/48633/3
   config = {
-    assertions = [
+    boot.zfs.extraPools = lib.uniqueStrings (builtins.attrNames cfg.pools);
-      {
-        assertion =
-          lib.any (x: x.poolName == null) (lib.mapAttrsToList (n: v: v) cfg.datasets)
-          -> cfg.defaultPoolName != null;
-        message = "Cannot have both datasets.poolName and defaultPoolName set to null";
-      }
-    ];
 
-    system.activationScripts = lib.mapAttrs' (
+    systemd.services =
-      name: cfg':
       let
-        dataset = (if cfg'.poolName != null then cfg'.poolName else cfg.defaultPoolName) + "/" + name;
+        mkPool =
+          poolName: poolCfg: lib.listToAttrs (lib.mapAttrsToList (mkDataset poolName) poolCfg.datasets);
+
+        mkDataset =
+          poolName: name: cfg':
+          let
+            dataset = poolName + "/" + name;
+          in
+          lib.attrsets.nameValuePair "zfs-create-${utils.escapeSystemdPath dataset}" {
+            # oneshot is used to make the systemd service wait on completion of the script.
+            serviceConfig.Type = "oneshot";
+            unitConfig.DefaultDependencies = false;
+            requiredBy = [ "local-fs.target" ];
+            before = [ "local-fs.target" ];
+            after = [
+              "zfs-import-${poolName}.service"
+              "zfs-mount.service"
+            ]
+            ++ map (n: "zfs-create-${poolName}-${n}.service") cfg'.after;
+
+            unitConfig.ConditionPathIsMountPoint = lib.mkIf (cfg'.path != "none") "!${cfg'.path}";
+
+            script = ''
+              ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
+                || ${pkgs.zfs}/bin/zfs create \
+                   -o mountpoint=none \
+                   ${dataset} || :
+
+              [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
+                || ${pkgs.zfs}/bin/zfs set \
+                   mountpoint="${cfg'.path}" \
+                   ${dataset}
+
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
+              chmod "${cfg'.mode}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
+              chown "${cfg'.owner}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
+              chown :"${cfg'.group}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
+              ${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
+            '';
+          };
+        mergeAttrs = lib.foldl lib.mergeAttrs { };
       in
-      lib.attrsets.nameValuePair "zfsCreate-${name}" {
+      mergeAttrs (lib.mapAttrsToList mkPool cfg.pools);
-        text = ''
-          ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
-            || ${pkgs.zfs}/bin/zfs create \
-               -o mountpoint=none \
-               ${dataset} || :
 
-          [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
+    system.preSwitchChecks."shb-zfs-snapshot" = lib.mkIf cfg.snapshotBeforeActivation.enable (
-            || ${pkgs.zfs}/bin/zfs set \
+      ''
-               mountpoint="${cfg'.path}" \
+        name="${cfg.snapshotBeforeActivation.template}"
-               ${dataset}
+      ''
-
+      + (
-        ''
+        let
-        + lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
+          recursiveFlag = lib.optionalString cfg.snapshotBeforeActivation.recursive "-r";
-          chmod "${cfg'.mode}" "${cfg'.path}"
+        in
-        ''
+        lib.concatMapStringsSep "\n" (ds: ''
-        + lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
+          echo "Taking ZFS snapshot of ${ds}@$name"
-          chown "${cfg'.owner}" "${cfg'.path}"
+          zfs snapshot ${recursiveFlag} ${ds}@"$name"
-        ''
+        '') (lib.uniqueStrings datasets)
-        + lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
+      )
-          chown :"${cfg'.group}" "${cfg'.path}"
+    );
-        ''
-        + lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
-          ${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
-        '';
-      }
-    ) cfg.datasets;
   };
 }

modules/blocks/zfs/docs/default.md

@@ -0,0 +1,138 @@
+# ZFS Block {#blocks-zfs}
+
+Defined in [`/modules/blocks/zfs.nix`](@REPO@/modules/blocks/zfs.nix):
+
+```nix
+{
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.zfs
+  ];
+}
+```
+
+This block creates ZFS datasets, optionally mounts them and sets permissions on the mount point.
+It also enables taking snapshots on activation.
+
+## Features {#blocks-zfs-features}
+
+- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
+- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
+- Backup of the files in the dataset [`shb.zfs.<name>.backup`][backup] through the [backup contract](./contracts-backup.html).
+- Backup of the dataset itself [`shb.zfs.<name>.datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html).
+- Take a snapshot [before activating or deploying][activationsnapshot] allowing a safe rollback.
+
+[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup
+[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup
+[activationsnapshot]: #blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable
+
+## Usage {#blocks-zfs-usage}
+
+Create a dataset at `root/safe/users` mounted on `/var/lib/nixos`:
+
+```nix
+shb.zfs.pools.root.datasets."safe/users".path = "/var/lib/nixos";
+```
+
+Create a dataset at `backup/syncoid` but do not mount it:
+
+```nix
+shb.zfs.pools.backup.datasets."syncoid".path = "none";
+```
+
+Create a dataset at `root/syncthing` and set custom permissions and ACL.
+Permission and ACL are only enforced for the mount point.
+
+```nix
+shb.zfs.pools.root.datasets."syncthing" = {
+  path = "/srv/syncthing";
+
+  mode = "ug=rwx,g+s,o=";
+  owner = "syncthing";
+  group = "syncthing";
+  defaultACLs = "g:syncthing:rwX";
+};
+```
+
+### Backup dataset {#blocks-zfs-usage-backup-dataset}
+
+To backup the dataset directly, use the [dataset backup contract](contracts-datasetbackup.html).
+For example, with the sanoid module as the dataset backup contract provider:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+    template = "yearly";
+  };
+}
+```
+
+See the [Sanoid block](blocks-sanoid.html) for more examples.
+
+### Backup files {#blocks-zfs-usage-backup-files}
+
+To backup the files in the dataset, use the [file backup contract](contracts-backup.html).
+For example, with the restic module as the file backup contract provider:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.restic.instances."myservice" = {
+    request = shb.zfs.pools.root.datasets.home.backup.request;
+  };
+}
+```
+
+See the [Restic block](blocks-restic.html) for more examples.
+
+### Snapshot before activation {#blocks-zfs-usage-snapshot-before-activation}
+
+To take a snapshot of all datasets before activating a new configuration:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation.enable = true;
+}
+```
+
+This does not take a recursive snapshot by default, to do it recursively, do:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation = {
+    enable = true;
+    recursive = true;
+  };
+}
+```
+
+We did not specify datasets manually, which means the datasets list comes from `boot.zfs.extraPools`.
+To specify the datasets manually, you can use:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation = {
+    enable = true;
+    datasets = [ "root" "root/var" ];
+  };
+}
+```
+
+## Options Reference {#blocks-zfs-options}
+
+```{=include=} options
+id-prefix: blocks-zfs-options-
+list-id: selfhostblocks-block-zfs-options
+source: @OPTIONS_JSON@
+```

modules/contracts/backup.nix

@@ -1,4 +1,9 @@
-{ lib, shb, ... }:
+{
+  lib,
+  shb,
+  pkgs,
+  ...
+}:
 let
   inherit (lib)
     concatStringsSep
@@ -187,12 +192,14 @@ in
 
                 ```bash
                 $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                <snapshot 1> <metadata>
+                <snapshot 2> <metadata>
                 ```
 
                 And restore the database with:
 
                 ```bash
-                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
                 ```
               '';
               type = str;
@@ -222,4 +229,56 @@ in
         };
       };
     };
+
+  passthru.mkRestoreScript =
+    {
+      name,
+      user,
+      sudoPreserveEnvs ? [ ],
+      secretsFile,
+      restoreCmd,
+      listCmd,
+    }:
+    pkgs.writeShellApplication {
+      inherit name;
+
+      text = ''
+        usage() {
+          echo "$0 snapshots"
+          echo "$0 restore <snapshot>"
+        }
+
+        sudocmd() {
+          sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
+        }
+
+        loadenv() {
+          set -a
+          # shellcheck disable=SC1090
+          source <(sudocmd cat "${secretsFile}")
+          set +a
+        }
+
+        if [ "$1" = "restore" ]; then
+          shift
+          snapshot="$1"
+
+          loadenv
+
+          echo "Will restore snapshot '$snapshot'"
+
+          sudocmd sh -c "${restoreCmd}"
+        elif [ "$1" = "snapshots" ]; then
+          shift
+
+          loadenv
+
+          sudocmd sh -c "${listCmd}"
+        else
+          usage
+          exit 1
+        fi
+      '';
+    };
+
 }

modules/contracts/backup/docs/default.md

@@ -20,27 +20,22 @@ source: @OPTIONS_JSON@
 ## Usage {#contract-backup-usage}
 
 A service that can be backed up will provide a `backup` option.
-Such a service is a `requester` providing a `request` for a module `provider` of this contract. 
 
-What this option defines is, from the user perspective - that is _you_ - an implementation detail
+Here is an example module defining such a `backup` option,
-but it will at least define what directories to backup,
+which defines what directories to backup (`sourceDirectories`)
-the user to backup with
+and the user to backup with (`user`).
-and possibly hooks to run before or after the backup job runs.
-
-Here is an example module defining such a `backup` option:
 
 ```nix
 {
   options = {
     myservice.backup = mkOption {
       type = lib.types.submodule {
-        options = contracts.backup.request;
+        options = shb.contracts.backup.mkRequester {
-      };
+          user = "nextcloud";
-      default = {
+          sourceDirectories = [
-        user = "myservice";
+            "/var/lib/nextcloud"
-        sourceDirectories = [
+          ];
-          "/var/lib/myservice"
+        };
-        ];
       };
     };
   };
@@ -50,13 +45,13 @@ Here is an example module defining such a `backup` option:
 Now, on the other side we have a service that uses this `backup` option and actually backs up files.
 This service is a `provider` of this contract and will provide a `result` option.
 
-Let's assume such a module is available under the `backupservice` option
+Let's assume such a module is available under the `backupService` option
-and that one can create multiple backup instances under `backupservice.instances`.
+and that one can create multiple backup instances under `backupService.instances`.
 Then, to actually backup the `myservice` service, one would write:
 
 ```nix
-backupservice.instances.myservice = {
+backupService.instances.myservice = {
-  request = myservice.backup;
+  request = myservice.backup.request;
   
   settings = {
     enable = true;
@@ -65,17 +60,17 @@ backupservice.instances.myservice = {
       path = "/srv/backup/myservice";
     };
 
-    # ... Other options specific to backupservice like scheduling.
+    # ... Other options specific to backupService like scheduling.
   };
 };
 ```
 
 It is advised to backup files to different location, to improve redundancy.
-Thanks to using contracts, this can be made easily either with the same `backupservice`:
+Thanks to using contracts, this can be made easily either with the same `backupService`:
 
 ```nix
-backupservice.instances.myservice_2 = {
+backupService.instances.myservice_2 = {
-  request = myservice.backup;
+  request = myservice.backup.request;
   
   settings = {
     enable = true;
@@ -87,12 +82,13 @@ backupservice.instances.myservice_2 = {
 };
 ```
 
-Or with another module `backupservice_2`!
+Or with another module `backupService_2`!
 
 ## Providers of the Backup Contract {#contract-backup-providers}
 
 - [Restic block](blocks-restic.html).
-- [Borgbackup block](blocks-borgbackup.html) [WIP].
+- [Borgbackup block](blocks-borgbackup.html).
+- [ZFS block](blocks-zfs.html).
 
 ## Requester Blocks and Services {#contract-backup-requesters}
 

modules/contracts/backup/test.nix

@@ -21,8 +21,8 @@ in
     "/opt/files/A"
     "/opt/files/B"
   ],
-  settings, # { repository, config } -> attrset
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
-  extraConfig ? null, # { username, config } -> attrset
+  extraConfig ? null, # { filesRoot, username, config } -> attrset
 }:
 shb.test.runNixOSTest {
   inherit name;
@@ -40,7 +40,7 @@ shb.test.runNixOSTest {
           };
           settings = settings {
             inherit config;
-            repository = "/opt/repos/${name}";
+            filesRoot = "/opt/files";
           };
         })
         (mkIf (username != "root") {
@@ -52,6 +52,7 @@ shb.test.runNixOSTest {
         })
         (optionalAttrs (extraConfig != null) (extraConfig {
           inherit username config;
+          filesRoot = "/opt/files";
         }))
       ];
     };
@@ -65,7 +66,9 @@ shb.test.runNixOSTest {
       provider = (getAttrFromPath providerRoot nodes.machine).result;
     in
     ''
+      from datetime import datetime, timedelta
       from dictdiffer import diff
+      import re
 
       username = "${username}"
       sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
@@ -103,9 +106,25 @@ shb.test.runNixOSTest {
                   f'{path}/fileB': 'repo_fileB_1',
               })
 
+      with subtest("Initial snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
       with subtest("First backup in repo"):
           print(machine.succeed("systemctl cat ${provider.backupService}"))
-          machine.succeed("systemctl start ${provider.backupService}")
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      # To accomodate for snapshot orchestrators which keep only a given amount
+      # of snapshots per unit of time, we set the time to now + 2h.
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
+      machine.succeed(f"timedatectl set-time '{new_date}'")
 
       with subtest("New content"):
           for path in sourceDirectories:
@@ -119,14 +138,37 @@ shb.test.runNixOSTest {
                   f'{path}/fileB': 'repo_fileB_2',
               })
 
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
       with subtest("Delete content"):
           for path in sourceDirectories:
               machine.succeed(f"""rm -r {path}/*""")
 
               assert_files(path, {})
 
-      with subtest("Restore initial content from repo"):
+      with subtest("Restore second backup"):
-          machine.succeed("""${provider.restoreScript} restore latest""")
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
 
           for path in sourceDirectories:
               assert_files(path, {

modules/contracts/databasebackup/docs/default.md

@@ -20,21 +20,21 @@ source: @OPTIONS_JSON@
 ## Usage {#contract-databasebackup-usage}
 
 A database that can be backed up will provide a `databasebackup` option.
-Such a service is a `requester` providing a `request` for a module `provider` of this contract. 
 
 What this option defines is, from the user perspective - that is _you_ - an implementation detail
 but it will at least define how to create a database dump,
 the user to backup with
 and how to restore from a database dump.
 
-Here is an example module defining such a `databasebackup` option:
+Here is an example module defining such a `databasebackup` option,
+which defines the user to backup with (`user`)
+and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
 
 ```nix
 {
   options = {
     myservice.databasebackup = mkOption {
-      type = contracts.databasebackup.request;
+      type = shb.contracts.databasebackup.mkRequester = {
-      default = {
         user = "myservice";
         backupCmd = ''
           ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
@@ -48,16 +48,16 @@ Here is an example module defining such a `databasebackup` option:
 };
 ```
 
-Now, on the other side we have a service that uses this `backup` option and actually backs up files.
+Now, on the other side we have a service that uses this `databasebackup` option and actually backs up files.
 This service is a `provider` of this contract and will provide a `result` option.
 
-Let's assume such a module is available under the `databasebackupservice` option
+Let's assume such a module is available under the `databaseBackupService` option
-and that one can create multiple backup instances under `databasebackupservice.instances`.
+and that one can create multiple backup instances under `databaseBackupService.instances`.
 Then, to actually backup the `myservice` service, one would write:
 
 ```nix
-databasebackupservice.instances.myservice = {
+databaseBackupService.instances.myservice = {
-  request = myservice.databasebackup;
+  request = myservice.databasebackup.request;
   
   settings = {
     enable = true;
@@ -72,11 +72,11 @@ databasebackupservice.instances.myservice = {
 ```
 
 It is advised to backup files to different location, to improve redundancy.
-Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
+Thanks to using contracts, this can be made easily either with the same `databaseBackupService`:
 
 ```nix
-databasebackupservice.instances.myservice_2 = {
+databaseBackupService.instances.myservice_2 = {
-  request = myservice.backup;
+  request = myservice.databasebackup.request;
   
   settings = {
     enable = true;
@@ -88,12 +88,12 @@ databasebackupservice.instances.myservice_2 = {
 };
 ```
 
-Or with another module `databasebackupservice_2`!
+Or with another module `databaseBackupService_2`!
 
 ## Providers of the Database Backup Contract {#contract-databasebackup-providers}
 
 - [Restic block](blocks-restic.html).
-- [Borgbackup block](blocks-borgbackup.html) [WIP].
+- [Borgbackup block](blocks-borgbackup.html).
 
 ## Requester Blocks and Services {#contract-databasebackup-requesters}
 

modules/contracts/databasebackup/test.nix

@@ -51,16 +51,18 @@ shb.test.runNixOSTest {
   testScript =
     { nodes, ... }:
     let
-      provider = getAttrFromPath providerRoot nodes.machine;
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
     in
     ''
       import csv
+      import re
 
       start_all()
       machine.wait_for_unit("postgresql.service")
+      machine.wait_for_unit("postgresql-setup.service")
       machine.wait_for_open_port(5432)
 
-      def peer_cmd(cmd, db="me"):
+      def peer_cmd(cmd, db="${database}"):
           return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
 
       def query(query):
@@ -73,30 +75,68 @@ shb.test.runNixOSTest {
               if len(diff) > 0:
                   raise Exception(i, diff)
 
-      table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
-
       with subtest("create fixture"):
           machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
-          machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
 
           res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
+          cmp_tables(res, table)
+
+      with subtest("Initial snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 0:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("backup"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("New content"):
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
           cmp_tables(res, table)
 
       with subtest("backup"):
-          print(machine.succeed("systemctl cat ${provider.result.backupService}"))
+          machine.succeed("systemctl start --wait ${provider.backupService}")
-          print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
+
-          machine.succeed("systemctl start ${provider.result.backupService}")
+      with subtest("Two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
 
       with subtest("drop database"):
           machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
           machine.fail(peer_cmd("SELECT * FROM test"))
 
-      with subtest("restore"):
+      with subtest("restore second snapshot"):
-          print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
-          machine.succeed("${provider.result.restoreScript} restore latest ")
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
 
-      with subtest("check restoration"):
           res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+          cmp_tables(res, table)
+
+      with subtest("restore first snapshot"):
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
           cmp_tables(res, table)
     '';
 }

modules/contracts/datasetbackup.nix

@@ -0,0 +1,113 @@
+{ lib, ... }:
+let
+  inherit (lib)
+    literalMD
+    mkOption
+    optionalAttrs
+    ;
+  inherit (lib.types)
+    submodule
+    str
+    ;
+in
+{
+  mkRequest =
+    {
+      dataset ? "",
+      datasetText ? null,
+    }:
+    mkOption {
+      description = ''
+        Request part of the backup contract.
+
+        Options set by the requester module
+        enforcing how to backup files.
+      '';
+
+      default = { };
+
+      type = submodule {
+        options = {
+          dataset =
+            mkOption {
+              description = ''
+                Dataset to backup, including the pool name.
+              '';
+              type = str;
+              example = "root/home";
+              default = dataset;
+            }
+            // optionalAttrs (datasetText != null) {
+              defaultText = literalMD datasetText;
+            };
+        };
+      };
+    };
+
+  mkResult =
+    {
+      restoreScript ? "restore",
+      restoreScriptText ? null,
+      backupService ? "backup.service",
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the backup contract.
+
+        Options set by the provider module that indicates the name of the backup and restore scripts.
+      '';
+      default = { };
+
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
+
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                <snapshot 1> <metadata>
+                <snapshot 2> <metadata>
+                ```
+
+                And restore the database with:
+
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
+                ```
+
+                It is not garanteed to be able to restore back to a snapshot in the future.
+                With the above example, it may not be possible to restore `<snapshot 2>`
+                after having restored `<snapshot 1>`.
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
+
+          backupService =
+            mkOption {
+              description = ''
+                Name of service backing up the database.
+
+                This script can be ran manually to backup the database:
+
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
+        };
+      };
+    };
+}

modules/contracts/datasetbackup/docs/default.md

@@ -0,0 +1,85 @@
+# Dataset Backup Contract {#contract-dataset-backup}
+
+This NixOS contract represents a backup job
+that will backup one ZFS dataset.
+
+It is a contract between a service that manages ZFS datasets
+and a service that backs up ZFS datasets.
+
+## Contract Reference {#contract-datatsetbackup-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-datasetbackup-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```
+
+## Usage {#contract-datasetbackup-usage}
+
+A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option.
+
+Here is an example module defining such a `backup` option,
+which defines what directories to backup (`sourceDirectories`)
+and the user to backup with (`user`).
+
+```nix
+{
+  options = {
+    myservice.datasetbackup = mkOption {
+      type = lib.types.submodule {
+        options = shb.contracts.datasetbackup.mkRequester {
+          dataset = "root/test";
+        };
+      };
+    };
+  };
+};
+```
+
+Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset.
+This service is a `provider` of this contract and will provide a `result` option.
+
+Let's assume such a module is available under the `backupService` option
+and that one can create multiple backup instances under `backupService.instances`.
+Then, to actually backup the `myservice` service, one would write:
+
+```nix
+backupService.instances.myservice = {
+  request = myservice.datasetbackup.request;
+  
+  settings = {
+    enable = true;
+
+    targetDataset = "backup/myservice";
+
+    # ... Other options specific to backupService like scheduling.
+  };
+};
+```
+
+It is advised to backup files to different location, to improve redundancy.
+Thanks to using contracts, this can be made easily either with the same `backupService`:
+
+```nix
+backupService.instances.myservice_2 = {
+  request = myservice.datasetbackup.request;
+  
+  settings = {
+    enable = true;
+  
+    targetDataset = "backup2/myservice";
+  };
+};
+```
+
+Or with another module `backupService_2`!
+
+## Providers of the Backup Contract {#contract-datasetbackup-providers}
+
+- [Sanoid block](blocks-sanoid.html).
+
+## Requester Blocks and Services {#contract-datasetbackup-requesters}
+
+- [ZFS](blocks-zfs.html#blocks-zfs-backup).

modules/contracts/datasetbackup/dummyModule.nix

@@ -0,0 +1,30 @@
+{ lib, shb, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) submodule;
+in
+{
+  imports = [
+    ../../../lib/module.nix
+  ];
+
+  options.shb.contracts.datasetbackup = mkOption {
+    description = ''
+      Contract for backing up ZFS datasets.
+
+      The requester communicates to the provider
+      the dataset to backup
+      through the `request` options.
+
+      The provider reads from the `request` options
+      and backs up the requested dataset.
+      It communicates to the requester what script is used
+      to backup and restore the files
+      through the `result` options.
+    '';
+
+    type = submodule {
+      options = shb.contracts.datasetbackup.contract;
+    };
+  };
+}

modules/contracts/datasetbackup/test.nix

@@ -0,0 +1,164 @@
+{
+  pkgs,
+  lib,
+  shb,
+}:
+let
+  inherit (lib)
+    concatMapStringsSep
+    getAttrFromPath
+    setAttrByPath
+    ;
+in
+{
+  name,
+  providerRoot,
+  modules ? [ ],
+  dataset ? "root/test",
+  sourceDirectories ? [
+    "/opt/files/A"
+    "/opt/files/B"
+  ],
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
+  extraConfig ? null, # { filesRoot, config } -> attrset
+}:
+shb.test.runNixOSTest {
+  inherit name;
+
+  nodes.machine =
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
+
+      config = lib.mkMerge [
+        (setAttrByPath providerRoot {
+          request = {
+            inherit dataset;
+          };
+          settings = settings {
+            inherit config;
+            filesRoot = "/opt/files";
+          };
+        })
+        (lib.optionalAttrs (extraConfig != null) (extraConfig {
+          inherit config;
+          filesRoot = "/opt/files";
+        }))
+      ];
+    };
+
+  extraPythonPackages = p: [ p.dictdiffer ];
+  skipTypeCheck = true;
+
+  testScript =
+    { nodes, ... }:
+    let
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      from datetime import datetime, timedelta
+      from dictdiffer import diff
+      import re
+
+      sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
+
+      def list_files(dir):
+          files_and_content = {}
+
+          files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
+
+          for f in files:
+              content = machine.succeed(f"""cat {f}""").strip()
+              files_and_content[f] = content
+
+          return files_and_content
+
+      def assert_files(dir, files):
+          result = list(diff(list_files(dir), files))
+          if len(result) > 0:
+              raise Exception("Unexpected files:", result)
+
+      with subtest("Create initial content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                  mkdir -p {path}
+                  echo repo_fileA_1 > {path}/fileA
+                  echo repo_fileB_1 > {path}/fileB
+              """)
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
+
+      with subtest("Initial snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("First backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      # To accomodate for snapshot orchestrators which keep only a given amount
+      # of snapshots per unit of time, we set the time to now + 2h.
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
+      machine.succeed(f"timedatectl set-time '{new_date}'")
+
+      with subtest("New content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                echo repo_fileA_2 > {path}/fileA
+                echo repo_fileB_2 > {path}/fileB
+                """)
+
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("Delete content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""rm -r {path}/*""")
+
+              assert_files(path, {})
+
+      with subtest("Restore second backup"):
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
+    '';
+}

modules/contracts/default.nix

@@ -54,12 +54,14 @@ let
         };
       };
     in
-    mkContractFunctions {
+    (mkContractFunctions {
       inherit (importedModule) mkRequest mkResult;
-    };
+    })
+    // (importedModule.passthru or { });
 
   contracts = {
     databasebackup = importContract ./databasebackup.nix;
+    datasetbackup = importContract ./datasetbackup.nix;
     dashboard = importContract ./dashboard.nix;
     backup = importContract ./backup.nix;
     mount = pkgs.callPackage ./mount.nix { };
@@ -68,6 +70,7 @@ let
     test = {
       secret = pkgs.callPackage ./secret/test.nix { inherit shb; };
       databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; };
+      datasetbackup = pkgs.callPackage ./datasetbackup/test.nix { inherit shb; };
       backup = pkgs.callPackage ./backup/test.nix { inherit shb; };
     };
   };

modules/services/home-assistant.nix

@@ -11,23 +11,13 @@ let
 
   fqdn = "${cfg.subdomain}.${cfg.domain}";
 
-  ldap_auth_script_repo = pkgs.fetchFromGitHub {
-    owner = "lldap";
-    repo = "lldap";
-    rev = "7d1f5abc137821c500de99c94f7579761fc949d8";
-    sha256 = "sha256-8D+7ww70Ja6Qwdfa+7MpjAAHewtCWNf/tuTAExoUrg0=";
-  };
-
-  ldap_auth_script = pkgs.writeShellScriptBin "ldap_auth.sh" ''
-    export PATH=${pkgs.gnused}/bin:${pkgs.curl}/bin:${pkgs.jq}/bin
-    exec ${pkgs.bash}/bin/bash ${ldap_auth_script_repo}/example_configs/lldap-ha-auth.sh $@
-  '';
-
   # Filter secrets from config. Secrets are those of the form { source = <path>; }
   secrets = lib.attrsets.filterAttrs (k: v: builtins.isAttrs v) cfg.config;
 
   nonSecrets = (lib.attrsets.filterAttrs (k: v: !(builtins.isAttrs v)) cfg.config);
 
+  lldap_ha_auth = pkgs.callPackage ./home-assistant/lldap_ha_auth.nix { };
+
   configWithSecretsIncludes = nonSecrets // (lib.attrsets.mapAttrs (k: v: "!secret ${k}") secrets);
 in
 {
@@ -263,7 +253,6 @@ in
       config = {
         # Includes dependencies for a basic setup
         # https://www.home-assistant.io/integrations/default_config/
-        default_config = { };
         http = {
           use_x_forwarded_for = true;
           server_host = "127.0.0.1";
@@ -281,9 +270,10 @@ in
               }
             ])
             ++ (lib.optionals cfg.ldap.enable [
+              # https://www.home-assistant.io/docs/authentication/providers/#command-line
               {
                 type = "command_line";
-                command = ldap_auth_script + "/bin/ldap_auth.sh";
+                command = lldap_ha_auth + "/bin/lldap-ha-auth";
                 args = [
                   "http://${cfg.ldap.host}:${toString cfg.ldap.port}"
                   cfg.ldap.userGroup

modules/services/home-assistant/lldap_ha_auth.nix

@@ -0,0 +1,37 @@
+{
+  lib,
+  pkgs,
+  stdenvNoCC,
+}:
+stdenvNoCC.mkDerivation {
+  name = "lldap-ha-auth";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "lldap";
+    repo = "lldap";
+    rev = "7d1f5abc137821c500de99c94f7579761fc949d8";
+    sha256 = "sha256-8D+7ww70Ja6Qwdfa+7MpjAAHewtCWNf/tuTAExoUrg0=";
+  };
+
+  nativeBuildInputs = [
+    pkgs.makeWrapper
+  ];
+
+  buildPhase = ''
+    mkdir -p $out/bin
+
+    cp example_configs/lldap-ha-auth.sh $out/bin/lldap-ha-auth
+    chmod a+x $out/bin/lldap-ha-auth
+  '';
+
+  installPhase = ''
+    wrapProgram $out/bin/lldap-ha-auth \
+      --prefix PATH : ${
+        lib.makeBinPath [
+          pkgs.gnused
+          pkgs.curl
+          pkgs.jq
+        ]
+      }
+  '';
+}

modules/services/mailserver.nix

@@ -14,7 +14,7 @@ in
       builtins.fetchGit {
         url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver.git";
         ref = "master";
-        rev = "7d433bf89882f61621f95082e90a4ab91eb0bdd3";
+        rev = "e33fbde199eaad513ef5d0746db19d5878150232";
       }
       + "/default.nix"
     )
@@ -105,7 +105,7 @@ in
               description = ''
                 Accounts to sync emails from using IMAP.
 
-                Emails will be stored under `''${config.mailserver.mailDirectory}/''${name}/''${username}`
+                Emails will be stored under `''${config.mailserver.storage.path}/''${name}/''${username}`
               '';
               type = lib.types.attrsOf (
                 lib.types.submodule {
@@ -137,7 +137,7 @@ in
                       type = lib.types.submodule {
                         options = shb.contracts.secret.mkRequester {
                           mode = "0400";
-                          owner = config.mailserver.vmailUserName;
+                          owner = config.mailserver.storage.owner;
                           restartUnits = [ "mbsync.service" ];
                         };
                       };
@@ -267,7 +267,7 @@ in
         Enabling this app will create a new LDAP configuration or update one that exists with
         the given host.
       '';
-      default = { };
+      default = null;
       type = lib.types.nullOr (
         lib.types.submodule {
           options = {
@@ -333,6 +333,26 @@ in
       );
     };
 
+    stateVersion = lib.mkOption {
+      type = lib.types.ints.positive;
+      # SHB started at stateVersion 3.
+      default = 4;
+      description = ''
+        Tracking stateful version changes as an incrementing number.
+
+        When a new release comes out we may require manual migration steps to
+        be completed, before the new version can be put into production.
+
+        If your `stateVersion` is too low one or multiple assertions may
+        trigger to give you instructions on what migrations steps are required
+        to continue. Increase the `stateVersion` as instructed by the assertion
+        message.
+
+        See https://nixos-mailserver.readthedocs.io/en/latest/release-notes.html
+        and https://nixos-mailserver.readthedocs.io/en/latest/migrations.html
+      '';
+    };
+
     backup = lib.mkOption {
       description = ''
         Backup emails, index and sieve.
@@ -340,16 +360,16 @@ in
       default = { };
       type = lib.types.submodule {
         options = shb.contracts.backup.mkRequester {
-          user = config.mailserver.vmailUserName;
+          user = config.mailserver.storage.owner;
           sourceDirectories = builtins.filter (x: x != null) [
             config.mailserver.indexDir
-            config.mailserver.mailDirectory
+            config.mailserver.storage.path
             config.mailserver.sieveDirectory
           ];
           sourceDirectoriesText = ''
             [
               config.mailserver.indexDir
-              config.mailserver.mailDirectory
+              config.mailserver.storage.path
               config.mailserver.sieveDirectory
             ]
           '';
@@ -367,7 +387,7 @@ in
           user = config.services.rspamd.user;
           userText = "services.rspamd.user";
           sourceDirectories = builtins.filter (x: x != null) [
-            config.mailserver.dkimKeyDirectory
+            config.mailserver.dkim.keyDirectory
           ];
           sourceDirectoriesText = ''
             [
@@ -385,14 +405,14 @@ in
       type = lib.types.attrsOf lib.types.str;
       default = {
         index = config.mailserver.indexDir;
-        mail = config.mailserver.mailDirectory;
+        mail = config.mailserver.storage.path;
         sieve = config.mailserver.sieveDirectory;
         dkim = config.mailserver.dkimKeyDirectory;
       };
       defaultText = lib.literalExpression ''
         {
           index = config.mailserver.indexDir;
-          mail = config.mailserver.mailDirectory;
+          mail = config.mailserver.storage.path;
           sieve = config.mailserver.sieveDirectory;
           dkim = config.mailserver.dkimKeyDirectory;
         }
@@ -417,8 +437,8 @@ in
     (lib.mkIf cfg.enable {
       mailserver = {
         enable = true;
-        stateVersion = 3;
         fqdn = "${cfg.subdomain}.${cfg.domain}";
+        inherit (cfg) stateVersion;
         domains = [ cfg.domain ];
 
         localDnsResolver = false;
@@ -433,18 +453,22 @@ in
         # Using / is needed for iOS mail.
         # Both following options are used to organize subfolders in subdirectories.
         hierarchySeparator = "/";
-        useFsLayout = true;
+        storage = {
+          directoryLayout = "fs";
+        };
       };
 
-      services.postfix.config = {
+      services.postfix.settings.main = {
         smtpd_tls_security_level = lib.mkForce "encrypt";
       };
 
       # Is probably needed for iOS mail.
-      services.dovecot2.extraConfig = ''
+      services.dovecot2.settings = {
-        ssl_min_protocol = TLSv1.2
+        ssl_min_protocol = "TLSv1.2";
-        ssl_cipher_list = HIGH:!aNULL:!MD5
+        # This conflicts with the default setting which seems much better than this one.
-      '';
+        # So I'm leaving the default settings and we'll see if something breaks.
+        # ssl_cipher_list = "HIGH:!aNULL:!MD5";
+      };
 
       services.nginx = {
         enable = true;
@@ -482,6 +506,8 @@ in
           in
           {
             forceSSL = true; # Redirect HTTP → HTTPS
+            sslCertificate = cfg.ssl.paths.cert;
+            sslCertificateKey = cfg.ssl.paths.key;
             root = "/var/www"; # Dummy root
             locations."/.well-known/autoconfig/mail/" = {
               alias = "${announce}/";
@@ -501,6 +527,8 @@ in
           in
           {
             forceSSL = true; # Redirect HTTP → HTTPS
+            sslCertificate = cfg.ssl.paths.cert;
+            sslCertificateKey = cfg.ssl.paths.key;
             root = "/var/www"; # Dummy root
             locations."/" = {
               alias = "${landingPage}/";
@@ -548,37 +576,44 @@ in
           uris = [
             "ldap://${cfg.ldap.host}:${toString cfg.ldap.port}"
           ];
-          searchBase = "ou=people,${cfg.ldap.dcdomain}";
+          base = "ou=people,${cfg.ldap.dcdomain}";
-          searchScope = "sub";
+          scope = "sub";
           bind = {
             dn = "uid=${cfg.ldap.adminName},ou=people,${cfg.ldap.dcdomain}";
             passwordFile = cfg.ldap.adminPassword.result.path;
           };
-          # Note that nixos simple mailserver sets auth_bind=yes
-          # which means authentication binds are used.
-          # https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_bind/#authentication-ldap-bind
           dovecot =
             let
               filter = "(&(objectClass=inetOrgPerson)(mail=%{user})(memberOf=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain}))";
             in
             {
-              passAttrs = "user=user";
               passFilter = filter;
-              userAttrs = lib.concatStringsSep "," [
-                "=home=${config.mailserver.mailDirectory}/${cfg.ldap.account}/%u"
-                # "mail=maildir:${config.mailserver.mailDirectory}/${cfg.ldap.account}/%u/mail"
-                "uid=${config.mailserver.vmailUserName}"
-                "gid=${config.mailserver.vmailGroupName}"
-              ];
               userFilter = filter;
             };
+          # username needs to be set to mail so postfix maps correctly the mail address.
+          # Otherwise we get error messages when sending
+          #   Sender address rejected: not owned by user
+          attributes = {
+            username = "mail";
+          };
           postfix = {
             filter = "(&(objectClass=inetOrgPerson)(mail=%s)(memberOf=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain}))";
-            mailAttribute = "mail";
-            uidAttribute = "mail";
           };
         };
       };
+      services.dovecot2.settings = {
+        "userdb ldap" = {
+          fields.home = lib.mkForce "${config.mailserver.storage.path}/${cfg.ldap.account}/%{user}";
+          fields.mail_index_path = lib.mkForce "${config.mailserver.storage.path}/${cfg.ldap.account}/%{user}";
+        };
+        "passdb ldap" = {
+          # LLDAP does not understand ldap user password setup.
+          fields.password = lib.mkForce null;
+          # We use bind DN auth.
+          # https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_bind/#authentication-ldap-bind
+          bind = true;
+        };
+      };
     })
     (lib.mkIf (cfg.enable && cfg.imapSync != null) {
       systemd.services.mbsync =
@@ -601,11 +636,11 @@ in
                 Account ${name}
 
                 MaildirStore ${name}-local
-                INBOX ${config.mailserver.mailDirectory}/${name}/${acct.username}/mail/
+                INBOX ${config.mailserver.storage.path}/${name}/${acct.username}/mail/
                 # Maps subfolders on far side to actual subfolders on disk.
                 # The other option is Maildir++ but then the mailserver.hierarchySeparator must be set to a dot '.'
                 SubFolders Verbatim
-                Path ${config.mailserver.mailDirectory}/${name}/${acct.username}/mail/
+                Path ${config.mailserver.storage.path}/${name}/${acct.username}/mail/
 
                 Channel ${name}-main
                 Far :${name}-remote:
@@ -672,7 +707,7 @@ in
           description = "Sync mailbox";
           serviceConfig = {
             Type = "oneshot";
-            User = config.mailserver.vmailUserName;
+            User = config.mailserver.storage.owner;
           };
           script =
             let
@@ -689,8 +724,8 @@ in
             name: acct:
             # The equal sign makes sure parent directories have the corret user and group too.
             [
-              "d '${config.mailserver.mailDirectory}/${name}' 0750 ${config.mailserver.vmailUserName} ${config.mailserver.vmailGroupName} - -"
+              "d '${config.mailserver.storage.path}/${name}' 0750 ${config.mailserver.storage.owner} ${config.mailserver.storage.group} - -"
-              "d '${config.mailserver.mailDirectory}/${name}/${acct.username}' 0750 ${config.mailserver.vmailUserName} ${config.mailserver.vmailGroupName} - -"
+              "d '${config.mailserver.storage.path}/${name}/${acct.username}' 0750 ${config.mailserver.storage.owner} ${config.mailserver.storage.group} - -"
             ];
         in
         lib.flatten (lib.mapAttrsToList mkAccount cfg.imapSync.accounts);

modules/services/mailserver/docs/default.md

@@ -128,6 +128,8 @@ in
 }
 ```
 
+With the example above, the mails are stored under `/var/vmail/fastmail/<email address>`.
+
 ### Secrets {#services-mailserver-usage-secrets}
 
 Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.

modules/services/open-webui.nix

@@ -59,12 +59,12 @@ in
           WEBUI_NAME = "SelfHostBlocks";
 
           OLLAMA_BASE_URL = "http://127.0.0.1:''${toString config.services.ollama.port}";
+          RAG_EMBEDDING_ENGINE = "ollama";
           RAG_EMBEDDING_MODEL = "nomic-embed-text:v1.5";
 
           ENABLE_OPENAI_API = "True";
           OPENAI_API_BASE_URL = "http://127.0.0.1:''${toString config.services.llama-cpp.port}";
           ENABLE_WEB_SEARCH = "True";
-          RAG_EMBEDDING_ENGINE = "openai";
         }
       '';
     };

patches/0001-selfhostblocks-never-onboard.patch

@@ -1,29 +1,24 @@
-From 6897dd86a41b336c7c03a466990f7e981c5c649c Mon Sep 17 00:00:00 2001
+From 8d38721322bc47ce089f8d246513be610e22c187 Mon Sep 17 00:00:00 2001
 From: ibizaman <ibizaman@tiserbox.com>
-Date: Tue, 23 Sep 2025 11:36:24 +0200
+Date: Sun, 10 May 2026 08:58:10 +0200
-Subject: [PATCH] selfhostblocks: never onboard
+Subject: [PATCH] never onboard
 
 ---
- backend/open_webui/main.py | 4 +---
+ backend/open_webui/main.py | 2 --
- 1 file changed, 1 insertion(+), 3 deletions(-)
+ 1 file changed, 2 deletions(-)
 
 diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
-index 5630a5883..5c7c88a64 100644
+index e05497c..826af7b 100644
 --- a/backend/open_webui/main.py
 +++ b/backend/open_webui/main.py
-@@ -1654,11 +1654,9 @@ async def get_app_config(request: Request):
+@@ -2391,8 +2391,6 @@ async def get_app_config(request: Request):
-             user = Users.get_user_by_id(data["id"])
+             user = await Users.get_user_by_id(data['id'])
  
-     user_count = Users.get_num_users()
-+    # Never onboard
      onboarding = False
- 
 -    if user is None:
--        onboarding = user_count == 0
+-        onboarding = not await Users.has_users()
--
+ 
-     return {
+     user_count = await Users.get_num_users() if app.state.LICENSE_METADATA else None
-         **({"onboarding": True} if onboarding else {}),
+ 
-         "status": True,
 -- 
-2.50.1
+2.53.0
-

test/blocks/authelia.nix

@@ -16,6 +16,7 @@ in
           (pkgs'.path + "/nixos/modules/profiles/qemu-guest.nix")
           ../../modules/blocks/authelia.nix
           ../../modules/blocks/hardcodedsecret.nix
+          ../../modules/blocks/ssl.nix
         ];
 
         networking.hosts = {
@@ -28,11 +29,29 @@ in
           ];
         };
 
+        shb.certs.cas.selfsigned.myca = {
+          name = "My CA";
+        };
+        shb.certs.certs.selfsigned = {
+          "machine.com" = {
+            ca = config.shb.certs.cas.selfsigned.myca;
+
+            domain = "*.machine.com";
+            group = "nginx";
+          };
+        };
+        systemd.services.nginx.after = [ config.shb.certs.certs.selfsigned."machine.com".systemdService ];
+        systemd.services.nginx.requires = [
+          config.shb.certs.certs.selfsigned."machine.com".systemdService
+        ];
+
         shb.lldap = {
           enable = true;
           dcdomain = "dc=example,dc=com";
           subdomain = "ldap";
           domain = "machine.com";
+          ssl = config.shb.certs.certs.selfsigned."machine.com";
+
           ldapUserPassword.result = config.shb.hardcodedsecret.ldapUserPassword.result;
           jwtSecret.result = config.shb.hardcodedsecret.jwtSecret.result;
         };
@@ -50,6 +69,8 @@ in
           enable = true;
           subdomain = "authelia";
           domain = "machine.com";
+          ssl = config.shb.certs.certs.selfsigned."machine.com";
+
           ldapHostname = "${config.shb.lldap.subdomain}.${config.shb.lldap.domain}";
           ldapPort = config.shb.lldap.ldapPort;
           dcdomain = config.shb.lldap.dcdomain;
@@ -136,10 +157,10 @@ in
             machine.wait_for_unit("authelia-authelia.machine.com.target")
             machine.wait_for_open_port(9091)
 
-            endpoints = json.loads(machine.succeed("curl -s http://machine.com/.well-known/openid-configuration"))
+            endpoints = json.loads(machine.succeed("curl -s https://authelia.machine.com/.well-known/openid-configuration"))
             auth_endpoint = endpoints['authorization_endpoint']
             print(f"auth_endpoint: {auth_endpoint}")
-            if auth_endpoint != "http://machine.com/api/oidc/authorization":
+            if auth_endpoint != "https://authelia.machine.com/api/oidc/authorization":
                 raise Exception("Unexpected auth_endpoint")
 
             resp = machine.succeed(

test/blocks/borgbackup.nix

@@ -143,7 +143,7 @@ let
               })
 
           with subtest("First backup in repo A"):
-              machine.succeed("systemctl start ${backupService}")
+              machine.succeed("systemctl start --wait ${backupService}")
 
           with subtest("New content"):
               machine.succeed("""
@@ -168,8 +168,12 @@ let
               assert_files("/opt/files", {})
 
           with subtest("Restore initial content from repo A"):
-              machine.succeed("""
+              snapshot = machine.succeed("""
-              ${restoreScript} restore latest
+              ${restoreScript} snapshots
+              """)
+              print(snapshot)
+              machine.succeed(f"""
+              ${restoreScript} restore {snapshot}
               """)
 
               assert_files("/opt/files", {

test/blocks/monitoring.nix

@@ -9,6 +9,9 @@ let
       { ... }:
       [
         "grafana.service"
+        "fluent-bit.service"
+        "loki.service"
+        "netdata.service"
       ];
     waitForPorts =
       { node, ... }:
@@ -101,7 +104,19 @@ let
               "page.get_by_role('button', name=re.compile('Accept')).click()"
               "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
               "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
-              "expect(page.get_by_text('Welcome to Grafana')).to_be_visible()"
+              ''
+                assert page.evaluate("""
+                    async () => {
+                        const r = await fetch('/api/user');
+                        if (!r.ok) return false;
+
+                        const u = await r.json();
+                        return u.login === 'alice'
+                            && u.email === 'alice@example.com'
+                            && u.orgId === 1;
+                    }
+                """)
+              ''
             ];
           }
           {
@@ -118,7 +133,19 @@ let
               "page.get_by_role('button', name=re.compile('Accept')).click()"
               "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
               "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
-              "expect(page.get_by_text('Welcome to Grafana')).to_be_visible()"
+              ''
+                assert page.evaluate("""
+                    async () => {
+                        const r = await fetch('/api/user');
+                        if (!r.ok) return false;
+
+                        const u = await r.json();
+                        return u.login === 'bob'
+                            && u.email === 'bob@example.com'
+                            && u.orgId === 1;
+                    }
+                """)
+              ''
             ];
           }
           {
@@ -132,8 +159,19 @@ let
             username = "charlie";
             password = "CharliePassword";
             nextPageExpect = [
-              "page.get_by_role('button', name=re.compile('Accept')).click()" # I don't understand why this is not needed. Maybe it keeps somewhere the previous token?
+              "page.get_by_role('button', name=re.compile('Accept')).click()"
-              "expect(page.get_by_text(re.compile('[Ll]ogin failed'))).to_be_visible(timeout=10000)"
+              "expect(page).to_have_url('https://${config.test.fqdn}/login', timeout=10000)"
+              ''
+                assert page.evaluate("""
+                    async () => {
+                        const r = await fetch('/api/user');
+                        if (r.status !== 401) return false;
+
+                        const u = await r.json();
+                        return u.message === 'Unauthorized';
+                    }
+                """)
+              ''
             ];
           }
         ];
@@ -182,7 +220,7 @@ let
               ''
               "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
               "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
-              "expect(page.get_by_text('Temperature history for each device')).to_be_visible()"
+              "expect(page.get_by_text('Temperature history for each device')).to_be_visible(timeout=20000)"
             ];
           }
           {
@@ -202,7 +240,7 @@ let
               ''
               "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
               "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
-              "expect(page.get_by_text('Temperature history for each device')).to_be_visible()"
+              "expect(page.get_by_text('Temperature history for each device')).to_be_visible(timeout=20000)"
             ];
           }
           {
@@ -216,8 +254,8 @@ let
             username = "charlie";
             password = "CharliePassword";
             nextPageExpect = [
-              "page.get_by_role('button', name=re.compile('Accept')).click()" # I don't understand why this is not needed. Maybe it keeps somewhere the previous token?
+              "expect(page.get_by_text(re.compile('Hi'))).to_be_visible(timeout=10000)" # This Hi is the Authelia profile page.
-              "expect(page.get_by_text(re.compile('[Ll]ogin failed'))).to_be_visible(timeout=10000)"
+              # "expect(page.get_by_text(re.compile('[Ll]ogin failed'))).to_be_visible(timeout=10000)"
             ];
           }
         ];

test/blocks/restic.nix

@@ -168,8 +168,12 @@ let
               assert_files("/opt/files", {})
 
           with subtest("Restore initial content from repo A"):
-              machine.succeed("""
+              snapshot = machine.succeed("""
-              ${restoreScript} restore latest
+              ${restoreScript} snapshots
+              """)
+              print(snapshot)
+              machine.succeed(f"""
+              ${restoreScript} restore {snapshot}
               """)
 
               assert_files("/opt/files", {

test/blocks/ssl.nix

@@ -3,6 +3,28 @@ let
   pkgs' = pkgs;
 in
 {
+  letsencryptDnsProvider = shb.test.runNixOSTest {
+    name = "ssl-letsencrypt-dns-provider";
+
+    nodes.server =
+      { config, ... }:
+      {
+        imports = [
+          shb.test.baseImports
+          ../../modules/blocks/ssl.nix
+        ];
+
+        shb.certs.certs.letsencrypt.dns = {
+          domain = "dns.example.com";
+          dnsProvider = "namecheap";
+          credentialsFile = "/run/this-file-does-not-exist-acme-env";
+          adminEmail = "admin@example.com";
+        };
+      };
+
+    testScript = "";
+  };
+
   test = shb.test.runNixOSTest {
     name = "ssl-test";
 

test/blocks/zfs.nix

@@ -0,0 +1,158 @@
+{
+  shb,
+  ...
+}:
+{
+  default = shb.test.runNixOSTest {
+    name = "zfs-default";
+
+    nodes.machine =
+      { config, pkgs, ... }:
+      {
+        imports = [
+          ../../modules/blocks/zfs.nix
+        ];
+
+        # Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix
+        networking.hostId = "deadbeef";
+        boot.supportedFilesystems = [ "zfs" ];
+
+        users.users.syncthing = {
+          isSystemUser = true;
+          group = "syncthing";
+        };
+        users.groups.syncthing = { };
+        virtualisation = {
+          emptyDiskImages = [
+            512
+            512
+          ];
+        };
+
+        systemd.services."zfs-zpool-create" = {
+          unitConfig.DefaultDependencies = false;
+          after = [ "systemd-modules-load.service" ];
+          requiredBy = [
+            "zfs-import-root.service"
+            "zfs-import-data.service"
+            "zfs-mount.service"
+          ];
+          before = [
+            "zfs-import-root.service"
+            "zfs-import-data.service"
+            "zfs-mount.service"
+          ];
+          script = ''
+            if [ ! -f /var/done ]; then
+              ${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl root /dev/vdb
+              ${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl data /dev/vdc
+              sync
+            fi
+            touch /var/done
+          '';
+        };
+
+        shb.zfs.pools.root.datasets.one.path = "/var/root/one";
+        shb.zfs.pools.root.datasets.two.path = "/var/root/two";
+        shb.zfs.pools.root.datasets.none.path = "none";
+        shb.zfs.pools.data.datasets.two = {
+          path = "/var/data/two";
+
+          mode = "ug=rwx,g+s,o=";
+          owner = "syncthing";
+          group = "syncthing";
+          defaultACLs = "g:syncthing:rwX";
+        };
+
+        shb.zfs.snapshotBeforeActivation = {
+          enable = true;
+          recursive = true;
+        };
+
+        specialisation = {
+          test.configuration = { };
+          test2.configuration = {
+            # A plus sign is not allowed as the snapshot name so it will make it fail.
+            shb.zfs.snapshotBeforeActivation.template = "+";
+          };
+        };
+      };
+
+    testScript =
+      { nodes, ... }:
+      let
+        specialisations = "${nodes.machine.system.build.toplevel}/specialisation";
+        switch = name: "${specialisations}/${name}/bin/switch-to-configuration test";
+      in
+      ''
+        import difflib
+
+        def assert_facl():
+            out = machine.succeed("getfacl /var/data/two").splitlines()
+            expect = """\
+        # file: var/data/two
+        # owner: syncthing
+        # group: syncthing
+        # flags: -s-
+        user::rwx
+        group::rwx
+        other::---
+        default:user::rwx
+        default:group::rwx
+        default:group:syncthing:rwx
+        default:mask::rwx
+        default:other::---
+
+        """.splitlines()
+            if out != expect:
+                diff = difflib.context_diff(expect, out)
+                raise Exception(f"Unexpected getfacl:\n{"\n".join(diff)}")
+
+        def assert_mounts():
+            out = sorted([l.split()[0] for l in machine.succeed("mount | grep /var/").splitlines()])
+            expect = ["data/two", "root/one", "root/two"]
+            if out != expect:
+                diff = difflib.context_diff(expect, out)
+                raise Exception(f"Unexpected mounts:\n{"\n".join(diff)}")
+
+        def assert_count_snapshots(name, count):
+            out = machine.succeed("zfs list -Ht snapshot root/one")
+            print(out)
+            if count != len(out.splitlines()):
+                raise Exception(f"Expected {count} dataset")
+
+        machine.start(allow_reboot=True)
+        machine.wait_for_unit("multi-user.target")
+
+        assert_facl()
+        assert_mounts()
+
+        with subtest("no snapshot yet"):
+            print(machine.succeed("zpool list"))
+            print(machine.succeed("zfs list"))
+            assert_count_snapshots("root/one", 0)
+            assert_count_snapshots("root/two", 0)
+            assert_count_snapshots("root/none", 0)
+            assert_count_snapshots("data/two", 0)
+
+        machine.succeed("${switch "test"}")
+
+        with subtest("snapshot created"):
+            print(machine.succeed("zpool list"))
+            print(machine.succeed("zfs list"))
+            assert_count_snapshots("root/one", 1)
+            assert_count_snapshots("root/two", 1)
+            assert_count_snapshots("root/none", 1)
+            assert_count_snapshots("data/two", 1)
+
+        machine.fail("${switch "test2"}")
+
+        # TODO: make this work. zpool import does not work after reboot.
+        # machine.crash()
+        # machine.wait_for_unit("multi-user.target")
+
+        # assert_facl()
+        # assert_mounts()
+      '';
+  };
+}

test/common.nix

@@ -152,22 +152,30 @@ let
           code, logs = server.execute("login_playwright")
           print(logs)
           try:
-              server.copy_from_vm("trace")
+              server.succeed("""
+                mkdir -p /tmp/shared/
+                cp -r trace /tmp/shared/
+              """)
+              server.copy_from_machine("trace")
           except:
               print("No trace found on server")
-          # if code != 0:
+          if code != 0:
-          #     raise Exception("login_playwright did not succeed")
+              raise Exception("login_playwright did not succeed")
     '')
     + (optionalString (hasAttr "test" nodes.client && hasAttr "login" nodes.client.test) ''
       with subtest("Login from client"):
           code, logs = client.execute("login_playwright")
           print(logs)
           try:
-              client.copy_from_vm("trace")
+              client.succeed("""
+                mkdir -p /tmp/shared/
+                cp -r trace /tmp/shared/
+              """)
+              client.copy_from_machine("trace")
           except:
               print("No trace found on client")
-          # if code != 0:
+          if code != 0:
-          #     raise Exception("login_playwright did not succeed")
+              raise Exception("login_playwright did not succeed")
     '')
   );
 
@@ -494,6 +502,19 @@ in
         settings.content = "jwtSecrets";
       };
 
+      shb.hardcodedsecret.alice = {
+        request = config.shb.lldap.ensureUsers.alice.password.request;
+        settings.content = "AlicePassword";
+      };
+      shb.hardcodedsecret.bob = {
+        request = config.shb.lldap.ensureUsers.bob.password.request;
+        settings.content = "BobPassword";
+      };
+      shb.hardcodedsecret.charlie = {
+        request = config.shb.lldap.ensureUsers.charlie.password.request;
+        settings.content = "CharliePassword";
+      };
+
       shb.lldap = {
         enable = true;
         inherit (config.test) domain;
@@ -509,7 +530,7 @@ in
           alice = {
             email = "alice@example.com";
             groups = [ "user_group" ];
-            password.result.path = pkgs.writeText "alicePassword" "AlicePassword";
+            password.result = config.shb.hardcodedsecret.alice.result;
           };
           bob = {
             email = "bob@example.com";
@@ -517,12 +538,12 @@ in
             # so we can make sure users only part admins
             # can also login normally.
             groups = [ "admin_group" ];
-            password.result.path = pkgs.writeText "bobPassword" "BobPassword";
+            password.result = config.shb.hardcodedsecret.bob.result;
           };
           charlie = {
             email = "charlie@example.com";
             groups = [ "other_group" ];
-            password.result.path = pkgs.writeText "charliePassword" "CharliePassword";
+            password.result = config.shb.hardcodedsecret.charlie.result;
           };
         };
 

test/contracts/backup.nix

@@ -14,12 +14,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };
@@ -49,12 +49,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };
@@ -84,12 +84,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };
@@ -119,12 +119,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };

test/contracts/datasetbackup.nix

@@ -0,0 +1,64 @@
+{ shb, pkgs, ... }:
+{
+  sanoid = shb.contracts.test.datasetbackup {
+    name = "sanoid";
+    providerRoot = [
+      "shb"
+      "sanoid"
+      "backup"
+      # This is the name of the dataset
+      "root/mytest"
+    ];
+    modules = [
+      ../../modules/blocks/sanoid.nix
+      # We use the zfs module to test the sanoid one
+      ../../modules/blocks/zfs.nix
+    ];
+    settings =
+      { ... }:
+      {
+        useTemplate = [ "test" ];
+      };
+    extraConfig =
+      { filesRoot, ... }:
+      {
+        # Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix
+        networking.hostId = "deadbeef";
+        boot.supportedFilesystems = [ "zfs" ];
+
+        virtualisation = {
+          emptyDiskImages = [
+            512
+          ];
+        };
+
+        # The test expects to keep one snapshot per hour.
+        services.sanoid.templates."test" = {
+          hourly = 1;
+          daily = 0;
+          monthly = 0;
+          yearly = 0;
+        };
+
+        systemd.services."zfs-zpool-create" = {
+          unitConfig.DefaultDependencies = false;
+          after = [ "systemd-modules-load.service" ];
+          requiredBy = [
+            "zfs-import-root.service"
+            "zfs-mount.service"
+          ];
+          before = [
+            "zfs-import-root.service"
+            "zfs-mount.service"
+          ];
+          script = ''
+            ${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl -O mountpoint=none root /dev/vdb
+          '';
+        };
+
+        shb.zfs.pools.root.datasets."mytest" = {
+          path = filesRoot;
+        };
+      };
+  };
+}

test/modules/davfs.nix

@@ -34,6 +34,7 @@ in
     expected = {
       services.davfs2.enable = false;
       systemd.mounts = [ ];
+      systemd.services = { };
     };
     expr = testConfig { };
   };

test/services/deluge.nix

@@ -25,6 +25,14 @@ let
       ''
         print(${node.name}.succeed('journalctl -n100 -u deluged'))
         print(${node.name}.succeed('systemctl status deluged'))
+        ${node.name}.succeed("""
+          restarts="$(systemctl show deluged.service -P NRestarts)"
+          if [ "$restarts" -gt 1 ]; then
+            systemctl status deluged.service --no-pager
+            journalctl -u deluged.service -b --no-pager
+            exit 1
+          fi
+        """)
         print(${node.name}.succeed('systemctl status delugeweb'))
 
         with subtest("web connect"):
@@ -108,6 +116,18 @@ let
         request = config.shb.deluge.localclientPassword.request;
         settings.content = "localpw";
       };
+
+      systemd.services.deluged = {
+        unitConfig = {
+          StartLimitIntervalSec = "30s";
+          StartLimitBurst = 2;
+        };
+
+        serviceConfig = {
+          Restart = lib.mkForce "on-failure";
+          RestartSec = "1s";
+        };
+      };
     };
 
   clientLogin =

test/services/home-assistant.nix

@@ -40,6 +40,36 @@ let
           unit_system = "metric";
         };
       };
+      services.home-assistant.extraComponents = [
+        # this is effecitvely default_config (2026.5.0), but with components
+        # skipped that would cause ERRORs in the sandbox
+        "bluetooth"
+        "cloud"
+        "conversation"
+        "dhcp"
+        "energy"
+        "file"
+        # Requires go2rtc service
+        # "go2rtc"
+        "history"
+        # Requires DNS and HTTP queries
+        # "homeassistant_alerts"
+        "logbook"
+        "media_source"
+        "mobile_app"
+        "my"
+        "ssdp"
+        "stream"
+        "sun"
+        "usage_prediction"
+        "usb"
+        "webhook"
+        "zeroconf"
+
+        # include some popular integrations, that absolutely shouldn't break
+        "knx"
+        "zha"
+      ];
     };
 
   clientLogin =
@@ -63,10 +93,10 @@ let
               "page.get_by_role('button', name=re.compile('Create my smart home')).click()"
 
               "expect(page.get_by_text('Create user')).to_be_visible()"
-              "page.get_by_label(re.compile('Name')).fill('Admin')"
+              ''page.get_by_role("textbox", name="Name*", exact=True).fill('Admin')''
-              "page.get_by_label(re.compile('Username')).fill('admin')"
+              ''page.get_by_role("textbox", name="Username*").fill('admin')''
-              "page.get_by_label(re.compile('Password')).fill('adminpassword')"
+              ''page.get_by_role("textbox", name="Password*", exact=True).fill('adminpassword')''
-              "page.get_by_label(re.compile('Confirm password')).fill('adminpassword')"
+              ''page.get_by_role("textbox", name="Confirm password*").fill('adminpassword')''
               "page.get_by_role('button', name=re.compile('Create account')).click()"
 
               "expect(page.get_by_text('All set!')).to_be_visible()"

test/services/jellyfin.nix

@@ -5,6 +5,36 @@ let
   adminUser = "jellyfin2";
   adminPassword = "admin";
 
+  commonExtraScript =
+    { node, ... }:
+    ''
+      server.wait_until_succeeds("journalctl --since -1m --unit jellyfin --grep 'Startup complete'")
+      headers = unline_with(" ", """
+          -H 'Content-Type: application/json'
+          -H 'Authorization: MediaBrowser Client="Android TV", Device="Nvidia Shield", DeviceId="ZQ9YQHHrUzk24vV", Version="0.15.3"'
+      """)
+      import time
+      with subtest("api login success"):
+          ok = False
+          for i in range(1, 5):
+              response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
+                  data="""{"Username": "${adminUser}", "Pw": "${adminPassword}"}""",
+                  extra=headers)
+              if response['code'] == 200:
+                  ok = True
+                  break
+              time.sleep(5)
+          if not ok:
+              raise Exception(f"Expected success, got: {response['code']}")
+
+      with subtest("api login failure"):
+          response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
+              data="""{"Username": "${adminUser}", "Pw": "badpassword"}""",
+              extra=headers)
+          if response['code'] != 401:
+              raise Exception(f"Expected failure, got: {response['code']}")
+    '';
+
   commonTestScript = shb.test.mkScripts {
     hasSSL = { node, ... }: !(isNull node.config.shb.jellyfin.ssl);
     waitForServices =
@@ -27,35 +57,7 @@ let
           status = 401;
         }
       ];
-    extraScript =
+    extraScript = commonExtraScript;
-      { node, ... }:
-      ''
-        server.wait_until_succeeds("journalctl --since -1m --unit jellyfin --grep 'Startup complete'")
-        headers = unline_with(" ", """
-            -H 'Content-Type: application/json'
-            -H 'Authorization: MediaBrowser Client="Android TV", Device="Nvidia Shield", DeviceId="ZQ9YQHHrUzk24vV", Version="0.15.3"'
-        """)
-        import time
-        with subtest("api login success"):
-            ok = False
-            for i in range(1, 5):
-                response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
-                    data="""{"Username": "${adminUser}", "Pw": "${adminPassword}"}""",
-                    extra=headers)
-                if response['code'] == 200:
-                    ok = True
-                    break
-                time.sleep(5)
-            if not ok:
-                raise Exception(f"Expected success, got: {response['code']}")
-
-        with subtest("api login failure"):
-            response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
-                data="""{"Username": "${adminUser}", "Pw": "badpassword"}""",
-                extra=headers)
-            if response['code'] != 401:
-                raise Exception(f"Expected failure, got: {response['code']}")
-      '';
   };
 
   basic =
@@ -452,12 +454,13 @@ in
 
     testScript = commonTestScript.access.override {
       extraScript =
-        {
+        args@{
           node,
           ...
         }:
+        (commonExtraScript args)
         # I have no idea why the LDAP Authentication_19.0.0.0 plugin disappears.
-        ''
+        + ''
           r = server.execute('cat "${node.config.services.jellyfin.dataDir}/plugins/LDAP Authentication_19.0.0.0/meta.json"')
           if r[0] != 0:
               print("meta.json for plugin LDAP Authentication_19.0.0.0 not found")

test/services/mailserver.nix

@@ -0,0 +1,278 @@
+{
+  lib,
+  pkgs,
+  shb,
+  ...
+}:
+let
+  domain = "example.com";
+  subdomain = "imap";
+  fqdn = "${subdomain}.${domain}";
+
+  trySendMail = pkgs.writeShellApplication {
+    name = "trySendMail";
+    runtimeInputs = [
+      pkgs.swaks
+    ];
+    text = ''
+      USER="''${1:-}"
+      PASSWORD="''${2:-}"
+
+      if [ -z "$USER" ]; then
+        echo "No user given"
+        exit 1
+      fi
+
+      if [ -z "$PASSWORD" ]; then
+        swaks \
+          --server ${fqdn} \
+          --port 465 \
+          --tls-on-connect \
+          --from "$USER" \
+          --to "$USER"
+      else
+        swaks \
+          --server ${fqdn} \
+          --port 465 \
+          --tls-on-connect \
+          --auth LOGIN \
+          --auth-user "$USER" \
+          --auth-password "$PASSWORD" \
+          --from "$USER" \
+          --to "$USER"
+      fi
+    '';
+  };
+
+  tryRcvMail = pkgs.writeShellApplication {
+    name = "tryRcvMail";
+    runtimeInputs = [
+      pkgs.curl
+    ];
+    text = ''
+      USER="''${1:-}"
+      PASSWORD="''${2:-}"
+
+      if [ -z "$USER" ]; then
+        echo "No user given"
+        exit 1
+      fi
+      if [ -z "$PASSWORD" ]; then
+        echo "No password given"
+        exit 1
+      fi
+
+      curl -s --url "imaps://imap.example.com/INBOX;UID=1" \
+        --user "$USER:$PASSWORD"
+    '';
+  };
+in
+{
+  basic = shb.test.runNixOSTest {
+    name = "mailserver_basic";
+
+    # Connect to the running VM started with driverInteractive:
+    #   ssh-keygen -R 'vsock-mux//run/user/1000/tmpcf3hs4yp/server_host.socket'; ssh -o User=root vsock-mux//run/user/1000/tmpcf3hs4yp/server_host.socket
+    interactive.sshBackdoor.enable = true;
+
+    nodes.server =
+      { config, ... }:
+      {
+        imports = [
+          ../../modules/blocks/ssl.nix
+          ../../modules/services/mailserver.nix
+        ];
+
+        networking.hosts = {
+          "127.0.0.1" = [ fqdn ];
+        };
+
+        shb.certs.cas.selfsigned.myca = {
+          name = "My CA";
+        };
+        shb.certs.certs.selfsigned = {
+          ${domain} = {
+            ca = config.shb.certs.cas.selfsigned.myca;
+
+            domain = "*.${domain}";
+            group = "nginx";
+          };
+        };
+
+        shb.mailserver = {
+          enable = true;
+          inherit subdomain domain;
+          stateVersion = 4;
+          ssl = config.shb.certs.certs.selfsigned.${domain};
+
+          # imapSync = {
+          #   # syncTimer = "10s";
+          #   # debug = false;
+          #   # accounts.fastmail = {
+          #   #   host = "imap.fastmail.com";
+          #   #   port = 993;
+          #   #   username = email;
+          #   #   password.result = config.shb.sops.secret."mailserver/imap/fastmail/password".result;
+          #   #   mapSpecialJunk = "Spam";
+          #   # };
+          # };
+
+          # smtpRelay = {
+          #   host = "smtp.fastmail.com";
+          #   port = 587;
+          #   username = email;
+          #   password.result = config.shb.sops.secret."mailserver/smtp/fastmail/password".result;
+          # };
+
+        };
+
+        mailserver.mailboxes = {
+          Junk = {
+            auto = "subscribe";
+            special_use = "\\Junk";
+          };
+        };
+
+        environment.systemPackages = [
+          pkgs.openssl
+          pkgs.swaks
+          trySendMail
+          tryRcvMail
+        ];
+
+        specialisation = {
+          ldap.configuration =
+            { config, ... }:
+            {
+              imports = [
+                ../../modules/blocks/hardcodedsecret.nix
+                ../../modules/blocks/lldap.nix
+              ];
+
+              networking.hosts = {
+                "127.0.0.1" = [ "ldap.${domain}" ];
+              };
+
+              environment.systemPackages = [
+                pkgs.openldap
+              ];
+
+              shb.hardcodedsecret.ldapUserPassword = {
+                request = config.shb.lldap.ldapUserPassword.request;
+                settings.content = "ldapUserPassword";
+              };
+              shb.hardcodedsecret.jwtSecret = {
+                request = config.shb.lldap.jwtSecret.request;
+                settings.content = "jwtSecrets";
+              };
+              shb.hardcodedsecret.alice = {
+                request = config.shb.lldap.ensureUsers.alice.password.request;
+                settings.content = "AlicePassword";
+              };
+              shb.hardcodedsecret.charlie = {
+                request = config.shb.lldap.ensureUsers.charlie.password.request;
+                settings.content = "CharliePassword";
+              };
+
+              shb.lldap = {
+                enable = true;
+                inherit domain;
+                subdomain = "ldap";
+                ldapPort = 3890;
+                webUIListenPort = 17170;
+                dcdomain = "dc=example,dc=com";
+                ldapUserPassword.result = config.shb.hardcodedsecret.ldapUserPassword.result;
+                jwtSecret.result = config.shb.hardcodedsecret.jwtSecret.result;
+                debug = true;
+
+                ensureUsers = {
+                  alice = {
+                    email = "alice@example.com";
+                    groups = [ "user_group" ];
+                    password.result = config.shb.hardcodedsecret.alice.result;
+                  };
+                  charlie = {
+                    email = "charlie@example.com";
+                    groups = [ "other_group" ];
+                    password.result = config.shb.hardcodedsecret.charlie.result;
+                  };
+                };
+
+                ensureGroups = {
+                  user_group = { };
+                  admin_group = { };
+                  other_group = { };
+                };
+              };
+
+              shb.mailserver = {
+                ldap = {
+                  enable = true;
+                  host = "127.0.0.1";
+                  port = config.shb.lldap.ldapPort;
+                  dcdomain = config.shb.lldap.dcdomain;
+                  adminName = "admin";
+                  adminPassword.result = config.shb.hardcodedsecret.ldapUserPassword.result;
+                  account = "fastmail";
+                  userGroup = "user_group";
+                };
+              };
+            };
+        };
+      };
+
+    testScript =
+      { nodes, ... }:
+      let
+        specialisations = "${nodes.server.system.build.toplevel}/specialisation";
+        switch = name: "${specialisations}/${name}/bin/switch-to-configuration test";
+
+        ldapSearch =
+          let
+            config = nodes.server.specialisation.ldap.configuration;
+          in
+          lib.concatStringsSep " " [
+            "ldapsearch"
+            "-H ldap://ldap.example.com:${toString config.shb.lldap.ldapPort}"
+          ];
+        ldapSearchAdmin =
+          let
+            config = nodes.server.specialisation.ldap.configuration;
+          in
+          lib.concatStringsSep " " [
+            "ldapsearch"
+            "-H ldap://ldap.example.com:${toString config.shb.lldap.ldapPort}"
+            "-D uid=admin,ou=people,${config.shb.lldap.dcdomain}"
+            "-w ${config.shb.hardcodedsecret.ldapUserPassword.settings.content}"
+          ];
+      in
+      ''
+        server.wait_for_unit("multi-user.target")
+        server.wait_for_unit("dovecot")
+
+        with subtest("ldap"):
+            server.succeed("${switch "ldap"}")
+
+            server.wait_for_unit("lldap")
+            print(server.succeed("${ldapSearch} -LLL -D uid=alice,ou=people,dc=example,dc=com -w AlicePassword -b cn=user_group,ou=groups,dc=example,dc=com | grep uniquemember | grep alice"))
+            print(server.succeed("${ldapSearchAdmin} -LLL -b cn=user_group,ou=groups,dc=example,dc=com | grep uniquemember | grep alice"))
+
+            server.wait_for_unit("dovecot")
+            print(server.fail("doveadm auth test auser@example.com apassword"))
+            print(server.fail("doveadm auth test charlie@example.com CharliePassword"))
+            print(server.succeed("doveadm auth test alice@example.com AlicePassword"))
+
+            server.wait_for_unit("postfix")
+            print(server.fail("trySendMail alice@example.com"))
+            print(server.fail("trySendMail auser@example.com apassword"))
+            print(server.fail("trySendMail charlie@example.com CharliePassword"))
+            print(server.succeed("trySendMail alice@example.com AlicePassword"))
+
+            print(server.fail("tryRcvMail charlie@example.com CharliePassword"))
+            print(server.succeed("tryRcvMail alice@example.com AlicePassword"))
+
+            print(server.succeed("find /var/vmail"))
+            print(server.succeed("find /var/vmail/fastmail/alice@example.com"))
+      '';
+  };
+}

test/services/open-webui.nix

@@ -35,7 +35,13 @@ let
         inherit (config.test) subdomain domain;
       };
       # Speeds up tests because models can't be downloaded anyway and that leads to retries.
-      services.open-webui.environment.OFFLINE_MODE = "true";
+      services.open-webui.environment = {
+        OFFLINE_MODE = "true";
+        BYPASS_EMBEDDING_AND_RETRIEVAL = "true";
+        RAG_EMBEDDING_ENGINE = "ollama";
+        RAG_EMBEDDING_MODEL = "dummy";
+        RAG_OLLAMA_BASE_URL = "http://127.0.0.1:9";
+      };
 
       networking.hosts = {
         "127.0.0.1" = [ "${config.test.subdomain}.${config.test.domain}" ];
@@ -91,7 +97,7 @@ let
             username = "alice";
             password = "NotAlicePassword";
             nextPageExpect = [
-              "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible()"
+              "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible(timeout=20000)"
             ];
           }
           {
@@ -101,14 +107,15 @@ let
               "page.get_by_role('button', name=re.compile('Accept')).click()"
               "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible()"
               "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
-              "expect(page.get_by_text('logged in')).to_be_visible()"
+              "expect(page).to_have_url(re.compile('https://o.example.com/$'), timeout=20000)"
+              "assert page.evaluate(\"async () => { const r = await fetch('/api/v1/auths/'); if (!r.ok) return false; const u = await r.json(); return u.email === 'alice@example.com'; }\")"
             ];
           }
           {
             username = "bob";
             password = "NotBobPassword";
             nextPageExpect = [
-              "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible()"
+              "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible(timeout=20000)"
             ];
           }
           {
@@ -118,14 +125,15 @@ let
               "page.get_by_role('button', name=re.compile('Accept')).click()"
               "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible()"
               "expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
-              "expect(page.get_by_text('logged in')).to_be_visible()"
+              "expect(page).to_have_url(re.compile('https://o.example.com/$'), timeout=20000)"
+              "assert page.evaluate(\"async () => { const r = await fetch('/api/v1/auths/'); if (!r.ok) return false; const u = await r.json(); return u.email === 'bob@example.com'; }\")"
             ];
           }
           {
             username = "charlie";
             password = "NotCharliePassword";
             nextPageExpect = [
-              "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible()"
+              "expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible(timeout=20000)"
             ];
           }
           {
@@ -133,7 +141,7 @@ let
             password = "CharliePassword";
             nextPageExpect = [
               "page.get_by_role('button', name=re.compile('Accept')).click()"
-              "expect(page.get_by_text('unauthorized')).to_be_visible()"
+              "expect(page.get_by_text('do not have permission')).to_be_visible(timeout=20000)"
             ];
           }
         ];
@@ -143,6 +151,8 @@ let
   sso =
     { config, ... }:
     {
+      virtualisation.memorySize = 4096;
+
       shb.open-webui = {
         sso = {
           enable = true;