open-webui: more consistent example

Open WebUI 0.9.6 still initializes its embedding function during startup, even when OFFLINE_MODE=true and BYPASS_EMBEDDING_AND_RETRIEVAL=true. With OFFLINE_MODE=true and no cached SentenceTransformer model, startup failed with:

ValueError: No embedding model is loaded. Set RAG_EMBEDDING_MODEL to a valid SentenceTransformer model name, or configure an external RAG_EMBEDDING_ENGINE (ollama, openai, azure_openai).

Configure the tests to use a dummy Ollama embedding endpoint so startup does not try to load or download a local embedding model. The endpoint is not meant to be contacted by these tests; it only keeps startup on the external embedding configuration path.

.github/workflows/auto-merge.yaml

@@ -23,7 +23,7 @@ on:
         required: false
 
 jobs:
-  automerge:
+  automerge-rebase:
     runs-on: ubuntu-latest
     steps:
       - uses: reitermarkus/automerge@v2
@@ -35,3 +35,16 @@ jobs:
           pull-request: ${{ github.event.inputs.pull-request }}
           review: ${{ github.event.inputs.review }}
           dry-run: false
+
+  automerge-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: reitermarkus/automerge@v2
+        with:
+          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          merge-method: merge
+          do-not-merge-labels: never-merge
+          required-labels: automerge-merge
+          pull-request: ${{ github.event.inputs.pull-request }}
+          review: ${{ github.event.inputs.review }}
+          dry-run: false

.github/workflows/build.yaml

@@ -24,9 +24,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
-      - uses: tj-actions/changed-files@v46
+      - uses: tj-actions/changed-files@v47
         id: filter
         with:
           files: |
@@ -51,20 +51,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@main
+        uses: cachix/install-nix-action@v31
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          extra-conf: "system-features = nixos-test benchmark big-parallel kvm"
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          enable_kvm: true
+          extra_nix_config: |
+            keep-outputs = true
+            keep-failed = true
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
       - name: Generate Matrix
         id: generate-matrix
         run: |
+          set -euox pipefail
+
           nix flake show --allow-import-from-derivation --json \
               | jq -c '.["checks"]["x86_64-linux"] | keys' > .output
 
@@ -78,14 +83,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@main
+        uses: cachix/install-nix-action@v31
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          extra-conf: "system-features = nixos-test benchmark big-parallel kvm"
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          enable_kvm: true
+          extra_nix_config: |
+            keep-outputs = true
+            keep-failed = true
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -101,32 +109,37 @@ jobs:
     runs-on: ubuntu-latest
     needs: [ "build-matrix" ]
     strategy:
+      fail-fast: false
       matrix:
         check: ${{ fromJson(needs.build-matrix.outputs.check) }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@main
+        uses: cachix/install-nix-action@v31
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          extra-conf: "system-features = nixos-test benchmark big-parallel kvm"
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          enable_kvm: true
+          extra_nix_config: |
+            keep-outputs = true
+            keep-failed = true
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
       - name: Build
         run: |
           echo "resultPath=$(nix eval .#checks.x86_64-linux.${{ matrix.check }} --raw)" >> $GITHUB_ENV
-          nix build --print-build-logs --show-trace --keep-outputs --keep-failed .#checks.x86_64-linux.${{ matrix.check }}
+          nix build --print-build-logs --show-trace .#checks.x86_64-linux.${{ matrix.check }}
       - name: Upload Build Result
-        uses: actions/upload-artifact@v4
-        if: always()
+        uses: actions/upload-artifact@v7
+        if: always() && startsWith(matrix.check, 'vm_')
         with:
           name: ${{ matrix.check }}
-          path: ${{ env.resultPath }}/**/*
+          path: ${{ env.resultPath }}/trace/*
           overwrite: true
+          if-no-files-found: ignore
 
   results:
     name: Final Results

.github/workflows/demo.yml

@@ -15,9 +15,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
-      - uses: tj-actions/changed-files@v46
+      - uses: tj-actions/changed-files@v47
         id: filter
         with:
           files: |
@@ -40,6 +40,7 @@ jobs:
     needs: [ "path-filter" ]
     if: needs.path-filter.outputs.changed == 'true'
     strategy:
+      fail-fast: false
       matrix:
         demo:
           - name:  homeassistant
@@ -54,16 +55,29 @@ jobs:
           - name:  nextcloud
             flake: sso
 
+          - name:  minimal
+            flake: minimal
+          - name:  minimal
+            flake: lowlevel
+          - name:  minimal
+            flake: sops
+
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
-      - name: Install nix
+      - name: Install Nix
         uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          enable_kvm: true
+          extra_nix_config: |
+            keep-outputs = true
+            keep-failed = true
 
-      - uses: cachix/cachix-action@v16
+      - uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -72,6 +86,11 @@ jobs:
         run: |
           cd demo/${{ matrix.demo.name }}
           nix flake update --override-input selfhostblocks ../.. selfhostblocks
+          nix \
+            --print-build-logs \
+            --option keep-going true \
+            --show-trace \
+            build .#nixosConfigurations.${{ matrix.demo.flake }}.config.system.build.toplevel
           nix \
             --print-build-logs \
             --option keep-going true \

.github/workflows/format.yaml

@@ -0,0 +1,24 @@
+name: "format"
+on:
+  pull_request:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Caching
+        uses: cachix/cachix-action@v17
+        with:
+          name: selfhostblocks
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+      - name: Check Formatting
+        run: |
+          find . -name '*.nix' | nix fmt -- --ci

.github/workflows/lock-update.yaml

@@ -1,24 +1,43 @@
+# See shb.skarabox.com/misc.html
 name: Update Flake Lock
 
 on:
-  workflow_dispatch:
   schedule:
-    - cron: '0 0 * * *' # runs daily at 00:00
+    - cron: '0 */3 * * *' # runs every 3 hours at :00
+
+  workflow_dispatch:
+    inputs:
+      bisect_future:
+        description: "Bisect in the future instead of the past"
+        required: false
+        default: "false"
 
 jobs:
   lockfile:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@main
+        uses: cachix/install-nix-action@v31
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          extra-conf: "system-features = nixos-test benchmark big-parallel kvm"
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Cache nixpkgs mirror
+        uses: actions/cache@v5
+        with:
+          path: .cache/nixpkgs.git
+          key: nixpkgs-mirror-v1
       - name: Update flake.lock
-        uses: DeterminateSystems/update-flake-lock@main
-        with:
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
-          pr-labels: |
-            automerge
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          BISECT_FUTURE: ${{ inputs.bisect_future }}
+        run: |
+          git config user.email "update-flake-lock-pr@selfhostblocks.com"
+          git config user.name "Update Flake Lock PR"
+          if [ "$BISECT_FUTURE" = false ]; then
+            BISECT_FUTURE=
+          elif [ -n "$BISECT_FUTURE" ]; then
+            BISECT_FUTURE=future
+          fi
+          nix run .#update-flake-lock-pr $BISECT_FUTURE
+

.github/workflows/pages.yml

@@ -29,13 +29,15 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
-      - name: Install nix
+      - name: Install Nix
         uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -57,13 +59,13 @@ jobs:
             public
 
       - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@v6
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v5
         with:
           path: ./public
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v5

.github/workflows/update-flake-lock-pr.nix

@@ -0,0 +1,223 @@
+{
+  gh,
+  writeShellApplication,
+}:
+writeShellApplication {
+  name = "update-flake-lock-pr";
+  runtimeInputs = [
+    gh
+  ];
+  text = ''
+    branch=ci/nixpkgs-update
+    if [ "''${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
+      branch=ci/nixpkgs-update-auto
+    fi
+
+    goto_future=no
+    if [ -n "''${1:-}" ]; then
+      goto_future=yes
+    fi
+
+    main () {
+      PR=$(get_pr)
+
+      if [ -z "$PR" ]; then
+        create_pr
+        exit 0
+      fi
+
+      case "$(get_pr_check_status "$PR")" in
+        pending)
+          echo "Checks are running on PR $PR, nothing to do yet."
+          exit 0
+          ;;
+
+        succeeded)
+          echo "Checks succeeded, nothing to do."
+          exit 0
+          ;;
+
+        failing)
+          echo "Checks failed, updating PR to midpoint commit."
+
+          local CURRENT_COMMIT
+          local LAST_WORKING_COMMIT
+          local FAILING_COMMIT
+          local NEW_COMMIT
+
+          CURRENT_COMMIT="$(get_main_branch_commit)"
+          if [ "$goto_future" = "no" ]; then
+            LAST_WORKING_COMMIT="$CURRENT_COMMIT"
+            FAILING_COMMIT="$(get_pr_commit "$PR")"
+          else
+            LAST_WORKING_COMMIT="$(get_pr_commit "$PR")"
+            FAILING_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+          NEW_COMMIT="$(midpoint_commit "$LAST_WORKING_COMMIT" "$FAILING_COMMIT")"
+
+          if [ "$NEW_COMMIT" = "$LAST_WORKING_COMMIT" ] || [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            NEW_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+
+          if [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            echo "Latest nixpkgs commit is still same failing commit, will retry later."
+            exit 0
+          fi
+
+          update_pr "$PR" "$CURRENT_COMMIT" "$NEW_COMMIT"
+          exit 0
+          ;;
+      esac
+    }
+
+    get_main_branch_commit() {
+      git fetch origin main
+
+      git show origin/main:flake.lock \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    midpoint_commit() {
+      local good="$1"
+      local bad="$2"
+
+      mkdir -p .cache
+      if [ ! -d .cache/nixpkgs.git ]; then
+        git clone --mirror https://github.com/NixOS/nixpkgs.git .cache/nixpkgs.git
+      else
+        git --git-dir=.cache/nixpkgs.git fetch origin
+      fi
+
+      local commits=()
+
+      mapfile -t commits < <(
+        git --git-dir=.cache/nixpkgs.git \
+          rev-list \
+          --reverse \
+          --ancestry-path \
+          "''${good}..''${bad}"
+      )
+
+      local count="''${#commits[@]}"
+
+      if [ "$count" -eq 0 ]; then
+        echo "$bad"
+        return
+      fi
+
+      local midpoint_index=$((count / 2))
+
+      echo "''${commits[$midpoint_index]}"
+    }
+
+    get_latest_nixpkgs_commit () {
+      git ls-remote https://github.com/NixOS/nixpkgs nixos-unstable | cut -f1
+    }
+
+    get_pr () {
+      gh pr list \
+        --head "$branch" \
+        --state open \
+        --json number \
+        --jq '.[0].number // empty'
+    }
+
+    create_pr() {
+      local test_commit
+
+      test_commit="$(get_latest_nixpkgs_commit)"
+      echo "Creating PR on nixpkgs' latest commit $test_commit"
+
+      git checkout -B "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push -u origin "$branch" --force
+
+      current_commit="$(get_main_branch_commit)"
+      gh pr create \
+        --title "update nixpkgs to $test_commit" \
+        --body "$(printf "%s\n\n%s" "Automated nixpkgs update. Latest tries:" " - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit")" \
+        --label automerge-merge
+    }
+
+    append_pr_body() {
+      local pr="$1"
+      local text="$2"
+
+      local current_body
+
+      current_body="$(gh pr view "$pr" --json body --jq .body)"
+
+      gh pr edit "$pr" \
+        --body "$(printf '%s%b' "$current_body" "$text")"
+    }
+
+    update_pr() {
+      local pr="$1"
+      local current_commit="$2"
+      local test_commit="$3"
+
+      echo "Updating PR to nixpkgs' commit $test_commit"
+
+      git checkout "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push --force-with-lease origin "$branch"
+
+      local future_text
+      if [ "$goto_future" = "yes" ]; then
+        future_text="bisected in the future"
+      else
+        future_text="bisected in the past"
+      fi
+
+      gh pr edit "$pr" \
+        --title "update nixpkgs to $test_commit"
+
+      append_pr_body "$pr" \
+        " ❌\n - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit ($future_text)"
+    }
+
+    get_pr_check_status() {
+      local pr="$1"
+
+      local status
+      status="$(gh pr checks "$pr" --json state --jq '.[].state' || true)"
+
+      if echo "$status" | grep -qE 'PENDING|QUEUED|IN_PROGRESS'; then
+        echo "pending"
+        return
+      fi
+
+      if echo "$status" | grep -qE 'FAILURE|TIMED_OUT|CANCELLED'; then
+        echo "failing"
+        return
+      fi
+
+      echo "success"
+    }
+
+    get_pr_commit() {
+      local pr="$1"
+
+      git fetch origin "$branch"
+
+      git show "origin/$branch:flake.lock" \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    main
+  '';
+}

.github/workflows/version.yaml

@@ -11,7 +11,7 @@ jobs:
   create-tag:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 1
       - name: Get version

.gitignore

@@ -1,3 +1,6 @@
 *.qcow2
 result
-result-*
+result-*
+docs/redirects.json.backup
+.nixos-test-history
+\#*#

CHANGELOG.md

@@ -2,10 +2,10 @@
 
 Template:
 
-## New Features
-
 ## Breaking Changes
 
+## New Features
+
 ## User Facing Backwards Compatible Changes
 
 ## Fixes
@@ -16,6 +16,296 @@ Template:
 
 # Upcoming Release
 
+# v0.9.0
+
+Commits: https://github.com/ibizaman/selfhostblocks/compare/v0.8.0...v0.9.0
+
+## Breaking Changes
+
+- Updated simple-nixos-mailserver. Update all options following this pattern:
+
+  ```diff
+  - mailserver.mailboxes.<name>.specialUse = "value"
+  + mailserver.mailboxes.<name>.special_use = "\\value"
+  ```
+
+  Note that simple-nixos-mailserver wanted to upgrade the location of the storage path
+  to include the ldap UID instead of the email but I kept the email address.
+  This means there is no migration to do.
+- Update nixpkgs [from 6201e2 to abd1ea](https://github.com/ibizaman/selfhostblocks/compare/6201e203d09599479a3b3450ed24fa81537ebc4e...abd1ea17fdbedd48cbca770847b39e3a7a09ab5b).
+
+## New Features
+
+- Add `shb.zfs.snapshotBeforeActivation` option to take a ZFS snapshot of given datasets before activation.
+  See [the manual](https://shb.skarabox.com/blocks-zfs.html) for more details.
+- Add [sanoid block](https://shb.skarabox.com/blocks-sanoid.html)
+- Add option to take snapshot before activation with ZFS block.
+
+## Fixes
+
+- Fix oidc_login build for Nextcloud
+- Fix mailserver build
+
+## Other Changes
+
+- Harden lldap configuration
+- Convert ZFS block to system and [add documentation](https://shb.skarabox.com/blocks-zfs.html)
+- Switch monitoring log fetching from deprecated promtail to fluent-bit
+- Add test to catch drift for Let's Encrypt DNS provider
+
+# [BROKEN] v0.8.0
+
+## Breaking Changes
+
+- Bump of Nextcloud version to 32 and 33 because of nixpkgs bump. All provided apps are verified compatible with Nextcloud 33 thanks to new tests.
+
+## New Features
+
+- Added Immich Public Proxy service
+- Add homepage service with dashboard contract implemented by all services
+- Add scrutiny service.
+- ZFS module now supports setting permissions
+- Add landing page for mailserver and dashboard contract integration
+
+## Bug Fixes
+
+- Use configurable dataDir in arr stack
+- Forgejo ensures ldap is setup when sso is configured
+- Add nixpkgs patches on aarch64-linux too
+- Self-signed certs are now idempotent
+- Prometheus scrapes metrics at 15s interval instead of 1m
+
+## Other Changes
+
+- Arr stack declares ldap groups, declare ApiKeys and bypasses auth for readarr when sso is enabled
+- Forgejo declares ldap group
+
+# v0.7.3
+
+## New Features
+
+- Add [mailserver module](https://shb.skarabox.com/services-mailserver.html) integrating with [Simple NixOS Mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver) and allowing full backup of an email provider.
+- Bump nixpkgs from https://github.com/NixOS/nixpkgs/commit/5e2a59a5b1a82f89f2c7e598302a9cacebb72a67 to https://github.com/NixOS/nixpkgs/commit/bfc1b8a4574108ceef22f02bafcf6611380c100d. [Full diff](https://github.com/nixos/nixpkgs/compare/5e2a59a5b1a82f89f2c7e598302a9cacebb72a67...bfc1b8a4574108ceef22f02bafcf6611380c100d).
+  On top of minor changes, the most notable one was:
+  - Updated Jellyfin LDAP and SSO plugins and configuration. @Codys-Wright
+
+## Bug Fixes
+
+- Fix Restic and Authelia modules referencing systemd services without the `.service` suffix and leading to 
+
+# v0.7.2
+
+## New Features
+
+- Forgejo uses secrets contract for smtp password.
+- Add [Firefly-iii](https://shb.skarabox.com/services-firefly-iii.html) service.
+- Jellyfin can [install plugins declaratively](https://shb.skarabox.com/services-jellyfin.html#services-jellyfin-options-shb.jellyfin.plugins).
+  (Support is quite crude and WIP).
+- Jellyfin configures LDAP and SSO fully declaratively, including installing necessary plugins.
+- Nextcloud 32 is fully supported thanks to tests for version 31 and 32.
+
+## Fixes
+
+- Revert Authelia to continue using dots in systemd service names.
+  This caused issue with nginx name resolution.
+
+## Other Changes
+
+- Authelia uses non deprecated `smtp.address` option.
+- Add documentation for Nginx block
+- Now a user which is only member of the admin LDAP group of a service can login.
+  Before, some services required a user to be member of both the user and admin LDAP group.
+  This is ensured by regression tests going forward.
+
+# v0.7.1
+
+## New Features
+
+- Add a Grafana dashboard showing SSL certificate renewal jobs
+
+## Fixes
+
+- Fix let's encrypt certificate renewal jobs by removing duplicated domain name.
+  Also adds an assertion to catch these kinds of errors.
+
+## Other Changes
+
+- Reduce number of late SSL renewal alert by merging all metrics corresponding to one CN.
+
+# v0.7.0
+
+## Breaking Changes
+
+- Fix pkgs overrides not being passed to users of SelfHostBlocks.
+  This will require to update your flake to follow the example in the [Usage](https://shb.skarabox.com/usage.html) section.
+
+## New Features
+
+- Add a Grafana dashboard showing stats on backup jobs
+  and also an alert if a backup job did not run in the last 24 hours or never succeeded in the last 24 hours.
+- Add SSO integration in Grafana.
+- Add Paperless service.
+
+## Fixes
+
+- Allow to upload big files in Immich.
+- Only enable php-fpm Prometheus exporter if Nextcloud is enabled.
+
+## Other Changes
+
+- Add recipe to setup DNS server with DNSSEC.
+
+# v0.6.1
+
+## New Features
+
+- Implement backup and databasebackup contracts with BorgBackup block.
+
+## Fixes
+
+- Add back erroneously removed Prometheus collectors.
+
+# v0.6.0
+
+## Breaking Changes
+
+- Removed Nextcloud 30, update to Nextcloud 31 then after to 32.
+- Removed the `sops` module in the `default` NixOS module. Removed the `all` NixOS module.
+
+## New Features
+
+- Meilisearch configured with production environment and master key.
+
+## Other Changes
+
+- Only import hardcodedsecret module in tests.
+- Better usage section in manual.
+- Added new demo for minimal SelfHostBlocks setup, which is tested in CI.
+- Format all files in repo and make sure they are formatted in CI.
+
+# v0.5.1
+
+## New Features
+
+- Added Karakeep service with SSO integration.
+- Add SelfHostBlocks' `lib` into `pkgs.lib.shb`. Integrates with [Skarabox](https://github.com/ibizaman/skarabox/blob/631ff5af0b5c850bb63a3b3df451df9707c0af4e/template/flake.nix#L42-L43) too.
+
+## Other Changes
+
+- Moved implementation guide under contributing section.
+
+# v0.5.0
+
+## Breaking Changes
+
+- Modules in the `nixosModules` output field do not anymore have the `system` in their path.
+  `selfhostblocks.nixosModules.x86_64-linux.home-assistant` becomes `selfhostblocks.nixosModules.home-assistant`
+  like it always should have been.
+
+## Fixes
+
+- Added test case making sure a user belonging to a not authorized LDAP group cannot login.
+  Fixed Open WebUI module.
+- Now importing a single module, like `selfhostblocks.nixosModules.home-assistant`, will
+  import all needed block modules at the same time.
+
+## Other Changes
+
+- Nextcloud module can now setup SSO integration without setting up LDAP integration.
+
+# v0.4.4
+
+## New Features
+
+- Added Pinchflat service with SSO integration. Declarative user creation only supported through SSO integration.
+- Added Immich service with SSO integration.
+- Added Open WebUI service with SSO integration.
+
+# v0.4.3
+
+## New Features
+
+- Allow user to change their SSO password in Authelia.
+- Make Audiobookshelf SSO integration respect admin users.
+
+## Fixes
+
+- Fix permission on Nextcloud systemd service.
+- Delete Forgejo backups correctly to avoid them piling up.
+
+## Other Changes
+
+- Add recipes section to the documentation.
+
+# v0.4.2
+
+## New Features
+
+- The LLDAP and Authelia modules gain a debug mode where a mitmdump instance is added so all traffic is printed.
+
+## Fixes
+
+- By default, LLDAP module only enforces groups declaratively. Users that are not defined declaratively
+  are not anymore deleted by inadvertence.
+- SSO integration with most services got fixed. A recent incompatible change in upstream Authelia broke most of them.
+- Fixed PostgreSQL and Home Assistant modules after nixpkgs updates.
+- Fixed Nextcloud module SSO integration with Authelia.
+- Make Nextcloud SSO integration respect admin users.
+
+# v0.4.1
+
+## New Features
+
+- LLDAP now manages users, groups, user attributes and group attributes declaratively.
+- Individual modules are exposed in the flake output for each block and service.
+- A mitmdump block is added that can be placed between two services and print all requests and responses.
+- The SSO setup for Audiobookshelf is now a bit more declarative.
+
+## Other Changes
+
+- Forgejo got a new playwright test to check the LDAP integration.
+- Some renaming options have been added retroactively for jellyfin and forgejo.
+
+# v0.4.0
+
+## Breaking Changes
+
+- Rename ldap module to lldap as well as option name `shb.ldap` to `shb.lldap`.
+
+## New Features
+
+- Jellyfin service now waits for Jellyfin server to be fully available before starting.
+- Add debug option for Jellyfin.
+- Allow to choose port for Jellyfin.
+- Make Jellyfin setup including initial admin user declarative.
+
+## Fixes
+
+- Fix Jellyfin redirect URI scheme after update.
+
+## Other Changes
+
+- Add documentation for LLDAP and Authelia block and link to it from other docs.
+
+# v0.3.1
+
+## Breaking Changes
+
+- Default version of Nextcloud is now 30.
+- Disable memories app on Nextcloud because it is broken.
+
+## New Features
+
+- Add patchNixpkgs function and pre-patched patchedNixpkgs output.
+
+## Fixes
+
+- Fix secrets passing to Nextcloud service after update.
+
+## Other Changes
+
+- Bump nixpkgs to https://github.com/NixOS/nixpkgs/commit/216207b1e58325f3590277d9102b45273afe9878
+
 # v0.3.0
 
 ## New Features

README.md

@@ -15,8 +15,19 @@
 
 # SelfHostBlocks
 
-*SelfHostBlocks is a NixOS based server management for self-hosting
-using building blocks and promoting best practices.*
+SelfHostBlocks is:
+
+- Your escape from the cloud, for privacy and data sovereignty enthusiast. [Why?](#why-self-hosting)
+- A groupware to self-host [all your data](#services): documents, pictures, calendars, contacts, etc.
+- An opinionated NixOS server management OS for a [safe self-hosting experience](#features).
+- A NixOS distribution making sure all services build and work correctly thanks to NixOS VM tests.
+- A collection of NixOS modules standardizing options so configuring services [look the same](#unified-interfaces).
+- A testing ground for [contracts](#contracts) which intents to make nixpkgs modules more modular.
+- [Upstreaming][] as much as possible.
+
+[upstreaming]: https://github.com/pulls?page=1&q=created%3A%3E2023-06-01+is%3Apr+author%3Aibizaman+archived%3Afalse+-repo%3Aibizaman%2Fselfhostblocks+-repo%3Aibizaman%2Fskarabox
+
+## Why Self-Hosting
 
 It is obvious by now that
 a deep dependency on proprietary service providers - "the cloud" -
@@ -55,9 +66,9 @@ that fit together to build their server.
 
 <!--toc:start-->
 - [Usage](#usage)
+  - [At a Glance](#at-a-glance)
   - [Existing Installation](#existing-installation)
   - [Installation From Scratch](#installation-from-scratch)
-  - [Full Example](#full-example)
 - [Features](#features)
   - [Services](#services)
   - [Blocks](#blocks)
@@ -79,12 +90,33 @@ that fit together to build their server.
 > production server, this is really just a one person effort for now and there are most certainly
 > bugs that I didn't discover yet.
 
-### Existing Installation
+To get started using SelfHostBlocks, the following snippet is enough:
 
-To get started using SelfHostBlocks,
-follow [the usage section](https://shb.skarabox.com/usage.html) of the manual.
-It goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
-and also goes over secrets management with [SOPS][].
+```nix
+{
+  inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks";
+
+  outputs = { selfhostblocks, ... }: let
+    system = "x86_64-linux";
+    nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
+  in
+    nixosConfigurations = {
+      myserver = nixpkgs'.nixosSystem {
+        inherit system;
+        modules = [
+          selfhostblocks.nixosModules.default
+          ./configuration.nix
+        ];
+      };
+    };
+}
+```
+
+SelfHostBlocks provides its own patched nixpkgs, so you are required to use it
+otherwise evaluation can quickly break.
+[The usage section](https://shb.skarabox.com/usage.html) of the manual has
+more details and goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
+and also how to handle secrets management with [SOPS][].
 
 [Colmena]: https://shb.skarabox.com/usage.html#usage-example-colmena
 [nixos-rebuild]: https://shb.skarabox.com/usage.html#usage-example-nixosrebuild
@@ -92,8 +124,9 @@ and also goes over secrets management with [SOPS][].
 [SOPS]: https://shb.skarabox.com/usage.html#usage-secrets
 
 Then, to actually configure services, you can choose which one interests you in
-[the services section](https://shb.skarabox.com/services.html) of the manual.
-Not all services have a corresponding manual page yet.
+the [services section](https://shb.skarabox.com/services.html) of the manual.
+
+The [recipes section](https://shb.skarabox.com/recipes.html) of the manual shows some other common use cases.
 
 Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org)
 for any remaining question, or just to say hi :)
@@ -113,12 +146,6 @@ which bootstraps a new server and sets up a few tools:
 
 [Skarabox]:  https://github.com/ibizaman/skarabox
 
-### Full Example
-
-See [full example][] in the manual.
-
-[full example]: https://shb.skarabox.com/usage.html#usage-complete-example
-
 ## Features
 
 SelfHostBlocks provides building blocks that take care of common self-hosting needs:
@@ -126,7 +153,7 @@ SelfHostBlocks provides building blocks that take care of common self-hosting ne
 - Backup for all services.
 - Automatic creation of ZFS datasets per service.
 - LDAP and SSO integration for most services.
-- Monitoring with Grafana and Prometheus stack with provided dashboards.
+- Monitoring with Grafana and Prometheus stack with provided dashboards and integration with Scrutiny.
 - Automatic reverse proxy and certificate management for HTTPS.
 - VPN and proxy tunneling services.
 
@@ -135,6 +162,9 @@ This translates into a test suite comprised of automated NixOS VM tests
 which includes playwright tests to verify some important workflow
 like logging in.
 
+This test suite also serves as a guaranty that all services provided by SelfHostBlocks
+all evaluate, build and work correctly together. It works similarly as a distribution but here it's all [automated](#automatic-updates).
+
 Also, the stack fits together nicely thanks to [contracts](#contracts).
 
 ### Services
@@ -144,11 +174,16 @@ Also, the stack fits together nicely thanks to [contracts](#contracts).
 - Nextcloud
 - Audiobookshelf
 - Deluge + *arr stack
+- Simple NixOS Mailserver
+- Firefly-iii
 - Forgejo
 - Grocy
 - Hledger
 - Home-Assistant
 - Jellyfin
+- Karakeep
+- Open WebUI
+- Pinchflat
 - Vaultwarden
 
 Like explained above, those services all benefit from
@@ -166,15 +201,14 @@ is to go to the [All Options][] section of the manual.
 
 ### Blocks
 
-The services above rely on the following [common blocks][]:
-
-[common blocks]: https://shb.skarabox.com/blocks.html
+The services above rely on the following [common blocks][]
+which altogether provides a solid foundation for self-hosting services:
 
 - Authelia
 - BorgBackup
 - Davfs
 - LDAP
-- Monitoring (Grafana - Prometheus - Loki stack)
+- Monitoring (Grafana - Prometheus - Loki stack + Scrutiny)
 - Nginx
 - PostgreSQL
 - Restic
@@ -185,11 +219,11 @@ The services above rely on the following [common blocks][]:
 - ZFS
 
 Those blocks can be used with services
-not provided by SelfHostBlocks.
+not provided by SelfHostBlocks as shown [in the manual][common blocks].
 
-Some blocks do not have an entry yet in the manual.
-To know options for those, the only way for now
-is to go to the [All Options][] section of the manual.
+[common blocks]: https://shb.skarabox.com/blocks.html
+
+The manual also provides documentation for each individual blocks.
 
 ### Unified Interfaces
 
@@ -214,16 +248,16 @@ shb.nextcloud = {
   apps.ldap = {
     enable = true;
     host = "127.0.0.1";
-    port = config.shb.ldap.ldapPort;
-    dcdomain = config.shb.ldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    port = config.shb.lldap.ldapPort;
+    dcdomain = config.shb.lldap.dcdomain;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
   apps.sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
+    secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
+    secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
   };
 };
 ```
@@ -239,17 +273,17 @@ shb.forgejo = {
   ldap = {
     enable = true;
     host = "127.0.0.1";
-    port = config.shb.ldap.ldapPort;
-    dcdomain = config.shb.ldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    port = config.shb.lldap.ldapPort;
+    dcdomain = config.shb.lldap.dcdomain;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
 
   sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."forgejo/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."forgejo/sso/secretForAuthelia".result;
+    secret.result = config.shb.sops.secret."forgejo/sso/secret".result;
+    secretForAuthelia.result = config.shb.sops.secret."forgejo/sso/secretForAuthelia".result;
   };
 };
 ```
@@ -257,7 +291,7 @@ shb.forgejo = {
 As you can see, they are pretty similar!
 This makes setting up a new service pretty easy and intuitive.
 
-SelfHostBlocks provides an ever growing list of [services](#provided-services)
+SelfHostBlocks provides an ever growing list of [services](#services)
 that are configured in the same way.
 
 ### Contracts
@@ -269,7 +303,7 @@ This lets you choose, for example,
 any reverse proxy you want or any database you want,
 without requiring work from maintainers of the services you want to self host.
 
-A [pre-RFC][] exists to upstream this concept into `nixpkgs`.
+An [RFC][] exists to upstream this concept into `nixpkgs`.
 The [manual][contracts] also provides an explanation of the why and how of contracts.
 
 Also, two videos exist of me presenting the topic,
@@ -277,7 +311,7 @@ the first at [NixCon North America in spring of 2024][NixConNA2024]
 and the second at [NixCon in Berlin in fall of 2024][NixConBerlin2024].
 
 [contracts]: https://shb.skarabox.com/contracts.html
-[pre-RFC]: https://discourse.nixos.org/t/pre-rfc-decouple-services-using-structured-typing/58257
+[RFC]: https://github.com/NixOS/rfcs/pull/189
 [NixConNA2024]: https://www.youtube.com/watch?v=lw7PgphB9qM
 [NixConBerlin2024]: https://www.youtube.com/watch?v=CP0hR6w1csc
 
@@ -285,7 +319,7 @@ and the second at [NixCon in Berlin in fall of 2024][NixConBerlin2024].
 
 Thanks to [contracts](#contracts), one can interface NixOS
 with systems on other OSes.
-The [pre-RFC][] explains how that works.
+The [RFC][] explains how that works.
 
 ### Sitting on the Shoulders of a Giant
 

VERSION

@@ -1 +1 @@
-0.3.0
+0.9.0

demo/homeassistant/configuration.nix

@@ -5,17 +5,17 @@ let
   targetPort = 2222;
 in
 {
-  imports =
-    [ # Include the results of the hardware scan.
-      ./hardware-configuration.nix
-    ];
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+  ];
 
   boot.loader.grub.enable = true;
   boot.kernelModules = [ "kvm-intel" ];
   system.stateVersion = "22.11";
 
   # Options above are generate by running nixos-generate-config on the VM.
-  
+
   # Needed otherwise deploy will say system won't be able to boot.
   boot.loader.grub.device = "/dev/vdb";
   # Needed to avoid getting into not available disk space in /boot.
@@ -26,7 +26,10 @@ in
 
   # Options above are needed to deploy in a VM.
 
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
 
   # We need to create the user we will deploy with.
   users.users.${targetUser} = {
@@ -41,9 +44,11 @@ in
 
   # The user we're deploying with must be able to run sudo without password.
   security.sudo.extraRules = [
-    { users = [ targetUser ];
+    {
+      users = [ targetUser ];
       commands = [
-        { command = "ALL";
+        {
+          command = "ALL";
           options = [ "NOPASSWD" ];
         }
       ];

demo/homeassistant/flake.lock

@@ -1,164 +0,0 @@
-{
-  "nodes": {
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1731533236,
-        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "nix-flake-tests": {
-      "locked": {
-        "lastModified": 1677844186,
-        "narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
-        "owner": "antifuchs",
-        "repo": "nix-flake-tests",
-        "rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
-        "type": "github"
-      },
-      "original": {
-        "owner": "antifuchs",
-        "repo": "nix-flake-tests",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1736883708,
-        "narHash": "sha256-uQ+NQ0/xYU0N1CnXsa2zghgNaOPxWpMJXSUJJ9W7140=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "eb62e6aa39ea67e0b8018ba8ea077efe65807dc8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1721524707,
-        "narHash": "sha256-5NctRsoE54N86nWd0psae70YSLfrOek3Kv1e8KoXe/0=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "556533a23879fc7e5f98dd2e0b31a6911a213171",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1721466660,
-        "narHash": "sha256-pFSxgSZqZ3h+5Du0KvEL1ccDZBwu4zvOil1zzrPNb3c=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "6e14bbce7bea6c4efd7adfa88a40dac750d80100",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nmdsrc": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1705050560,
-        "narHash": "sha256-x3zzcdvhJpodsmdjqB4t5mkVW22V3wqHLOun0KRBzUI=",
-        "ref": "refs/heads/master",
-        "rev": "66d9334933119c36f91a78d565c152a4fdc8d3d3",
-        "revCount": 66,
-        "type": "git",
-        "url": "https://git.sr.ht/~rycee/nmd"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.sr.ht/~rycee/nmd"
-      }
-    },
-    "root": {
-      "inputs": {
-        "selfhostblocks": "selfhostblocks",
-        "sops-nix": "sops-nix"
-      }
-    },
-    "selfhostblocks": {
-      "inputs": {
-        "flake-utils": "flake-utils",
-        "nix-flake-tests": "nix-flake-tests",
-        "nixpkgs": "nixpkgs",
-        "nmdsrc": "nmdsrc"
-      },
-      "locked": {
-        "lastModified": 1742237336,
-        "narHash": "sha256-KQlWwQ0XoLhjobgfK9CGrIPA4KofnUVO1Btid1Ca+OU=",
-        "owner": "ibizaman",
-        "repo": "selfhostblocks",
-        "rev": "023991b4b7bc8edc0276b3a45d1842ed2da7ae76",
-        "type": "github"
-      },
-      "original": {
-        "owner": "ibizaman",
-        "repo": "selfhostblocks",
-        "type": "github"
-      }
-    },
-    "sops-nix": {
-      "inputs": {
-        "nixpkgs": "nixpkgs_2",
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
-      "locked": {
-        "lastModified": 1723501126,
-        "narHash": "sha256-N9IcHgj/p1+2Pvk8P4Zc1bfrMwld5PcosVA0nL6IGdE=",
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "rev": "be0eec2d27563590194a9206f551a6f73d52fa34",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "type": "github"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

demo/homeassistant/flake.nix

@@ -6,151 +6,160 @@
     sops-nix.url = "github:Mic92/sops-nix";
   };
 
-  outputs = inputs@{ self, selfhostblocks, sops-nix }:
+  outputs =
+    inputs@{
+      self,
+      selfhostblocks,
+      sops-nix,
+    }:
     let
       system = "x86_64-linux";
-      originPkgs = selfhostblocks.inputs.nixpkgs;
+      nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
 
-      nixpkgs' = originPkgs.legacyPackages.${system}.applyPatches {
-        name = "nixpkgs-patched";
-        src = originPkgs;
-        patches = selfhostblocks.patches.${system};
-      };
-      nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
+      basic =
+        { config, ... }:
+        {
+          imports = [
+            ./configuration.nix
+            selfhostblocks.nixosModules.authelia
+            selfhostblocks.nixosModules.home-assistant
+            selfhostblocks.nixosModules.sops
+            selfhostblocks.nixosModules.ssl
+            sops-nix.nixosModules.default
+          ];
 
-      basic = { config, ...  }: {
-        imports = [
-          ./configuration.nix
-          selfhostblocks.nixosModules.x86_64-linux.default
-          sops-nix.nixosModules.default
-        ];
+          sops.defaultSopsFile = ./secrets.yaml;
 
-        sops.defaultSopsFile = ./secrets.yaml;
+          shb.home-assistant = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "ha";
+            config = {
+              name = "SHB Home Assistant";
+              country.source = config.shb.sops.secret."home-assistant/country".result.path;
+              latitude.source = config.shb.sops.secret."home-assistant/latitude".result.path;
+              longitude.source = config.shb.sops.secret."home-assistant/longitude".result.path;
+              time_zone.source = config.shb.sops.secret."home-assistant/time_zone".result.path;
+              unit_system = "metric";
+            };
+          };
+          shb.sops.secret."home-assistant/country".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+          shb.sops.secret."home-assistant/latitude".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+          shb.sops.secret."home-assistant/longitude".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+          shb.sops.secret."home-assistant/time_zone".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
 
-        shb.home-assistant = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "ha";
-          config = {
-            name = "SHB Home Assistant";
-            country.source = config.shb.sops.secret."home-assistant/country".result.path;
-            latitude.source = config.shb.sops.secret."home-assistant/latitude".result.path;
-            longitude.source = config.shb.sops.secret."home-assistant/longitude".result.path;
-            time_zone.source = config.shb.sops.secret."home-assistant/time_zone".result.path;
-            unit_system = "metric";
+          nixpkgs.config.permittedInsecurePackages = [
+            "openssl-1.1.1w"
+          ];
+        };
+
+      ldap =
+        { config, ... }:
+        {
+          shb.lldap = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "ldap";
+            ldapPort = 3890;
+            webUIListenPort = 17170;
+            dcdomain = "dc=example,dc=com";
+            ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
+            jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
+          };
+          shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
+          shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
+
+          shb.home-assistant.ldap = {
+            enable = true;
+            host = "127.0.0.1";
+            port = config.shb.lldap.webUIListenPort;
+            userGroup = "homeassistant_user";
           };
         };
-        shb.sops.secret."home-assistant/country".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-        shb.sops.secret."home-assistant/latitude".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-        shb.sops.secret."home-assistant/longitude".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-        shb.sops.secret."home-assistant/time_zone".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-
-        nixpkgs.config.permittedInsecurePackages = [
-          "openssl-1.1.1w"
-        ];
-      };
-
-      ldap = { config, ...  }: {
-        shb.ldap = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "ldap";
-          ldapPort = 3890;
-          webUIListenPort = 17170;
-          dcdomain = "dc=example,dc=com";
-          ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
-          jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
-        };
-        shb.sops.secret."lldap/user_password".request = config.shb.ldap.ldapUserPassword.request;
-        shb.sops.secret."lldap/jwt_secret".request = config.shb.ldap.jwtSecret.request;
-
-        shb.home-assistant.ldap = {
-          enable = true;
-          host = "127.0.0.1";
-          port = config.shb.ldap.webUIListenPort;
-          userGroup = "homeassistant_user";
-        };
-      };
 
       sopsConfig = {
         sops.age.keyFile = "/etc/sops/my_key";
         environment.etc."sops/my_key".source = ./keys.txt;
       };
     in
-      {
-        nixosConfigurations = {
-          basic = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              basic
-              sopsConfig
-            ];
-          };
-
-          ldap = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              basic
-              ldap
-              sopsConfig
-            ];
-          };
+    {
+      nixosConfigurations = {
+        basic = nixpkgs'.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            basic
+            sopsConfig
+          ];
         };
 
-        colmena = {
-          meta = {
-            nixpkgs = import nixpkgs' {
-              system = "x86_64-linux";
-            };
-            specialArgs = inputs;
-          };
-
-          basic = { config, ... }: {
-            imports = [
-              basic
-            ];
-
-            # Used by colmena to know which target host to deploy to.
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
-
-          ldap = { config, ... }: {
-            imports = [
-              basic
-              ldap
-            ];
-
-            # Used by colmena to know which target host to deploy to.
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
+        ldap = nixpkgs'.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            basic
+            ldap
+            sopsConfig
+          ];
         };
       };
+
+      colmena = {
+        meta = {
+          nixpkgs = import nixpkgs' {
+            system = "x86_64-linux";
+          };
+          specialArgs = inputs;
+        };
+
+        basic =
+          { config, ... }:
+          {
+            imports = [
+              basic
+            ];
+
+            # Used by colmena to know which target host to deploy to.
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+
+        ldap =
+          { config, ... }:
+          {
+            imports = [
+              basic
+              ldap
+            ];
+
+            # Used by colmena to know which target host to deploy to.
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+      };
+    };
 }

demo/homeassistant/hardware-configuration.nix

@@ -3,52 +3,65 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
 
 {
-  imports =
-    [ (modulesPath + "/profiles/qemu-guest.nix")
-    ];
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "floppy" "sr_mod" "virtio_blk" ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "floppy"
+    "sr_mod"
+    "virtio_blk"
+  ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
 
-  fileSystems."/" =
-    { device = "/dev/vda";
-      fsType = "ext4";
-    };
+  fileSystems."/" = {
+    device = "/dev/vda";
+    fsType = "ext4";
+  };
 
-  fileSystems."/nix/.ro-store" =
-    { device = "nix-store";
-      fsType = "9p";
-    };
+  fileSystems."/nix/.ro-store" = {
+    device = "nix-store";
+    fsType = "9p";
+  };
 
-  fileSystems."/nix/.rw-store" =
-    { device = "tmpfs";
-      fsType = "tmpfs";
-    };
+  fileSystems."/nix/.rw-store" = {
+    device = "tmpfs";
+    fsType = "tmpfs";
+  };
 
-  fileSystems."/tmp/shared" =
-    { device = "shared";
-      fsType = "9p";
-    };
+  fileSystems."/tmp/shared" = {
+    device = "shared";
+    fsType = "9p";
+  };
 
-  fileSystems."/tmp/xchg" =
-    { device = "xchg";
-      fsType = "9p";
-    };
+  fileSystems."/tmp/xchg" = {
+    device = "xchg";
+    fsType = "9p";
+  };
 
-  fileSystems."/nix/store" =
-    { device = "overlay";
-      fsType = "overlay";
-    };
+  fileSystems."/nix/store" = {
+    device = "overlay";
+    fsType = "overlay";
+  };
 
-  fileSystems."/boot" =
-    { device = "/dev/vdb2";
-      fsType = "vfat";
-    };
+  fileSystems."/boot" = {
+    device = "/dev/vdb2";
+    fsType = "vfat";
+  };
 
   swapDevices = [ ];
 

demo/minimal/flake.nix

@@ -0,0 +1,223 @@
+{
+  description = "Minimal example to setup SelfHostBlocks";
+
+  inputs = {
+    selfhostblocks.url = "github:ibizaman/selfhostblocks";
+
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
+    };
+  };
+
+  outputs =
+    {
+      self,
+      selfhostblocks,
+      sops-nix,
+    }:
+    {
+      nixosConfigurations =
+        let
+          system = "x86_64-linux";
+          nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
+
+          # This module makes the assertions happy and the build succeed.
+          # This is of course wrong and will not work on any real system.
+          filesystemModule = {
+            fileSystems."/" = {
+              device = "/dev/null";
+              fsType = "none";
+            };
+            boot.loader.grub.devices = [ "/dev/null" ];
+          };
+        in
+        {
+          # Test with:
+          #   nix build .#nixosConfigurations.minimal.config.system.build.toplevel
+          minimal = nixpkgs'.nixosSystem {
+            inherit system;
+            modules = [
+              selfhostblocks.nixosModules.default
+              filesystemModule
+              # This modules showcases the use of SHB's lib.
+              (
+                {
+                  config,
+                  lib,
+                  shb,
+                  ...
+                }:
+                {
+                  options.myOption = lib.mkOption {
+                    # Using provided nixosSystem directly.
+                    # SHB's lib is available under `shb` thanks to the overlay.
+                    type = shb.secretFileType;
+                  };
+                  config = {
+                    myOption.source = "/a/path";
+                    # Use the option.
+                    environment.etc.myOption.text = config.myOption.source;
+                  };
+                }
+              )
+            ];
+          };
+
+          # Test with:
+          #   nix build .#nixosConfigurations.sops.config.system.build.toplevel
+          #   nix eval .#nixosConfigurations.sops.config.myOption
+          sops = nixpkgs'.nixosSystem {
+            inherit system;
+            modules = [
+              selfhostblocks.nixosModules.default
+              selfhostblocks.nixosModules.sops
+              sops-nix.nixosModules.default
+              filesystemModule
+              # This modules showcases the use of SHB's lib.
+              (
+                {
+                  config,
+                  lib,
+                  shb,
+                  ...
+                }:
+                {
+                  options.myOption = lib.mkOption {
+                    # Using provided nixosSystem directly.
+                    # SHB's lib is available under `shb` thanks to the overlay.
+                    type = shb.secretFileType;
+                  };
+                  config = {
+                    myOption.source = "/a/path";
+                    # Use the option.
+                    environment.etc.myOption.text = config.myOption.source;
+                  };
+                }
+              )
+            ];
+          };
+
+          # This example shows how to import the nixosSystem patches to nixpkgs manually.
+          #
+          # Test with:
+          #   nix build .#nixosConfigurations.lowlevel.config.system.build.toplevel
+          #   nix eval .#nixosConfigurations.lowlevel.config.myOption
+          lowlevel =
+            let
+              # We must import nixosSystem directly from the patched nixpkgs
+              # otherwise we do not get the patches.
+              nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
+            in
+            nixosSystem' {
+              inherit system;
+              modules = [
+                selfhostblocks.nixosModules.default
+                filesystemModule
+                # This modules showcases the use of SHB's lib.
+                (
+                  {
+                    config,
+                    lib,
+                    shb,
+                    ...
+                  }:
+                  {
+                    options.myOption = lib.mkOption {
+                      # Using provided nixosSystem directly.
+                      # SHB's lib is available under `shb` thanks to the overlay.
+                      type = shb.secretFileType;
+                    };
+                    config = {
+                      myOption.source = "/a/path";
+                      # Use the option.
+                      environment.etc.myOption.text = config.myOption.source;
+                    };
+                  }
+                )
+              ];
+            };
+
+          # This example shows how to apply patches to nixpkgs manually.
+          #
+          # Test with:
+          #   nix build .#nixosConfigurations.manual.config.system.build.toplevel
+          #   nix eval .#nixosConfigurations.manual.config.myOption
+          manual =
+            let
+              pkgs = import selfhostblocks.inputs.nixpkgs {
+                inherit system;
+              };
+              nixpkgs' = pkgs.applyPatches {
+                name = "nixpkgs-patched";
+                src = selfhostblocks.inputs.nixpkgs;
+                patches = selfhostblocks.lib.${system}.patches;
+              };
+              # We must import nixosSystem directly from the patched nixpkgs
+              # otherwise we do not get the patches.
+              nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
+            in
+            nixosSystem' {
+              inherit system;
+              modules = [
+                selfhostblocks.nixosModules.default
+                filesystemModule
+                # This modules showcases the use of SHB's lib.
+                (
+                  {
+                    config,
+                    lib,
+                    shb,
+                    ...
+                  }:
+                  {
+                    options.myOption = lib.mkOption {
+                      # Using provided nixosSystem directly.
+                      # SHB's lib is available under `shb` thanks to the overlay.
+                      type = shb.secretFileType;
+                    };
+                    config = {
+                      myOption.source = "/a/path";
+                      # Use the option.
+                      environment.etc.myOption.text = config.myOption.source;
+                    };
+                  }
+                )
+              ];
+            };
+
+          # Test with:
+          #   nix build .#nixosConfigurations.contractsDirect.config.system.build.toplevel
+          contractsDirect =
+            let
+              nixosSystem' = import "${selfhostblocks.inputs.nixpkgs}/nixos/lib/eval-config.nix";
+            in
+            nixosSystem' {
+              inherit system;
+              modules = [
+                filesystemModule
+                (import "${selfhostblocks}/lib/module.nix")
+                (
+                  {
+                    config,
+                    lib,
+                    shb,
+                    ...
+                  }:
+                  {
+                    options.myOption = lib.mkOption {
+                      # Using provided nixosSystem directly.
+                      # SHB's lib is available under `shb` thanks to the overlay.
+                      type = shb.secretFileType;
+                    };
+                    config = {
+                      myOption.source = "/a/path";
+                      # Use the option.
+                      environment.etc.myOption.text = config.myOption.source;
+                    };
+                  }
+                )
+              ];
+            };
+        };
+    };
+}

demo/nextcloud/README.md

@@ -187,8 +187,9 @@ This section corresponds to the `sso` section of the [Nextcloud
 manual](services-nextcloud.html#services-nextcloudserver-usage-oidc).
 ::::
 
-At this point, it is assumed you already deployed the `sso` demo. There is no host to add to
-`/etc/hosts` here. Instead, there is a `dnsmasq` server running in the VM and you must create a
+At this point, it is assumed you already deployed the `sso` demo. This time, we cannot simply edit local
+`/etc/hosts`, because Nextcloud SSO addon must be able to connect to Authelia by domain name
+(`auth.example.com`). Instead, there is a `dnsmasq` server running in the VM and you must create a
 SOCKS proxy to connect to it like so:
 
 ```bash

demo/nextcloud/configuration.nix

@@ -5,17 +5,17 @@ let
   targetPort = 2222;
 in
 {
-  imports =
-    [ # Include the results of the hardware scan.
-      ./hardware-configuration.nix
-    ];
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+  ];
 
   boot.loader.grub.enable = true;
   boot.kernelModules = [ "kvm-intel" ];
   system.stateVersion = "22.11";
 
   # Options above are generate by running nixos-generate-config on the VM.
-  
+
   # Needed otherwise deploy will say system won't be able to boot.
   boot.loader.grub.device = "/dev/vdb";
   # Needed to avoid getting into not available disk space in /boot.
@@ -27,7 +27,10 @@ in
 
   # Options above are needed to deploy in a VM.
 
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
 
   # We need to create the user we will deploy with.
   users.users.${targetUser} = {
@@ -42,9 +45,11 @@ in
 
   # The user we're deploying with must be able to run sudo without password.
   security.sudo.extraRules = [
-    { users = [ targetUser ];
+    {
+      users = [ targetUser ];
       commands = [
-        { command = "ALL";
+        {
+          command = "ALL";
           options = [ "NOPASSWD" ];
         }
       ];

demo/nextcloud/flake.lock

@@ -1,164 +0,0 @@
-{
-  "nodes": {
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1731533236,
-        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "nix-flake-tests": {
-      "locked": {
-        "lastModified": 1677844186,
-        "narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
-        "owner": "antifuchs",
-        "repo": "nix-flake-tests",
-        "rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
-        "type": "github"
-      },
-      "original": {
-        "owner": "antifuchs",
-        "repo": "nix-flake-tests",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1736883708,
-        "narHash": "sha256-uQ+NQ0/xYU0N1CnXsa2zghgNaOPxWpMJXSUJJ9W7140=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "eb62e6aa39ea67e0b8018ba8ea077efe65807dc8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1721524707,
-        "narHash": "sha256-5NctRsoE54N86nWd0psae70YSLfrOek3Kv1e8KoXe/0=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "556533a23879fc7e5f98dd2e0b31a6911a213171",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1725194671,
-        "narHash": "sha256-tLGCFEFTB5TaOKkpfw3iYT9dnk4awTP/q4w+ROpMfuw=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "b833ff01a0d694b910daca6e2ff4a3f26dee478c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nmdsrc": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1705050560,
-        "narHash": "sha256-x3zzcdvhJpodsmdjqB4t5mkVW22V3wqHLOun0KRBzUI=",
-        "ref": "refs/heads/master",
-        "rev": "66d9334933119c36f91a78d565c152a4fdc8d3d3",
-        "revCount": 66,
-        "type": "git",
-        "url": "https://git.sr.ht/~rycee/nmd"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.sr.ht/~rycee/nmd"
-      }
-    },
-    "root": {
-      "inputs": {
-        "selfhostblocks": "selfhostblocks",
-        "sops-nix": "sops-nix"
-      }
-    },
-    "selfhostblocks": {
-      "inputs": {
-        "flake-utils": "flake-utils",
-        "nix-flake-tests": "nix-flake-tests",
-        "nixpkgs": "nixpkgs",
-        "nmdsrc": "nmdsrc"
-      },
-      "locked": {
-        "lastModified": 1742039945,
-        "narHash": "sha256-DMKGfQce0Jzv3ylln1LBBA6b9h2O4W2YCvyr+ZKIWNk=",
-        "owner": "ibizaman",
-        "repo": "selfhostblocks",
-        "rev": "9ac30326cd46e2760d5cd31a443fd2e61f1b88c0",
-        "type": "github"
-      },
-      "original": {
-        "owner": "ibizaman",
-        "repo": "selfhostblocks",
-        "type": "github"
-      }
-    },
-    "sops-nix": {
-      "inputs": {
-        "nixpkgs": "nixpkgs_2",
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
-      "locked": {
-        "lastModified": 1725540166,
-        "narHash": "sha256-htc9rsTMSAY5ek+DB3tpntdD/es0eam2hJgO92bWSys=",
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "rev": "d9d781523a1463965cd1e1333a306e70d9feff07",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "type": "github"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

demo/nextcloud/flake.nix

@@ -1,243 +1,260 @@
 {
-  description = "Home Assistant example for Self Host Blocks";
+  description = "Nextcloud example for Self Host Blocks";
 
   inputs = {
     selfhostblocks.url = "github:ibizaman/selfhostblocks";
     sops-nix.url = "github:Mic92/sops-nix";
   };
 
-  outputs = inputs@{ self, selfhostblocks, sops-nix }:
+  outputs =
+    inputs@{
+      self,
+      selfhostblocks,
+      sops-nix,
+    }:
     let
       system = "x86_64-linux";
-      originPkgs = selfhostblocks.inputs.nixpkgs;
+      nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
 
-      nixpkgs' = originPkgs.legacyPackages.${system}.applyPatches {
-        name = "nixpkgs-patched";
-        src = originPkgs;
-        patches = selfhostblocks.patches.${system};
-      };
-      nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
+      basic =
+        { config, ... }:
+        {
+          imports = [
+            ./configuration.nix
+            selfhostblocks.nixosModules.authelia
+            selfhostblocks.nixosModules.nextcloud-server
+            selfhostblocks.nixosModules.nginx
+            selfhostblocks.nixosModules.sops
+            selfhostblocks.nixosModules.ssl
+            sops-nix.nixosModules.default
+          ];
 
-      basic = { config, ... }: {
-        imports = [
-          ./configuration.nix
-          selfhostblocks.nixosModules.x86_64-linux.default
-          sops-nix.nixosModules.default
-        ];
+          sops.defaultSopsFile = ./secrets.yaml;
 
-        sops.defaultSopsFile = ./secrets.yaml;
+          shb.nextcloud = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "n";
+            dataDir = "/var/lib/nextcloud";
+            tracing = null;
+            defaultPhoneRegion = "US";
 
-        shb.nextcloud = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "n";
-          dataDir = "/var/lib/nextcloud";
-          tracing = null;
-          defaultPhoneRegion = "US";
+            # This option is only needed because we do not access Nextcloud at the default port in the VM.
+            port = 8080;
 
-          # This option is only needed because we do not access Nextcloud at the default port in the VM.
-          port = 8080;
+            adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result;
 
-          adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result;
-
-          apps = {
-            previewgenerator.enable = true;
-          };
-        };
-        shb.sops.secret."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
-
-        # Set to true for more debug info with `journalctl -f -u nginx`.
-        shb.nginx.accessLog = true;
-        shb.nginx.debugLog = false;
-      };
-
-      ldap = { config, ... }: {
-        shb.ldap = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "ldap";
-          ldapPort = 3890;
-          webUIListenPort = 17170;
-          dcdomain = "dc=example,dc=com";
-          ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
-          jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
-        };
-        shb.sops.secret."lldap/user_password".request = config.shb.ldap.ldapUserPassword.request;
-        shb.sops.secret."lldap/jwt_secret".request = config.shb.ldap.jwtSecret.request;
-
-        shb.nextcloud.apps.ldap = {
-          enable = true;
-          host = "127.0.0.1";
-          port = config.shb.ldap.ldapPort;
-          dcdomain = config.shb.ldap.dcdomain;
-          adminName = "admin";
-          adminPassword.result = config.shb.sops.secret."nextcloud/ldap_admin_password".result;
-          userGroup = "nextcloud_user";
-        };
-        shb.sops.secret."nextcloud/ldap_admin_password" = {
-          request = config.shb.nextcloud.apps.ldap.adminPassword.request;
-          settings.key = "lldap/user_password";
-        };
-      };
-
-      sso = { config, lib, ... }: {
-        shb.certs = {
-          cas.selfsigned.myca = {
-            name = "My CA";
-          };
-          certs.selfsigned = {
-            n = {
-              ca = config.shb.certs.cas.selfsigned.myca;
-              domain = "*.example.com";
-              group = "nginx";
+            apps = {
+              previewgenerator.enable = true;
             };
           };
-        };
-        shb.nextcloud = {
-          port = lib.mkForce null;
-          ssl = config.shb.certs.certs.selfsigned.n;
-        };
-        shb.ldap.ssl = config.shb.certs.certs.selfsigned.n;
+          shb.sops.secret."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
 
-        services.dnsmasq = {
-          enable = true;
-          settings = {
-            domain-needed = true;
-            # no-resolv = true;
-            bogus-priv = true;
-            address =
-              map (hostname: "/${hostname}/127.0.0.1") [
+          # Set to true for more debug info with `journalctl -f -u nginx`.
+          shb.nginx.accessLog = true;
+          shb.nginx.debugLog = false;
+        };
+
+      ldap =
+        { config, ... }:
+        {
+          shb.lldap = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "ldap";
+            ldapPort = 3890;
+            webUIListenPort = 17170;
+            dcdomain = "dc=example,dc=com";
+            ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
+            jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
+          };
+          shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
+          shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
+
+          shb.nextcloud.apps.ldap = {
+            enable = true;
+            host = "127.0.0.1";
+            port = config.shb.lldap.ldapPort;
+            dcdomain = config.shb.lldap.dcdomain;
+            adminName = "admin";
+            adminPassword.result = config.shb.sops.secret."nextcloud/ldap_admin_password".result;
+            userGroup = "nextcloud_user";
+          };
+          shb.sops.secret."nextcloud/ldap_admin_password" = {
+            request = config.shb.nextcloud.apps.ldap.adminPassword.request;
+            settings.key = "lldap/user_password";
+          };
+        };
+
+      sso =
+        { config, lib, ... }:
+        {
+          shb.certs = {
+            cas.selfsigned.myca = {
+              name = "My CA";
+            };
+            certs.selfsigned = {
+              n = {
+                ca = config.shb.certs.cas.selfsigned.myca;
+                domain = "*.example.com";
+                group = "nginx";
+              };
+            };
+          };
+          shb.nextcloud = {
+            port = lib.mkForce null;
+            ssl = config.shb.certs.certs.selfsigned.n;
+          };
+          shb.lldap.ssl = config.shb.certs.certs.selfsigned.n;
+
+          services.dnsmasq = {
+            enable = true;
+            settings = {
+              domain-needed = true;
+              # no-resolv = true;
+              bogus-priv = true;
+              address = map (hostname: "/${hostname}/127.0.0.1") [
                 "example.com"
                 "n.example.com"
                 "ldap.example.com"
                 "auth.example.com"
               ];
+            };
+          };
+
+          shb.authelia = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "auth";
+            ssl = config.shb.certs.certs.selfsigned.n;
+            ldapPort = config.shb.lldap.ldapPort;
+            ldapHostname = "127.0.0.1";
+            dcdomain = config.shb.lldap.dcdomain;
+
+            secrets = {
+              jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
+              ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
+              sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
+              storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
+              identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
+              identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
+            };
+          };
+          shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
+          shb.sops.secret."authelia/ldap_admin_password" = {
+            request = config.shb.authelia.secrets.ldapAdminPassword.request;
+            settings.key = "lldap/user_password";
+          };
+          shb.sops.secret."authelia/session_secret".request =
+            config.shb.authelia.secrets.sessionSecret.request;
+          shb.sops.secret."authelia/storage_encryption_key".request =
+            config.shb.authelia.secrets.storageEncryptionKey.request;
+          shb.sops.secret."authelia/hmac_secret".request =
+            config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
+          shb.sops.secret."authelia/private_key".request =
+            config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
+
+          shb.nextcloud.apps.sso = {
+            enable = true;
+            endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+            clientID = "nextcloud";
+            fallbackDefaultAuth = true;
+
+            secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
+            secretForAuthelia.result = config.shb.sops.secret."authelia/nextcloud_sso_secret".result;
+          };
+          shb.sops.secret."nextcloud/sso/secret".request = config.shb.nextcloud.apps.sso.secret.request;
+          shb.sops.secret."authelia/nextcloud_sso_secret" = {
+            request = config.shb.nextcloud.apps.sso.secretForAuthelia.request;
+            settings.key = "nextcloud/sso/secret";
           };
         };
 
-        shb.authelia = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "auth";
-          ssl = config.shb.certs.certs.selfsigned.n;
-          ldapPort = config.shb.ldap.ldapPort;
-          ldapHostname = "127.0.0.1";
-          dcdomain = config.shb.ldap.dcdomain;
-
-          secrets = {
-            jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
-            ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
-            sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
-            storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
-            identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
-            identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
-          };
-        };
-        shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
-        shb.sops.secret."authelia/ldap_admin_password" = {
-          request = config.shb.authelia.secrets.ldapAdminPassword.request;
-          settings.key = "lldap/user_password";
-        };
-        shb.sops.secret."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
-        shb.sops.secret."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
-        shb.sops.secret."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
-        shb.sops.secret."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
-
-        shb.nextcloud.apps.sso = {
-          enable = true;
-          endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
-          clientID = "nextcloud";
-          fallbackDefaultAuth = true;
-
-          secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
-          secretForAuthelia.result = config.shb.sops.secret."authelia/nextcloud_sso_secret".result;
-        };
-        shb.sops.secret."nextcloud/sso/secret".request = config.shb.nextcloud.apps.sso.secret.request;
-        shb.sops.secret."authelia/nextcloud_sso_secret" = {
-          request = config.shb.nextcloud.apps.sso.secretForAuthelia.request;
-          settings.key = "nextcloud/sso/secret";
-        };
-      };
-
       sopsConfig = {
         sops.age.keyFile = "/etc/sops/my_key";
         environment.etc."sops/my_key".source = ./keys.txt;
       };
     in
-      {
-        nixosConfigurations = {
-          basic = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              sopsConfig
-              basic
-            ];
-          };
-          ldap = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              sopsConfig
-              basic
-              ldap
-            ];
-          };
-          sso = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              sopsConfig
-              basic
-              ldap
-              sso
-            ];
-          };
+    {
+      nixosConfigurations = {
+        basic = nixpkgs'.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            sopsConfig
+            basic
+          ];
         };
-
-        colmena = {
-          meta = {
-            nixpkgs = import nixpkgs' {
-              system = "x86_64-linux";
-            };
-            specialArgs = inputs;
-          };
-
-          basic = { config, ... }: {
-            imports = [
-              basic
-            ];
-
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
-
-          ldap = { config, ... }: {
-            imports = [
-              basic
-              ldap
-            ];
-
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
-
-          sso = { config, ... }: {
-            imports = [
-              basic
-              ldap
-              sso
-            ];
-
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
+        ldap = nixpkgs'.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            sopsConfig
+            basic
+            ldap
+          ];
+        };
+        sso = nixpkgs'.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            sopsConfig
+            basic
+            ldap
+            sso
+          ];
         };
       };
+
+      colmena = {
+        meta = {
+          nixpkgs = import nixpkgs' {
+            system = "x86_64-linux";
+          };
+          specialArgs = inputs;
+        };
+
+        basic =
+          { config, ... }:
+          {
+            imports = [
+              basic
+            ];
+
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+
+        ldap =
+          { config, ... }:
+          {
+            imports = [
+              basic
+              ldap
+            ];
+
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+
+        sso =
+          { config, ... }:
+          {
+            imports = [
+              basic
+              ldap
+              sso
+            ];
+
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+      };
+    };
 }

demo/nextcloud/hardware-configuration.nix

@@ -3,52 +3,65 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
 
 {
-  imports =
-    [ (modulesPath + "/profiles/qemu-guest.nix")
-    ];
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "floppy" "sr_mod" "virtio_blk" ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "floppy"
+    "sr_mod"
+    "virtio_blk"
+  ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
 
-  fileSystems."/" =
-    { device = "/dev/vda";
-      fsType = "ext4";
-    };
+  fileSystems."/" = {
+    device = "/dev/vda";
+    fsType = "ext4";
+  };
 
-  fileSystems."/nix/.ro-store" =
-    { device = "nix-store";
-      fsType = "9p";
-    };
+  fileSystems."/nix/.ro-store" = {
+    device = "nix-store";
+    fsType = "9p";
+  };
 
-  fileSystems."/nix/.rw-store" =
-    { device = "tmpfs";
-      fsType = "tmpfs";
-    };
+  fileSystems."/nix/.rw-store" = {
+    device = "tmpfs";
+    fsType = "tmpfs";
+  };
 
-  fileSystems."/tmp/shared" =
-    { device = "shared";
-      fsType = "9p";
-    };
+  fileSystems."/tmp/shared" = {
+    device = "shared";
+    fsType = "9p";
+  };
 
-  fileSystems."/tmp/xchg" =
-    { device = "xchg";
-      fsType = "9p";
-    };
+  fileSystems."/tmp/xchg" = {
+    device = "xchg";
+    fsType = "9p";
+  };
 
-  fileSystems."/nix/store" =
-    { device = "overlay";
-      fsType = "overlay";
-    };
+  fileSystems."/nix/store" = {
+    device = "overlay";
+    fsType = "overlay";
+  };
 
-  fileSystems."/boot" =
-    { device = "/dev/vdb2";
-      fsType = "vfat";
-    };
+  fileSystems."/boot" = {
+    device = "/dev/vdb2";
+    fsType = "vfat";
+  };
 
   swapDevices = [ ];
 

docs/blocks.md

@@ -5,48 +5,70 @@ Blocks help you self-host apps or services. They implement a specific function l
 access through a subdomain. Each block is designed to be usable on its own and to fit nicely with
 others.
 
-In practice, a block implements a [contract](contracts.html) that must be followed to implement a
-specific self-hosting function. It also comes with a unit test and NixOS VM test suite to ensure any
-implementation follows the contract.
+All blocks are implemented under the blocks folder [in the repository](@REPO@/modules/blocks).
 
-As an example, let's take the HTTPS access block which allows for a service to be accessible through
-a specific subdomain. In Nix terms, this block defines at minimum the inputs:
+All services in SHB document how to setup the various blocks provided here.
+For custom services or those not provided by SHB,
+the [Expose a service Recipe](recipes-exposeService.html) explains how to use the blocks here.
 
-- subdomain,
-- domain,
-- and upstream address of the service.
+## Authentication {#blocks-category-authentication}
 
-It defines no outputs but has one major side effect:
-
-- the service should be accessible through HTTPS at `https://subdomain.domain`.
-
-Anything that provides the inputs and expected outputs and side effects defined by the block can be
-used to fulfill its contract. In this example, we could use any of Nginx, Caddy, Haproxy or others.
-
-Self Host Blocks provides at least one implementation for each block and allows you to use your own
-implementation if you want to, as long as it passes the tests. You can then use blocks to improve
-services you already have deployed.
-
-::: {.note}
-Not all blocks are yet documented. You can find all available blocks [in the repository](@REPO@/modules/blocks).
-:::
-
-```{=include=} chapters html:into-file=//blocks-sops.html
-modules/blocks/sops/docs/default.md
+```{=include=} chapters html:into-file=//blocks-authelia.html
+modules/blocks/authelia/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//blocks-ssl.html
-modules/blocks/ssl/docs/default.md
+```{=include=} chapters html:into-file=//blocks-lldap.html
+modules/blocks/lldap/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//blocks-postgresql.html
-modules/blocks/postgresql/docs/default.md
+## Backup {#blocks-category-backup}
+
+```{=include=} chapters html:into-file=//blocks-borgbackup.html
+modules/blocks/borgbackup/docs/default.md
 ```
 
 ```{=include=} chapters html:into-file=//blocks-restic.html
 modules/blocks/restic/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//blocks-sanoid.html
+modules/blocks/sanoid/docs/default.md
+```
+
+## Database {#blocks-category-database}
+
+```{=include=} chapters html:into-file=//blocks-postgresql.html
+modules/blocks/postgresql/docs/default.md
+```
+
+## Secrets {#blocks-category-secrets}
+
+```{=include=} chapters html:into-file=//blocks-sops.html
+modules/blocks/sops/docs/default.md
+```
+
+## Filesystem {#blocks-category-filesystem}
+
+```{=include=} chapters html:into-file=//blocks-zfs.html
+modules/blocks/zfs/docs/default.md
+```
+
+## Network {#blocks-category-network}
+
+```{=include=} chapters html:into-file=//blocks-ssl.html
+modules/blocks/ssl/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//blocks-nginx.html
+modules/blocks/nginx/docs/default.md
+```
+
+## Introspection {#blocks-category-introspection}
+
 ```{=include=} chapters html:into-file=//blocks-monitoring.html
 modules/blocks/monitoring/docs/default.md
 ```
+
+```{=include=} chapters html:into-file=//blocks-mitmdump.html
+modules/blocks/mitmdump/docs/default.md
+```

docs/contracts.md

@@ -1,6 +1,13 @@
 <!-- Read these docs at https://shb.skarabox.com -->
 # Contracts {#contracts}
 
+::: {.note}
+An [RFC][] has been created which is the most up-to-date version of contracts.
+The text here is still relevant although the implementation itself has changed a little bit.
+
+[RFC]: https://github.com/NixOS/rfcs/pull/189
+:::
+
 A contract decouples modules that use a functionality from modules that provide it. A first
 intuition for contracts is they are generally related to accessing a shared resource.
 
@@ -30,14 +37,21 @@ Provided contracts are:
 - [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
   Two providers are implemented: self-signed and Let's Encrypt.
 - [Backup contract][] to backup directories.
-  One provider is implemented: [Restic][].
+  Two providers are implemented: [BorgBackup][] and [Restic][].
 - [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
-  One provider is implemented: [Restic][].
-- [Secret contract](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
+  Two providers are implemented: [BorgBackup][] and [Restic][].
+- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets.
+  One provider is implemented: [Sanoid][]
+- [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
   One provider is implemented: [SOPS][].
+- [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard.
+  One provider is implemented: [Homepage][].
 
 [backup contract]: contracts-backup.html
+[borgbackup]: blocks-borgbackup.html
+[homepage]: services-homepage.html
 [restic]: blocks-restic.html
+[sanoid]: blocks-sanoid.html
 [sops]: blocks-sops.html
 
 ```{=include=} chapters html:into-file=//contracts-ssl.html
@@ -52,10 +66,18 @@ modules/contracts/backup/docs/default.md
 modules/contracts/databasebackup/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//contracts-datasetbackup.html
+modules/contracts/datasetbackup/docs/default.md
+```
+
 ```{=include=} chapters html:into-file=//contracts-secret.html
 modules/contracts/secret/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//contracts-dashboard.html
+modules/contracts/dashboard/docs/default.md
+```
+
 ## Problem Statement {#contracts-why}
 
 Currently in nixpkgs, every module accessing a shared resource
@@ -438,7 +460,7 @@ A simplified test for a secret contract would look like the following.
 First, there is the generic test:
 
 ```nix
-{ pkgs, lib, ... }:
+{ pkgs, lib, shb, ... }:
 let
   inherit (lib) getAttrFromPath setAttrByPath;
 in
@@ -448,7 +470,7 @@ in
     modules ? [],
     owner ? "root",
     content ? "secretPasswordA",
-  }: pkgs.testers.runNixOSTest {
+  }: shb.test.runNixOSTest {
     inherit name;
     
     nodes.machine = { config, ... }: {

docs/contributing.md

@@ -53,15 +53,7 @@ $ nix build .#checks.${system}.modules
 $ nix build .#checks.${system}.vm_postgresql_peerAuth
 ```
 
-Run one VM test interactively:
-
-```bash
-$ nix run .#checks.${system}.vm_postgresql_peerAuth.driverInteractive
-```
-
-When you get to the shell, run either `start_all()` or `test_script()`. The former just starts all
-the VMs and service, then you can introspect. The latter also starts the VMs if they are not yet and
-then will run the test script.
+### Playwright Tests {#contributing-playwright-tests}
 
 If the test includes playwright tests, you can see the playwright trace with:
 
@@ -69,6 +61,60 @@ If the test includes playwright tests, you can see the playwright trace with:
 $ nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
 ```
 
+If the test fails, you won't have the output directory available.
+Instead, run the test with `--keep-failed` and at the end you will see a line saying
+
+```
+Keeping build directory '/nix/var/nix/builds/nix-31776-4270051001/build'
+```
+
+Copy the path and run:
+
+```bash
+sudo cp -r /nix/var/nix/builds/nix-31776-4270051001/build/shared-xchg/trace trace && sudo chown -R $USER: trace
+```
+
+Now, open that file with playwright:
+
+```bash
+nix run .#playwright -- show-trace trace/0.zip
+```
+
+### Debug Tests {#contributing-debug-tests}
+
+Run the test in driver interactive mode:
+
+```bash
+$ nix run .#checks.${system}.vm_postgresql_peerAuth.driverInteractive
+```
+
+When you get to the shell, start the server and/or client with one of the following commands:
+
+```bash
+server.start()
+client.start()
+start_all()
+```
+
+To run the test from the shell, use `test_script()`.
+Note that if the test script ends in error,
+the shell will exit and you will need to restart the VMs.
+
+After the shell started, you will see lines like so:
+
+```
+SSH backdoor enabled, the machines can be accessed like this:
+Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer).
+    client:  ssh -o User=root vsock/3
+    server:  ssh -o User=root vsock/4
+```
+
+With the following command, you can directly access the server's nginx instance with your browser at `http://localhost:8000`:
+
+```bash
+ssh-keygen -R vsock/4; ssh -o User=root -L 8000:localhost:80 vsock/4
+```
+
 ## Upload test results to CI {#contributing-upload}
 
 Github actions do now have hardware acceleration, so running them there is not slow anymore. If
@@ -80,6 +126,18 @@ After running the `nix-fast-build` command from the previous section, run:
 $ find . -type l -name "result-vm_*" | xargs readlink | nix run nixpkgs#cachix -- push selfhostblocks
 ```
 
+## Upload package to CI {#contributing-upload-package}
+
+In the rare case where a package must be built but cannot in CI,
+for example because of not enough memory,
+you can push the package directly to the cache with:
+
+```bash
+nix build .#checks.x86_64-linux.vm_karakeep_backup.nodes.server.services.karakeep.package
+readlink result | nix run nixpkgs#cachix -- push selfhostblocks
+
+```
+
 ## Deploy using colmena {#contributing-deploy-colmena}
 
 ```bash
@@ -164,6 +222,12 @@ $ nix run nixpkgs#nvd -- diff \
 $ nix run nixpkgs#openssl -- rand -hex 64
 ```
 
+## Write code {#contributing-code}
+
+```{=include=} chapters html:into-file=//service-implementation-guide.html
+service-implementation-guide.md
+```
+
 ## Links that helped {#contributing-links}
 
 While creating NixOS tests:

docs/default.nix

@@ -1,35 +1,49 @@
 # Taken nearly verbatim from https://github.com/nix-community/home-manager/pull/4673
 # Read these docs online at https://shb.skarabox.com.
-{ pkgs
-, buildPackages
-, lib
-, nmdsrc
-, stdenv
-, documentation-highlighter
-, nixos-render-docs
+{
+  pkgs,
+  buildPackages,
+  lib,
+  nmdsrc,
+  stdenv,
+  documentation-highlighter,
+  nixos-render-docs,
 
-, release
-, allModules
+  release,
+  allModules,
+
+  version ? builtins.readFile ../VERSION,
+  substituteVersionIn,
+
+  modules,
 }:
 
 let
   shbPath = toString ./..;
 
-  gitHubDeclaration = user: repo: subpath:
-    let urlRef = "main";
-        end = if subpath == "" then "" else "/" + subpath;
-    in {
+  gitHubDeclaration =
+    user: repo: subpath:
+    let
+      urlRef = "main";
+      end = if subpath == "" then "" else "/" + subpath;
+    in
+    {
       url = "https://github.com/${user}/${repo}/blob/${urlRef}${end}";
       name = "<${repo}${end}>";
     };
 
   ghRoot = (gitHubDeclaration "ibizaman" "selfhostblocks" "").url;
 
-  buildOptionsDocs = args@{ modules, ... }:
+  buildOptionsDocs =
+    {
+      modules,
+      filterOptionPath ? null,
+    }:
+    args:
     let
       config = {
         _module.check = false;
-        _module.args = {};
+        _module.args = { };
         system.stateVersion = "22.11";
       };
 
@@ -46,34 +60,57 @@ let
         };
       };
 
-      options = lib.filterAttrs (name: v: name == "shb") eval.options;
-    in buildPackages.nixosOptionsDoc ({
-      inherit options;
+      options = lib.setAttrByPath filterOptionPath (lib.getAttrFromPath filterOptionPath eval.options);
+    in
+    buildPackages.nixosOptionsDoc (
+      {
+        inherit options;
 
-      transformOptions = opt:
-        opt // {
-          # Clean up declaration sites to not refer to the Home Manager
-          # source tree.
-          declarations = map (decl:
-            gitHubDeclaration "ibizaman" "selfhostblocks"
-              (lib.removePrefix "/" (lib.removePrefix shbPath (toString decl)))) opt.declarations;
-        };
-    } // builtins.removeAttrs args [ "modules" "includeModuleSystemOptions" ]);
+        transformOptions =
+          opt:
+          opt
+          // {
+            # Clean up declaration sites to not refer to the Home Manager
+            # source tree.
+            declarations = map (
+              decl:
+              gitHubDeclaration "ibizaman" "selfhostblocks" (
+                lib.removePrefix "/" (lib.removePrefix shbPath (toString decl))
+              )
+            ) opt.declarations;
+          };
+      }
+      // builtins.removeAttrs args [ "includeModuleSystemOptions" ]
+    );
 
   scrubbedModule = {
     _module.args.pkgs = lib.mkForce (nmd.scrubDerivations "pkgs" pkgs);
     _module.check = false;
   };
 
-  allOptionsDocs = paths: (buildOptionsDocs {
-    modules = paths ++ allModules ++ [ scrubbedModule ];
-    variablelistId = "selfhostblocks-options";
-  }).optionsJSON;
+  allOptionsDocs =
+    paths:
+    (buildOptionsDocs
+      {
+        modules = paths ++ allModules ++ [ scrubbedModule ];
+        filterOptionPath = [ "shb" ];
+      }
+      {
+        variablelistId = "selfhostblocks-options";
+      }
+    ).optionsJSON;
 
-  individualModuleOptionsDocs = paths: (buildOptionsDocs {
-    modules = paths ++ [ scrubbedModule ];
-    variablelistId = "selfhostblocks-options";
-  }).optionsJSON;
+  individualModuleOptionsDocs =
+    filterOptionPath: paths:
+    (buildOptionsDocs
+      {
+        modules = paths ++ [ scrubbedModule ];
+        inherit filterOptionPath;
+      }
+      {
+        variablelistId = "selfhostblocks-options";
+      }
+    ).optionsJSON;
 
   nmd = import nmdsrc {
     inherit lib;
@@ -81,15 +118,15 @@ let
     # `nmd` uses to work around the broken stylesheets in
     # `docbook-xsl-ns`, so we restore the patched version here.
     pkgs = pkgs // {
-      docbook-xsl-ns =
-        pkgs.docbook-xsl-ns.override { withManOptDedupPatch = true; };
+      docbook-xsl-ns = pkgs.docbook-xsl-ns.override { withManOptDedupPatch = true; };
     };
   };
 
   outputPath = "share/doc/selfhostblocks";
 
-  manpage-urls = pkgs.writeText "manpage-urls.json" ''{}'';
-in stdenv.mkDerivation {
+  manpage-urls = pkgs.writeText "manpage-urls.json" "{}";
+in
+stdenv.mkDerivation {
   name = "self-host-blocks-manual";
 
   nativeBuildInputs = [ nixos-render-docs ];
@@ -121,100 +158,47 @@ in stdenv.mkDerivation {
       ${nmdsrc}/static/highlightjs/highlight.min.js \
       ${nmdsrc}/static/highlightjs/highlight.load.js
 
-    substituteInPlace ./manual.md \
-      --replace \
-        '@VERSION@' \
-        ${builtins.readFile ../VERSION}
-
-    substituteInPlace ./usage.md \
-      --replace \
-        '@VERSION@' \
-        ${builtins.readFile ../VERSION}
-
+  ''
+  + lib.concatStringsSep "\n" (
+    map (m: ''
+      substituteInPlace ${m} --replace '@VERSION@' ${version}
+    '') substituteVersionIn
+  )
+  + ''
     substituteInPlace ./options.md \
       --replace \
         '@OPTIONS_JSON@' \
-        ${allOptionsDocs [
-          (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
-        ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/blocks/ssl/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-        ${individualModuleOptionsDocs [ ../modules/blocks/ssl.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/blocks/monitoring/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/blocks/monitoring.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/blocks/postgresql/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-        ${individualModuleOptionsDocs [ ../modules/blocks/postgresql.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/blocks/restic/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-        ${individualModuleOptionsDocs [ ../modules/blocks/restic.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/blocks/sops/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-        ${individualModuleOptionsDocs [ ../modules/blocks/sops.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/services/arr/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/services/arr.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/services/home-assistant/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/services/home-assistant.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/services/nextcloud-server/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/services/nextcloud-server.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/services/vaultwarden/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/services/vaultwarden.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/services/forgejo/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [
-         ../modules/services/forgejo.nix
-         (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
-       ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/contracts/backup/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/contracts/backup/dummyModule.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/contracts/databasebackup/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/contracts/databasebackup/dummyModule.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/contracts/secret/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/contracts/secret/dummyModule.nix ]}/share/doc/nixos/options.json
-
-    substituteInPlace ./modules/contracts/ssl/docs/default.md \
-      --replace \
-        '@OPTIONS_JSON@' \
-       ${individualModuleOptionsDocs [ ../modules/contracts/ssl/dummyModule.nix ]}/share/doc/nixos/options.json
-
+        ${
+          allOptionsDocs [
+            (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
+          ]
+        }/share/doc/nixos/options.json
+  ''
+  + lib.concatStringsSep "\n" (
+    lib.mapAttrsToList (
+      name: cfg':
+      let
+        cfg = if builtins.isAttrs cfg' then cfg' else { module = cfg'; };
+        module = if builtins.isList cfg.module then cfg.module else [ cfg.module ];
+        optionRoot =
+          cfg.optionRoot or [
+            "shb"
+            (lib.last (lib.splitString "/" name))
+          ];
+      in
+      ''
+        substituteInPlace ./modules/${name}/docs/default.md \
+          --replace-fail \
+            '@OPTIONS_JSON@' \
+            ${individualModuleOptionsDocs optionRoot module}/share/doc/nixos/options.json
+      ''
+    ) modules
+  )
+  + ''
     find . -name "*.md" -print0 | \
       while IFS= read -r -d ''' f; do
         substituteInPlace "''${f}" \
-          --replace \
+          --replace-quiet \
             '@REPO@' \
             "${ghRoot}" 2>/dev/null
       done

docs/manual.md

@@ -24,6 +24,14 @@ contracts.md
 blocks.md
 ```
 
+```{=include=} chapters html:into-file=//recipes.html
+recipes.md
+```
+
+```{=include=} chapters html:into-file=//misc.html
+misc.md
+```
+
 ```{=include=} chapters html:into-file=//demos.html
 demos.md
 ```
@@ -32,10 +40,6 @@ demos.md
 contributing.md
 ```
 
-```{=include=} chapters html:into-file=//service-implementation-guide.html
-service-implementation-guide.md
-```
-
 ```{=include=} appendix html:into-file=//options.html
 options.md
 ```

docs/misc.md

@@ -0,0 +1,39 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Miscellaneous {#misc}
+
+Here goes meta-discussions around the repository and other topics not related directly to the usage of SHB.
+
+## Lock file update {#misc-lock-file-update}
+
+*SHB has an unusual strategy to keep up with its flake inputs. This section explains why.*
+
+SHB only depends on `nixpkgs` as far as flake inputs goes.
+There are others but they only matter for tasks around taking care of the repository itself.
+So only the `nixpkgs` input is important for downstream consumers of SHB.
+
+To keep up to date with nixpkgs unstable,
+initially a job was updating the nixpkgs input by simply updating the lock file and creating a PR.
+Now, this job failed most of the time because the tip of unstable often breaks modules and packages, resulting in failing SHB tests.
+To fix this, we usually needed to choose a commit around 2 weeks prior to the latest unstable commit.
+This usually did not lead to successful tests but the reason was then not
+because of failing packages but because nixpkgs modules evolved and SHB needs to adapt to that.
+The net effect was SHB was lagging too far behind unstable and updating it was becoming annoying.
+
+The new strategy now is as follows. On every job run:
+
+- If there is no PR, update nixpkgs input to latest unstable commit and create a PR which auto-merges when tests are successful.
+- If there is a PR:
+  - If the checks succeeded, do nothing because the PR will auto-merge.
+  - If the checks are pending, do nothing yet.
+  - If the checks failed, bisect and update nixpkgs input to between the commit on main and the commit in the PR.
+
+The effect here is as long as the tests are failing, we'll try a commit further in the past from nixpkgs unstable
+and closer to the tip of main branch.
+
+This could be made smarter by distinguishing between types of failures.
+We will see if this is needed.
+
+One can also run the bisect manually with `nix run .#update-flake-lock-pr`.
+This will create a PR on a different branch than the one used for the automated workflow.
+It accepts also one argument `future` which instead of trying a commit in between the tip of SHB main and the PR's failing one,
+it tries a commit between the PR's failing one and the nixpkgs unstable latest commit.

docs/preface.md

@@ -10,22 +10,203 @@ Feel free to join the dedicated Matrix room
 [matrix.org#selfhostblocks](https://matrix.to/#/#selfhostblocks:matrix.org).
 :::
 
-Self Host Blocks intends to help you self host any service you would like
-with best practices out of the box.
+SelfHostBlocks is:
 
-Compared to the stock nixpkgs experience, Self Host Blocks provides
-an unified interface to setup common dependencies, called blocks
-in this project:
+- Your escape from the cloud, for privacy and data sovereignty enthusiast. [Why?](#preface-why-self-hosting)
+- A groupware to self-host [all your data](#preface-services): documents, pictures, calendars, contacts, etc.
+- An opinionated NixOS server management OS for a [safe self-hosting experience](#preface-features).
+- A NixOS distribution making sure all services build and work correctly thanks to NixOS VM tests.
+- A collection of NixOS modules standardizing options so configuring services [look the same](#preface-unified-interfaces).
+- A testing ground for [contracts](#preface-contracts) which intents to make nixpkgs modules more modular.
+- [Upstreaming][] as much as possible.
 
-- reverse proxy
-- TLS certificate management
-- serving service under subdomain
-- backup
+[upstreaming]: https://github.com/pulls?page=1&q=created%3A%3E2023-06-01+is%3Apr+author%3Aibizaman+archived%3Afalse+-repo%3Aibizaman%2Fselfhostblocks+-repo%3Aibizaman%2Fskarabox
+
+## Why Self-Hosting {#preface-why-self-hosting}
+
+It is obvious by now that
+a deep dependency on proprietary service providers - "the cloud" -
+is a significant liability.
+One aspect often talked about is privacy
+which is inherently not guaranteed when using a proprietary service
+and is a valid concern.
+A more punishing issue is having your account closed or locked
+without prior warning
+When that happens,
+you get an instantaneous sinking feeling in your stomach
+at the realization you lost access to your data,
+possibly without recourse.
+
+Hosting services yourself is the obvious alternative
+to alleviate those concerns
+but it tends to require a lot of technical skills and time.
+SelfHostBlocks (together with its sibling project [Skarabox][])
+aims to lower the bar to self-hosting,
+and provides an opinionated server management system based on NixOS modules
+embedding best practices.
+Contrary to other server management projects,
+its main focus is ease of long term maintenance
+before ease of installation.
+To achieve this, it provides building blocks to setup services.
+Some are already provided out of the box,
+and customizing or adding additional ones is done easily.
+
+The building blocks fit nicely together thanks to [contracts](#contracts)
+which SelfHostBlocks sets out to introduce into nixpkgs.
+This will increase modularity, code reuse
+and empower end users to assemble components
+that fit together to build their server.
+
+## Usage {#preface-usage}
+
+> **Caution:** You should know that although I am using everything in this repo for my personal
+> production server, this is really just a one person effort for now and there are most certainly
+> bugs that I didn't discover yet.
+
+To get started using SelfHostBlocks, the following snippet is enough:
+
+```nix
+{
+  inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks";
+
+  outputs = { selfhostblocks, ... }: let
+    system = "x86_64-linux";
+    nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
+  in
+    nixosConfigurations = {
+      myserver = nixpkgs'.nixosSystem {
+        inherit system;
+        modules = [
+          selfhostblocks.nixosModules.default
+          ./configuration.nix
+        ];
+      };
+    };
+}
+```
+
+SelfHostBlocks provides its own patched nixpkgs, so you are required to use it
+otherwise evaluation can quickly break.
+[The usage section](https://shb.skarabox.com/usage.html) of the manual has
+more details and goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
+and also how to handle secrets management with [SOPS][].
+
+[Colmena]: https://shb.skarabox.com/usage.html#usage-example-colmena
+[nixos-rebuild]: https://shb.skarabox.com/usage.html#usage-example-nixosrebuild
+[deploy-rs]: https://shb.skarabox.com/usage.html#usage-example-deployrs
+[SOPS]: https://shb.skarabox.com/usage.html#usage-secrets
+
+Then, to actually configure services, you can choose which one interests you in
+the [services section](https://shb.skarabox.com/services.html) of the manual.
+
+The [recipes section](https://shb.skarabox.com/recipes.html) of the manual shows some other common use cases.
+
+Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org)
+for any remaining question, or just to say hi :)
+
+### Installation From Scratch {#preface-usage-installation-from-scratch}
+
+I do recommend for this my sibling project [Skarabox][]
+which bootstraps a new server and sets up a few tools:
+
+- Create a bootable ISO, installable on an USB key.
+- Handles one or two (in raid 1) SSDs for root partition.
+- Handles two (in raid 1) or more hard drives for data partition.
+- [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) to install NixOS headlessly.
+- [disko](https://github.com/nix-community/disko) to format the drives using native ZFS encryption with remote unlocking through ssh.
+- [sops-nix](https://github.com/Mic92/sops-nix) to handle secrets.
+- [deploy-rs](https://github.com/serokell/deploy-rs) to deploy updates.
+
+[Skarabox]:  https://github.com/ibizaman/skarabox
+
+## Features {#preface-features}
+
+SelfHostBlocks provides building blocks that take care of common self-hosting needs:
+
+- Backup for all services.
+- Automatic creation of ZFS datasets per service.
+- LDAP and SSO integration for most services.
+- Monitoring with Grafana and Prometheus stack with provided dashboards.
+- Automatic reverse proxy and certificate management for HTTPS.
+- VPN and proxy tunneling services.
+
+Great care is taken to make the proposed stack robust.
+This translates into a test suite comprised of automated NixOS VM tests
+which includes playwright tests to verify some important workflow
+like logging in.
+
+This test suite also serves as a guaranty that all services provided by SelfHostBlocks
+all evaluate, build and work correctly together. It works similarly as a distribution but here it's all [automated](#preface-updates).
+
+Also, the stack fits together nicely thanks to [contracts](#preface-contracts).
+
+### Services {#preface-services}
+
+[Provided services](https://shb.skarabox.com/services.html) are:
+
+- Nextcloud
+- Audiobookshelf
+- Deluge + *arr stack
+- Forgejo
+- Grocy
+- Hledger
+- Home-Assistant
+- Jellyfin
+- Karakeep
+- Open WebUI
+- Pinchflat
+- Vaultwarden
+
+Like explained above, those services all benefit from
+out of the box backup,
+LDAP and SSO integration,
+monitoring with Grafana,
+reverse proxy and certificate management
+and VPN integration for the *arr suite.
+
+Some services do not have an entry yet in the manual.
+To know options for those, the only way for now
+is to go to the [All Options][] section of the manual.
+
+[All Options]: https://shb.skarabox.com/options.html
+
+### Blocks {#preface-blocks}
+
+The services above rely on the following [common blocks][]
+which altogether provides a solid foundation for self-hosting services:
+
+- Authelia
+- BorgBackup
+- Davfs
 - LDAP
-- SSO.
+- Monitoring (Grafana - Prometheus - Loki stack)
+- Nginx
+- PostgreSQL
+- Restic
+- Sops
+- SSL
+- Tinyproxy
+- VPN
+- ZFS
+
+Those blocks can be used with services
+not provided by SelfHostBlocks as shown [in the manual][common blocks].
+
+[common blocks]: https://shb.skarabox.com/blocks.html
+
+The manual also provides documentation for each individual blocks.
+
+### Unified Interfaces {#preface-unified-interfaces}
+
+Thanks to the blocks,
+SelfHostBlocks provides an unified configuration interface
+for the services it provides.
 
 Compare the configuration for Nextcloud and Forgejo.
-The following snippets focus on similitudes and assume the relevant blocks are configured off-screen.
+The following snippets focus on similitudes and assume the relevant blocks - like secrets - are configured off-screen.
+It also does not show specific options for each service.
+These are still complete snippets that configure HTTPS,
+subdomain serving the service, LDAP and SSO integration.
 
 ```nix
 shb.nextcloud = {
@@ -38,16 +219,16 @@ shb.nextcloud = {
   apps.ldap = {
     enable = true;
     host = "127.0.0.1";
-    port = config.shb.ldap.ldapPort;
-    dcdomain = config.shb.ldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    port = config.shb.lldap.ldapPort;
+    dcdomain = config.shb.lldap.dcdomain;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
   apps.sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
+    secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
+    secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
   };
 };
 ```
@@ -63,60 +244,162 @@ shb.forgejo = {
   ldap = {
     enable = true;
     host = "127.0.0.1";
-    port = config.shb.ldap.ldapPort;
-    dcdomain = config.shb.ldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    port = config.shb.lldap.ldapPort;
+    dcdomain = config.shb.lldap.dcdomain;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
 
   sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."forgejo/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."forgejo/sso/secretForAuthelia".result;
+    secret.result = config.shb.sops.secret."forgejo/sso/secret".result;
+    secretForAuthelia.result = config.shb.sops.secret."forgejo/sso/secretForAuthelia".result;
   };
 };
 ```
 
-SHB facilitates testing NixOS and slowly switching an existing installation to NixOS.
+As you can see, they are pretty similar!
+This makes setting up a new service pretty easy and intuitive.
 
-To achieve this, SHB pioneers [contracts][]
-which allows you, the final user, to be more in control of which piece go where.
+SelfHostBlocks provides an ever growing list of [services](#preface-services)
+that are configured in the same way.
+
+### Contracts {#preface-contracts}
+
+To make building blocks that fit nicely together,
+SelfHostBlocks pioneers [contracts][] which allows you, the final user,
+to be more in control of which piece goes where.
 This lets you choose, for example,
 any reverse proxy you want or any database you want,
 without requiring work from maintainers of the services you want to self host.
 
-[contracts]: contracts.html
+An [RFC][] exists to upstream this concept into `nixpkgs`.
+The [manual][contracts] also provides an explanation of the why and how of contracts.
 
-To achieve this, Self Host Blocks provides building blocks
-which each provide part of what a self hosted app should do (SSO, HTTPS, etc.).
-It also provides some services that are already integrated with all those building blocks.
+Also, two videos exist of me presenting the topic,
+the first at [NixCon North America in spring of 2024][NixConNA2024]
+and the second at [NixCon in Berlin in fall of 2024][NixConBerlin2024].
 
-- You are new to self hosting and want pre-configured services to deploy easily.
-  Look at the [services section](services.html).
-- You are a seasoned self-hoster but want to enhance some services you deploy already.
-  Go to the [blocks section](blocks.html).
-- You are a user of Self Host Blocks but would like to use your own implementation for a block.
-  Go to the [contracts section](https://shb.skarabox.com/contracts.html).
+[contracts]: https://shb.skarabox.com/contracts.html
+[RFC]: https://github.com/NixOS/rfcs/pull/189
+[NixConNA2024]: https://www.youtube.com/watch?v=lw7PgphB9qM
+[NixConBerlin2024]: https://www.youtube.com/watch?v=CP0hR6w1csc
 
-Self Host Blocks uses the full power of NixOS modules to achieve these goals.
-Blocks and service are both NixOS modules.
+### Interfacing With Other OSes {#preface-interface}
 
-## Next Steps {#next-steps}
+Thanks to [contracts](#contracts), one can interface NixOS
+with systems on other OSes.
+The [RFC][] explains how that works.
 
-To get started using SelfHostBlocks,
-follow [the usage section](https://shb.skarabox.com/usage.html) of the manual.
-It goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
-and also goes over secrets management with [SOPS][].
+### Sitting on the Shoulders of a Giant {#preface-giants}
 
-[Colmena]: https://shb.skarabox.com/usage.html#usage-example-colmena
-[nixos-rebuild]: https://shb.skarabox.com/usage.html#usage-example-nixosrebuild
-[deploy-rs]: https://shb.skarabox.com/usage.html#usage-example-deployrs
-[SOPS]: https://shb.skarabox.com/usage.html#usage-secrets
+By using SelfHostBlocks, you get all the benefits of NixOS
+which are, for self hosted applications specifically:
 
-Then, to actually configure services, you can choose which one interests you in
-[the services section](https://shb.skarabox.com/services.html) of the manual.
-Not all services have a corresponding manual page yet.
+- declarative configuration;
+- atomic configuration rollbacks;
+- real programming language to define configurations;
+- create your own higher level abstractions on top of SelfHostBlocks;
+- integration with the rest of nixpkgs;
+- much fewer "works on my machine" type of issues.
 
-Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org)
-for any remaining question, or just to say hi :)
+### Automatic Updates {#preface-updates}
+
+SelfHostBlocks follows nixpkgs unstable branch closely.
+There is a GitHub action running every couple of days that updates
+the `nixpkgs` input in the root `flakes.nix`,
+runs the tests and merges the PR automatically
+if the tests pass.
+
+A release is then made every few commits,
+whenever deemed sensible.
+On your side, to update I recommend pinning to a release
+with the following command,
+replacing the RELEASE with the one you want:
+
+```bash
+RELEASE=0.2.4
+nix flake update \
+  --override-input selfhostblocks github:ibizaman/selfhostblocks/$RELEASE \
+  selfhostblocks
+```
+
+### Demos {#preface-demos}
+
+Demos that start and deploy a service
+on a Virtual Machine on your computer are located
+under the [demo](./demo/) folder.
+
+These show the onboarding experience you would get
+if you deployed one of the services on your own server.
+
+## Roadmap {#preface-roadmap}
+
+Currently, the Nextcloud and Vaultwarden services
+and the SSL and backup blocks
+are the most advanced and most documented.
+
+Documenting all services and blocks will be done
+as I make all blocks and services use the contracts.
+
+Upstreaming changes is also on the roadmap.
+
+Check the [issues][] and the [milestones]() to see planned work.
+Feel free to add more or to contribute!
+
+[issues]: (https://github.com/ibizaman/selfhostblocks/issues)
+[milestones]: https://github.com/ibizaman/selfhostblocks/milestones
+
+All blocks and services have NixOS tests.
+Also, I am personally using all the blocks and services in this project, so they do work to some extent.
+
+## Community {#preface-community}
+
+This project has been the main focus
+of my (non work) life for the past 3 year now
+and I intend to continue working on this for a long time.
+
+All issues and PRs are welcome:
+
+- Use this project. Something does not make sense? Something's not working?
+- Documentation. Something is not clear?
+- New services. Have one of your preferred service not integrated yet?
+- Better patterns. See something weird in the code?
+
+For PRs, if they are substantial changes, please open an issue to
+discuss the details first. More details in [the contributing section](https://shb.skarabox.com/contributing.html)
+of the manual.
+
+Issues that are being worked on are labeled with the [in progress][] label.
+Before starting work on those, you might want to talk about it in the issue tracker
+or in the [matrix][] channel.
+
+The prioritized issues are those belonging to the [next milestone][milestone].
+Those issues are not set in stone and I'd be very happy to solve
+an issue an user has before scratching my own itch.
+
+[in progress]: https://github.com/ibizaman/selfhostblocks/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22in%20progress%22
+[matrix]: https://matrix.to/#/%23selfhostblocks%3Amatrix.org
+[milestone]: https://github.com/ibizaman/selfhostblocks/milestones
+
+One aspect that's close to my heart is I intent to make SelfHostBlocks the lightest layer on top of nixpkgs as
+possible. I want to upstream as much as possible. I will still take some time to experiment here but
+when I'm satisfied with how things look, I'll upstream changes.
+
+## Funding {#preface-funding}
+
+I was lucky to [obtain a grant][nlnet] from NlNet which is an European fund,
+under [NGI Zero Core][NGI0],
+to work on this project.
+This also funds the contracts RFC.
+
+Go apply for a grant too!
+
+[nlnet]: https://nlnet.nl/project/SelfHostBlocks
+[NGI0]: https://nlnet.nl/core/
+
+## License {#preface-license}
+
+I'm following the [Nextcloud](https://github.com/nextcloud/server) license which is AGPLv3.
+See [this article](https://www.fsf.org/bulletin/2021/fall/the-fundamentals-of-the-agplv3) from the FSF that explains what this license adds to the GPL one.

docs/recipes.md

@@ -0,0 +1,16 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Recipes {#recipes}
+
+This section of the manual gives you easy to follow recipes for common use cases. 
+
+```{=include=} chapters html:into-file=//recipes-dnsServer.html
+recipes/dnsServer.md
+```
+
+```{=include=} chapters html:into-file=//recipes-exposeService.html
+recipes/exposeService.md
+```
+
+```{=include=} chapters html:into-file=//recipes-serveStaticPages.html
+recipes/serveStaticPages.md
+```

docs/recipes/dnsServer.md

@@ -0,0 +1,186 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Self-Host a DNS server {#recipes-dnsServer}
+
+This recipe will show how to setup [dnsmasq][] as a local DNS server
+that forwards all queries to your own domain `example.com` to a local IP - your server running SelfHostBlocks for example.
+
+[dnsmasq]: https://dnsmasq.org/doc.html
+
+Other DNS queries will be forwarded to an external DNS server
+using [DNSSEC][] to encrypt your queries.
+
+[DNSSEC]: https://en.wikipedia.org/wiki/Domain_Name_System_Security_Extensions
+
+For this to work, you must configure the DHCP server of your network
+to set the DNS server to the IP of the host where the DNS server is running.
+Usually, your ISP's router can do this but probably easier is to disable completely that DHCP server
+and also self-host the DHCP server.
+This recipe shows how to do that too.
+
+## Why {#recipes-dnsServer-why}
+
+_You want to hide your DNS queries from your ISP or other prying eyes._
+
+Even if you use HTTPS to access an URL,
+DNS queries are by default made in plain text.
+Crazy, right?
+So, even if the actual communication is encrypted,
+everyone can see which site you're trying to access.
+Using DNSSEC means encrypting the traffic to your preferred external DNS server.
+Of course, that server will see what domain names you're trying to resolve,
+but at least intermediary hops will not be able to anymore.
+
+_You want more control on which DNS queries can be made._
+
+Self-hosting your own DNS server means you can block some domains or subdomains.
+This is done in practice by instructing your DNS server
+to fail resolving some domains or subdomains.
+Want to block Facebook for every host in the house?
+That's the way to go.
+
+Some routers allow this level of fine-tuning but if not,
+self-hosting your own DNS server is the way to go.
+
+## Drawbacks {#recipes-dnsServer-drawbacks}
+
+Although it has some nice advantages,
+self-hosting your own DNS server has one major drawback:
+if it goes down, the whole household will be impacted.
+By experience, it takes up to 5 minutes for others to notice something is wrong with internet.
+
+So be wary when you deploy a new config.
+
+## Recipe {#recipes-dnsServer-recipe}
+
+The following snippet:
+
+- Opens UDP port 53 in the firewall which is the ubiquitous (and hardcoded, crazy I know) port for DNS queries.
+- Disables the default DNS resolver.
+- Sets up dnsmasq as the DNS server.
+- Optionally sets up dnsmasq as the DHCP server.
+- Answers all DNS requests to your domain with the internal IP of the server.
+- Forwards all other DNS requests to an external DNS server using DNSSEC.
+  This is done using [stubby][].
+
+[stubby]: https://dnsprivacy.org/dns_privacy_daemon_-_stubby/
+
+For more information about options, read the dnsmasq [manual][].
+
+[manual]: https://dnsmasq.org/docs/dnsmasq-man.html
+
+```nix
+let
+  # Replace these values with what matches your network.
+  domain = "example.com";
+  serverIP = "192.168.1.30";
+
+  # This port is used internally for dnsmasq to talk to stubby on the loopback interface.
+  # Only change this if that port is already taken.
+  stubbyPort = 53000;
+in
+{
+  networking.firewall.allowedUDPPorts = [ 53 ];
+
+  services.resolved.enable = false;
+  services.dnsmasq = {
+    enable = true;
+    settings = {
+      inherit domain;
+
+      # Redirect queries to the stubby instance.
+      server = [
+        "127.0.0.1#${stubbyPort}"
+        "::1#${stubbyPort}"
+      ];
+      # We do trust our own instance of stubby
+      # so we can proxy DNSSEC stuff.
+      # I'm not sure how useful this is.
+      proxy-dnssec = true;
+
+      # Log all queries.
+      # This produces a lot of log lines
+      # and looking at those can be scary!
+      log-queries = true;
+
+      # Do not look at /etc/resolv.conf
+      no-resolv = true;
+
+      # Do not forward externally reverse DNS lookups for internal IPs.
+      bogus-priv = true;
+
+      address = [
+        "/.${domain}/${serverIP}"
+        # You can redirect anything anywhere too.
+        "/pikvm.${domain}/192.168.1.31"
+      ];
+    };
+  };
+
+  services.stubby = {
+    enable = true;
+    # It's a bit weird but default values comes from the examples settings hosted at
+    # https://github.com/getdnsapi/stubby/blob/develop/stubby.yml.example
+    settings = pkgs.stubby.passthru.settingsExample // {
+      listen_addresses = [
+        "127.0.0.1@${stubbyPort}"
+        "0::1@${stubbyPort}"
+      ];
+
+      # For more example of good DNS resolvers,
+      # head to https://dnsprivacy.org/public_resolvers/
+      #
+      # The digest comes from https://nixos.wiki/wiki/Encrypted_DNS#Stubby
+      upstream_recursive_servers = [
+        {
+          address_data = "9.9.9.9";
+          tls_auth_name = "dns.quad9.net";
+          tls_pubkey_pinset = [
+            {
+              digest = "sha256";
+              value = "i2kObfz0qIKCGNWt7MjBUeSrh0Dyjb0/zWINImZES+I=";
+            }
+          ];
+        }
+        {
+          address_data = "149.112.112.112";
+          tls_auth_name = "dns.quad9.net";
+          tls_pubkey_pinset = [
+            {
+              digest = "sha256";
+              value = "i2kObfz0qIKCGNWt7MjBUeSrh0Dyjb0/zWINImZES+I=";
+            }
+          ];
+        }
+      ];
+    };
+  };
+}
+```
+
+Optionally, to use dnsmasq as the DHCP server too,
+use the following snippet:
+
+```nix
+services.dnsmasq = {
+  settings = {
+    # When switching DNS server, accept old leases from previous server.
+    dhcp-authoritative = true;
+
+    # Adapt to your needs
+    # <ip-from>,<ip-to>,<mask>,<lease-ttl>
+    dhcp-range = "192.168.1.101,192.168.1.150,255.255.255.0,6h";
+
+    # Static DNS leases if needed.
+    # Choose an IP outside of the DHCP range
+    # <mac-address>,<DNS name>,<ip>,<lease-ttl>
+    dhcp-host = [
+      "12:34:56:78:9a:bc,server,192.168.1.50,infinite"
+    ];
+
+    # Set default route to the router that can acccess the internet.
+    dhcp-option = [
+      "3,192.168.1.1"
+    ];
+  };
+};
+```

docs/recipes/exposeService.md

@@ -0,0 +1,156 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Expose a service {#recipes-exposeService}
+
+Let's see how one can use most of the blocks provided by SelfHostBlocks to make a service
+accessible through a reverse proxy with LDAP and SSO integration as well as backing up
+this service and creating a ZFS dataset to store the service's data.
+
+We'll use an hypothetical well made service found under `services.awesome` as our example.
+We're purposely not using a real service to avoid needing to deal with uninteresting particularities.
+
+## Service setup {#recipes-exposeService-service}
+
+Let's say our domain name is `example.com`,
+and we want to reach our service under the `awesome` subdomain:
+
+```nix
+let
+  domain = "example.com";
+  subdomain = "awesome";
+  fqdn = "${subdomain}.${domain}";
+  listenPort = 9000;
+  dataDir = "/var/lib/awesome";
+  ldapGroup = "awesome_user";
+in
+```
+
+We then `enable` the service and explicitly set the `listenPort` and `dataDir`,
+assuming those options exist:
+
+```nix
+services.awesome = {
+  enable = true;
+  inherit dataDir listenPort;
+};
+```
+
+## SSL Certificate {#recipes-exposeService-ssl}
+
+Requesting an SSL certificate from Let's Encrypt is done by adding an entry to
+the `extraDomains` option:
+
+```nix
+shb.certs.certs.letsencrypt.${domain}.extraDomains = [ fqdn ];
+```
+
+This assumes the `shb.certs` block has been configured:
+
+```nix
+shb.certs.certs.letsencrypt.${domain} = {
+  inherit domain;
+  group = "nginx";
+  reloadServices = [ "nginx.service" ];
+  adminEmail = "admin@${domain}";
+};
+```
+
+## LDAP group {#recipes-exposeService-ldap}
+
+We want only users of the group `calibre_user` to be able to access this subdomain.
+The following snippet creates the LDAP group:
+
+```nix
+shb.lldap.ensureGroups = {
+  calibre_user = {};
+};
+```
+
+## Reverse Proxy with Forward Auth {#recipes-exposeService-nginx}
+
+If our service does not integrate with OIDC, we can still protect it with SSO
+with forward authentication by letting the reverse proxy handle authentication.
+This is done by adding an entry to `shb.nginx.vhosts`:
+
+```nix
+shb.nginx.vhosts = [
+  {
+    inherit subdomain domain;
+    ssl = config.shb.certs.certs.letsencrypt.${domain};
+    upstream = "http://127.0.0.1:${toString listenPort}";
+    authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+    autheliaRules = [{
+      policy = "one_factor";
+      subject = [ "group:${ldapGroup}" ];
+    }];
+  }
+];
+```
+
+## ZFS support {#recipes-exposeService-zfs}
+
+If you use ZFS, you can use SelfHostBlocks to create a dataset for you:
+
+```nix
+shb.zfs.datasets."safe/awesome".path = config.services.awesome.dataDir;
+```
+
+## Debugging {#recipes-exposeService-debug}
+
+Usually, the log level of the service can be increased with some option they provide.
+
+With SelfHostBlocks, you can also introspect any HTTP service by adding an
+`mitmdump` instance between the reverse proxy and the `awesome` service:
+
+```nix
+shb.mitmdump.awesome = {
+  inherit listenPort;
+  upstreamPort = listenPort + 1;
+};
+services.awesome.listenPort = lib.mkForce (listenPort + 1);
+```
+
+This creates a `mitmdump-awesome.service` systemd service which prints the requests' and responses' headers and bodies.
+
+## Backup {#recipes-exposeService-backup}
+
+The following snippet uses the `shb.restic` block to backup the `services.awesome.dataDir` directory:
+
+```nix
+shb.restic.instances.awesome = {
+  request.user = "awesome";
+  request.sourceDirectories = [ dataDir ];
+  settings.enable = true;
+  settings.passphrase.result = config.shb.sops.secret.awesome.result;
+  settings.repository.path = config.services.awesome.dataDir;
+};
+
+shb.sops.secret."awesome" = {
+  request = config.shb.restic.instances.awesome.settings.passphrase.request;
+};
+```
+
+## Impermanence {#recipes-exposeService-impermanence}
+
+To save the data folder in an impermanence setup, add:
+
+```nix
+{
+  shb.zfs.datasets."safe/awesome".path = config.services.awesome.dataDir;
+}
+```
+
+## Application Dashboard {#recipes-exposeService-applicationdashboard}
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.MyServices.services.Awesome = {
+    sortOrder = 1;
+    dashboard.request = {
+      externalUrl = "https://${fqdn}";
+      internalUrl = "http://127.0.0.1:${toString listenPort}";
+    };
+  };
+}
+```

docs/recipes/serveStaticPages.md

@@ -0,0 +1,75 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Serve Static Pages {#recipes-serveStaticPages}
+
+This recipe shows how to use SelfHostBlocks blocks to serve static web pages using the Nginx reverse proxy with SSL termination.
+
+In this recipe, we'll assume the pages to serve are found under the `/srv/my-website` path and will be served under the `my-website.example.com` fqdn.
+
+```nix
+let
+  name = "my-website";
+  subdomain = name;
+  domain = "example.com";
+  fqdn = "${subdomain}.${domain}";
+  user = "me";
+in
+```
+
+We also assume the static web pages are owned and updated by the user named `me`.
+
+## ZFS dataset {#recipes-serveStaticPages-zfs}
+
+We can create a ZFS dataset with:
+
+```nix
+shb.zfs.datasets."safe/${name}".path = "/srv/${name}";
+```
+
+## SSL Certificate {#recipes-serveStaticPages-ssl}
+
+Requesting an SSL certificate from Let's Encrypt is done by adding an entry to
+the `extraDomains` option:
+
+```nix
+shb.certs.certs.letsencrypt.${domain}.extraDomains = [ fqdn ];
+```
+
+This assumes the `shb.certs` block has been configured:
+
+```nix
+shb.certs.certs.letsencrypt.${domain} = {
+  inherit domain;
+  group = "nginx";
+  reloadServices = [ "nginx.service" ];
+  adminEmail = "admin@${domain}";
+};
+```
+
+## Reverse Proxy {#recipes-serveStaticPages-nginx}
+
+First, we make the parent directory owned by the user which will upload them and `nginx`:
+
+```nix
+systemd.tmpfiles.rules = lib.mkBefore [
+  "d '/srv/${name}' 0750 ${user} nginx - -"
+];
+```
+
+Now, we can setup nginx. The following snippet serves files from the `/srv/${name}/` directory.
+
+```nix
+services.nginx.enable = true;
+
+services.nginx.virtualHosts."skarabox.${domain}" = {
+  forceSSL = true;
+  sslCertificate = config.shb.certs.certs.letsencrypt."${domain}".paths.cert;
+  sslCertificateKey = config.shb.certs.certs.letsencrypt."${domain}".paths.key;
+  locations."/" = {
+    root = "/srv/${name}/";
+    extraConfig = ''
+      add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
+      add_header Cache-Control "max-age=604800, stale-while-revalidate=86400, stale-if-error=86400, must-revalidate, public";
+    '';
+  };
+};
+```

docs/service-implementation-guide.md

@@ -71,7 +71,6 @@ Location: `modules/services/servicename.nix`
 let
   cfg = config.shb.servicename;
   contracts = pkgs.callPackage ../contracts {};
-  shblib = pkgs.callPackage ../../lib {};
   fqdn = "${cfg.subdomain}.${cfg.domain}";
   
   # Choose appropriate format based on service config
@@ -296,12 +295,12 @@ let
   
 in {
   # Test variants (all 6 required)
-  basic = pkgs.testers.runNixOSTest { ... };
-  backup = pkgs.testers.runNixOSTest { ... };
-  https = pkgs.testers.runNixOSTest { ... };
-  ldap = pkgs.testers.runNixOSTest { ... };
-  monitoring = pkgs.testers.runNixOSTest { ... };
-  sso = pkgs.testers.runNixOSTest { ... };
+  basic = lib.shb.test.runNixOSTest { ... };
+  backup = lib.shb.test.runNixOSTest { ... };
+  https = lib.shb.test.runNixOSTest { ... };
+  ldap = lib.shb.test.runNixOSTest { ... };
+  monitoring = lib.shb.test.runNixOSTest { ... };
+  sso = lib.shb.test.runNixOSTest { ... };
 }
 ```
 
@@ -381,14 +380,16 @@ source: @OPTIONS_JSON@
 
 Note: Replace `servicename` with your actual service name (e.g., `nzbget`, `jellyfin`).
 
-For the `@OPTIONS_JSON@` to work, the following snippet must be added
-in the `docs/default.nix` file.
+For the `@OPTIONS_JSON@` to work, a line must be added
+in the `flake.nix` file:
 
 ```nix
-substituteInPlace ./modules/services/servicename/docs/default.md \
-  --replace \
-    '@OPTIONS_JSON@' \
-   ${individualModuleOptionsDocs [ ../modules/services/servicename.nix ]}/share/doc/nixos/options.json
+packages.manualHtml = pkgs.callPackage ./docs {
+  modules = {
+    "blocks/authelia" = ./modules/blocks/authelia.nix;
+    // Add line and keep in alphabetical order.
+  };
+};
 ```
 
 ### 7. Update Redirects Automatically {#update-redirects-automatically}
@@ -460,6 +461,13 @@ nix flake check
 nix build .#manualHtml
 ```
 
+To continuously rebuild the documentation of file change, run the following command.
+To exit, you'll need to do Ctrl-C twice in a row.
+
+```bash
+nix run .#manualHtml-watch
+```
+
 ### Iterative Development Approach {#iterative-development-approach}
 
 1. **Start with basic functionality** - get core service working

docs/services.md

@@ -1,8 +1,10 @@
 <!-- Read these docs at https://shb.skarabox.com -->
 # Services {#services}
 
-Services are usually web applications that SHB help you self-host. Configuration of those is
-purposely made opinionated and require as few nix options as can make sense. That is possible thanks to the extensive use of blocks provided by SHB.
+Services are usually web applications that SHB help you self-host some of your data.
+Configuration of those is purposely made more opinionated than the upstream nixpkgs modules
+in exchange for an uniformized configuration experience.
+That is possible thanks to the extensive use of blocks provided by SHB.
 
 ::: {.note}
 Not all services are yet documented. You can find all available services [in the repository](@REPO@/modules/services).
@@ -11,13 +13,20 @@ Not all services are yet documented. You can find all available services [in the
 The following table summarizes for each documented service what features it provides. More
 information is provided in the respective manual sections.
 
-| Service               | Backup | Reverse Proxy | SSO | LDAP  | Monitoring | Profiling |
-|-----------------------|--------|---------------|-----|-------|------------|-----------|
-| [*Arr][]             | Y (1)  | Y             | Y   | Y (4) | Y (2)      | N         |
-| [Forgejo][]          | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
-| [Home-Assistant][]   | Y (1)  | Y             | N   | Y     | Y (2)      | N         |
-| [Nextcloud Server][] | Y (1)  | Y             | Y   | Y     | Y (2)      | P (3)     |
-| [Vaultwarden][]      | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| Service                     | Backup | Reverse Proxy | SSO | LDAP  | Monitoring | Profiling |
+|-----------------------------|--------|---------------|-----|-------|------------|-----------|
+| [*Arr][]                    | Y (1)  | Y             | Y   | Y (4) | Y (2)      | N         |
+| [Firefly-iii][]             | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Forgejo][]                 | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Home-Assistant][]          | Y (1)  | Y             | N   | Y     | Y (2)      | N         |
+| [Homepage][]                | Y (1)  | Y             | N   | Y     | Y (2)      | N         |
+| [Jellyfin][]                | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Karakeep][]                | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Nextcloud Server][]        | Y (1)  | Y             | Y   | Y     | Y (2)      | P (3)     |
+| [Open WebUI][]              | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Pinchflat][]               | Y      | Y             | Y   | Y (4) | Y (5)      | N         |
+| [Simple NixOS Mailserver][] | Y      | Y             | N   | Y     | Y          | N         |
+| [Vaultwarden][]             | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
 
 Legend: **N**: no but WIP; **P**: partial; **Y**: yes
 
@@ -29,27 +38,80 @@ Legend: **N**: no but WIP; **P**: partial; **Y**: yes
 4. Uses LDAP indirectly through forward auth.
 
 [*Arr]: services-arr.html
+[Firefly-iii]: services-firefly-iii.html
 [Forgejo]: services-forgejo.html
 [Home-Assistant]: services-home-assistant.html
+[Homepage]: services-homepage.html
+[Jellyfin]: services-jellyfin.html
+[Karakeep]: services-karakeep.html
 [Nextcloud Server]: services-nextcloud.html
+[Open WebUI]: services-open-webui.html
+[Pinchflat]: services-pinchflat.html
+[Simple NixOS Mailserver]: services-mailserver.html
 [Vaultwarden]: services-vaultwarden.html
 
-```{=include=} chapters html:into-file=//services-arr.html
-modules/services/arr/docs/default.md
+## Dashboard {#services-category-dashboard}
+
+```{=include=} chapters html:into-file=//services-homepage.html
+modules/services/homepage/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//services-forgejo.html
-modules/services/forgejo/docs/default.md
-```
-
-```{=include=} chapters html:into-file=//services-home-assistant.html
-modules/services/home-assistant/docs/default.md
-```
+## Documents {#services-category-documents}
 
 ```{=include=} chapters html:into-file=//services-nextcloud.html
 modules/services/nextcloud-server/docs/default.md
 ```
 
+## Emails {#services-category-emails}
+
+```{=include=} chapters html:into-file=//services-mailserver.html
+modules/services/mailserver/docs/default.md
+```
+
+## Passwords {#services-category-passwords}
+
 ```{=include=} chapters html:into-file=//services-vaultwarden.html
 modules/services/vaultwarden/docs/default.md
 ```
+
+## Automation {#services-category-automation}
+
+```{=include=} chapters html:into-file=//services-home-assistant.html
+modules/services/home-assistant/docs/default.md
+```
+
+## AI {#services-category-ai}
+
+```{=include=} chapters html:into-file=//services-karakeep.html
+modules/services/karakeep/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//services-open-webui.html
+modules/services/open-webui/docs/default.md
+```
+
+## Code {#services-category-code}
+
+```{=include=} chapters html:into-file=//services-forgejo.html
+modules/services/forgejo/docs/default.md
+```
+
+## Media {#services-category-media}
+
+```{=include=} chapters html:into-file=//services-arr.html
+modules/services/arr/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//services-jellyfin.html
+modules/services/jellyfin/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//services-pinchflat.html
+modules/services/pinchflat/docs/default.md
+```
+
+## Finance {#services-category-finance}
+
+```{=include=} chapters html:into-file=//services-firefly-iii.html
+modules/services/firefly-iii/docs/default.md
+```

flake.lock

@@ -20,11 +20,11 @@
     },
     "nix-flake-tests": {
       "locked": {
-        "lastModified": 1677844186,
-        "narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
+        "lastModified": 1775571237,
+        "narHash": "sha256-1f5Uvgcy3gx/eyRp4rqfDRBjcZc9uiHoIlfcFIL7GvI=",
         "owner": "antifuchs",
         "repo": "nix-flake-tests",
-        "rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
+        "rev": "86b789cff8aecbd7c1bed7cd421dd837c3f62201",
         "type": "github"
       },
       "original": {
@@ -35,11 +35,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1742069588,
-        "narHash": "sha256-C7jVfohcGzdZRF6DO+ybyG/sqpo1h6bZi9T56sxLy+k=",
+        "lastModified": 1781074563,
+        "narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "c80f6a7e10b39afcc1894e02ef785b1ad0b0d7e5",
+        "rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca",
         "type": "github"
       },
       "original": {
@@ -52,11 +52,11 @@
     "nmdsrc": {
       "flake": false,
       "locked": {
-        "lastModified": 1705050560,
-        "narHash": "sha256-x3zzcdvhJpodsmdjqB4t5mkVW22V3wqHLOun0KRBzUI=",
+        "lastModified": 1757277932,
+        "narHash": "sha256-eW2AzE4HeJXHKxSGDcp35/VQyxeESBt4p9i8QIFrSE8=",
         "ref": "refs/heads/master",
-        "rev": "66d9334933119c36f91a78d565c152a4fdc8d3d3",
-        "revCount": 66,
+        "rev": "055e814ac35379dece164a8413d336b57c5d2e0c",
+        "revCount": 67,
         "type": "git",
         "url": "https://git.sr.ht/~rycee/nmd"
       },

flake.nix

@@ -11,225 +11,273 @@
     };
   };
 
-  outputs = { nixpkgs, nix-flake-tests, flake-utils, nmdsrc, ... }: flake-utils.lib.eachDefaultSystem (system:
+  outputs =
+    inputs@{
+      self,
+      nixpkgs,
+      nix-flake-tests,
+      flake-utils,
+      nmdsrc,
+      ...
+    }:
     let
-      originPkgs = nixpkgs.legacyPackages.${system};
-      patches = originPkgs.lib.optionals (system == "x86_64-linux") [
-        # Leaving commented out for an example.
-        # (originPkgs.fetchpatch {
-        #   url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
-        #   hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
-        # })
-      ];
-      patchedNixpkgs = originPkgs.applyPatches {
-        name = "nixpkgs-patched";
-        src = nixpkgs;
-        inherit patches;
-      };
-      pkgs = import patchedNixpkgs {
-        inherit system;
+      shbPatches =
+        system:
+        nixpkgs.legacyPackages.${system}.lib.optionals
+          (system == "x86_64-linux" || system == "aarch64-linux")
+          [
+            # Get rid of lldap patches when https://github.com/NixOS/nixpkgs/pull/425923 is merged.
+            ./patches/lldap.patch
+            ./patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch
 
-        config = {
-          permittedInsecurePackages = [
-            # https://github.com/NixOS/nixpkgs/issues/360592
-            "aspnetcore-runtime-6.0.36"
-            # TODO: https://github.com/NixOS/nixpkgs/issues/326335
-            "dotnet-sdk-6.0.428"
+            # Leaving commented out as an example.
+            # (originPkgs.fetchpatch {
+            #   url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
+            #   hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
+            # })
           ];
+
+      patchNixpkgs =
+        {
+          nixpkgs,
+          patches,
+          system,
+        }:
+        nixpkgs.legacyPackages.${system}.applyPatches {
+          name = "nixpkgs-patched";
+          src = nixpkgs;
+          inherit patches;
+        };
+      patchedNixpkgs =
+        system:
+        let
+          patched = patchNixpkgs {
+            nixpkgs = inputs.nixpkgs;
+            patches = shbPatches system;
+            inherit system;
+          };
+        in
+        patched
+        // {
+          nixosSystem = args: import "${patched}/nixos/lib/eval-config.nix" args;
+        };
+      pkgs' =
+        system:
+        import (patchedNixpkgs system) {
+          inherit system;
+          config.allowUnfree = true;
         };
-
-        overlays = [
-          (final: prev: {
-            exiftool = prev.exiftool.overrideAttrs (f: p: {
-              version = "12.70";
-              src = pkgs.fetchurl {
-                url = "https://exiftool.org/Image-ExifTool-12.70.tar.gz";
-                hash = "sha256-TLJSJEXMPj870TkExq6uraX8Wl4kmNerrSlX3LQsr/4=";
-              };
-            });
-          })
-          (final: prev: {
-            grocy = prev.grocy.overrideAttrs (f: p: {
-              patches = p.patches ++ [
-                ./patches/grocy.patch
-              ];
-            });
-          })
-        ];
-      };
-
-      allModules = [
-        modules/blocks/authelia.nix
-        modules/blocks/davfs.nix
-        modules/blocks/hardcodedsecret.nix
-        modules/blocks/ldap.nix
-        modules/blocks/monitoring.nix
-        modules/blocks/nginx.nix
-        modules/blocks/postgresql.nix
-        modules/blocks/restic.nix
-        modules/blocks/ssl.nix
-        modules/blocks/sops.nix
-        modules/blocks/tinyproxy.nix
-        modules/blocks/vpn.nix
-        modules/blocks/zfs.nix
-
-        modules/services/arr.nix
-        modules/services/audiobookshelf.nix
-        modules/services/deluge.nix
-        modules/services/forgejo.nix
-        modules/services/grocy.nix
-        modules/services/hledger.nix
-        modules/services/home-assistant.nix
-        modules/services/jellyfin.nix
-        modules/services/nextcloud-server.nix
-        modules/services/vaultwarden.nix
-      ];
-
-      # The contract dummies are used to show options for contracts.
-      contractDummyModules = [
-        modules/contracts/backup/dummyModule.nix
-        modules/contracts/ssl/dummyModule.nix
-      ];
     in
-      {
-        inherit patches;
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = pkgs' system;
 
-        nixosModules.default = { config, ... }: {
-          imports = allModules;
-        };
+        # The contract dummies are used to show options for contracts.
+        contractDummyModules = [
+          modules/contracts/backup/dummyModule.nix
+          modules/contracts/dashboard/dummyModule.nix
+          modules/contracts/databasebackup/dummyModule.nix
+          modules/contracts/datasetbackup/dummyModule.nix
+          modules/contracts/secret/dummyModule.nix
+          modules/contracts/ssl/dummyModule.nix
+        ];
+      in
+      {
+        formatter = pkgs.nixfmt-tree;
 
         packages.manualHtml = pkgs.callPackage ./docs {
           inherit nmdsrc;
-          allModules = allModules ++ contractDummyModules;
+          allModules =
+            self.nixosModules.default.imports
+            ++ [
+              self.nixosModules.sops
+            ]
+            ++ contractDummyModules;
           release = builtins.readFile ./VERSION;
+
+          substituteVersionIn = [
+            "./manual.md"
+            "./usage.md"
+          ];
+          modules = {
+            "blocks/authelia" = ./modules/blocks/authelia.nix;
+            "blocks/borgbackup" = ./modules/blocks/borgbackup.nix;
+            "blocks/lldap" = ./modules/blocks/lldap.nix;
+            "blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
+            "blocks/monitoring" = ./modules/blocks/monitoring.nix;
+            "blocks/nginx" = ./modules/blocks/nginx.nix;
+            "blocks/postgresql" = ./modules/blocks/postgresql.nix;
+            "blocks/restic" = ./modules/blocks/restic.nix;
+            "blocks/sanoid" = ./modules/blocks/sanoid.nix;
+            "blocks/sops" = ./modules/blocks/sops.nix;
+            "blocks/ssl" = {
+              module = ./modules/blocks/ssl.nix;
+              optionRoot = [
+                "shb"
+                "certs"
+              ];
+            };
+            "blocks/zfs" = ./modules/blocks/zfs.nix;
+
+            "services/arr" = ./modules/services/arr.nix;
+            "services/firefly-iii" = ./modules/services/firefly-iii.nix;
+            "services/forgejo" = [
+              ./modules/services/forgejo.nix
+              (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
+            ];
+            "services/home-assistant" = ./modules/services/home-assistant.nix;
+            "services/homepage" = ./modules/services/homepage.nix;
+            "services/jellyfin" = ./modules/services/jellyfin.nix;
+            "services/karakeep" = ./modules/services/karakeep.nix;
+            "services/mailserver" = ./modules/services/mailserver.nix;
+            "services/nextcloud-server" = {
+              module = ./modules/services/nextcloud-server.nix;
+              optionRoot = [
+                "shb"
+                "nextcloud"
+              ];
+            };
+            "services/open-webui" = ./modules/services/open-webui.nix;
+            "services/pinchflat" = ./modules/services/pinchflat.nix;
+            "services/vaultwarden" = ./modules/services/vaultwarden.nix;
+
+            "contracts/backup" = {
+              module = ./modules/contracts/backup/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "backup"
+              ];
+            };
+            "contracts/dashboard" = {
+              module = ./modules/contracts/dashboard/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "dashboard"
+              ];
+            };
+            "contracts/databasebackup" = {
+              module = ./modules/contracts/databasebackup/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "databasebackup"
+              ];
+            };
+            "contracts/datasetbackup" = {
+              module = ./modules/contracts/datasetbackup/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "datasetbackup"
+              ];
+            };
+            "contracts/secret" = {
+              module = ./modules/contracts/secret/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "secret"
+              ];
+            };
+            "contracts/ssl" = {
+              module = ./modules/contracts/ssl/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "ssl"
+              ];
+            };
+          };
         };
 
         # Documentation redirect generation tool - scans HTML files for anchor mappings
-        packages.generateRedirects = 
-        let
-          # Python patch to inject redirect collector
-          pythonPatch = pkgs.writeText "nixos-render-docs-patch.py" ''
-            # Load redirect collector patch
-            try:
-                import sys, os
-                sys.path.insert(0, os.path.dirname(__file__) + '/..')
-                import missing_refs_collector
-            except Exception as e:
-                print(f"Warning: Failed to load redirect collector: {e}", file=sys.stderr)
-          '';
-
-          # Patched nixos-render-docs that collects redirects during HTML generation
-          nixos-render-docs-patched = pkgs.writeShellApplication {
-            name = "nixos-render-docs";
-            runtimeInputs = [ pkgs.nixos-render-docs ];
-            text = ''
-              TEMP_DIR=$(mktemp -d); trap 'rm -rf "$TEMP_DIR"' EXIT
-              
-              cp -r ${pkgs.nixos-render-docs}/${pkgs.python3.sitePackages}/nixos_render_docs "$TEMP_DIR/"
-              chmod -R +w "$TEMP_DIR"
-              cp ${./docs/generate-redirects-nixos-render-docs.py} "$TEMP_DIR/missing_refs_collector.py"
-              echo '{}' > "$TEMP_DIR/empty_redirects.json"
-              cat ${pythonPatch} >> "$TEMP_DIR/nixos_render_docs/__init__.py"
-              
-              ARGS=()
-              while [[ $# -gt 0 ]]; do
-                case $1 in
-                  --redirects) ARGS+=("$1" "$TEMP_DIR/empty_redirects.json"); shift 2 ;;
-                  *) ARGS+=("$1"); shift ;;
-                esac
-              done
-              
-              export PYTHONPATH="$TEMP_DIR:''${PYTHONPATH:-}"
-              nixos-render-docs "''${ARGS[@]}"
-            '';
-          };
-        in
-        (pkgs.callPackage ./docs {
-          inherit nmdsrc;
-          allModules = allModules ++ contractDummyModules;
-          release = builtins.readFile ./VERSION;
-          nixos-render-docs = nixos-render-docs-patched;
-        }).overrideAttrs (old: {
-          installPhase = ''
-            ${old.installPhase}
-            ln -sf share/doc/selfhostblocks/redirects.json $out/redirects.json
-          '';
-        });
-
-        lib =
-          (pkgs.callPackage ./lib {})
-          // {
-            contracts = pkgs.callPackage ./modules/contracts {};
-          };
-
-        checks =
+        packages.generateRedirects =
           let
-            inherit (pkgs.lib) foldl foldlAttrs mergeAttrs optionalAttrs;
+            # Python patch to inject redirect collector
+            pythonPatch = pkgs.writeText "nixos-render-docs-patch.py" ''
+              # Load redirect collector patch
+              try:
+                  import sys, os
+                  sys.path.insert(0, os.path.dirname(__file__) + '/..')
+                  import missing_refs_collector
+              except Exception as e:
+                  print(f"Warning: Failed to load redirect collector: {e}", file=sys.stderr)
+            '';
 
-            importFiles = files:
-              map (m: pkgs.callPackage m {}) files;
+            # Patched nixos-render-docs that collects redirects during HTML generation
+            nixos-render-docs-patched = pkgs.writeShellApplication {
+              name = "nixos-render-docs";
+              runtimeInputs = [ pkgs.nixos-render-docs ];
+              text = ''
+                TEMP_DIR=$(mktemp -d); trap 'rm -rf "$TEMP_DIR"' EXIT
 
-            mergeTests = foldl mergeAttrs {};
+                cp -r ${pkgs.nixos-render-docs}/${pkgs.python3.sitePackages}/nixos_render_docs "$TEMP_DIR/"
+                chmod -R +w "$TEMP_DIR"
+                cp ${./docs/generate-redirects-nixos-render-docs.py} "$TEMP_DIR/missing_refs_collector.py"
+                echo '{}' > "$TEMP_DIR/empty_redirects.json"
+                cat ${pythonPatch} >> "$TEMP_DIR/nixos_render_docs/__init__.py"
 
-            flattenAttrs = root: attrset: foldlAttrs (acc: name: value: acc // {
-              "${root}_${name}" = value;
-            }) {} attrset;
+                ARGS=()
+                while [[ $# -gt 0 ]]; do
+                  case $1 in
+                    --redirects) ARGS+=("$1" "$TEMP_DIR/empty_redirects.json"); shift 2 ;;
+                    *) ARGS+=("$1"); shift ;;
+                  esac
+                done
 
-            vm_test = name: path: flattenAttrs "vm_${name}" (
-              import path {
-                inherit pkgs;
-                inherit (pkgs) lib;
-              }
-            );
-
-            shblib = pkgs.callPackage ./lib {};
-          in (optionalAttrs (system == "x86_64-linux") ({
-            modules = shblib.check {
-              inherit pkgs;
-              tests =
-                mergeTests (importFiles [
-                  ./test/modules/arr.nix
-                  ./test/modules/davfs.nix
-                  # TODO: Make this not use IFD
-                  ./test/modules/lib.nix
-                  ./test/modules/nginx.nix
-                  ./test/modules/postgresql.nix
-                ]);
+                export PYTHONPATH="$TEMP_DIR:''${PYTHONPATH:-}"
+                nixos-render-docs "''${ARGS[@]}"
+              '';
             };
+          in
+          (self.packages.${system}.manualHtml.override {
+            nixos-render-docs = nixos-render-docs-patched;
+          }).overrideAttrs
+            (old: {
+              installPhase = ''
+                ${old.installPhase}
+                ln -sf share/doc/selfhostblocks/redirects.json $out/redirects.json
+              '';
+            });
 
-            # TODO: Make this not use IFD
-            lib = nix-flake-tests.lib.check {
-              inherit pkgs;
-              tests = pkgs.callPackage ./test/modules/lib.nix {};
-            };
-          }
-          // (vm_test "arr" ./test/services/arr.nix)
-          // (vm_test "audiobookshelf" ./test/services/audiobookshelf.nix)
-          // (vm_test "deluge" ./test/services/deluge.nix)
-          // (vm_test "forgejo" ./test/services/forgejo.nix)
-          // (vm_test "grocy" ./test/services/grocy.nix)
-          // (vm_test "hledger" ./test/services/hledger.nix)
-          // (vm_test "homeassistant" ./test/services/home-assistant.nix)
-          // (vm_test "jellyfin" ./test/services/jellyfin.nix)
-          // (vm_test "monitoring" ./test/services/monitoring.nix)
-          // (vm_test "nextcloud" ./test/services/nextcloud.nix)
-          // (vm_test "vaultwarden" ./test/services/vaultwarden.nix)
+        packages.manualHtml-watch = pkgs.writeShellApplication {
+          name = "manualHtml-watch";
+          runtimeInputs = [
+            pkgs.findutils
+            pkgs.entr
+          ];
+          text = ''
+            while sleep 1; do
+              find . -name "*.nix" -o -name "*.md" \
+                | entr -d sh -c '(nix run --offline .#update-redirects && nix build --offline .#manualHtml)' || :
+            done
+          '';
+        };
 
-          // (vm_test "authelia" ./test/blocks/authelia.nix)
-          // (vm_test "ldap" ./test/blocks/ldap.nix)
-          // (vm_test "lib" ./test/blocks/lib.nix)
-          // (vm_test "postgresql" ./test/blocks/postgresql.nix)
-          // (vm_test "restic" ./test/blocks/restic.nix)
-          // (vm_test "ssl" ./test/blocks/ssl.nix)
+        packages.update-flake-lock-pr = pkgs.callPackage ./.github/workflows/update-flake-lock-pr.nix { };
 
-          // (vm_test "contracts-backup" ./test/contracts/backup.nix)
-          // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
-          // (vm_test "contracts-secret" ./test/contracts/secret.nix)
-          ));
+        lib = (pkgs.callPackage ./lib { }) // {
+          test = pkgs.callPackage ./test/common.nix { };
+          contracts = pkgs.callPackage ./modules/contracts {
+            shb = self.lib.${system};
+          };
+          patches = shbPatches system;
+          inherit patchNixpkgs;
+          patchedNixpkgs = patchedNixpkgs system;
+        };
 
-        # Run nix .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
-        packages.playwright =
-          pkgs.callPackage ({ stdenvNoCC, makeWrapper, playwright }: stdenvNoCC.mkDerivation {
+        # To see the traces, run:
+        #   nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
+        packages.playwright = pkgs.callPackage (
+          {
+            stdenvNoCC,
+            makeWrapper,
+            playwright,
+          }:
+          stdenvNoCC.mkDerivation {
             name = "playwright";
 
             src = playwright;
@@ -244,35 +292,223 @@
                 --set PLAYWRIGHT_BROWSERS_PATH ${pkgs.playwright-driver.browsers} \
                 --set PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS true
             '';
-          }) {};
+          }
+        ) { };
 
         # Run "nix run .#update-redirects" to regenerate docs/redirects.json
         apps.update-redirects = {
           type = "app";
-          program = "${pkgs.writeShellApplication {
-            name = "update-redirects";
-            runtimeInputs = [ pkgs.nix pkgs.jq ];
-            text = ''
-              echo "=== SelfHostBlocks Redirects Updater ==="
-              echo "Generating fresh ./docs/redirects.json..."
-              
-              nix build .#generateRedirects || { echo "Error: Failed to generate redirects" >&2; exit 1; }
-              [[ -f result/redirects.json ]] || { echo "Error: Generated redirects file not found" >&2; exit 1; }
-              
-              echo "Generated $(jq 'keys | length' result/redirects.json) redirects"
-              echo
-              read -p "Update docs/redirects.json? This will backup the current file [y/N] " -r response
-              [[ "$response" =~ ^[Yy] ]] || { echo "Aborted - no changes made"; exit 0; }
-              
-              [[ -f docs/redirects.json ]] && cp docs/redirects.json docs/redirects.json.backup && echo "Created backup"
-              cp result/redirects.json docs/redirects.json
-              echo "  Updated docs/redirects.json"
-              echo "To verify: nix build .#manualHtml"
-            '';
-          }}/bin/update-redirects";
+          program = "${
+            pkgs.writeShellApplication {
+              name = "update-redirects";
+              runtimeInputs = [
+                pkgs.nix
+                pkgs.jq
+              ];
+              text = ''
+                echo "=== SelfHostBlocks Redirects Updater ==="
+                echo "Generating fresh ./docs/redirects.json..."
+
+                nix build .#generateRedirects || { echo "Error: Failed to generate redirects" >&2; exit 1; }
+                [[ -f result/redirects.json ]] || { echo "Error: Generated redirects file not found" >&2; exit 1; }
+
+                echo "Generated $(jq 'keys | length' result/redirects.json) redirects"
+
+                [[ -f docs/redirects.json ]] && cp docs/redirects.json docs/redirects.json.backup && echo "Created backup"
+                cp result/redirects.json docs/redirects.json
+                echo "  Updated docs/redirects.json"
+                echo "To verify: nix build .#manualHtml"
+              '';
+            }
+          }/bin/update-redirects";
         };
       }
-  ) // {
-    herculesCI.ciSystems = [ "x86_64-linux" ];
-  };
+    )
+    // flake-utils.lib.eachSystem [ "x86_64-linux" ] (
+      system:
+      let
+        pkgs = pkgs' system;
+      in
+      {
+        checks =
+          let
+            inherit (pkgs.lib)
+              foldl
+              foldlAttrs
+              mergeAttrs
+              ;
+
+            importFiles =
+              files:
+              map (
+                m:
+                pkgs.callPackage m {
+                  shb = self.lib.${system};
+                }
+              ) files;
+
+            mergeTests = foldl mergeAttrs { };
+
+            flattenAttrs =
+              root: attrset:
+              foldlAttrs (
+                acc: name: value:
+                acc
+                // {
+                  "${root}_${name}" = value;
+                }
+              ) { } attrset;
+
+            vm_test =
+              name: path:
+              flattenAttrs "vm_${name}" (
+                removeAttrs
+                  (pkgs.callPackage path {
+                    shb = self.lib.${system};
+                  })
+                  [
+                    "override"
+                    "overrideDerivation"
+                  ]
+              );
+          in
+          (
+            {
+              modules = self.lib.${system}.check {
+                inherit pkgs;
+                tests = mergeTests (importFiles [
+                  ./test/modules/davfs.nix
+                  # TODO: Make this not use IFD
+                  ./test/modules/lib.nix
+                ]);
+              };
+
+              # TODO: Make this not use IFD
+              lib = nix-flake-tests.lib.check {
+                inherit pkgs;
+                tests = pkgs.callPackage ./test/modules/lib.nix {
+                  shb = self.lib.${system};
+                };
+              };
+            }
+            // (vm_test "arr" ./test/services/arr.nix)
+            // (vm_test "audiobookshelf" ./test/services/audiobookshelf.nix)
+            // (vm_test "deluge" ./test/services/deluge.nix)
+            // (vm_test "firefly-iii" ./test/services/firefly-iii.nix)
+            // (vm_test "forgejo" ./test/services/forgejo.nix)
+            // (vm_test "grocy" ./test/services/grocy.nix)
+            // (vm_test "hledger" ./test/services/hledger.nix)
+            // (vm_test "immich" ./test/services/immich.nix)
+            // (vm_test "homeassistant" ./test/services/home-assistant.nix)
+            // (vm_test "homepage" ./test/services/homepage.nix)
+            // (vm_test "jellyfin" ./test/services/jellyfin.nix)
+            // (vm_test "karakeep" ./test/services/karakeep.nix)
+            // (vm_test "mailserver" ./test/services/mailserver.nix)
+            // (vm_test "nextcloud" ./test/services/nextcloud.nix)
+            // (vm_test "open-webui" ./test/services/open-webui.nix)
+            // (vm_test "paperless" ./test/services/paperless.nix)
+            // (vm_test "pinchflat" ./test/services/pinchflat.nix)
+            // (vm_test "vaultwarden" ./test/services/vaultwarden.nix)
+
+            // (vm_test "authelia" ./test/blocks/authelia.nix)
+            // (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
+            // (vm_test "lib" ./test/blocks/lib.nix)
+            // (vm_test "lldap" ./test/blocks/lldap.nix)
+            // (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
+            // (vm_test "monitoring" ./test/blocks/monitoring.nix)
+            // (vm_test "postgresql" ./test/blocks/postgresql.nix)
+            // (vm_test "restic" ./test/blocks/restic.nix)
+            // (vm_test "ssl" ./test/blocks/ssl.nix)
+            // (vm_test "zfs" ./test/blocks/zfs.nix)
+
+            // (vm_test "contracts-backup" ./test/contracts/backup.nix)
+            // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
+            // (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix)
+            // (vm_test "contracts-secret" ./test/contracts/secret.nix)
+          );
+
+      }
+    )
+    // {
+      herculesCI.ciSystems = [ "x86_64-linux" ];
+
+      nixosModules.default = {
+        imports = [
+          # blocks
+          self.nixosModules.authelia
+          self.nixosModules.borgbackup
+          self.nixosModules.davfs
+          self.nixosModules.hardcodedsecret
+          self.nixosModules.lldap
+          self.nixosModules.mitmdump
+          self.nixosModules.monitoring
+          self.nixosModules.nginx
+          self.nixosModules.postgresql
+          self.nixosModules.restic
+          self.nixosModules.sanoid
+          self.nixosModules.ssl
+          self.nixosModules.tinyproxy
+          self.nixosModules.vpn
+          self.nixosModules.zfs
+
+          # services
+          self.nixosModules.arr
+          self.nixosModules.audiobookshelf
+          self.nixosModules.deluge
+          self.nixosModules.firefly-iii
+          self.nixosModules.forgejo
+          self.nixosModules.grocy
+          self.nixosModules.hledger
+          self.nixosModules.immich
+          self.nixosModules.home-assistant
+          self.nixosModules.homepage
+          self.nixosModules.jellyfin
+          self.nixosModules.karakeep
+          self.nixosModules.mailserver
+          self.nixosModules.nextcloud-server
+          self.nixosModules.open-webui
+          self.nixosModules.pinchflat
+          self.nixosModules.paperless
+          self.nixosModules.vaultwarden
+        ];
+      };
+
+      nixosModules.lib = lib/module.nix;
+
+      nixosModules.authelia = modules/blocks/authelia.nix;
+      nixosModules.borgbackup = modules/blocks/borgbackup.nix;
+      nixosModules.davfs = modules/blocks/davfs.nix;
+      nixosModules.hardcodedsecret = modules/blocks/hardcodedsecret.nix;
+      nixosModules.lldap = modules/blocks/lldap.nix;
+      nixosModules.mitmdump = modules/blocks/mitmdump.nix;
+      nixosModules.monitoring = modules/blocks/monitoring.nix;
+      nixosModules.nginx = modules/blocks/nginx.nix;
+      nixosModules.postgresql = modules/blocks/postgresql.nix;
+      nixosModules.restic = modules/blocks/restic.nix;
+      nixosModules.sanoid = modules/blocks/sanoid.nix;
+      nixosModules.sops = modules/blocks/sops.nix;
+      nixosModules.ssl = modules/blocks/ssl.nix;
+      nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
+      nixosModules.vpn = modules/blocks/vpn.nix;
+      nixosModules.zfs = modules/blocks/zfs.nix;
+
+      nixosModules.arr = modules/services/arr.nix;
+      nixosModules.audiobookshelf = modules/services/audiobookshelf.nix;
+      nixosModules.deluge = modules/services/deluge.nix;
+      nixosModules.firefly-iii = modules/services/firefly-iii.nix;
+      nixosModules.forgejo = modules/services/forgejo.nix;
+      nixosModules.grocy = modules/services/grocy.nix;
+      nixosModules.hledger = modules/services/hledger.nix;
+      nixosModules.immich = modules/services/immich.nix;
+      nixosModules.home-assistant = modules/services/home-assistant.nix;
+      nixosModules.homepage = modules/services/homepage.nix;
+      nixosModules.jellyfin = modules/services/jellyfin.nix;
+      nixosModules.karakeep = modules/services/karakeep.nix;
+      nixosModules.mailserver = modules/services/mailserver.nix;
+      nixosModules.nextcloud-server = modules/services/nextcloud-server.nix;
+      nixosModules.open-webui = modules/services/open-webui.nix;
+      nixosModules.paperless = modules/services/paperless.nix;
+      nixosModules.pinchflat = modules/services/pinchflat.nix;
+      nixosModules.vaultwarden = modules/services/vaultwarden.nix;
+    };
 }

lib/default.nix

@@ -1,299 +1,460 @@
 { pkgs, lib }:
 let
   inherit (builtins) isAttrs hasAttr;
-  inherit (lib) any concatMapStringsSep concatStringsSep mapAttrsToList;
-in
-rec {
-  # Replace secrets in a file.
-  # - userConfig is an attrset that will produce a config file.
-  # - resultPath is the location the config file should have on the filesystem.
-  # - generator is a function taking two arguments name and value and returning path in the nix
-  #   nix store where the
-  replaceSecrets = { userConfig, resultPath, generator, user ? null, permissions ? "u=r,g=r,o=" }:
-    let
-      configWithTemplates = withReplacements userConfig;
+  inherit (lib)
+    any
+    concatMapStringsSep
+    concatStringsSep
+    escapeShellArg
+    ;
+  shb = rec {
+    # Replace secrets in a file.
+    # - userConfig is an attrset that will produce a config file.
+    # - resultPath is the location the config file should have on the filesystem.
+    # - generator is a function taking two arguments name and value and returning path in the nix
+    #   nix store where the
+    replaceSecrets =
+      {
+        userConfig,
+        resultPath,
+        generator,
+        user ? null,
+        permissions ? "u=r,g=r,o=",
+      }:
+      let
+        configWithTemplates = withReplacements userConfig;
 
-      nonSecretConfigFile = generator "template" configWithTemplates;
+        nonSecretConfigFile = generator "template" configWithTemplates;
 
-      replacements = getReplacements userConfig;
-    in
+        replacements = getReplacements userConfig;
+      in
       replaceSecretsScript {
         file = nonSecretConfigFile;
         inherit resultPath replacements;
         inherit user permissions;
       };
 
-  replaceSecretsFormatAdapter = format: format.generate;
-  replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value);
+    replaceSecretsFormatAdapter = format: format.generate;
+    replaceSecretsGeneratorAdapter =
+      generator: name: value:
+      pkgs.writeText "generator " (generator value);
+    toEnvVar = replaceSecretsGeneratorAdapter (
+      v: (lib.generators.toINIWithGlobalSection { } { globalSection = v; })
+    );
 
-  template = file: newPath: replacements: replaceSecretsScript {
-    inherit file replacements;
-    resultPath = newPath;
-  };
+    template =
+      file: newPath: replacements:
+      replaceSecretsScript {
+        inherit file replacements;
+        resultPath = newPath;
+      };
 
-  genReplacement = secret:
-    let
-      t = { transform ? null, ... }: if isNull transform then x: x else transform;
-    in
+    genReplacement =
+      secret:
+      let
+        t =
+          {
+            transform ? null,
+            ...
+          }:
+          if isNull transform then x: x else transform;
+      in
       lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
 
-  replaceSecretsScript = { file, resultPath, replacements, user ? null, permissions ? "u=r,g=r,o=" }:
-    let
-      templatePath = resultPath + ".template";
+    replaceSecretsScript =
+      {
+        file,
+        resultPath,
+        replacements,
+        user ? null,
+        permissions ? "u=r,g=r,o=",
+      }:
+      let
+        templatePath = resultPath + ".template";
 
-      # We check that the files containing the secrets have the
-      # correct permissions for us to read them in this separate
-      # step. Otherwise, the $(cat ...) commands inside the sed
-      # replacements could fail but not fail individually but
-      # not fail the whole script.
-      checkPermissions = concatMapStringsSep "\n" (pattern: "cat ${pattern.source} > /dev/null") replacements;
+        # We check that the files containing the secrets have the
+        # correct permissions for us to read them in this separate
+        # step. Otherwise, the $(cat ...) commands inside the sed
+        # replacements could fail but not fail individually but
+        # not fail the whole script.
+        checkPermissions = concatMapStringsSep "\n" (
+          pattern: "cat ${pattern.source} > /dev/null"
+        ) replacements;
 
-      sedPatterns = concatMapStringsSep " " (pattern: "-e \"s|${pattern.name}|${pattern.value}|\"") (map genReplacement replacements);
+        generatedReplacements = map genReplacement replacements;
 
-      sedCmd = if replacements == []
-               then "cat"
-               else "${pkgs.gnused}/bin/sed ${sedPatterns}";
-    in
+        replaceScript = pkgs.writers.writePython3 "replace-secret" { } ''
+          import argparse
+          import pathlib
+
+
+          def parse_args() -> argparse.Namespace:
+              parser = argparse.ArgumentParser()
+              parser.add_argument("--file", required=True, type=pathlib.Path)
+              parser.add_argument("--marker", required=True)
+              parser.add_argument("--replacement", required=True)
+              return parser.parse_args()
+
+
+          args = parse_args()
+          content = args.file.read_text()
+          args.file.write_text(content.replace(args.marker, args.replacement))
+        '';
+
+        replaceCommands = concatMapStringsSep "\n" (pattern: ''
+          ${replaceScript} \
+            --file ${escapeShellArg resultPath} \
+            --marker ${escapeShellArg pattern.name} \
+            --replacement "${pattern.value}"
+        '') generatedReplacements;
+
+        replaceCmd =
+          if replacements == [ ] then
+            "cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}"
+          else
+            ''
+              cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}
+              ${replaceCommands}
+            '';
+      in
       ''
-      set -euo pipefail
+        set -euo pipefail
 
-      ${checkPermissions}
+        ${checkPermissions}
 
-      mkdir -p $(dirname ${templatePath})
-      ln -fs ${file} ${templatePath}
-      rm -f ${resultPath}
-      touch ${resultPath}
-      '' + (lib.optionalString (user != null) ''
-      chown ${user} ${resultPath}
-      '') + ''
-      ${sedCmd} ${templatePath} > ${resultPath}
-      chmod ${permissions} ${resultPath}
+        mkdir -p $(dirname ${templatePath})
+        ln -fs ${file} ${templatePath}
+        rm -f ${resultPath}
+        touch ${resultPath}
+      ''
+      + (lib.optionalString (user != null) ''
+        chown ${user} ${resultPath}
+      '')
+      + ''
+        ${replaceCmd}
+        chmod ${permissions} ${resultPath}
       '';
 
-  secretFileType = lib.types.submodule {
-    options = {
-      source = lib.mkOption {
-        type = lib.types.path;
-        description = "File containing the value.";
-      };
+    secretFileType = lib.types.submodule {
+      options = {
+        source = lib.mkOption {
+          type = lib.types.path;
+          description = "File containing the value.";
+        };
 
-      transform = lib.mkOption {
-        type = lib.types.raw;
-        description = "An optional function to transform the secret.";
-        default = null;
-        example = lib.literalExpression ''
-        v: "prefix-$${v}-suffix"
-        '';
+        transform = lib.mkOption {
+          type = lib.types.raw;
+          description = "An optional function to transform the secret.";
+          default = null;
+          example = lib.literalExpression ''
+            v: "prefix-$${v}-suffix"
+          '';
+        };
       };
     };
-  };
 
-  secretName = names:
-    "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) names)}%";
+    secretName =
+      names: "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) names)}%";
 
-  withReplacements = attrs:
-    let
-      valueOrReplacement = name: value:
-        if !(builtins.isAttrs value && value ? "source")
-        then value
-        else secretName name;
-    in
-      mapAttrsRecursiveCond (v: ! v ? "source") valueOrReplacement attrs;
+    withReplacements =
+      attrs:
+      let
+        valueOrReplacement =
+          name: value: if !(builtins.isAttrs value && value ? "source") then value else secretName name;
+      in
+      mapAttrsRecursiveCond (v: !v ? "source") valueOrReplacement attrs;
 
-  getReplacements = attrs:
-    let
-      addNameField = name: value:
-        if !(builtins.isAttrs value && value ? "source")
-        then value
-        else value // { name = name; };
+    getReplacements =
+      attrs:
+      let
+        addNameField =
+          name: value:
+          if !(builtins.isAttrs value && value ? "source") then value else value // { name = name; };
 
-      secretsWithName = mapAttrsRecursiveCond (v: ! v ? "source") addNameField attrs;
-    in
+        secretsWithName = mapAttrsRecursiveCond (v: !v ? "source") addNameField attrs;
+      in
       collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
-      
-  # Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
-  mapAttrsRecursiveCond =
-    # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
-    cond:
-    # A function, given a list of attribute names and a value, returns a new value.
-    f:
-    # Attribute set or list to recursively map over.
-    set:
-    let
-      recurse = path: val:
-        if builtins.isAttrs val && cond val
-        then lib.attrsets.mapAttrs (n: v: recurse (path ++ [n]) v) val
-        else if builtins.isList val && cond val
-        then lib.lists.imap0 (i: v: recurse (path ++ [(builtins.toString i)]) v) val
-        else f path val;
-    in recurse [] set;
 
-  # Like lib.attrsets.collect but also recurses on lists.
-  collect =
-  # Given an attribute's value, determine if recursion should stop.
-  pred:
-  # The attribute set to recursively collect.
-  attrs:
-    if pred attrs then
-      [ attrs ]
-    else if builtins.isAttrs attrs then
-      lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
-    else if builtins.isList attrs then
-      lib.lists.concatMap (collect pred) attrs
-    else
-      [];
+    # Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
+    mapAttrsRecursiveCond =
+      # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
+      cond:
+      # A function, given a list of attribute names and a value, returns a new value.
+      f:
+      # Attribute set or list to recursively map over.
+      set:
+      let
+        recurse =
+          path: val:
+          if builtins.isAttrs val && cond val then
+            lib.attrsets.mapAttrs (n: v: recurse (path ++ [ n ]) v) val
+          else if builtins.isList val && cond val then
+            lib.lists.imap0 (i: v: recurse (path ++ [ (builtins.toString i) ]) v) val
+          else
+            f path val;
+      in
+      recurse [ ] set;
 
-  indent = i: str: lib.concatMapStringsSep "\n" (x: (lib.strings.replicate i " ") + x) (lib.splitString "\n" str);
+    # Like lib.attrsets.collect but also recurses on lists.
+    collect =
+      # Given an attribute's value, determine if recursion should stop.
+      pred:
+      # The attribute set to recursively collect.
+      attrs:
+      if pred attrs then
+        [ attrs ]
+      else if builtins.isAttrs attrs then
+        lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
+      else if builtins.isList attrs then
+        lib.lists.concatMap (collect pred) attrs
+      else
+        [ ];
+
+    indent =
+      i: str:
+      lib.concatMapStringsSep "\n" (x: (lib.strings.replicate i " ") + x) (lib.splitString "\n" str);
+
+    # Generator for XML
+    formatXML =
+      {
+        enclosingRoot ? null,
+      }:
+      {
+        type =
+          with lib.types;
+          let
+            valueType =
+              nullOr (oneOf [
+                bool
+                int
+                float
+                str
+                path
+                (attrsOf valueType)
+                (listOf valueType)
+              ])
+              // {
+                description = "XML value";
+              };
+          in
+          valueType;
+
+        generate =
+          name: value:
+          pkgs.callPackage (
+            { runCommand, python3 }:
+            runCommand "config"
+              {
+                value = builtins.toJSON (if enclosingRoot == null then value else { ${enclosingRoot} = value; });
+                passAsFile = [ "value" ];
+              }
+              (
+                pkgs.writers.writePython3 "dict2xml"
+                  {
+                    libraries = with python3.pkgs; [
+                      python
+                      dict2xml
+                    ];
+                  }
+                  ''
+                    import os
+                    import json
+                    from dict2xml import dict2xml
+
+                    with open(os.environ["valuePath"]) as f:
+                        content = json.loads(f.read())
+                        if content is None:
+                            print("Could not parse env var valuePath as json")
+                            os.exit(2)
+                        with open(os.environ["out"], "w") as out:
+                            out.write(dict2xml(content))
+                  ''
+              )
+          ) { };
 
-  # Generator for XML
-  formatXML = {
-    enclosingRoot ? null
-  }: {
-    type = with lib.types; let
-      valueType = nullOr (oneOf [
-        bool
-        int
-        float
-        str
-        path
-        (attrsOf valueType)
-        (listOf valueType)
-      ]) // {
-        description = "XML value";
       };
-    in valueType;
 
-    generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
-      value = builtins.toJSON (
-        if enclosingRoot == null then
-          value
-        else
-          { ${enclosingRoot} = value; });
-      passAsFile = [ "value" ];
-    } (pkgs.writers.writePython3 "dict2xml" {
-      libraries = with python3.pkgs; [ python dict2xml ];
-    } ''
-      import os
-      import json
-      from dict2xml import dict2xml
-
-      with open(os.environ["valuePath"]) as f:
-          content = json.loads(f.read())
-          if content is None:
-              print("Could not parse env var valuePath as json")
-              os.exit(2)
-          with open(os.environ["out"], "w") as out:
-              out.write(dict2xml(content))
-    '')) {};
-
-  };
-
-  parseXML = xml:
-    let
-      xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
-        inherit xml;
-        passAsFile = [ "xml" ];
-      } (pkgs.writers.writePython3 "xml2json" {
-        libraries = with python3.pkgs; [ python ];
-      } ''
-        import os
-        import json
-        from collections import ChainMap
-        from xml.etree import ElementTree
-        
-
-        def xml_to_dict_recursive(root):
-            all_descendants = list(root)
-            if len(all_descendants) == 0:
-                return {root.tag: root.text}
-            else:
-                merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
-                return {root.tag: dict(merged_dict)}
+    parseXML =
+      xml:
+      let
+        xmlToJsonFile = pkgs.callPackage (
+          { runCommand, python3 }:
+          runCommand "config"
+            {
+              inherit xml;
+              passAsFile = [ "xml" ];
+            }
+            (
+              pkgs.writers.writePython3 "xml2json"
+                {
+                  libraries = with python3.pkgs; [ python ];
+                }
+                ''
+                  import os
+                  import json
+                  from collections import ChainMap
+                  from xml.etree import ElementTree
 
 
-        with open(os.environ["xmlPath"]) as f:
-            root = ElementTree.XML(f.read())
-            xml = xml_to_dict_recursive(root)
-            j = json.dumps(xml)
+                  def xml_to_dict_recursive(root):
+                      all_descendants = list(root)
+                      if len(all_descendants) == 0:
+                          return {root.tag: root.text}
+                      else:
+                          merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
+                          return {root.tag: dict(merged_dict)}
 
-            with open(os.environ["out"], "w") as out:
-                out.write(j)
-      '')) {};
-    in
+
+                  with open(os.environ["xmlPath"]) as f:
+                      root = ElementTree.XML(f.read())
+                      xml = xml_to_dict_recursive(root)
+                      j = json.dumps(xml)
+
+                      with open(os.environ["out"], "w") as out:
+                          out.write(j)
+                ''
+            )
+        ) { };
+      in
       builtins.fromJSON (builtins.readFile xmlToJsonFile);
 
-  renameAttrName = attrset: from: to:
-    (lib.attrsets.filterAttrs (name: v: name == from) attrset) // {
-      ${to} = attrset.${from};
-    };
+    renameAttrName =
+      attrset: from: to:
+      (lib.attrsets.filterAttrs (name: v: name == from) attrset)
+      // {
+        ${to} = attrset.${from};
+      };
 
-  # Taken from https://github.com/antifuchs/nix-flake-tests/blob/main/default.nix
-  # with a nicer diff display function.
-  check = { pkgs, tests }:
-    let
-      formatValue = val:
-        if (builtins.isList val || builtins.isAttrs val) then builtins.toJSON val
-        else builtins.toString val;
+    # Taken from https://github.com/antifuchs/nix-flake-tests/blob/main/default.nix
+    # with a nicer diff display function.
+    check =
+      { pkgs, tests }:
+      let
+        formatValue =
+          val:
+          if (builtins.isList val || builtins.isAttrs val) then
+            builtins.toJSON val
+          else
+            builtins.toString val;
 
-      resultToString = { name, expected, result }:
-        builtins.readFile (pkgs.runCommand "nix-flake-tests-error" {
-          expected = formatValue expected;
-          result = formatValue result;
-          passAsFile = [ "expected" "result" ];
-        } ''
-          echo "${name} failed (- expected, + result)" > $out
-          cp ''${expectedPath} ''${expectedPath}.json
-          cp ''${resultPath} ''${resultPath}.json
-          ${pkgs.deepdiff}/bin/deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
-        '');
+        resultToString =
+          {
+            name,
+            expected,
+            result,
+          }:
+          builtins.readFile (
+            pkgs.runCommand "nix-flake-tests-error"
+              {
+                expected = formatValue expected;
+                result = formatValue result;
+                passAsFile = [
+                  "expected"
+                  "result"
+                ];
 
-      results = pkgs.lib.runTests tests;
-    in
-    if results != [ ] then
-      builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
-    else
-      pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
+                nativeBuildInputs = [
+                  (pkgs.python3.withPackages (ps: [ ps.deepdiff ] ++ ps.deepdiff.optional-dependencies.cli))
+                ];
+              }
+              ''
+                echo "${name} failed (- expected, + result)" > $out
+                cp ''${expectedPath} ''${expectedPath}.json
+                cp ''${resultPath} ''${resultPath}.json
+                deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
+              ''
+          );
 
+        results = pkgs.lib.runTests tests;
+      in
+      if results != [ ] then
+        builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
+      else
+        pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
 
-  genConfigOutOfBandSystemd = { config, configLocation, generator, user ? null, permissions ? "u=r,g=r,o=" }:
-    {
-      loadCredentials = getLoadCredentials "source" config;
-      preStart = lib.mkBefore (replaceSecrets {
-        userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
-        resultPath = configLocation;
-        inherit generator;
-        inherit user permissions;
-      });
-    };
+    genConfigOutOfBandSystemd =
+      {
+        config,
+        configLocation,
+        generator,
+        user ? null,
+        permissions ? "u=r,g=r,o=",
+      }:
+      {
+        loadCredentials = getLoadCredentials "source" config;
+        preStart = lib.mkBefore (replaceSecrets {
+          userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
+          resultPath = configLocation;
+          inherit generator;
+          inherit user permissions;
+        });
+      };
 
-  updateToLoadCredentials = sourceField: rootDir: attrs:
-    let
-      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+    updateToLoadCredentials =
+      sourceField: rootDir: attrs:
+      let
+        hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
 
-      valueOrLoadCredential = path: value:
-        if ! (hasPlaceholderField value)
-        then value
-        else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
-    in
-      mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
+        valueOrLoadCredential =
+          path: value:
+          if !(hasPlaceholderField value) then
+            value
+          else
+            value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
+      in
+      mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) valueOrLoadCredential attrs;
 
-  getLoadCredentials = sourceField: attrs:
-    let
-      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+    getLoadCredentials =
+      sourceField: attrs:
+      let
+        hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
 
-      addPathField = path: value:
-        if ! (hasPlaceholderField value)
-        then value
-        else value // { inherit path; };
+        addPathField =
+          path: value: if !(hasPlaceholderField value) then value else value // { inherit path; };
 
-      secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
+        secretsWithPath = mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) addPathField attrs;
 
-      allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
+        allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
 
-      genLoadCredentials = secret:
-        "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
-    in
+        genLoadCredentials = secret: "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
+      in
       map genLoadCredentials allSecrets;
 
-  anyNotNull = any (x: x != null);
+    anyNotNull = any (x: x != null);
+
+    mkJellyfinPlugin =
+      {
+        pname,
+        version,
+        hash,
+        url,
+      }:
+      pkgs.callPackage (
+        { stdenv, fetchzip }:
+        stdenv.mkDerivation (finalAttrs: {
+          inherit pname version;
+
+          src = fetchzip {
+            inherit url hash;
+            stripRoot = false;
+          };
+
+          dontBuild = true;
+
+          installPhase = ''
+            mkdir $out
+            cp -r . $out
+          '';
+        })
+      ) { };
+
+    update =
+      attr: fn: attrset:
+      attrset // { ${attr} = fn attrset.${attr}; };
+
+  };
+in
+shb
+// {
+  homepage = pkgs.callPackage ./homepage.nix { inherit shb; };
 }

lib/homepage.nix

@@ -0,0 +1,92 @@
+{ lib, shb }:
+let
+  sort =
+    attr: vs:
+    map (v: { ${v.name} = v.${attr}; }) (
+      lib.sortOn (v: v.sortOrder) (lib.mapAttrsToList (n: v: v // { name = n; }) vs)
+    );
+
+  slufigy = builtins.replaceStrings [ "-" ] [ "_" ];
+
+  mkService =
+    groupName: serviceName:
+    {
+      request,
+      ...
+    }:
+    apiKey: settings:
+    lib.recursiveUpdate (
+      {
+        href = request.externalUrl;
+        siteMonitor = if (request.internalUrl == null) then null else request.internalUrl;
+        icon = "sh-${lib.toLower serviceName}";
+      }
+      // lib.optionalAttrs (apiKey != null) {
+        widget = {
+          # Duplicating because widgets call the api key various names
+          # and duplicating is a hacky but easy solution.
+          key = "{{HOMEPAGE_FILE_${slufigy groupName}_${slufigy serviceName}}}";
+          password = "{{HOMEPAGE_FILE_${slufigy groupName}_${slufigy serviceName}}}";
+          type = lib.toLower serviceName;
+          url = if (request.internalUrl != null) then request.internalUrl else request.externalUrl;
+        };
+      }
+    ) settings;
+
+  asServiceGroup =
+    cfg:
+    sort "services" (
+      lib.mapAttrs (
+        groupName: groupCfg:
+        shb.update "services" (
+          services:
+          sort "dashboard" (
+            lib.mapAttrs (
+              serviceName: serviceCfg:
+              shb.update "dashboard" (
+                dashboard:
+                (mkService groupName serviceName) dashboard serviceCfg.apiKey (serviceCfg.settings or { })
+              ) serviceCfg
+            ) services
+          )
+        ) groupCfg
+      ) cfg
+    );
+
+  allKeys =
+    cfg:
+    let
+      flat = lib.flatten (
+        lib.mapAttrsToList (
+          groupName: groupCfg:
+          lib.mapAttrsToList (
+            serviceName: serviceCfg:
+            lib.optionalAttrs (serviceCfg.apiKey != null) {
+              inherit serviceName groupName;
+              inherit (serviceCfg.apiKey.result) path;
+            }
+          ) groupCfg.services
+        ) cfg
+      );
+
+      flatWithApiKey = builtins.filter (v: v != { }) flat;
+    in
+    builtins.listToAttrs (
+      map (
+        {
+          groupName,
+          serviceName,
+          path,
+        }:
+        lib.nameValuePair "${slufigy groupName}_${slufigy serviceName}" path
+      ) flatWithApiKey
+    );
+in
+{
+  inherit
+    allKeys
+    asServiceGroup
+    mkService
+    sort
+    ;
+}

lib/module.nix

@@ -0,0 +1,10 @@
+{ pkgs, lib, ... }:
+let
+  shb = (import ./default.nix { inherit pkgs lib; });
+in
+{
+  _module.args.shb = shb // {
+    test = pkgs.callPackage ../test/common.nix { };
+    contracts = pkgs.callPackage ../modules/contracts { inherit shb; };
+  };
+}

modules/blocks/authelia.nix

@@ -1,20 +1,33 @@
-{ config, options, pkgs, lib, ... }:
+{
+  config,
+  options,
+  pkgs,
+  lib,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.authelia;
   opt = options.shb.authelia;
 
-  contracts = pkgs.callPackage ../contracts {};
-  shblib = pkgs.callPackage ../../lib {};
-
   fqdn = "${cfg.subdomain}.${cfg.domain}";
   fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";
 
   autheliaCfg = config.services.authelia.instances.${fqdn};
 
   inherit (lib) hasPrefix;
+
+  listenPort = if cfg.debug then 9090 else 9091;
 in
 {
+  imports = [
+    ../../lib/module.nix
+    ./lldap.nix
+    ./mitmdump.nix
+    ./postgresql.nix
+  ];
+
   options.shb.authelia = {
     enable = lib.mkEnableOption "selfhostblocks.authelia";
 
@@ -38,7 +51,7 @@ in
 
     ssl = lib.mkOption {
       description = "Path to SSL files";
-      type = lib.types.nullOr contracts.ssl.certs;
+      type = lib.types.nullOr shb.contracts.ssl.certs;
       default = null;
     };
 
@@ -73,50 +86,50 @@ in
           jwtSecret = lib.mkOption {
             description = "JWT secret.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           ldapAdminPassword = lib.mkOption {
             description = "LDAP admin user password.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           sessionSecret = lib.mkOption {
             description = "Session secret.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           storageEncryptionKey = lib.mkOption {
-            description = "Storage encryption key.";
+            description = "Storage encryption key. Must be >= 20 characters.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           identityProvidersOIDCHMACSecret = lib.mkOption {
-            description = "Identity provider OIDC HMAC secret.";
+            description = "Identity provider OIDC HMAC secret. Must be >= 40 characters.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
@@ -127,10 +140,10 @@ in
               Generate one with `nix run nixpkgs#openssl -- genrsa -out keypair.pem 2048`
             '';
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
@@ -138,6 +151,30 @@ in
       };
     };
 
+    extraOidcClaimsPolicies = lib.mkOption {
+      description = "Extra OIDC claims policies.";
+      type = lib.types.attrsOf lib.types.attrs;
+      default = { };
+    };
+
+    extraOidcScopes = lib.mkOption {
+      description = "Extra OIDC scopes.";
+      type = lib.types.attrsOf lib.types.attrs;
+      default = { };
+    };
+
+    extraOidcAuthorizationPolicies = lib.mkOption {
+      description = "Extra OIDC authorization policies.";
+      type = lib.types.attrsOf lib.types.attrs;
+      default = { };
+    };
+
+    extraDefinitions = lib.mkOption {
+      description = "Extra definitions.";
+      type = lib.types.attrsOf lib.types.attrs;
+      default = { };
+    };
+
     oidcClients = lib.mkOption {
       description = "OIDC clients";
       default = [
@@ -147,68 +184,92 @@ in
           client_secret.source = pkgs.writeText "dummy.secret" "dummy_client_secret";
           public = false;
           authorization_policy = "one_factor";
-          redirect_uris = [];
+          redirect_uris = [ ];
         }
       ];
-      type = lib.types.listOf (lib.types.submodule {
-        freeformType = lib.types.attrsOf lib.types.anything;
+      type = lib.types.listOf (
+        lib.types.submodule {
+          freeformType = lib.types.attrsOf lib.types.anything;
 
-        options = {
-          client_id = lib.mkOption {
-            type = lib.types.str;
-            description = "Unique identifier of the OIDC client.";
+          options = {
+            client_id = lib.mkOption {
+              type = lib.types.str;
+              description = "Unique identifier of the OIDC client.";
+            };
+
+            client_name = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = "Human readable description of the OIDC client.";
+              default = null;
+            };
+
+            client_secret = lib.mkOption {
+              type = shb.secretFileType;
+              description = ''
+                File containing the shared secret with the OIDC client.
+
+                Generate with:
+
+                ```
+                nix run nixpkgs#authelia -- \
+                    crypto hash generate pbkdf2 \
+                    --variant sha512 \
+                    --random \
+                    --random.length 72 \
+                    --random.charset rfc3986
+                ```
+              '';
+            };
+
+            public = lib.mkOption {
+              type = lib.types.bool;
+              description = "If the OIDC client is public or not.";
+              default = false;
+              apply = v: if v then "true" else "false";
+            };
+
+            authorization_policy = lib.mkOption {
+              type = lib.types.enum (
+                [
+                  "one_factor"
+                  "two_factor"
+                ]
+                ++ lib.attrNames cfg.extraOidcAuthorizationPolicies
+              );
+              description = "Require one factor (password) or two factor (device) authentication.";
+              default = "one_factor";
+            };
+
+            redirect_uris = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = "List of uris that are allowed to be redirected to.";
+            };
+
+            scopes = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = "Scopes to ask for. See https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims";
+              example = [
+                "openid"
+                "profile"
+                "email"
+                "groups"
+              ];
+              default = [ ];
+            };
+
+            claims_policy = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Claim policy.
+
+                Defaults to 'default' to provide a backwards compatible experience.
+                Read [this document](https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims/#restore-functionality-prior-to-claims-parameter) for more information.
+              '';
+              default = "default";
+            };
           };
-
-          client_name = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
-            description = "Human readable description of the OIDC client.";
-            default = null;
-          };
-
-          client_secret = lib.mkOption {
-            type = shblib.secretFileType;
-            description = ''
-            File containing the shared secret with the OIDC client.
-
-            Generate with:
-
-            ```
-            nix run nixpkgs#authelia -- \
-                crypto hash generate pbkdf2 \
-                --variant sha512 \
-                --random \
-                --random.length 72 \
-                --random.charset rfc3986
-            ```
-            '';
-          };
-
-          public = lib.mkOption {
-            type = lib.types.bool;
-            description = "If the OIDC client is public or not.";
-            default = false;
-            apply = v: if v then "true" else "false";
-          };
-
-          authorization_policy = lib.mkOption {
-            type = lib.types.enum [ "one_factor" "two_factor" ];
-            description = "Require one factor (password) or two factor (device) authentication.";
-            default = "one_factor";
-          };
-
-          redirect_uris = lib.mkOption {
-            type = lib.types.listOf lib.types.str;
-            description = "List of uris that are allowed to be redirected to.";
-          };
-
-          scopes = lib.mkOption {
-            type = lib.types.listOf lib.types.str;
-            description = "Scopes to ask for";
-            example = [ "openid" "profile" "email" "groups" ];
-            default = [];
-          };
-        };
-      });
+        }
+      );
     };
 
     smtp = lib.mkOption {
@@ -221,54 +282,65 @@ in
       default = "/tmp/authelia-notifications";
       type = lib.types.oneOf [
         lib.types.str
-        (lib.types.nullOr (lib.types.submodule {
-          options = {
-            from_address = lib.mkOption {
-              type = lib.types.str;
-              description = "SMTP address from which the emails originate.";
-              example = "authelia@mydomain.com";
-            };
-            from_name = lib.mkOption {
-              type = lib.types.str;
-              description = "SMTP name from which the emails originate.";
-              default = "Authelia";
-            };
-            host = lib.mkOption {
-              type = lib.types.str;
-              description = "SMTP host to send the emails to.";
-            };
-            port = lib.mkOption {
-              type = lib.types.port;
-              description = "SMTP port to send the emails to.";
-              default = 25;
-            };
-            username = lib.mkOption {
-              type = lib.types.str;
-              description = "Username to connect to the SMTP host.";
-            };
-            password = lib.mkOption {
-              description = "File containing the password to connect to the SMTP host.";
-              type = lib.types.submodule {
-                options = contracts.secret.mkRequester {
-                  mode = "0400";
-                  owner = cfg.autheliaUser;
-                  restartUnits = [ "authelia-${fqdn}" ];
+        (lib.types.nullOr (
+          lib.types.submodule {
+            options = {
+              from_address = lib.mkOption {
+                type = lib.types.str;
+                description = "SMTP address from which the emails originate.";
+                example = "authelia@mydomain.com";
+              };
+              from_name = lib.mkOption {
+                type = lib.types.str;
+                description = "SMTP name from which the emails originate.";
+                default = "Authelia";
+              };
+              scheme = lib.mkOption {
+                description = "The protocl must be smtp, submission, or submissions. The only difference between these schemes are the default ports and submissions requires a TLS transport per SMTP Ports Security Measures, whereas submission and smtp use a standard TCP transport and typically enforce StartTLS.";
+                type = lib.types.enum [
+                  "smtp"
+                  "submission"
+                  "submissions"
+                ];
+                default = "smtp";
+              };
+              host = lib.mkOption {
+                type = lib.types.str;
+                description = "SMTP host to send the emails to.";
+              };
+              port = lib.mkOption {
+                type = lib.types.port;
+                description = "SMTP port to send the emails to.";
+                default = 25;
+              };
+              username = lib.mkOption {
+                type = lib.types.str;
+                description = "Username to connect to the SMTP host.";
+              };
+              password = lib.mkOption {
+                description = "File containing the password to connect to the SMTP host.";
+                type = lib.types.submodule {
+                  options = shb.contracts.secret.mkRequester {
+                    mode = "0400";
+                    owner = cfg.autheliaUser;
+                    restartUnits = [ "authelia-${fqdn}.service" ];
+                  };
                 };
               };
             };
-          };
-        }))
+          }
+        ))
       ];
     };
 
     rules = lib.mkOption {
       type = lib.types.listOf lib.types.anything;
       description = "Rule based clients";
-      default = [];
+      default = [ ];
     };
 
     mount = lib.mkOption {
-      type = contracts.mount;
+      type = shb.contracts.mount;
       description = ''
         Mount configuration. This is an output option.
 
@@ -282,12 +354,16 @@ in
         ```
       '';
       readOnly = true;
-      default = { path = "/var/lib/authelia-authelia.${cfg.domain}"; };
-      defaultText = { path = "/var/lib/authelia-authelia.example.com"; };
+      default = {
+        path = "/var/lib/authelia-authelia.${cfg.domain}";
+      };
+      defaultText = {
+        path = "/var/lib/authelia-authelia.example.com";
+      };
     };
 
     mountRedis = lib.mkOption {
-      type = contracts.mount;
+      type = shb.contracts.mount;
       description = ''
         Mount configuration for Redis. This is an output option.
 
@@ -301,7 +377,32 @@ in
         ```
       '';
       readOnly = true;
-      default = { path = "/var/lib/redis-authelia"; };
+      default = {
+        path = "/var/lib/redis-authelia";
+      };
+    };
+
+    debug = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Set logging level to debug and add a mitmdump instance
+        to see exactly what Authelia receives and sends back.
+      '';
+    };
+
+    dashboard = lib.mkOption {
+      description = ''
+        Dashboard contract consumer
+      '';
+      default = { };
+      type = lib.types.submodule {
+        options = shb.contracts.dashboard.mkRequester {
+          externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
+          externalUrlText = "https://\${config.shb.authelia.subdomain}.\${config.shb.authelia.domain}";
+          internalUrl = "http://127.0.0.1:${toString listenPort}";
+        };
+      };
     };
   };
 
@@ -319,7 +420,7 @@ in
 
     # Overriding the user name so we don't allow any weird characters anywhere. For example, postgres users do not accept the '.'.
     users = {
-      groups.${autheliaCfg.user} = {};
+      groups.${autheliaCfg.user} = { };
       users.${autheliaCfg.user} = {
         isSystemUser = true;
         group = autheliaCfg.user;
@@ -333,46 +434,37 @@ in
       secrets = {
         jwtSecretFile = cfg.secrets.jwtSecret.result.path;
         storageEncryptionKeyFile = cfg.secrets.storageEncryptionKey.result.path;
+        sessionSecretFile = cfg.secrets.sessionSecret.result.path;
+        oidcIssuerPrivateKeyFile = cfg.secrets.identityProvidersOIDCIssuerPrivateKey.result.path;
+        oidcHmacSecretFile = cfg.secrets.identityProvidersOIDCHMACSecret.result.path;
       };
       # See https://www.authelia.com/configuration/methods/secrets/
       environmentVariables = {
         AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD_FILE = toString cfg.secrets.ldapAdminPassword.result.path;
-        AUTHELIA_SESSION_SECRET_FILE = toString cfg.secrets.sessionSecret.result.path;
         # Not needed since we use peer auth.
         # AUTHELIA_STORAGE_POSTGRES_PASSWORD_FILE = "/run/secrets/authelia/postgres_password";
-        AUTHELIA_STORAGE_ENCRYPTION_KEY_FILE = toString cfg.secrets.storageEncryptionKey.result.path;
-        AUTHELIA_IDENTITY_PROVIDERS_OIDC_HMAC_SECRET_FILE = toString cfg.secrets.identityProvidersOIDCHMACSecret.result.path;
-        AUTHELIA_IDENTITY_PROVIDERS_OIDC_ISSUER_PRIVATE_KEY_FILE = toString cfg.secrets.identityProvidersOIDCIssuerPrivateKey.result.path;
-
-        AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE = lib.mkIf (!(builtins.isString cfg.smtp)) (toString cfg.smtp.password.result.path);
+        AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE = lib.mkIf (!(builtins.isString cfg.smtp)) (
+          toString cfg.smtp.password.result.path
+        );
+        X_AUTHELIA_CONFIG_FILTERS = "template";
       };
       settings = {
-        server.address = "tcp://127.0.0.1:9091";
+        server.address = "tcp://127.0.0.1:${toString listenPort}";
 
         # Inspired from https://github.com/lldap/lldap/blob/7d1f5abc137821c500de99c94f7579761fc949d8/example_configs/authelia_config.yml
         authentication_backend = {
           refresh_interval = "5m";
-          password_reset = {
-            disable = "false";
-          };
+          # We allow password reset and change because the ldap user we use allows it.
+          password_reset.disable = "false";
+          password_change.disable = "false";
           ldap = {
-            implementation = "custom";
+            implementation = "lldap";
             address = "ldap://${cfg.ldapHostname}:${toString cfg.ldapPort}";
             timeout = "5s";
             start_tls = "false";
             base_dn = cfg.dcdomain;
-            additional_users_dn = "ou=people";
-            # Sign in with username or email.
-            users_filter = "(&(|({username_attribute}={input})({mail_attribute}={input}))(objectClass=person))";
-            additional_groups_dn = "ou=groups";
-            groups_filter = "(member={dn})";
+            # TODO: use user with less privilege and with lldap_password_manager group to be able to change passwords.
             user = "uid=admin,ou=people,${cfg.dcdomain}";
-            attributes = {
-              username = "uid";
-              group_name = "cn";
-              mail = "mail";
-              display_name = "displayName";
-            };
           };
         };
         totp = {
@@ -387,10 +479,12 @@ in
         # Inspired from https://www.authelia.com/configuration/session/introduction/ and https://www.authelia.com/configuration/session/redis
         session = {
           name = "authelia_session";
-          cookies = [{
-            domain = if isNull cfg.port then cfg.domain else "${cfg.domain}:${toString cfg.port}";
-            authelia_url = "https://${cfg.subdomain}.${cfg.domain}";
-          }];
+          cookies = [
+            {
+              domain = if isNull cfg.port then cfg.domain else "${cfg.domain}:${toString cfg.port}";
+              authelia_url = "https://${cfg.subdomain}.${cfg.domain}";
+            }
+          ];
           same_site = "lax";
           expiration = "1h";
           inactivity = "5m";
@@ -414,8 +508,7 @@ in
             filename = cfg.smtp;
           };
           smtp = lib.mkIf (!(builtins.isString cfg.smtp)) {
-            host = cfg.smtp.host;
-            port = cfg.smtp.port;
+            address = "${cfg.smtp.scheme}://${cfg.smtp.host}:${toString cfg.smtp.port}";
             username = cfg.smtp.username;
             sender = "${cfg.smtp.from_name} <${cfg.smtp.from_address}>";
             subject = "[Authelia] {title}";
@@ -427,7 +520,11 @@ in
           networks = [
             {
               name = "internal";
-              networks = [ "10.0.0.0/8" "172.16.0.0/12" "192.168.0.0/18" ];
+              networks = [
+                "10.0.0.0/8"
+                "172.16.0.0/12"
+                "192.168.0.0/18"
+              ];
             }
           ];
           rules = [
@@ -438,7 +535,8 @@ in
                 "^/api/.*"
               ];
             }
-          ] ++ cfg.rules;
+          ]
+          ++ cfg.rules;
         };
         telemetry = {
           metrics = {
@@ -446,6 +544,28 @@ in
             address = "tcp://127.0.0.1:9959";
           };
         };
+
+        log.level = if cfg.debug then "debug" else "info";
+      }
+      // {
+        identity_providers.oidc = {
+          claims_policies = {
+            # This default claim should go away at some point.
+            # https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims/#restore-functionality-prior-to-claims-parameter
+            default.id_token = [
+              "email"
+              "preferred_username"
+              "name"
+              "groups"
+            ];
+          }
+          // cfg.extraOidcClaimsPolicies;
+          scopes = cfg.extraOidcScopes;
+          authorization_policies = cfg.extraOidcAuthorizationPolicies;
+        };
+      }
+      // lib.optionalAttrs (cfg.extraDefinitions != { }) {
+        definitions = cfg.extraDefinitions;
       };
 
       settingsFiles = [ "/var/lib/authelia-${fqdn}/oidc_clients.yaml" ];
@@ -453,18 +573,22 @@ in
 
     systemd.services."authelia-${fqdn}".preStart =
       let
-        mkCfg = clients:
-          shblib.replaceSecrets {
+        mkCfg =
+          clients:
+          shb.replaceSecrets {
             userConfig = {
               identity_providers.oidc.clients = clients;
             };
             resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
-            generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {});
+            generator = shb.replaceSecretsGeneratorAdapter (lib.generators.toYAML { });
           };
       in
-        lib.mkBefore (mkCfg cfg.oidcClients + ''
-        ${pkgs.bash}/bin/bash -c '(while ! ${pkgs.netcat-openbsd}/bin/nc -z -v -w1 ${cfg.ldapHostname} ${toString cfg.ldapPort}; do echo "Waiting for port ${cfg.ldapHostname}:${toString cfg.ldapPort} to open..."; sleep 2; done); sleep 2'
-          '');
+      lib.mkBefore (
+        mkCfg cfg.oidcClients
+        + ''
+          ${pkgs.bash}/bin/bash -c '(while ! ${pkgs.netcat-openbsd}/bin/nc -z -v -w1 ${cfg.ldapHostname} ${toString cfg.ldapPort}; do echo "Waiting for port ${cfg.ldapHostname}:${toString cfg.ldapPort} to open..."; sleep 2; done); sleep 2'
+        ''
+      );
 
     services.nginx.virtualHosts.${fqdn} = {
       forceSSL = !(isNull cfg.ssl);
@@ -485,6 +609,8 @@ in
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $http_host;
+        proxy_set_header X-Forwarded-Uri $request_uri;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
@@ -497,7 +623,7 @@ in
             error_page 403 = /error/403;
             error_page 404 = /error/404;
         }
-        '';
+      '';
 
       locations."/api/verify".extraConfig = ''
         add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
@@ -510,7 +636,22 @@ in
 
         proxy_set_header Host $http_x_forwarded_host;
         proxy_pass http://127.0.0.1:9091;
-        '';
+      '';
+    };
+
+    # I would like this to live outside of the Authelia module.
+    # This will require a reverse proxy contract.
+    # Actually, not sure a full reverse proxy contract is needed.
+    shb.mitmdump.instances."authelia-${fqdn}" = lib.mkIf cfg.debug {
+      listenPort = 9091;
+      upstreamPort = 9090;
+      timeout = 30;
+      after = [ "authelia-${fqdn}.service" ];
+      enabledAddons = [ config.shb.mitmdump.addons.logger ];
+      extraArgs = [
+        "--set"
+        "verbose_pattern=/api"
+      ];
     };
 
     services.redis.servers.authelia = {
@@ -530,7 +671,7 @@ in
         job_name = "authelia";
         static_configs = [
           {
-            targets = ["127.0.0.1:9959"];
+            targets = [ "127.0.0.1:9959" ];
             labels = {
               "hostname" = config.networking.hostName;
               "domain" = cfg.domain;
@@ -539,5 +680,21 @@ in
         ];
       }
     ];
+
+    systemd.targets."authelia-${fqdn}" =
+      let
+        services = [
+          "authelia-${fqdn}.service"
+        ]
+        ++ lib.optionals cfg.debug [
+          config.shb.mitmdump.instances."authelia-${fqdn}".serviceName
+        ];
+      in
+      {
+        after = services;
+        requires = services;
+
+        wantedBy = [ "multi-user.target" ];
+      };
   };
 }

modules/blocks/authelia/docs/default.md

@@ -0,0 +1,244 @@
+# Authelia Block {#blocks-authelia}
+
+Defined in [`/modules/blocks/authelia.nix`](@REPO@/modules/blocks/authelia.nix).
+
+This block sets up an [Authelia][] service for Single-Sign On integration.
+
+[Authelia]: https://www.authelia.com/
+
+Compared to the upstream nixpkgs module, this module is tightly integrated
+with SHB which allows easy configuration of SSO with [OIDC integration](#blocks-authelia-shb-oidc)
+as well as some extensive [troubleshooting](#blocks-authelia-troubleshooting) features.
+
+Note that forward authentication is configured with the [nginx block](blocks-nginx.html#blocks-nginx-usage-shbforwardauth).
+
+## Features {#services-authelia-features}
+
+- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-authelia-usage-applicationdashboard)
+
+## Usage {#services-authelia-usage}
+
+### Initial Configuration {#blocks-authelia-usage-configuration}
+
+Authelia cannot work without SSL and LDAP.
+So setting up the Authelia block requires to setup the [SSL block][] first
+and the [LLDAP block][] first.
+
+[SSL block]: blocks-ssl.html
+[LLDAP block]: blocks-lldap.html
+
+SSL is required to encrypt the communication and LDAP is used to handle users and group assignments.
+Authelia will allow access to a given resource only if the user that is authenticated
+is a member of the corresponding LDAP group.
+
+Afterwards, assuming the LDAP service runs on the same machine,
+the Authelia configuration can be done with:
+
+```nix
+shb.authelia = {
+  enable = true;
+  domain = "example.com";
+  subdomain = "auth";
+  ssl = config.shb.certs.certs.letsencrypt."example.com";
+
+  ldapHostname = "127.0.0.1";
+  ldapPort = config.shb.lldap.ldapPort;
+  dcdomain = config.shb.lldap.dcdomain;
+
+  smtp = {
+    host = "smtp.eu.mailgun.org";
+    port = 587;
+    username = "postmaster@mg.example.com";
+    from_address = "authelia@example.com";
+    password.result = config.shb.sops.secret."authelia/smtp_password".result;
+  };
+
+  secrets = {
+    jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
+    ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
+    sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
+    storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
+    identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
+    identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
+  };
+};
+
+shb.certs.certs.letsencrypt."example.com".extraDomains = [ "auth.example.com" ];
+
+shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
+shb.sops.secret."authelia/ldap_admin_password" = {
+  request = config.shb.authelia.secrets.ldapAdminPassword.request;
+  settings.key = "lldap/user_password";
+};
+shb.sops.secret."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
+shb.sops.secret."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
+shb.sops.secret."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
+shb.sops.secret."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
+shb.sops.secret."authelia/smtp_password".request = config.shb.authelia.smtp.password.request;
+```
+
+This assumes secrets are setup with SOPS
+as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual.
+It's a bit annoying to setup all those secrets but it's only necessary once.
+Use `nix run nixpkgs#openssl -- rand -hex 64` to generate them.
+
+Crucially, the `shb.authelia.secrets.ldapAdminPasswordFile` must be the same
+as the `shb.lldap.ldapUserPassword` defined for the [LLDAP block][].
+This is done using Sops' `key` option.
+
+### Application Dashboard {#services-authelia-usage-applicationdashboard}
+
+Integration with the [dashboard contract](contracts-dashboard.html) is provided
+by the [dashboard option](#blocks-authelia-options-shb.authelia.dashboard).
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.Authelia = {
+    sortOrder = 2;
+    dashboard.request = config.shb.authelia.dashboard.request;
+  };
+}
+```
+
+## SHB OIDC integration {#blocks-authelia-shb-oidc}
+
+For services [provided by SelfHostBlocks][services] that handle [OIDC integration][OIDC],
+integrating with this block is done by configuring the service itself
+and linking it to this Authelia block through the `endpoint` option
+and by sharing a secret:
+
+[services]: services.html
+[OIDC]: https://openid.net/developers/how-connect-works/
+
+```nix
+shb.<service>.sso = {
+  enable = true;
+  endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+
+  secret.result = config.shb.sops.secret."<service>/sso/secret".result;
+  secretForAuthelia.result = config.shb.sops.secret."<service>/sso/secretForAuthelia".result;
+};
+
+shb.sops.secret."<service>/sso/secret".request = config.shb.<service>.sso.secret.request;
+shb.sops.secret."<service>/sso/secretForAuthelia" = {
+  request = config.shb.<service>.sso.secretForAuthelia.request;
+  settings.key = "<service>/sso/secret";
+};
+```
+
+To share a secret between the service and Authelia,
+we generate a secret with `nix run nixpkgs#openssl -- rand -hex 64` under `<service>/sso/secret`
+then we ask Sops to use the same password for `<service>/sso/secretForAuthelia`
+thanks to the `settings.key` option.
+The difference between both secrets is one if owned by the `authelia` user
+while the other is owned by the user of the `<service`> we are configuring.
+
+## OIDC Integration {#blocks-authelia-oidc}
+
+To integrate a service handling OIDC integration not provided by SelfHostBlocks with this Authelia block,
+the necessary configuration is:
+
+```nix
+shb.authelia.oidcClients = [
+  {
+    client_id = "<service>";
+    client_secret.source = config.shb.sops.secret."<service>/sso/secretForAuthelia".response.path;
+    scopes = [ "openid" "email" "profile" ];
+    redirect_uris = [
+      "<provided by service documentation>"
+    ];
+  }
+];
+
+shb.sops.secret."<service>/sso/secret".request = {
+  owner = "<service_user>";
+};
+shb.sops.secret."<service>/sso/secretForAuthelia" = {
+  request.owner = "authelia";
+  settings.key = "<service>/sso/secret";
+};
+```
+
+As in the previous section, we create a shared secret using Sops'
+`settings.key` option.
+
+The configuration for the service itself is much dependent on the service itself.
+For example for [open-webui][], the configuration looks like so:
+
+[open-webui]: https://search.nixos.org/options?query=services.open-webui
+
+```nix
+services.open-webui.environment = {
+  ENABLE_SIGNUP = "False";
+  WEBUI_AUTH = "True";
+  ENABLE_FORWARD_USER_INFO_HEADERS = "True";
+  ENABLE_OAUTH_SIGNUP = "True";
+  OAUTH_UPDATE_PICTURE_ON_LOGIN = "True";
+  OAUTH_CLIENT_ID = "open-webui";
+  OAUTH_CLIENT_SECRET = "<raw secret>";
+  OPENID_PROVIDER_URL = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}/.well-known/openid-configuration";
+  OAUTH_PROVIDER_NAME = "Single Sign-On";
+  OAUTH_SCOPES = "openid email profile";
+  OAUTH_ALLOWED_ROLES = "open-webui_user";
+  OAUTH_ADMIN_ROLES = "open-webui_admin";
+  ENABLE_OAUTH_ROLE_MANAGEMENT = "True";
+};
+
+shb.authelia.oidcClients = [
+  {
+    client_id = "open-webui";
+    client_secret.source = config.shb.sops.secret."open-webui/sso/secretForAuthelia".response.path;
+    scopes = [ "openid" "email" "profile" ];
+    redirect_uris = [
+      "<provided by service documentation>"
+    ];
+  }
+];
+
+shb.sops.secret."open-webui/sso/secret".request = {
+  owner = "open-webui";
+};
+shb.sops.secret."open-webui/sso/secretForAuthelia" = {
+  request.owner = "authelia";
+  settings.key = "open-webui/sso/secret";
+};
+```
+
+Here, there is no way to give a path for the `OAUTH_CLIENT_SECRET`,
+we are obligated to pass the raw secret which is a very bad idea.
+There are ways around this but they are out of scope for this section.
+Inspiration can be taken from SelfHostBlocks' source code.
+
+To access the UI, we will need to create an `open-webui_user` and
+`open-webui_admin` LDAP group and assign our user to it.
+
+## Forward Auth {#blocks-authelia-forward-auth}
+
+Forward authentication is provided by the [nginx block](blocks-nginx.html#blocks-nginx-usage-ssl).
+
+## Troubleshooting {#blocks-authelia-troubleshooting}
+
+Set the [debug][opt-debug] option to `true` to:
+
+[opt-debug]: #blocks-authelia-options-shb.authelia.debug
+
+- Set logging level to `"debug"`.
+- Add an [shb.mitmdump][] instance in front of Authelia
+  which prints all requests and responses headers and body
+  to the systemd service `mitmdump-authelia-${config.shb.authelia.subdomain}.${config.shb.authelia.domain}.service`.
+
+[shb.mitmdump]: ./blocks-mitmdump.html
+
+## Tests {#blocks-authelia-tests}
+
+Specific integration tests are defined in [`/test/blocks/authelia.nix`](@REPO@/test/blocks/authelia.nix).
+
+## Options Reference {#blocks-authelia-options}
+
+```{=include=} options
+id-prefix: blocks-authelia-options-
+list-id: selfhostblocks-block-authelia-options
+source: @OPTIONS_JSON@
+```

modules/blocks/borgbackup.nix

@@ -1,145 +1,241 @@
-{ config, pkgs, lib, utils, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  utils,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.borgbackup;
 
-  instanceOptions = {
-    enable = lib.mkEnableOption "shb borgbackup";
+  inherit (lib)
+    concatStringsSep
+    filterAttrs
+    flatten
+    literalExpression
+    optionals
+    listToAttrs
+    mapAttrsToList
+    mkOption
+    mkMerge
+    ;
+  inherit (lib)
+    mkIf
+    nameValuePair
+    optionalAttrs
+    removePrefix
+    ;
+  inherit (lib.types)
+    attrsOf
+    int
+    oneOf
+    nonEmptyStr
+    nullOr
+    str
+    submodule
+    ;
 
-    keySopsFile = lib.mkOption {
-      description = "Sops file that holds this instance's repository key and passphrase.";
-      type = lib.types.path;
-      example = "secrets/backup.yaml";
-    };
+  commonOptions =
+    {
+      name,
+      prefix,
+      config,
+      ...
+    }:
+    {
+      enable = lib.mkEnableOption ''
+        SelfHostBlocks' BorgBackup block;
 
-    encryptionKeyFile = lib.mkOption {
-      description = "Encryption key for the backup.";
-      type = lib.types.path;
-    };
+        A disabled instance will not backup data anymore
+        but still provides the helper tool to restore snapshots
+      '';
 
-    encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}";
-    borg_keys_directory = "/run/secrets/borgmatic/keys";
-
-    sourceDirectories = lib.mkOption {
-      description = "Source directories.";
-      type = lib.types.nonEmptyListOf lib.types.str;
-    };
-
-    excludePatterns = lib.mkOption {
-      description = "Exclude patterns.";
-      type = lib.types.listOf lib.types.str;
-      default = [];
-    };
-
-    secretName = lib.mkOption {
-      description = "Secret name, if null use the name of the backup instance.";
-      type = lib.types.nullOr lib.types.str;
-      default = null;
-    };
-
-    repositories = lib.mkOption {
-      description = "Repositories to back this instance to.";
-      type = lib.types.nonEmptyListOf (lib.types.submodule {
-        options = {
-          path = lib.mkOption {
-            type = lib.types.str;
-            description = "Repository location";
-          };
-
-          timerConfig = lib.mkOption {
-            type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption;
-            default = {
-              OnCalendar = "daily";
-              Persistent = true;
-            };
-            description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
-            example = {
-              OnCalendar = "00:05";
-              RandomizedDelaySec = "5h";
-              Persistent = true;
-            };
-          };
-        };
-      });
-    };
-
-    retention = lib.mkOption {
-      description = "Retention options.";
-      type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
-      default = {
-        keep_within = "1d";
-        keep_hourly = 24;
-        keep_daily = 7;
-        keep_weekly = 4;
-        keep_monthly = 6;
-      };
-    };
-
-    consistency = lib.mkOption {
-      description = "Consistency frequency options.";
-      type = lib.types.attrsOf lib.types.nonEmptyStr;
-      default = {};
-      example = {
-        repository = "2 weeks";
-        archives = "1 month";
-      };
-    };
-
-    hooks = lib.mkOption {
-      description = "Hooks to run before or after the backup.";
-      default = {};
-      type = lib.types.submodule {
-        options = {
-          beforeBackup = lib.mkOption {
-            description = "Hooks to run before backup";
-            type = lib.types.listOf lib.types.str;
-            default = [];
-          };
-
-          afterBackup = lib.mkOption {
-            description = "Hooks to run after backup";
-            type = lib.types.listOf lib.types.str;
-            default = [];
+      passphrase = lib.mkOption {
+        description = "Encryption key for the backup repository.";
+        type = lib.types.submodule {
+          options = shb.contracts.secret.mkRequester {
+            mode = "0400";
+            owner = config.request.user;
+            ownerText = "[shb.borgbackup.${prefix}.<name>.request.user](#blocks-borgbackup-options-shb.borgbackup.${prefix}._name_.request.user)";
+            restartUnits = [ (fullName name config.settings.repository) ];
+            restartUnitsText = "[ [shb.borgbackup.${prefix}.<name>.settings.repository](#blocks-borgbackup-options-shb.borgbackup.${prefix}._name_.settings.repository) ]";
           };
         };
       };
+
+      repository = lib.mkOption {
+        description = "Repository to send the backups to.";
+        type = submodule {
+          options = {
+            path = mkOption {
+              type = str;
+              description = "Repository location";
+            };
+
+            secrets = mkOption {
+              type = attrsOf shb.secretFileType;
+              default = { };
+              description = ''
+                Secrets needed to access the repository where the backups will be stored.
+
+                See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+                and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
+
+              '';
+              example = literalExpression ''
+                {
+                  AWS_ACCESS_KEY_ID.source = <path/to/secret>;
+                  AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
+                }
+              '';
+            };
+
+            timerConfig = mkOption {
+              type = attrsOf utils.systemdUtils.unitOptions.unitOption;
+              default = {
+                OnCalendar = "daily";
+                Persistent = true;
+              };
+              description = "When to run the backup. See {manpage}`systemd.timer(5)` for details.";
+              example = {
+                OnCalendar = "00:05";
+                RandomizedDelaySec = "5h";
+                Persistent = true;
+              };
+            };
+          };
+        };
+      };
+
+      retention = lib.mkOption {
+        description = "Retention options. See {command}`borg help prune` for the available options.";
+        type = attrsOf (oneOf [
+          int
+          nonEmptyStr
+        ]);
+        default = {
+          within = "1d";
+          hourly = 24;
+          daily = 7;
+          weekly = 4;
+          monthly = 6;
+        };
+      };
+
+      consistency = lib.mkOption {
+        description = "Consistency frequency options.";
+        type = lib.types.attrsOf lib.types.nonEmptyStr;
+        default = { };
+        example = {
+          repository = "2 weeks";
+          archives = "1 month";
+        };
+      };
+
+      limitUploadKiBs = mkOption {
+        type = nullOr int;
+        description = "Limit upload bandwidth to the given KiB/s amount.";
+        default = null;
+        example = 8000;
+      };
+
+      stateDir = mkOption {
+        type = nullOr lib.types.str;
+        description = ''
+          Override the directory in which {command}`borg` stores its
+          configuration and cache. By default it uses the user's
+          home directory but is some cases this can cause conflicts.
+        '';
+        default = null;
+      };
     };
 
-    environmentFile = lib.mkOption {
-      type = lib.types.bool;
-      description = "Add environment file to be read by the systemd service.";
-      default = false;
-      example = true;
-    };
-  };
-
-  repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name);
-
+  repoSlugName = name: builtins.replaceStrings [ "/" ":" ] [ "_" "_" ] (removePrefix "/" name);
+  fullName = name: repository: "borgbackup-job-${name}_${repoSlugName repository.path}";
 in
 {
+  imports = [
+    ../../lib/module.nix
+    ../blocks/monitoring.nix
+  ];
+
   options.shb.borgbackup = {
-    user = lib.mkOption {
-      description = "Unix user doing the backups.";
-      type = lib.types.str;
-      default = "backup";
+    enableDashboard = lib.mkEnableOption "the Backups SHB dashboard" // {
+      default = true;
     };
 
-    group = lib.mkOption {
-      description = "Unix group doing the backups.";
-      type = lib.types.str;
-      default = "backup";
+    instances = mkOption {
+      description = "Files to backup following the [backup contract](./shb.contracts-backup.html).";
+      default = { };
+      type = attrsOf (
+        submodule (
+          { name, config, ... }:
+          {
+            options = shb.contracts.backup.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the BorgBackup provider.
+                '';
+
+                type = submodule {
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "instances";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
+            };
+          }
+        )
+      );
     };
 
-    instances = lib.mkOption {
-      description = "Each instance is a backup setting";
-      default = {};
-      type = lib.types.attrsOf (lib.types.submodule {
-        options = instanceOptions;
-      });
+    databases = mkOption {
+      description = "Databases to backup following the [database backup contract](./shb.contracts-databasebackup.html).";
+      default = { };
+      type = attrsOf (
+        submodule (
+          { name, config, ... }:
+          {
+            options = shb.contracts.databasebackup.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the BorgBackup provider.
+                '';
+
+                type = submodule {
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "databases";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
+            };
+          }
+        )
+      );
     };
 
     borgServer = lib.mkOption {
-      description = "Add borgbackup package so external backups can use this server as a remote.";
+      description = "Add borgbackup package to `environment.systemPackages` so external backups can use this server as a remote.";
       default = false;
       example = true;
       type = lib.types.bool;
@@ -148,7 +244,7 @@ in
     # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
     performance = lib.mkOption {
       description = "Reduce performance impact of backup jobs.";
-      default = {};
+      default = { };
       type = lib.types.submodule {
         options = {
           niceness = lib.mkOption {
@@ -157,7 +253,11 @@ in
             default = 15;
           };
           ioSchedulingClass = lib.mkOption {
-            type = lib.types.enum [ "idle" "best-effort" "realtime" ];
+            type = lib.types.enum [
+              "idle"
+              "best-effort"
+              "realtime"
+            ];
             description = "ionice scheduling class, defaults to best-effort IO.";
             default = "best-effort";
           };
@@ -171,125 +271,231 @@ in
     };
   };
 
-  config = lib.mkIf (cfg.instances != {}) (
+  config = lib.mkIf (cfg.instances != { } || cfg.databases != { }) (
     let
-      enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
-    in lib.mkMerge [
-      # Secrets configuration
+      enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances;
+      enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases;
+    in
+    lib.mkMerge [
       {
-        users.users = {
-          ${cfg.user} = {
-            name = cfg.user;
-            group = cfg.group;
-            home = "/var/lib/backup";
-            createHome = true;
-            isSystemUser = true;
-            extraGroups = [ "keys" ];
-          };
-        };
-        users.groups = {
-          ${cfg.group} = {
-            name = cfg.group;
-          };
-        };
-
-        sops.secrets =
-          let
-            mkSopsSecret = name: instance: (
-              [
-                {
-                  "${instance.backend}/passphrases/${if isNull instance.secretName then name else instance.secretName}" = {
-                    sopsFile = instance.keySopsFile;
-                    mode = "0440";
-                    owner = cfg.user;
-                    group = cfg.group;
-                  };
-                }
-              ] ++ lib.optional ((lib.filter ({path, ...}: lib.strings.hasPrefix "s3" path) instance.repositories) != []) {
-                "${instance.backend}/environmentfiles/${if isNull instance.secretName then name else instance.secretName}" = {
-                  sopsFile = instance.keySopsFile;
-                  mode = "0440";
-                  owner = cfg.user;
-                  group = cfg.group;
-                };
-              } ++ lib.optionals (instance.backend == "borgmatic") (lib.flatten (map ({path, ...}: {
-                "${instance.backend}/keys/${repoSlugName path}" = {
-                  key = "${instance.backend}/keys/${if isNull instance.secretName then name else instance.secretName}";
-                  sopsFile = instance.keySopsFile;
-                  mode = "0440";
-                  owner = cfg.user;
-                  group = cfg.group;
-                };
-              }) instance.repositories))
-            );
-          in
-            lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSopsSecret enabledInstances));
+        environment.systemPackages =
+          optionals (cfg.borgServer || enabledInstances != { } || enabledDatabases != { })
+            [
+              pkgs.borgbackup
+            ];
       }
-      # Borgmatic configuration
       {
-        systemd.timers.borgmatic = lib.mkIf (enabledInstances != {}) {
-          timerConfig = {
-            OnCalendar = "hourly";
-          };
-        };
-
-        systemd.services.borgmatic = lib.mkIf (enabledInstances != {}) {
-          serviceConfig = {
-            User = cfg.user;
-            Group = cfg.group;
-            ExecStartPre = [ "" ]; # Do not sleep before starting.
-            ExecStart = [ "" "${pkgs.borgmatic}/bin/borgmatic --verbosity -1 --syslog-verbosity 1" ];
-            # For borgmatic, since we have only one service, we need to merge all environmentFile
-            # from all instances.
-            EnvironmentFile = lib.mapAttrsToList (name: value: value.environmentFile) enabledInstances;
-          };
-        };
-
-        systemd.packages = lib.mkIf (enabledInstances != {}) [ pkgs.borgmatic ];
-        environment.systemPackages = (
-          lib.optionals cfg.borgServer [ pkgs.borgbackup ]
-          ++ lib.optionals (enabledInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ]
-        );
-
-        environment.etc =
+        services.borgbackup.jobs =
           let
-            mkSettings = name: instance: {
-              "borgmatic.d/${name}.yaml".text = lib.generators.toYAML {} {
-                location =
-                  {
-                    source_directories = instance.sourceDirectories;
-                    repositories = map ({path, ...}: path) instance.repositories;
-                  }
-                  // (lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
-                    excludePatterns = instance.excludePatterns;
-                  });
+            mkJob = name: instance: {
+              "${name}_${repoSlugName instance.settings.repository.path}" = {
+                inherit (instance.request) user;
 
-                storage = {
-                  encryption_passcommand = "cat ${instance.encryptionKeyFile}";
-                  borg_keys_directory = "/run/secrets/borgmatic/keys";
-                };
+                repo = instance.settings.repository.path;
 
-                retention = instance.retention;
-                consistency.checks =
-                  let
-                    mkCheck = name: frequency: {
-                      inherit name frequency;
-                    };
-                  in
-                    lib.attrsets.mapAttrsToList mkCheck instance.consistency;
+                paths = instance.request.sourceDirectories;
 
-                # hooks = lib.mkMerge [
-                #   lib.optionalAttrs (builtins.length instance.hooks.beforeBackup > 0) {
-                #     inherit (instance.hooks) beforeBackup;
-                #   }
-                #   lib.optionalAttrs (builtins.length instance.hooks.afterBackup > 0) {
-                #     inherit (instance.hooks) afterBackup;
-                #   }
-                # ];
+                encryption.mode = "repokey-blake2";
+                # We do not set encryption.passphrase here, we set BORG_PASSPHRASE_FD further down.
+                encryption.passCommand = "cat ${instance.settings.passphrase.result.path}";
+
+                doInit = true;
+                failOnWarnings = true;
+                stateDir = instance.settings.stateDir;
+
+                persistentTimer = instance.settings.repository.timerConfig.Persistent or false;
+                startAt = ""; # Some non-empty string value tricks the upstream module in creating the systemd timer.
+
+                prune.keep = instance.settings.retention;
+
+                preHook = concatStringsSep "\n" instance.request.hooks.beforeBackup;
+
+                postHook = concatStringsSep "\n" instance.request.hooks.afterBackup;
+
+                extraArgs = (
+                  optionals (instance.settings.limitUploadKiBs != null) [
+                    "--upload-ratelimit=${toString instance.settings.limitUploadKiBs}"
+                  ]
+                );
+
+                exclude = instance.request.excludePatterns;
               };
             };
           in
-            lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings enabledInstances);
+          mkMerge (mapAttrsToList mkJob enabledInstances);
       }
-    ]);
+      {
+        services.borgbackup.jobs =
+          let
+            mkJob = name: instance: {
+              "${name}_${repoSlugName instance.settings.repository.path}" = {
+                inherit (instance.request) user;
+
+                repo = instance.settings.repository.path;
+
+                dumpCommand = lib.getExe (
+                  pkgs.writeShellApplication {
+                    name = "dump-command";
+                    text = instance.request.backupCmd;
+                  }
+                );
+
+                encryption.mode = "repokey-blake2";
+                # We do not set encryption.passphrase here, we set BORG_PASSPHRASE_FD further down.
+                encryption.passCommand = "cat ${instance.settings.passphrase.result.path}";
+
+                doInit = true;
+                failOnWarnings = true;
+                stateDir = instance.settings.stateDir;
+
+                persistentTimer = instance.settings.repository.timerConfig.Persistent or false;
+                startAt = ""; # Some non-empty list value that tricks upstream in creating the systemd timer.
+
+                prune.keep = instance.settings.retention;
+
+                extraArgs = (
+                  optionals (instance.settings.limitUploadKiBs != null) [
+                    "--upload-ratelimit=${toString instance.settings.limitUploadKiBs}"
+                  ]
+                );
+              };
+            };
+          in
+          mkMerge (mapAttrsToList mkJob enabledDatabases);
+      }
+      {
+        systemd.timers =
+          let
+            mkTimer = name: instance: {
+              ${fullName name instance.settings.repository} = {
+                timerConfig = lib.mkForce instance.settings.repository.timerConfig;
+              };
+            };
+          in
+          mkMerge (mapAttrsToList mkTimer (enabledInstances // enabledDatabases));
+      }
+      {
+        systemd.services =
+          let
+            mkSettings =
+              name: instance:
+              let
+                serviceName = fullName name instance.settings.repository;
+              in
+              {
+                ${serviceName} = mkMerge [
+                  {
+                    serviceConfig = {
+                      # Purposely not a oneshot systemd service otherwise
+                      # the service waits on the completion the backup before deactivating.
+                      # This seems like a nice property at first but there is one annoying
+                      # edge case when deploying. If a backup job somehow is started when
+                      # the deploy happens, the deploy will wait on the service to finish
+                      # before considering the deploy done. And worse, it will consider the
+                      # deploy as failed if the backup fails, which is not what you want.
+                      Nice = lib.mkForce cfg.performance.niceness;
+                      IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass;
+                      IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority;
+                      # BindReadOnlyPaths = instance.sourceDirectories;
+                    };
+                  }
+                  (optionalAttrs (instance.settings.repository.secrets != { }) {
+                    serviceConfig.EnvironmentFile = [
+                      "/run/secrets_borgbackup/${serviceName}"
+                    ];
+                    after = [ "${serviceName}-pre.service" ];
+                    requires = [ "${serviceName}-pre.service" ];
+                  })
+                ];
+
+                "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != { }) (
+                  let
+                    script = shb.genConfigOutOfBandSystemd {
+                      config = instance.settings.repository.secrets;
+                      configLocation = "/run/secrets_borgbackup/${serviceName}";
+                      generator = shb.toEnvVar;
+                      user = instance.request.user;
+                    };
+                  in
+                  {
+                    script = script.preStart;
+                    # Makes the systemd service wait for the backup to be done before changing state to inactive.
+                    serviceConfig.Type = "oneshot";
+                    serviceConfig.LoadCredential = script.loadCredentials;
+                  }
+                );
+              };
+          in
+          mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
+      }
+      {
+        systemd.services =
+          let
+            mkEnv =
+              name: instance:
+              nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
+                enable = true;
+                wantedBy = [ "multi-user.target" ];
+                serviceConfig.Type = "oneshot";
+                script = (
+                  shb.replaceSecrets {
+                    userConfig = instance.settings.repository.secrets // {
+                      BORG_PASSCOMMAND = ''"cat ${instance.settings.passphrase.result.path}"'';
+                      BORG_REPO = instance.settings.repository.path;
+                    };
+                    resultPath = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
+                    generator = shb.toEnvVar;
+                    user = instance.request.user;
+                  }
+                );
+              };
+          in
+          listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases)));
+      }
+      {
+        environment.systemPackages =
+          let
+            mkBorgBackupBinary =
+              name: instance:
+              shb.contracts.backup.mkRestoreScript {
+                name = fullName name instance.settings.repository;
+                user = instance.request.user;
+                sudoPreserveEnvs = [
+                  "BORG_REPO"
+                  "BORG_PASSCOMMAND"
+                ];
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
+                restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
+              };
+          in
+          flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
+      }
+      {
+        environment.systemPackages =
+          let
+            mkBorgBackupBinary =
+              name: instance:
+              shb.contracts.backup.mkRestoreScript {
+                name = fullName name instance.settings.repository;
+                user = instance.request.user;
+                sudoPreserveEnvs = [
+                  "BORG_REPO"
+                  "BORG_PASSCOMMAND"
+                ];
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
+                restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
+              };
+          in
+          flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);
+      }
+
+      (lib.mkIf (cfg.enableDashboard && (cfg.instances != { } || cfg.databases != { })) {
+        shb.monitoring.dashboards = [
+          ./backup/dashboard/Backups.json
+        ];
+      })
+    ]
+  );
 }

modules/blocks/borgbackup/docs/default.md

@@ -0,0 +1,302 @@
+# Borgbackup Block {#blocks-borgbackup}
+
+Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix).
+
+This block sets up a backup job using [BorgBackup][].
+It is heavily based on the nixpkgs BorgBackup module.
+
+[borgbackup]: https://www.borgbackup.org/
+
+## Provider Contracts {#blocks-borgbackup-contract-provider}
+
+This block provides the following contracts:
+
+- [backup contract](contracts-backup.html) under the [`shb.borgbackup.instances`][instances] option.
+  It is tested with [contract tests][backup contract tests].
+- [database backup contract](contracts-databasebackup.html) under the [`shb.borgbackup.databases`][databases] option.
+  It is tested with [contract tests][database backup contract tests].
+
+[instances]: #blocks-borgbackup-options-shb.borgbackup.instances
+[databases]: #blocks-borgbackup-options-shb.borgbackup.databases
+[backup contract tests]: @REPO@/test/contracts/backup.nix
+[database backup contract tests]: @REPO@/test/contracts/databasebackup.nix
+
+As requested by those two contracts, when setting up a backup with BorgBackup,
+a backup Systemd service and a [restore script](#blocks-borgbackup-maintenance) are provided.
+
+## Usage {#blocks-borgbackup-usage}
+
+The following examples assume usage of the [sops block][] to provide secrets
+although any blocks providing the [secrets contract][] works too.
+
+[sops block]: ./blocks-sops.html
+[secrets contract]: ./contracts-secrets.html
+
+### One folder backed up manually {#blocks-borgbackup-usage-provider-manual}
+
+The following snippet shows how to configure
+the backup of 1 folder to 1 repository.
+We assume that the folder `/var/lib/myfolder` of the service `myservice` must be backed up.
+
+```nix
+shb.borgbackup.instances."myservice" = {
+  request = {
+    user = "myservice";
+
+    sourceDirectories = [
+      "/var/lib/myfolder"
+    ];
+  };
+
+  settings = {
+    enable = true;
+
+    passphrase.result = config.shb.sops.secret."passphrase".result;
+
+    repository = {
+      path = "/srv/backups/myservice";
+      timerConfig = {
+        OnCalendar = "00:00:00";
+        RandomizedDelaySec = "3h";
+      };
+    };
+
+    retention = {
+      within = "1d";
+      hourly = 24;
+      daily = 7;
+      weekly = 4;
+      monthly = 6;
+    };
+  };
+};
+
+shb.sops.secret."passphrase".request =
+  config.shb.borgbackup.instances."myservice".settings.passphrase.request;
+```
+
+### One folder backed up with contract {#blocks-borgbackup-usage-provider-contract}
+
+With the same example as before but assuming the `myservice` service
+has a `myservice.backup` option that is a requester for the backup contract,
+the snippet above becomes:
+
+```nix
+shb.borgbackup.instances."myservice" = {
+  request = config.myservice.backup;
+
+  settings = {
+    enable = true;
+
+    passphrase.result = config.shb.sops.secret."passphrase".result;
+
+    repository = {
+      path = "/srv/backups/myservice";
+      timerConfig = {
+        OnCalendar = "00:00:00";
+        RandomizedDelaySec = "3h";
+      };
+    };
+
+    retention = {
+      within = "1d";
+      hourly = 24;
+      daily = 7;
+      weekly = 4;
+      monthly = 6;
+    };
+  };
+};
+
+shb.sops.secret."passphrase".request =
+  config.shb.borgbackup.instances."myservice".settings.passphrase.request;
+```
+
+### One folder backed up to S3 {#blocks-borgbackup-usage-provider-remote}
+
+Here we will only highlight the differences with the previous configuration.
+
+This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/).
+
+```diff
+  shb.test.backup.instances.myservice = {
+
+    repository = {
+-     path = "/srv/pool1/backups/myfolder";
++     path = "s3:s3.us-west-000.backblazeb2.com/backups/myfolder";
+      timerConfig = {
+        OnCalendar = "00:00:00";
+        RandomizedDelaySec = "3h";
+      };
+
++     extraSecrets = {
++       AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
++       AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
++     };
+    };
+  }
+```
+
+### Multiple directories to multiple destinations {#blocks-borgbackup-usage-multiple}
+
+The following snippet shows how to configure backup of any number of folders to 3 repositories,
+each happening at different times to avoid I/O contention.
+
+We will also make sure to be able to re-use as much as the configuration as possible.
+
+A few assumptions:
+- 2 hard drive pools used for backup are mounted respectively on `/srv/pool1` and `/srv/pool2`.
+- You have a backblaze account.
+
+First, let's define a variable to hold all the repositories we want to back up to:
+
+```nix
+repos = [
+  {
+    path = "/srv/pool1/backups";
+    timerConfig = {
+      OnCalendar = "00:00:00";
+      RandomizedDelaySec = "3h";
+    };
+  }
+  {
+    path = "/srv/pool2/backups";
+    timerConfig = {
+      OnCalendar = "08:00:00";
+      RandomizedDelaySec = "3h";
+    };
+  }
+  {
+    path = "s3:s3.us-west-000.backblazeb2.com/backups";
+    timerConfig = {
+      OnCalendar = "16:00:00";
+      RandomizedDelaySec = "3h";
+    };
+  }
+];
+```
+
+Compared to the previous examples, we do not include the name of what we will back up in the
+repository paths.
+
+Now, let's define a function to create a backup configuration. It will take a list of repositories,
+a name identifying the backup and a list of folders to back up.
+
+```nix
+backupcfg = repositories: name: sourceDirectories {
+  enable = true;
+
+  backend = "borgbackup";
+
+  keySopsFile = ../secrets/backup.yaml;
+
+  repositories = builtins.map (r: {
+    path = "${r.path}/${name}";
+    inherit (r) timerConfig;
+  }) repositories;
+
+  inherit sourceDirectories;
+
+  retention = {
+    within = "1d";
+    hourly = 24;
+    daily = 7;
+    weekly = 4;
+    monthly = 6;
+  };
+
+  environmentFile = true;
+};
+```
+
+Now, we can define multiple backup jobs to backup different folders:
+
+```nix
+shb.test.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"];
+shb.test.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"];
+```
+
+The difference between the above snippet and putting all the folders into one configuration (shown
+below) is the former splits the backups into sub-folders on the repositories.
+
+```nix
+shb.test.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
+```
+
+## Monitoring {#blocks-borgbackup-monitoring}
+
+A generic dashboard for all backup solutions is provided.
+See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backup) section in the monitoring chapter.
+
+## Maintenance {#blocks-borgbackup-maintenance}
+
+### Manual Backup {#blocks-borgbackup-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'borgbackup*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start borgbackup<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-borgbackup-maintenance-restore}
+
+One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
+
+The restore script has all the secrets needed to access the repo,
+it will run `sudo` automatically
+and the user running it needs to have correct permissions for privilege escalation
+
+In the [multiple directories example](#blocks-borgbackup-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
+
+```bash
+borgbackup-job-myfolder1_srv_pool1_backups
+borgbackup-job-myfolder1_srv_pool2_backups
+borgbackup-job-myfolder1_s3_s3.us-west-000.backblazeb2.com_backups
+borgbackup-job-myfolder2_srv_pool1_backups
+borgbackup-job-myfolder2_srv_pool2_backups
+borgbackup-job-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups
+```
+
+Discovering those is easy thanks to tab-completion.
+
+One can then restore a backup from a given repository with:
+
+```bash
+borgbackup-job-myfolder1_srv_pool1_backups restore latest
+```
+
+### Troubleshooting {#blocks-borgbackup-maintenance-troubleshooting}
+
+In case something bad happens with a backup, the [official documentation](https://borgbackup.readthedocs.io/en/stable/077_troubleshooting.html) has a lot of tips.
+
+## Tests {#blocks-borgbackup-tests}
+
+Specific integration tests are defined in [`/test/blocks/borgbackup.nix`](@REPO@/test/blocks/borgbackup.nix).
+
+## Options Reference {#blocks-borgbackup-options}
+
+```{=include=} options
+id-prefix: blocks-borgbackup-options-
+list-id: selfhostblocks-block-borgbackup-options
+source: @OPTIONS_JSON@
+```

modules/blocks/davfs.nix

@@ -1,80 +1,104 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, ... }:
 
 let
   cfg = config.shb.davfs;
-
-  shblib = pkgs.callPackage ../../lib {};
 in
 {
   options.shb.davfs = {
     mounts = lib.mkOption {
       description = "List of mounts.";
-      default = [];
-      type = lib.types.listOf (lib.types.submodule {
-        options = {
-          remoteUrl = lib.mkOption {
-            type = lib.types.str;
-            description = "Webdav endpoint to connect to.";
-            example = "https://my.domain.com/dav";
-          };
+      default = [ ];
+      type = lib.types.listOf (
+        lib.types.submodule {
+          options = {
+            remoteUrl = lib.mkOption {
+              type = lib.types.str;
+              description = "Webdav endpoint to connect to.";
+              example = "https://my.domain.com/dav";
+            };
 
-          mountPoint = lib.mkOption {
-            type = lib.types.str;
-            description = "Mount point to mount the webdav endpoint on.";
-            example = "/mnt";
-          };
+            mountPoint = lib.mkOption {
+              type = lib.types.str;
+              description = "Mount point to mount the webdav endpoint on.";
+              example = "/mnt";
+            };
 
-          username = lib.mkOption {
-            type = lib.types.str;
-            description = "Username to connect to the webdav endpoint.";
-          };
+            username = lib.mkOption {
+              type = lib.types.str;
+              description = "Username to connect to the webdav endpoint.";
+            };
 
-          passwordFile = lib.mkOption {
-            type = lib.types.str;
-            description = "Password to connect to the webdav endpoint.";
-          };
+            passwordFile = lib.mkOption {
+              type = lib.types.str;
+              description = "Password to connect to the webdav endpoint.";
+            };
 
-          uid = lib.mkOption {
-            type = lib.types.nullOr lib.types.int;
-            description = "User owner of the mount point.";
-            example = 1000;
-            default = null;
-          };
+            uid = lib.mkOption {
+              type = lib.types.nullOr lib.types.int;
+              description = "User owner of the mount point.";
+              example = 1000;
+              default = null;
+            };
 
-          gid = lib.mkOption {
-            type = lib.types.nullOr lib.types.int;
-            description = "Group owner of the mount point.";
-            example = 1000;
-            default = null;
-          };
+            gid = lib.mkOption {
+              type = lib.types.nullOr lib.types.int;
+              description = "Group owner of the mount point.";
+              example = 1000;
+              default = null;
+            };
 
-          fileMode = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
-            description = "File creation mode";
-            example = "0664";
-            default = null;
-          };
+            fileMode = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = "File creation mode";
+              example = "0664";
+              default = null;
+            };
 
-          directoryMode = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
-            description = "Directory creation mode";
-            example = "2775";
-            default = null;
-          };
+            directoryMode = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = "Directory creation mode";
+              example = "2775";
+              default = null;
+            };
 
-          automount = lib.mkOption {
-            type = lib.types.bool;
-            description = "Create a systemd automount unit";
-            default = true;
+            automount = lib.mkOption {
+              type = lib.types.bool;
+              description = "Create a systemd automount unit";
+              default = true;
+            };
           };
-        };
-      });
+        }
+      );
     };
   };
 
   config = {
     services.davfs2.enable = builtins.length cfg.mounts > 0;
 
+    systemd.services = lib.optionalAttrs (builtins.length cfg.mounts > 0) {
+      davfs2-password-generation = {
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig.Type = "oneshot";
+        script = ''
+          mkdir -p /etc/davfs2
+          [ -f /etc/davfs2/secrets ] && mv /etc/davfs2/secrets /etc/davfs2/secrets.prev
+          touch /etc/davfs2/secrets
+          chown root: /etc/davfs2
+          chown root: /etc/davfs2/secrets
+          chmod 700 /etc/davfs2
+          chmod 600 /etc/davfs2/secrets
+        ''
+        + (
+          let
+            mkPasswordCmd = cfg': ''
+              printf "%s %s %s\n" "${cfg'.mountPoint}" "${cfg'.username}" "$(cat ${cfg'.passwordFile})" >> /etc/davfs2/secrets
+            '';
+          in
+          lib.concatStringsSep "\n" (map mkPasswordCmd cfg.mounts)
+        );
+      };
+    };
+
     systemd.mounts =
       let
         mkMountCfg = c: {
@@ -82,6 +106,7 @@ in
           description = "Webdav mount point";
           after = [ "network-online.target" ];
           wants = [ "network-online.target" ];
+          wantedBy = [ "multi-user.target" ];
 
           what = c.remoteUrl;
           where = c.mountPoint;
@@ -95,6 +120,6 @@ in
           mountConfig.TimeoutSec = 15;
         };
       in
-        map mkMountCfg cfg.mounts;
+      map mkMountCfg cfg.mounts;
   };
 }

modules/blocks/hardcodedsecret.nix

@@ -1,84 +1,105 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  ...
+}:
 let
   cfg = config.shb.hardcodedsecret;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   inherit (lib) mapAttrs' mkOption nameValuePair;
-  inherit (lib.types) attrsOf nullOr str submodule;
+  inherit (lib.types)
+    attrsOf
+    nullOr
+    str
+    submodule
+    ;
   inherit (pkgs) writeText;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.hardcodedsecret = mkOption {
-    default = {};
+    default = { };
     description = ''
       Hardcoded secrets. These should only be used in tests.
     '';
     example = lib.literalExpression ''
-    {
-      mySecret = {
-        request = {
-          user = "me";
-          mode = "0400";
-          restartUnits = [ "myservice.service" ];
+      {
+        mySecret = {
+          request = {
+            user = "me";
+            mode = "0400";
+            restartUnits = [ "myservice.service" ];
+          };
+          settings.content = "My Secret";
         };
-        settings.content = "My Secret";
-      };
-    }
+      }
     '';
-    type = attrsOf (submodule ({ name, ... }: {
-      options = contracts.secret.mkProvider {
-        settings = mkOption {
-          description = ''
-            Settings specific to the hardcoded secret module.
+    type = attrsOf (
+      submodule (
+        { name, ... }:
+        {
+          options = shb.contracts.secret.mkProvider {
+            settings = mkOption {
+              description = ''
+                Settings specific to the hardcoded secret module.
 
-            Give either `content` or `source`.
-          '';
+                Give either `content` or `source`.
+              '';
 
-          type = submodule {
-            options = {
-              content = mkOption {
-                type = nullOr str;
-                description = ''
-                  Content of the secret as a string.
+              type = submodule {
+                options = {
+                  content = mkOption {
+                    type = nullOr str;
+                    description = ''
+                      Content of the secret as a string.
 
-                  This will be stored in the nix store and should only be used for testing or maybe in dev.
-                '';
-                default = null;
-              };
+                      This will be stored in the nix store and should only be used for testing or maybe in dev.
+                    '';
+                    default = null;
+                  };
 
-              source = mkOption {
-                type = nullOr str;
-                description = ''
-                  Source of the content of the secret as a path in the nix store.
-                '';
-                default = null;
+                  source = mkOption {
+                    type = nullOr str;
+                    description = ''
+                      Source of the content of the secret as a path in the nix store.
+                    '';
+                    default = null;
+                  };
+                };
               };
             };
-          };
-        };
 
-        resultCfg = {
-          path = "/run/hardcodedsecrets/hardcodedsecret_${name}";
-        };
-      };
-    }));
+            resultCfg = {
+              path = "/run/hardcodedsecrets/hardcodedsecret_${name}";
+            };
+          };
+        }
+      )
+    );
   };
 
   config = {
-    system.activationScripts = mapAttrs' (n: cfg':
+    system.activationScripts = mapAttrs' (
+      n: cfg':
       let
-        source = if cfg'.settings.source != null
-                 then cfg'.settings.source
-                 else writeText "hardcodedsecret_${n}_content" cfg'.settings.content;
+        source =
+          if cfg'.settings.source != null then
+            cfg'.settings.source
+          else
+            writeText "hardcodedsecret_${n}_content" cfg'.settings.content;
       in
-        nameValuePair "hardcodedsecret_${n}" ''
-          mkdir -p "$(dirname "${cfg'.result.path}")"
-          touch "${cfg'.result.path}"
-          chmod ${cfg'.request.mode} "${cfg'.result.path}"
-          chown ${cfg'.request.owner}:${cfg'.request.group} "${cfg'.result.path}"
-          cp ${source} "${cfg'.result.path}"
-        ''
+      nameValuePair "hardcodedsecret_${n}" ''
+        mkdir -p "$(dirname "${cfg'.result.path}")"
+        touch "${cfg'.result.path}"
+        chmod ${cfg'.request.mode} "${cfg'.result.path}"
+        chown ${cfg'.request.owner}:${cfg'.request.group} "${cfg'.result.path}"
+        cp ${source} "${cfg'.result.path}"
+      ''
     ) cfg;
   };
 }

modules/blocks/ldap.nix

@@ -1,175 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  cfg = config.shb.ldap;
-
-  contracts = pkgs.callPackage ../contracts {};
-
-  fqdn = "${cfg.subdomain}.${cfg.domain}";
-in
-{
-  options.shb.ldap = {
-    enable = lib.mkEnableOption "the LDAP service";
-
-    dcdomain = lib.mkOption {
-      type = lib.types.str;
-      description = "dc domain to serve.";
-      example = "dc=mydomain,dc=com";
-    };
-
-    subdomain = lib.mkOption {
-      type = lib.types.str;
-      description = "Subdomain under which the LDAP service will be served.";
-      example = "grafana";
-    };
-
-    domain = lib.mkOption {
-      type = lib.types.str;
-      description = "Domain under which the LDAP service will be served.";
-      example = "mydomain.com";
-    };
-
-    ldapPort = lib.mkOption {
-      type = lib.types.port;
-      description = "Port on which the server listens for the LDAP protocol.";
-      default = 3890;
-    };
-
-    ssl = lib.mkOption {
-      description = "Path to SSL files";
-      type = lib.types.nullOr contracts.ssl.certs;
-      default = null;
-    };
-
-    webUIListenPort = lib.mkOption {
-      type = lib.types.port;
-      description = "Port on which the web UI is exposed.";
-      default = 17170;
-    };
-
-    ldapUserPassword = lib.mkOption {
-      description = "LDAP admin user secret.";
-      type = lib.types.submodule {
-        options = contracts.secret.mkRequester {
-          mode = "0440";
-          owner = "lldap";
-          group = "lldap";
-          restartUnits = [ "lldap.service" ];
-        };
-      };
-    };
-
-    jwtSecret = lib.mkOption {
-      description = "JWT secret.";
-      type = lib.types.submodule {
-        options = contracts.secret.mkRequester {
-          mode = "0440";
-          owner = "lldap";
-          group = "lldap";
-          restartUnits = [ "lldap.service" ];
-        };
-      };
-    };
-
-    restrictAccessIPRange = lib.mkOption {
-      type = lib.types.nullOr lib.types.str;
-      description = "Set a local network range to restrict access to the UI to only those IPs.";
-      example = "192.168.1.1/24";
-      default = null;
-    };
-
-    debug = lib.mkOption {
-      description = "Enable debug logging.";
-      type = lib.types.bool;
-      default = false;
-    };
-
-    mount = lib.mkOption {
-      type = contracts.mount;
-      description = ''
-        Mount configuration. This is an output option.
-
-        Use it to initialize a block implementing the "mount" contract.
-        For example, with a zfs dataset:
-
-        ```
-        shb.zfs.datasets."ldap" = {
-          poolName = "root";
-        } // config.shb.ldap.mount;
-        ```
-      '';
-      readOnly = true;
-      default = { path = "/var/lib/lldap"; };
-    };
-
-    backup = lib.mkOption {
-      description = ''
-        Backup configuration.
-      '';
-      type = lib.types.submodule {
-        options = contracts.backup.mkRequester {
-          # TODO: is there a workaround that avoid needing to use root?
-          # root because otherwise we cannot access the private StateDiretory
-          user = "root";
-          # /private because the systemd service uses DynamicUser=true
-          sourceDirectories = [
-            "/var/lib/private/lldap"
-          ];
-        };
-      };
-    };
-  };
-
-  
-  config = lib.mkIf cfg.enable {
-    services.nginx = {
-      enable = true;
-
-      virtualHosts.${fqdn} = {
-        forceSSL = !(isNull cfg.ssl);
-        sslCertificate = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.cert;
-        sslCertificateKey = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.key;
-        locations."/" = {
-          extraConfig = ''
-            proxy_set_header Host $host;
-          '' + (if isNull cfg.restrictAccessIPRange then "" else ''
-            allow ${cfg.restrictAccessIPRange};
-            deny all;
-          '');
-          proxyPass = "http://${toString config.services.lldap.settings.http_host}:${toString config.services.lldap.settings.http_port}/";
-        };
-      };
-    };
-
-    users.users.lldap = {
-      name = "lldap";
-      group = "lldap";
-      isSystemUser = true;
-    };
-    users.groups.lldap = {};
-
-    services.lldap = {
-      enable = true;
-
-      environment = {
-        LLDAP_JWT_SECRET_FILE = toString cfg.jwtSecret.result.path;
-        LLDAP_LDAP_USER_PASS_FILE = toString cfg.ldapUserPassword.result.path;
-
-        RUST_LOG = lib.mkIf cfg.debug "debug";
-      };
-
-      settings = {
-        http_url = "https://${fqdn}";
-        http_host = "127.0.0.1";
-        http_port = cfg.webUIListenPort;
-
-        ldap_host = "127.0.0.1";
-        ldap_port = cfg.ldapPort;
-
-        ldap_base_dn = cfg.dcdomain;
-
-        verbose = cfg.debug;
-      };
-    };
-  };
-}

modules/blocks/lldap.nix

@@ -0,0 +1,467 @@
+{
+  config,
+  pkgs,
+  lib,
+  shb,
+  ...
+}:
+
+let
+  cfg = config.shb.lldap;
+
+  fqdn = "${cfg.subdomain}.${cfg.domain}";
+
+  inherit (lib) mkOption types;
+
+  ensureFormat = pkgs.formats.json { };
+
+  ensureFieldsOptions = name: {
+    name = mkOption {
+      type = types.str;
+      description = "Name of the field.";
+      default = name;
+    };
+
+    attributeType = mkOption {
+      type = types.enum [
+        "STRING"
+        "INTEGER"
+        "JPEG"
+        "DATE_TIME"
+      ];
+      description = "Attribute type.";
+    };
+
+    isEditable = mkOption {
+      type = types.bool;
+      description = "Is field editable.";
+      default = true;
+    };
+
+    isList = mkOption {
+      type = types.bool;
+      description = "Is field a list.";
+      default = false;
+    };
+
+    isVisible = mkOption {
+      type = types.bool;
+      description = "Is field visible in UI.";
+      default = true;
+    };
+  };
+in
+{
+  imports = [
+    ../../lib/module.nix
+    ./mitmdump.nix
+
+    (lib.mkRenamedOptionModule [ "shb" "ldap" ] [ "shb" "lldap" ])
+  ];
+
+  options.shb.lldap = {
+    enable = lib.mkEnableOption "the LDAP service";
+
+    dcdomain = lib.mkOption {
+      type = lib.types.str;
+      description = "dc domain to serve.";
+      example = "dc=mydomain,dc=com";
+    };
+
+    subdomain = lib.mkOption {
+      type = lib.types.str;
+      description = "Subdomain under which the LDAP service will be served.";
+      example = "grafana";
+    };
+
+    domain = lib.mkOption {
+      type = lib.types.str;
+      description = "Domain under which the LDAP service will be served.";
+      example = "mydomain.com";
+    };
+
+    ldapPort = lib.mkOption {
+      type = lib.types.port;
+      description = "Port on which the server listens for the LDAP protocol.";
+      default = 3890;
+    };
+
+    ssl = lib.mkOption {
+      description = "Path to SSL files";
+      type = lib.types.nullOr shb.contracts.ssl.certs;
+      default = null;
+    };
+
+    webUIListenPort = lib.mkOption {
+      type = lib.types.port;
+      description = "Port on which the web UI is exposed.";
+      default = 17170;
+    };
+
+    ldapUserPassword = lib.mkOption {
+      description = "LDAP admin user secret. Must be >= 8 characters.";
+      type = lib.types.submodule {
+        options = shb.contracts.secret.mkRequester {
+          mode = "0440";
+          owner = "lldap";
+          group = "lldap";
+          restartUnits = [ "lldap.service" ];
+        };
+      };
+    };
+
+    jwtSecret = lib.mkOption {
+      description = "JWT secret.";
+      type = lib.types.submodule {
+        options = shb.contracts.secret.mkRequester {
+          mode = "0440";
+          owner = "lldap";
+          group = "lldap";
+          restartUnits = [ "lldap.service" ];
+        };
+      };
+    };
+
+    restrictAccessIPRange = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      description = "Set a local network range to restrict access to the UI to only those IPs.";
+      example = "192.168.1.1/24";
+      default = null;
+    };
+
+    debug = lib.mkOption {
+      description = "Enable debug logging.";
+      type = lib.types.bool;
+      default = false;
+    };
+
+    mount = lib.mkOption {
+      type = shb.contracts.mount;
+      description = ''
+        Mount configuration. This is an output option.
+
+        Use it to initialize a block implementing the "mount" contract.
+        For example, with a zfs dataset:
+
+        ```
+        shb.zfs.datasets."ldap" = {
+          poolName = "root";
+        } // config.shb.lldap.mount;
+        ```
+      '';
+      readOnly = true;
+      default = {
+        path = "/var/lib/lldap";
+      };
+    };
+
+    backup = lib.mkOption {
+      description = ''
+        Backup configuration.
+      '';
+      type = lib.types.submodule {
+        options = shb.contracts.backup.mkRequester {
+          # TODO: is there a workaround that avoid needing to use root?
+          # root because otherwise we cannot access the private StateDiretory
+          user = "root";
+          # /private because the systemd service uses DynamicUser=true
+          sourceDirectories = [
+            "/var/lib/private/lldap"
+          ];
+        };
+      };
+    };
+
+    ensureUsers = mkOption {
+      description = ''
+        Create the users defined here on service startup.
+
+        If `enforceUsers` option is `true`, the groups
+        users belong to must be present in the `ensureGroups` option.
+
+        Non-default options must be added to the `ensureGroupFields` option.
+      '';
+      default = { };
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            freeformType = ensureFormat.type;
+
+            options = {
+              id = mkOption {
+                type = types.str;
+                description = "Username.";
+                default = name;
+              };
+
+              email = mkOption {
+                type = types.str;
+                description = "Email.";
+              };
+
+              password = mkOption {
+                description = "Password.";
+                type = lib.types.submodule {
+                  options = shb.contracts.secret.mkRequester {
+                    mode = "0440";
+                    owner = "lldap";
+                    group = "lldap";
+                    restartUnits = [ "lldap.service" ];
+                  };
+                };
+              };
+
+              displayName = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "Display name.";
+              };
+
+              firstName = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "First name.";
+              };
+
+              lastName = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "Last name.";
+              };
+
+              avatar_file = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "Avatar file. Must be a valid path to jpeg file (ignored if avatar_url specified)";
+              };
+
+              avatar_url = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "Avatar url. must be a valid URL to jpeg file (ignored if gravatar_avatar specified)";
+              };
+
+              gravatar_avatar = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "Get avatar from Gravatar using the email.";
+              };
+
+              weser_avatar = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = "Convert avatar retrieved by gravatar or the URL.";
+              };
+
+              groups = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = "Groups the user would be a member of (all the groups must be specified in group config files).";
+              };
+            };
+          }
+        )
+      );
+    };
+
+    ensureGroups = mkOption {
+      description = ''
+        Create the groups defined here on service startup.
+
+        Non-default options must be added to the `ensureGroupFields` option.
+      '';
+      default = { };
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            freeformType = ensureFormat.type;
+
+            options = {
+              name = mkOption {
+                type = types.str;
+                description = "Name of the group.";
+                default = name;
+              };
+            };
+          }
+        )
+      );
+    };
+
+    ensureUserFields = mkOption {
+      description = "Extra fields for users";
+      default = { };
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = ensureFieldsOptions name;
+          }
+        )
+      );
+    };
+
+    ensureGroupFields = mkOption {
+      description = "Extra fields for groups";
+      default = { };
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = ensureFieldsOptions name;
+          }
+        )
+      );
+    };
+
+    enforceUsers = mkOption {
+      description = "Delete users not set declaratively.";
+      type = types.bool;
+      default = false;
+    };
+
+    enforceUserMemberships = mkOption {
+      description = "Remove users from groups not set declaratively.";
+      type = types.bool;
+      default = false;
+    };
+
+    enforceGroups = mkOption {
+      description = "Remove groups not set declaratively.";
+      type = types.bool;
+      default = false;
+    };
+
+    dashboard = lib.mkOption {
+      description = ''
+        Dashboard contract consumer
+      '';
+      default = { };
+      type = lib.types.submodule {
+        options = shb.contracts.dashboard.mkRequester {
+          externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
+          externalUrlText = "https://\${config.shb.lldap.subdomain}.\${config.shb.lldap.domain}";
+          internalUrl = "http://127.0.0.1:${toString cfg.webUIListenPort}";
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    services.nginx = {
+      enable = true;
+
+      virtualHosts.${fqdn} = {
+        forceSSL = !(isNull cfg.ssl);
+        sslCertificate = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.cert;
+        sslCertificateKey = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.key;
+        locations."/" = {
+          extraConfig = ''
+            proxy_set_header Host $host;
+          ''
+          + (
+            if isNull cfg.restrictAccessIPRange then
+              ""
+            else
+              ''
+                allow ${cfg.restrictAccessIPRange};
+                deny all;
+              ''
+          );
+          proxyPass = "http://${toString config.services.lldap.settings.http_host}:${toString config.shb.lldap.webUIListenPort}/";
+        };
+      };
+    };
+
+    users.users.lldap = {
+      name = "lldap";
+      group = "lldap";
+      isSystemUser = true;
+    };
+    users.groups.lldap = { };
+
+    services.lldap = {
+      enable = true;
+
+      inherit (cfg) enforceUsers enforceUserMemberships enforceGroups;
+
+      environment = {
+        RUST_LOG = lib.mkIf cfg.debug "debug";
+      };
+
+      settings = {
+        http_url = "https://${fqdn}";
+        http_host = "127.0.0.1";
+        http_port = if !cfg.debug then cfg.webUIListenPort else cfg.webUIListenPort + 1;
+
+        ldap_host = "127.0.0.1";
+        ldap_port = cfg.ldapPort; # Would be great to be able to inspect this but it requires tcpdump instead of mitmproxy.
+        ldap_base_dn = cfg.dcdomain;
+
+        ldap_user_pass_file = toString cfg.ldapUserPassword.result.path;
+        force_ldap_user_pass_reset = "always";
+        jwt_secret_file = toString cfg.jwtSecret.result.path;
+
+        verbose = cfg.debug;
+      };
+
+      inherit (cfg) ensureGroups ensureUserFields ensureGroupFields;
+      ensureUsers = lib.mapAttrs (
+        n: v:
+        (lib.removeAttrs v [ "password" ])
+        // {
+          "password_file" = toString v.password.result.path;
+        }
+      ) cfg.ensureUsers;
+    };
+
+    # Harden lldap following https://github.com/NixOS/nixpkgs/pull/487933
+    systemd.services.lldap.serviceConfig = {
+      RemoveIPC = true;
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      RestrictAddressFamilies = [
+        "AF_UNIX"
+        "AF_INET"
+        "AF_INET6"
+      ];
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+        "~@resources"
+      ];
+      SystemCallArchitectures = "native";
+      CapabilityBoundingSet = "";
+      LockPersonality = true;
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      PrivateDevices = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectSystem = "strict";
+      ProtectProc = "invisible";
+      ProcSubset = "pid";
+      MemoryDenyWriteExecute = true;
+    };
+
+    shb.mitmdump.instances."lldap-web" = lib.mkIf cfg.debug {
+      listenPort = config.shb.lldap.webUIListenPort;
+      upstreamPort = config.shb.lldap.webUIListenPort + 1;
+      after = [ "lldap.service" ];
+      enabledAddons = [ config.shb.mitmdump.addons.logger ];
+      extraArgs = [
+        "--set"
+        "verbose_pattern=/api"
+      ];
+    };
+  };
+}

modules/blocks/lldap/docs/default.md

@@ -0,0 +1,212 @@
+# LLDAP Block {#blocks-lldap}
+
+Defined in [`/modules/blocks/lldap.nix`](@REPO@/modules/blocks/lldap.nix).
+
+This block sets up an [LLDAP][] service for user and group management
+across services.
+
+[LLDAP]: https://github.com/lldap/lldap
+
+## Features {#blocks-lldap-features}
+
+- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#blocks-lldap-usage-applicationdashboard)
+
+## Usage {#blocks-lldap-usage}
+
+### Initial Configuration {#blocks-lldap-usage-configuration}
+
+```nix
+shb.lldap = {
+  enable = true;
+  subdomain = "ldap";
+  domain = "example.com";
+  dcdomain = "dc=example,dc=com";
+
+  ldapPort = 3890;
+  webUIListenPort = 17170;
+
+  jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
+  ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
+};
+shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
+shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
+```
+
+This assumes secrets are setup with SOPS
+as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual.
+
+### SSL {#blocks-lldap-usage-ssl}
+
+Using SSL is an important security practice, like always.
+Using the [SSL block][], the configuration to add to the one above is:
+
+[SSL block]: blocks-ssl.html
+
+```nix
+shb.certs.certs.letsencrypt.${domain}.extraDomains = [
+  "${config.shb.lldap.subdomain}.${config.shb.lldap.domain}"
+];
+
+shb.lldap.ssl = config.shb.certs.certs.letsencrypt.${config.shb.lldap.domain};
+```
+
+### Restrict Access By IP {#blocks-lldap-usage-restrict-access-by-ip}
+
+For added security, you can restrict access to the LLDAP UI
+by adding the following line:
+
+```nix
+shb.lldap.restrictAccessIPRange = "192.168.50.0/24";
+```
+
+### Application Dashboard {#blocks-lldap-usage-applicationdashboard}
+
+Integration with the [dashboard contract](contracts-dashboard.html) is provided
+by the [dashboard option](#blocks-lldap-options-shb.lldap.dashboard).
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.LLDAP = {
+    sortOrder = 2;
+    dashboard.request = config.shb.lldap.dashboard.request;
+  };
+}
+```
+
+## Manage Groups {#blocks-lldap-manage-groups}
+
+The following snippet will create group named "family" if it does not exist yet.
+Also, all other groups will be deleted and only the "family" group will remain.
+
+Note that the `lldap_admin`, `lldap_password_manager` and `lldap_strict_readonly` groups, which are internal to LLDAP, will always exist.
+
+If you want existing groups not declared in the `shb.lldap.ensureGroups` to be deleted,
+set [`shb.lldap.enforceGroups`](#blocks-lldap-options-shb.lldap.enforceGroups) to `true`.
+
+```nix
+{
+  shb.lldap.ensureGroups = {
+   family = {};
+  };
+}
+```
+
+Changing the configuration to the following will add a new group "friends":
+
+```nix
+{
+  shb.lldap.ensureGroups = {
+    family = {};
+    friends = {};
+  };
+}
+```
+
+Switching back the configuration to the previous one will delete the group "friends":
+
+```nix
+{
+  shb.lldap.ensureGroups = {
+    family = {};
+  };
+}
+```
+
+Custom fields can be added to groups as long as they are added to the `ensureGroupFields` field:
+
+```nix
+shb.lldap = {
+  ensureGroupFields = {
+    mygroupattribute = {
+      attributeType = "STRING";
+    };
+  };
+
+  ensureGroups = {
+    family = {
+      mygroupattribute = "Managed by NixOS";
+    };
+  };
+};
+```
+
+## Manage Users {#blocks-lldap-manage-users}
+
+The following snippet creates a user and makes it a member of the "family" group.
+
+Note the following behavior:
+
+- New users will be created following the `shb.lldap.ensureUsers` option.
+- Existing users will be updated, their password included, if they are mentioned in the `shb.lldap.ensureUsers` option.
+- Existing users not declared in the `shb.lldap.ensureUsers` will be left as-is.
+- User memberships to groups not declared in their respective `shb.lldap.ensureUsers.<name>.groups`.
+
+If you want existing users not declared in the `shb.lldap.ensureUsers` to be deleted,
+set [`shb.lldap.enforceUsers`](#blocks-lldap-options-shb.lldap.enforceUsers) to `true`.
+
+If you want memberships to groups not declared in the respective
+`shb.lldap.ensureUsers.<name>.groups` option to be deleted,
+set [`shb.lldap.enforceUserMemberships`](#blocks-lldap-options-shb.lldap.enforceUserMemberships) `true`.
+
+```nix
+{
+  shb.lldap.ensureUsers = {
+    dad = {
+      email = "dad@example.com";
+      displayName = "Dad";
+      firstName = "First Name";
+      lastName = "Last Name";
+      groups = [ "family" ];
+      password.result = config.shb.sops.secret."dad".result;
+    };
+  };
+
+  shb.sops.secret."dad".request =
+    config.shb.lldap.ensureUsers.dad.password.request;
+}
+```
+
+The password field assumes usage of the [sops block][] to provide secrets
+although any blocks providing the [secrets contract][] works too.
+
+[sops block]: blocks-sops.html
+[secrets contract]: contracts-secrets.html
+
+The user is still editable through the UI.
+That being said, any change will be overwritten next time the configuration is applied.
+
+## Troubleshooting {#blocks-lldap-troubleshooting}
+
+To increase logging verbosity and see the trace of the GraphQL queries, add:
+
+```nix
+shb.lldap.debug = true;
+```
+
+Note that verbosity is truly verbose here
+so you will want to revert this at some point.
+
+To see the logs, then run `journalctl -u lldap.service`.
+
+Setting the `debug` option to `true` will also
+add an [shb.mitmdump][] instance in front of the LLDAP [web UI port](#blocks-lldap-options-shb.lldap.webUIListenPort)
+which prints all requests and responses headers and body
+to the systemd service `mitmdump-lldap.service`. Note the you won't
+see the query done using something like `ldapsearch` since those
+go through the [`LDAP` port](#blocks-lldap-options-shb.lldap.ldapPort).
+
+[shb.mitmdump]: ./blocks-mitmdump.html
+
+## Tests {#blocks-lldap-tests}
+
+Specific integration tests are defined in [`/test/blocks/lldap.nix`](@REPO@/test/blocks/lldap.nix).
+
+## Options Reference {#blocks-lldap-options}
+
+```{=include=} options
+id-prefix: blocks-lldap-options-
+list-id: selfhostblocks-block-lldap-options
+source: @OPTIONS_JSON@
+```

modules/blocks/mitmdump.nix

@@ -0,0 +1,307 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  inherit (lib)
+    mapAttrs'
+    mkOption
+    nameValuePair
+    types
+    ;
+  inherit (types)
+    attrsOf
+    listOf
+    port
+    submodule
+    str
+    ;
+
+  cfg = config.shb.mitmdump;
+
+  mitmdumpScript =
+    pkgs.writers.writePython3Bin "mitmdump"
+      {
+        libraries =
+          let
+            p = pkgs.python3Packages;
+          in
+          [
+            p.systemd-python
+            p.mitmproxy
+          ];
+        flakeIgnore = [ "E501" ];
+      }
+      ''
+        from systemd.daemon import notify
+        import argparse
+        import logging
+        import os
+        import subprocess
+        import socket
+        import sys
+        import time
+
+
+        logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+
+        def wait_for_port(host, port, timeout):
+            deadline = time.time() + timeout
+            while time.time() < deadline:
+                try:
+                    with socket.create_connection((host, port), timeout=0.5):
+                        return True
+                except Exception:
+                    time.sleep(0.1)
+            return False
+
+
+        def flatten(xss):
+            return [x for xs in xss for x in xs]
+
+
+        parser = argparse.ArgumentParser()
+        parser.add_argument("--listen_host", default="127.0.0.1", help="Host mitmdump will listen on")
+        parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
+        parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
+        parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
+        parser.add_argument("--timeout", required=False, type=int, default=10, help="Timeout to wait for port availability")
+        args, rest = parser.parse_known_args()
+
+        MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
+        if MITMDUMP_BIN is None:
+            raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
+
+        logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
+        wait_for_port(args.upstream_host, args.upstream_port, timeout=args.timeout)
+        logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
+
+        proc = subprocess.Popen(
+            [
+                MITMDUMP_BIN,
+                "--listen-host", args.listen_host,
+                "-p", args.listen_port,
+                "--mode", f"reverse:{args.upstream_host}:{args.upstream_port}",
+            ] + rest,
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+        )
+
+        logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
+        if wait_for_port("127.0.0.1", args.listen_port, timeout=args.timeout):
+            logging.info(f"Mitmdump is started on port {args.listen_port}.")
+            notify("READY=1")
+        else:
+            print(f"Mitmdump instance did not start before the timeout of {args.timeout} seconds, consider increasing the timeout")
+            proc.terminate()
+            exit(1)
+
+        proc.wait()
+      '';
+
+  logger = toString (
+    pkgs.writers.writeText "loggerAddon.py" ''
+      import logging
+      from collections.abc import Sequence
+      from mitmproxy import ctx, http
+      import re
+
+
+      logger = logging.getLogger(__name__)
+
+
+      class RegexLogger:
+          def __init__(self):
+              self.verbose_patterns = None
+
+          def load(self, loader):
+              loader.add_option(
+                  name="verbose_pattern",
+                  typespec=Sequence[str],
+                  default=[],
+                  help="Regex patterns for verbose logging",
+              )
+
+          def response(self, flow: http.HTTPFlow):
+              if self.verbose_patterns is None:
+                  self.verbose_patterns = [re.compile(p) for p in ctx.options.verbose_pattern]
+
+              matched = any(p.search(flow.request.path) for p in self.verbose_patterns)
+              if matched:
+                  logger.info(format_flow(flow))
+
+
+      def format_flow(flow: http.HTTPFlow) -> str:
+          return (
+              "\n"
+              "RequestHeaders:\n"
+              f"    {format_headers(flow.request.headers.items())}\n"
+              f"RequestBody:     {flow.request.get_text()}\n"
+              f"Status:          {flow.response.data.status_code}\n"
+              "ResponseHeaders:\n"
+              f"    {format_headers(flow.response.headers.items())}\n"
+              f"ResponseBody:    {flow.response.get_text()}\n"
+          )
+
+
+      def format_headers(headers) -> str:
+          return "\n    ".join(k + ": " + v for k, v in headers)
+
+
+      addons = [RegexLogger()]
+    ''
+  );
+in
+{
+  options.shb.mitmdump = {
+    addons = mkOption {
+      type = attrsOf str;
+      default = [ ];
+      description = ''
+        Addons available to the be added to the mitmdump instance.
+
+        To enabled them, add them to the `enabledAddons` option.
+      '';
+    };
+
+    instances = mkOption {
+      default = { };
+      description = "Mitmdump instance.";
+      type = attrsOf (
+        submodule (
+          { name, ... }:
+          {
+            options = {
+              package = lib.mkPackageOption pkgs "mitmproxy" { };
+
+              serviceName = mkOption {
+                type = str;
+                description = ''
+                  Name of the mitmdump system service.
+                '';
+                default = "mitmdump-${name}.service";
+                readOnly = true;
+              };
+
+              listenHost = mkOption {
+                type = str;
+                default = "127.0.0.1";
+                description = ''
+                  Host the mitmdump instance will connect on.
+                '';
+              };
+
+              listenPort = mkOption {
+                type = port;
+                description = ''
+                  Port the mitmdump instance will listen on.
+
+                  The upstream port from the client's perspective.
+                '';
+              };
+
+              upstreamHost = mkOption {
+                type = str;
+                default = "http://127.0.0.1";
+                description = ''
+                  Host the mitmdump instance will connect to.
+
+                  If only an IP or domain is provided,
+                  mitmdump will default to connect using HTTPS.
+                  If this is not wanted, prefix the IP or domain with the 'http://' protocol.
+                '';
+              };
+
+              upstreamPort = mkOption {
+                type = port;
+                description = ''
+                  Port the mitmdump instance will connect to.
+
+                  The port the server is listening on.
+                '';
+              };
+
+              timeout = mkOption {
+                type = types.int;
+                default = 30;
+                description = ''
+                  Time to wait for upstream to start and mitmdump to start.
+                '';
+              };
+
+              after = mkOption {
+                type = listOf str;
+                default = [ ];
+                description = ''
+                  Systemd services that must be started before this mitmdump proxy instance.
+
+                  You are guaranteed the mitmdump is listening on the `listenPort`
+                  when its systemd service has started.
+                '';
+              };
+
+              enabledAddons = mkOption {
+                type = listOf str;
+                default = [ ];
+                description = ''
+                  Addons to enable on this mitmdump instance.
+                '';
+                example = lib.literalExpression "[ config.shb.mitmdump.addons.logger ]";
+              };
+
+              extraArgs = mkOption {
+                type = listOf str;
+                default = [ ];
+                description = ''
+                  Extra arguments to pass to the mitmdump instance.
+
+                  See upstream [manual](https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) for all possible options.
+                '';
+                example = lib.literalExpression ''[ "--set" "verbose_pattern=/api" ]'';
+              };
+            };
+          }
+        )
+      );
+    };
+  };
+
+  config = {
+    systemd.services = mapAttrs' (
+      name: cfg':
+      nameValuePair "mitmdump-${name}" {
+        environment = {
+          "HOME" = "/var/lib/private/mitmdump-${name}";
+          "MITMDUMP_BIN" = "${cfg'.package}/bin/mitmdump";
+        };
+        serviceConfig = {
+          Type = "notify";
+          Restart = "on-failure";
+          StandardOutput = "journal";
+          StandardError = "journal";
+
+          DynamicUser = true;
+          WorkingDirectory = "/var/lib/mitmdump-${name}";
+          StateDirectory = "mitmdump-${name}";
+
+          ExecStart =
+            let
+              addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
+              extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
+            in
+            "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} --timeout ${toString cfg'.timeout} ${addons} ${extraArgs}";
+        };
+        requires = cfg'.after;
+        after = cfg'.after;
+        wantedBy = [ "multi-user.target" ];
+      }
+    ) cfg.instances;
+
+    shb.mitmdump.addons = {
+      inherit logger;
+    };
+  };
+}

modules/blocks/mitmdump/docs/default.md

@@ -0,0 +1,178 @@
+# Mitmdump Block {#blocks-mitmdump}
+
+Defined in [`/modules/blocks/mitmdump.nix`](@REPO@/modules/blocks/mitmdump.nix).
+
+This block sets up an [Mitmdump][] service in [reverse proxy][] mode.
+In other words, you can put this block between a client and a server to inspect all the network traffic.
+
+[Mitmdump]: https://plattner.me/mp-docs/#mitmdump
+[reverse proxy]: https://plattner.me/mp-docs/concepts-modes/#reverse-proxy
+
+Multiple instances of mitmdump all listening on different ports
+and proxying to different upstream servers can be created.
+
+The systemd service is made so it is started only when the mitmdump instance
+has started listening on the expected port.
+
+Also, addons can be enabled with the `enabledAddons` option.
+
+## Usage {#blocks-mitmdump-usage}
+
+Put mitmdump in front of a HTTP server listening on port 8000 on the same machine:
+
+```nix
+shb.mitmdump.instances."my-instance" = {
+  listenPort = 8001;
+  upstreamHost = "http://127.0.0.1";
+  upstreamPort = 8000;
+  after = [ "server.service" ];
+};
+```
+
+`upstreamHost` has its default value here and can be left out.
+
+Put mitmdump in front of a HTTP server listening on port 8000 on another machine:
+
+```nix
+shb.mitmdump.instances."my-instance" = {
+  listenPort = 8001;
+  upstreamHost = "http://otherhost";
+  upstreamPort = 8000;
+  after = [ "server.service" ];
+};
+```
+
+### Handle Upstream TLS {#blocks-mitmdump-usage-https}
+
+Replace `http` with `https` if the server expects an HTTPS connection.
+
+### Accept Connections from Anywhere {#blocks-mitmdump-usage-anywhere}
+
+By default, `mitmdump` is configured to listen only for connections from localhost.
+Add `listenHost=0.0.0.0` to make `mitmdump` accept connections from anywhere.
+
+### Extra Logging {#blocks-mitmdump-usage-logging}
+
+To print request and response bodies and more, increase the logging with:
+
+```nix
+extraArgs = [
+    "--set" "flow_detail=3"
+    "--set" "content_view_lines_cutoff=2000"
+];
+```
+
+The default `flow_details` is 1. See the [manual][] for more explanations on the option.
+
+[manual]: (https://docs.mitmproxy.org/stable/concepts/options/#flow_detail)
+
+This will change the verbosity for all requests and responses.
+If you need more fine grained logging, configure instead the [Logger Addon][].
+
+[Logger Addon]: #blocks-mitmdump-addons-logger
+
+## Addons {#blocks-mitmdump-addons}
+
+All provided addons can be found under the `shb.mitmproxy.addons` option.
+
+To enable one for an instance, add it to the `enabledAddons` option. For example:
+
+```nix
+shb.mitmdump.instances."my-instance" = {
+    enabledAddons = [ config.shb.mitmdump.addons.logger ]
+}
+```
+
+### Fine Grained Logger {#blocks-mitmdump-addons-logger}
+
+The Fine Grained Logger addon is found under `shb.mitmproxy.addons.logger`.
+Enabling this addon will add the `mitmdump` option `verbose_pattern` which takes a regex and if it matches,
+prints the request and response headers and body.
+If it does not match, it will just print the response status.
+
+For example, with the `extraArgs`:
+
+```nix
+extraArgs = [
+  "--set" "verbose_pattern=/verbose"
+];
+```
+
+A `GET` request to `/notverbose` will print something similar to:
+
+```
+mitmdump[972]: 127.0.0.1:53586: GET http://127.0.0.1:8000/notverbose HTTP/1.1
+mitmdump[972]:      << HTTP/1.0 200 OK 16b
+```
+
+While a `GET` request to `/verbose` will print something similar to:
+
+```
+mitmdump[972]: [22:42:58.840]
+mitmdump[972]: RequestHeaders:
+mitmdump[972]:     Host: 127.0.0.1:8000
+mitmdump[972]:     User-Agent: curl/8.14.1
+mitmdump[972]:     Accept: */*
+mitmdump[972]: RequestBody:
+mitmdump[972]: Status:          200
+mitmdump[972]: ResponseHeaders:
+mitmdump[972]:     Server: BaseHTTP/0.6 Python/3.13.4
+mitmdump[972]:     Date: Sun, 03 Aug 2025 22:42:58 GMT
+mitmdump[972]:     Content-Type: text/plain
+mitmdump[972]:     Content-Length: 13
+mitmdump[972]: ResponseBody:    test2/verbose
+mitmdump[972]: 127.0.0.1:53602: GET http://127.0.0.1:8000/verbose HTTP/1.1
+mitmdump[972]:      << HTTP/1.0 200 OK 13b
+```
+
+## Example {#blocks-mitmdump-example}
+
+Let's assume a server is listening on port 8000
+which responds a plain text response `test1`
+and its related systemd service is named `test1.service`.
+Sorry, creative naming is not my forte.
+
+Let's put an mitmdump instance in front of it, like so:
+
+```nix
+shb.mitmdump.instances."test1" = {
+  listenPort = 8001;
+  upstreamPort = 8000;
+  after = [ "test1.service" ];
+  extraArgs = [
+    "--set" "flow_detail=3"
+    "--set" "content_view_lines_cutoff=2000"
+  ];
+};
+```
+
+This creates an `mitmdump-test1.service` systemd service.
+We can then use `journalctl -u mitmdump-test1.service` to see the output.
+
+If we make a `curl` request to it: `curl -v http://127.0.0.1:8001`,
+we will get the following output:
+
+```
+mitmdump-test1[971]: 127.0.0.1:40878: GET http://127.0.0.1:8000/ HTTP/1.1
+mitmdump-test1[971]:     Host: 127.0.0.1:8000
+mitmdump-test1[971]:     User-Agent: curl/8.14.1
+mitmdump-test1[971]:     Accept: */*
+mitmdump-test1[971]:  << HTTP/1.0 200 OK 5b
+mitmdump-test1[971]:     Server: BaseHTTP/0.6 Python/3.13.4
+mitmdump-test1[971]:     Date: Thu, 31 Jul 2025 20:55:16 GMT
+mitmdump-test1[971]:     Content-Type: text/plain
+mitmdump-test1[971]:     Content-Length: 5
+mitmdump-test1[971]:     test1
+```
+
+## Tests {#blocks-mitmdump-tests}
+
+Specific integration tests are defined in [`/test/blocks/mitmdump.nix`](@REPO@/test/blocks/mitmdump.nix).
+
+## Options Reference {#blocks-mitmdump-options}
+
+```{=include=} options
+id-prefix: blocks-mitmdump-options-
+list-id: selfhostblocks-block-mitmdump-options
+source: @OPTIONS_JSON@
+```

modules/blocks/monitoring/dashboards/Health.json

@@ -0,0 +1,872 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "panels": [],
+      "repeat": "hostname",
+      "title": "${hostname}",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "footer": {
+              "reducers": []
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 3,
+      "maxDataPoints": 400,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "node_os_info",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "OS Versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {
+            "keepLabels": [
+              "build_id",
+              "domain",
+              "hostname",
+              "id",
+              "instance",
+              "job",
+              "name",
+              "pretty_name",
+              "version",
+              "version_codename",
+              "version_id"
+            ],
+            "mode": "columns"
+          }
+        },
+        {
+          "id": "groupBy",
+          "options": {
+            "fields": {
+              "Time": {
+                "aggregations": [
+                  "firstNotNull"
+                ],
+                "operation": "aggregate"
+              },
+              "pretty_name": {
+                "aggregations": [],
+                "operation": "groupby"
+              }
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "expr": "node_hwmon_temp_celsius{hostname=~\"$hostname\"}",
+          "instant": false,
+          "legendFormat": "{{chip}} - {{sensor}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Temperature",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 0
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "id": 5,
+      "interval": "1m",
+      "maxDataPoints": 200,
+      "options": {
+        "colWidth": 1,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "rowHeight": 0.8,
+        "showValue": "never",
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "none",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "repeat": "hostname",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "node_zfs_zpool_state{hostname=~\"$hostname\", state=\"online\"} > 0",
+          "legendFormat": "{{zpool}} - {{state}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "ZFS Pools",
+      "type": "status-history"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "line+area"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              },
+              {
+                "color": "transparent",
+                "value": 604808
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "expr": "ssl_certificate_expiry_seconds",
+          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Certificate Remaining Validity",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "years"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "editorMode": "builder",
+          "expr": "scrutiny_smart_power_on_hours{hostname=~\"$hostname\"} / (24 * 365)",
+          "legendFormat": "{{device_name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Operating Years",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-YlRd"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 100,
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 21
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "width": 400
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum by(hostname, domain, mountpoint, device) (node_filesystem_free_bytes{hostname=~\"$hostname\"})",
+          "fullMetaSearch": false,
+          "hide": true,
+          "includeNullMetadata": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "builder",
+          "expr": "sum by(hostname, domain, mountpoint, device) (node_filesystem_size_bytes{hostname=~\"$hostname\"})",
+          "hide": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "name": "Expression",
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "(1 - $A / $B) * 100",
+          "refId": "Disk Full",
+          "type": "math"
+        }
+      ],
+      "title": "Filesystem Disk Usage",
+      "transformations": [
+        {
+          "id": "joinByField",
+          "options": {
+            "byField": "Time",
+            "mode": "outer"
+          }
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 21
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "editorMode": "builder",
+          "expr": "scrutiny_smart_power_on_hours{hostname=~\"$hostname\"}",
+          "legendFormat": "{{device_name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Operating Years",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "footer": {
+              "reducers": []
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 29
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "scrutiny_device_info{hostname=~\"$hostname\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{device_name}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Disk Info",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "domain": true,
+              "hostname": true,
+              "instance": true,
+              "job": true,
+              "wwn": false
+            },
+            "includeByName": {},
+            "indexByName": {},
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 42,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "baryum",
+          "value": "baryum"
+        },
+        "definition": "label_values(up,hostname)",
+        "name": "hostname",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(up,hostname)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "regexApplyTo": "value",
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Node Health",
+  "uid": "edhuvl28vpjwge",
+  "version": 25,
+  "weekStart": ""
+}

modules/blocks/monitoring/dashboards/SSL.json

@@ -1,143 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": 16,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "line+area"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "transparent",
-                "value": 604808
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 12,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true,
-          "sortBy": "Last *",
-          "sortDesc": false
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
-          },
-          "editorMode": "code",
-          "expr": "ssl_certificate_expiry_seconds",
-          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Certificate Remaining Validity",
-      "type": "timeseries"
-    }
-  ],
-  "preload": false,
-  "schemaVersion": 40,
-  "tags": [],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "browser",
-  "title": "SSL Certificates",
-  "uid": "ae818js0bvw8wb",
-  "version": 8,
-  "weekStart": ""
-}

modules/blocks/monitoring/dashboards/Scraping_Jobs.json

@@ -21,6 +21,50 @@
   "id": 5,
   "links": [],
   "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "{unit=~\"prometheus-.*-exporter.service\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Exporter Logs",
+      "type": "logs"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -29,7 +73,8 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "fixedColor": "red",
+            "mode": "shades"
           },
           "custom": {
             "axisBorderShow": false,
@@ -55,6 +100,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -70,7 +116,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -96,11 +142,12 @@
           "showLegend": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
-      "pluginVersion": "11.4.0",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -127,9 +174,10 @@
         "defaults": {
           "color": {
             "fixedColor": "red",
-            "mode": "fixed"
+            "mode": "thresholds"
           },
           "custom": {
+            "axisPlacement": "auto",
             "fillOpacity": 70,
             "hideFrom": {
               "legend": false,
@@ -146,11 +194,11 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 1
               }
             ]
           }
@@ -175,11 +223,12 @@
         "rowHeight": 0.9,
         "showValue": "never",
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
-      "pluginVersion": "11.4.0",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -200,7 +249,7 @@
             "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
           },
           "editorMode": "code",
-          "expr": "label_replace(increase((sum by(dialer_name) (net_conntrack_dialer_conn_failed_total{hostname=~\"$hostname\"}))[15m:1m]), \"config\", \"$1\", \"dialer_name\", \"(.*)\") > 10",
+          "expr": "label_replace(increase((sum by(dialer_name) (net_conntrack_dialer_conn_failed_total{hostname=~\"$hostname\"}))[15m:1m]), \"config\", \"$1\", \"dialer_name\", \"(.*)\")",
           "hide": false,
           "instant": false,
           "legendFormat": "{{dialer_name}}",
@@ -249,7 +298,7 @@
   ],
   "preload": false,
   "refresh": "",
-  "schemaVersion": 40,
+  "schemaVersion": 42,
   "tags": [],
   "templating": {
     "list": [
@@ -281,6 +330,5 @@
   "timezone": "",
   "title": "Scraping Jobs",
   "uid": "debb763d-77aa-47bd-9290-2e02583c8ed2",
-  "version": 15,
-  "weekStart": ""
+  "version": 24
 }

modules/blocks/monitoring/docs/default.md

@@ -8,38 +8,70 @@ This block sets up the monitoring stack for Self Host Blocks. It is composed of:
 - Prometheus as the database for metrics.
 - Loki as the database for logs.
 
-## Configuration {#blocks-monitoring-configuration}
+## Features {#services-monitoring-features}
+
+- Declarative [LDAP](#blocks-monitoring-options-shb.monitoring.ldap) Configuration.
+  - Needed LDAP groups are created automatically.
+- Declarative [SSO](#blocks-monitoring-options-shb.monitoring.sso) Configuration.
+  - When SSO is enabled, login with user and password is disabled.
+  - Registration is enabled through SSO.
+- Access through [subdomain](#blocks-monitoring-options-shb.monitoring.subdomain) using reverse proxy.
+- Access through [HTTPS](#blocks-monitoring-options-shb.monitoring.ssl) using reverse proxy.
+- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#blocks-monitoring-usage-applicationdashboard)
+- Out of the box integration with [Scrutiny](https://github.com/AnalogJ/scrutiny) service for Hard Drives monitoring. [Manual](#blocks-monitoring-usage-scrutiny)
+
+## Usage {#blocks-monitoring-usage}
+
+### Initial Configuration {#blocks-monitoring-usage-configuration}
+
+The following snippet assumes a few blocks have been setup already:
+
+- the [secrets block](usage.html#usage-secrets) with SOPS,
+- the [`shb.ssl` block](blocks-ssl.html#usage),
+- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
+- the [`shb.authelia` block](blocks-authelia.html#blocks-sso-global-setup).
 
 ```nix
-shb.monitoring = {
-  enable = true;
-  subdomain = "grafana";
-  inherit domain;
-  contactPoints = [ "me@example.com" ];
-  adminPassword.result = config.sops.secrets."monitoring/admin_password".result;
-  secretKey.result = config.sops.secrets."monitoring/secret_key".result;
-};
+{
+  shb.monitoring = {
+    enable = true;
+    subdomain = "grafana";
+    inherit domain;
+    contactPoints = [ "me@example.com" ];
+    adminPassword.result = config.shb.sops.secret."monitoring/admin_password".result;
+    secretKey.result = config.shb.sops.secret."monitoring/secret_key".result;
 
-sops.secrets."monitoring/admin_password" = {
-  sopsFile = ./secrets.yaml;
-  mode = "0400";
-  owner = "grafana";
-  group = "grafana";
-  restartUnits = [ "grafana.service" ];
-};
-sops.secrets."monitoring/secret_key" = {
-  sopsFile = ./secrets.yaml;
-  mode = "0400";
-  owner = "grafana";
-  group = "grafana";
-  restartUnits = [ "grafana.service" ];
+    sso = {
+      enable = true;
+      authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+
+      sharedSecret.result = config.shb.sops.secret."monitoring/oidcSecret".result;
+      sharedSecretForAuthelia.result = config.shb.sops.secret."monitoring/oidcAutheliaSecret".result;
+    };
+  };
+  
+  shb.sops.secret."monitoring/admin_password".request = config.shb.monitoring.adminPassword.request;
+  shb.sops.secret."monitoring/secret_key".request = config.shb.monitoring.secretKey.request;
+  shb.sops.secret."monitoring/oidcSecret".request = config.shb.monitoring.sso.sharedSecret.request;
+  shb.sops.secret."monitoring/oidcAutheliaSecret" = {
+    request = config.shb.monitoring.sso.sharedSecretForAuthelia.request;
+    settings.key = "monitoring/oidcSecret";
+  };
 };
 ```
 
-With that, Grafana, Prometheus, Loki and Promtail are setup! You can access `Grafana` at
-`grafana.example.com` with user `admin` and password ``.
+Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.
 
-I recommend adding a STMP server configuration so you receive alerts by email:
+With that, Grafana, Prometheus, Loki and Promtail are setup! You can access `Grafana` at
+`grafana.example.com` with user `admin` and the password from the sops key `monitoring/admin_password`.
+
+The [user](#blocks-monitoring-options-shb.monitoring.ldap.userGroup)
+and [admin](#blocks-monitoring-options-shb.monitoring.ldap.adminGroup)
+LDAP groups are created automatically.
+
+### SMTP {#blocks-monitoring-usage-smtp}
+
+I recommend adding an SMTP server configuration so you receive alerts by email:
 
 ```nix
 shb.monitoring.smtp = {
@@ -60,6 +92,8 @@ sops.secrets."monitoring/secret_key" = {
 };
 ```
 
+### Log Optimization {#blocks-monitoring-usage-log-optimization}
+
 Since all logs are now stored in Loki, you can probably reduce the systemd journal retention
 time with:
 
@@ -73,6 +107,49 @@ MaxFileSec=day
 '';
 ```
 
+Other options are accessible through the upstream services modules.
+You might for example want to update the metrics retention time with:
+
+```nix
+services.prometheus.retentionTime = "60d";
+```
+
+### Application Dashboard {#blocks-monitoring-usage-applicationdashboard}
+
+Integration with the [dashboard contract](contracts-dashboard.html) is provided
+by the [dashboard option](#blocks-monitoring-options-shb.monitoring.dashboard).
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.Grafana = {
+    sortOrder = 10;
+    dashboard.request = config.shb.monitoring.dashboard.request;
+  };
+}
+```
+
+There is also an integration for the scrutiny service, see next section.
+
+### Scrutiny {#blocks-monitoring-usage-scrutiny}
+
+Integration with the [Scrutiny](https://github.com/AnalogJ/scrutiny) service is enabled by default and setup automatically.
+
+The web interface will be served under the [scrutiny.subdomain](#blocks-monitoring-options-shb.monitoring.scrutiny.subdomain) option.
+If you don't want the web interface, set the option to `null`.
+
+For integration with the [dashboard contract](contracts-dashboard.html):
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.Scrutiny = {
+    sortOrder = 11;
+    dashboard.request = config.shb.monitoring.scrutiny.dashboard.request;
+  };
+}
+```
+
 ## Provisioning {#blocks-monitoring-provisioning}
 
 Self Host Blocks will create automatically the following resources:
@@ -123,6 +200,43 @@ This dashboard is used to monitor a [deluge](./services-deluge.html) instance.
 ![Deluge Dashboard Top Part](./assets/dashboards_Deluge_1.png)
 ![Deluge Dashboard Bottom Part](./assets/dashboards_Deluge_2.png)
 
+## Backups Dashboard and Alert {#blocks-monitoring-backup}
+
+This dashboard shows Restic and BorgBackup backup jobs, or any job with "backup" in the systemd service name.
+
+### Dashboard {#blocks-monitoring-backup-dashboard}
+
+Variables:
+
+- The "Job" variable allows to select one or more backup jobs. "All" is the default.
+- The "mountpoints" variable allows to select only relevant mountpoints for backup. "All" is the default.
+
+The most important graphs are the first three:
+
+- "Backup Jobs in the Past Week": Shows stats on all backup jobs that ran in the past.
+  It is sorted by the "Failed" column in descending order.
+  This way, one can directly see when a job has failures.
+- "Schedule": Shows when a job will run.
+  The unit is "Datetime from Now" meaning it shows when a job ran or will run relative to the current time.
+  An annotation will show up when the "Late Backups" alert fired or resolved.
+- "Backup jobs": Shows when a backup job ran.
+  Normally, jobs running for less than 15 seconds will not show up in the graph.
+  We crafted a query that still shows them but the length is 15 seconds, even if the backup job
+  took less time to run.
+
+![Backups Dashboard Top Part](./assets/dashboards_Backups_1.png)
+![Backups Dashboard Middle Part](./assets/dashboards_Backups_2.png)
+![Backups Dashboard Bottom Part](./assets/dashboards_Backups_3.png)
+
+### Alerts {#blocks-monitoring-backup-alerts}
+
+- The "Late Backups" alert will fire if a backup job did not run at all in the last 24 hours
+  or if all runs were failures in the last 24 hours.
+  It will show up as annotations in the "Schedule" panel of the dashboard.
+
+![Late Backups Alert Firing](./assets/alert_rules_LateBackups_1.png)
+![Backups Alert Showing Up In Dashboard](./assets/dashboards_Backups_alert.png)
+
 ## Requests Error Budget Alert {#blocks-monitoring-budget-alerts}
 
 This alert will fire when the ratio between number of requests getting a 5XX response from a service
@@ -131,6 +245,55 @@ and the total requests to that service exceeds 1%.
 ![Error Dashboard Top Part](./assets/alert_rules_5xx_1.png)
 ![Error Dashboard Bottom Part](./assets/alert_rules_5xx_2.png)
 
+## SSL Certificates Dashboard and Alert {#blocks-monitoring-ssl}
+
+This dashboard shows Let's Encrypt renewal and setup jobs,
+or any job starting with "acme-" in the systemd service name.
+
+### Dashboard {#blocks-monitoring-ssl-dashboard}
+
+Variables:
+
+- The "Job" variable allows to focus on one or more certificate. "All" is the default.
+
+Graphs:
+
+- "Certificate Remaining Validity": Shows in how long will certificates expire.
+  It shows all files under `/var/lib/acme`.
+  An annotation will show up when the "Certificate Did Not Renew" alert fired or resolved.
+- "Schedule": Shows when a job will run.
+  The unit is "Datetime from Now" meaning it shows when a job ran or will run relative to the current time.
+- "Jobs in the Past Week": Shows stats on all renewal jobs that ran in the past.
+  It is sorted by the "Failed" column in descending order.
+  This way, one can directly see when a job has failures.
+  Note, the stats is not accurate because detecting jobs taking taking less than 15 seconds
+  is not supported well.
+- "Job Runs": Shows when a renewal job ran.
+  Normally, jobs running for less than 15 seconds will not show up in the graph.
+  We crafted a query that still shows them but the length is 100 seconds, even if the job
+  took less time to run.
+
+![SSL Dashboard No Filter](./assets/dashboards_SSL_all.png)
+![SSL Dashboard Filter Failing](./assets/dashboards_SSL_fail.png)
+
+### Alerts {#blocks-monitoring-ssl-alerts}
+
+- The "Certificate Did Not Renew" alert will fire if a backup job did not run at all in the last 24 hours
+    or if all runs were failures in the last 24 hours.
+  It will show up as annotations in the "Schedule" panel of the dashboard.
+
+![Late SSL Jobs Alert Firing](./assets/alert_rules_LateSSL_1.png)
+
+## Impermanence {#blocks-monitoring-impermanence}
+
+To save the fluent-bit folder in an impermanence setup, add:
+
+```nix
+{
+  shb.zfs.datasets."safe/monitoring-fluent-bit".path = config.shb.jellyfin.impermanence.fluent-bit;
+}
+```
+
 ## Options Reference {#blocks-monitoring-options}
 
 ```{=include=} options

modules/blocks/monitoring/rules.json

@@ -146,7 +146,7 @@
             "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
           },
           "editorMode": "code",
-          "expr": "ssl_certificate_expiry_seconds",
+          "expr": "min by(subject) (ssl_certificate_expiry_seconds)",
           "interval": "",
           "intervalMs": 15000,
           "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
@@ -254,5 +254,133 @@
       "role": "sysadmin"
     },
     "isPaused": false
+  },
+  {
+    "uid": "df4doj5pomhvkf",
+    "title": "Late Backups",
+    "condition": "C",
+    "data": [
+      {
+        "refId": "A",
+        "relativeTimeRange": {
+          "from": 10800,
+          "to": 0
+        },
+        "datasourceUid": "df80f9f5-97d7-4112-91d8-72f523a02b09",
+        "model": {
+          "adhocFilters": [],
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "(\n  # Timer triggered at least once in the last 24h\n  label_replace((\n    time()\n      -\n    systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"}\n  ) < 24*60*60, \"name\", \"$1.service\", \"name\", \"(.*).timer\")\n  AND on(name)\n  # At least one failure in the last 24h\n  (\n    max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"failed\"}[24h]) > 0\n  )\n  AND on(name)\n  # No successes in the last 24h\n  (\n    max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"inactive\"}[24h]) == 0\n  )\n)",
+          "instant": false,
+          "interval": "",
+          "intervalMs": 15000,
+          "legendFormat": "{{name}}",
+          "maxDataPoints": 43200,
+          "range": true,
+          "refId": "A"
+        }
+      },
+      {
+        "refId": "B",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        },
+        "datasourceUid": "__expr__",
+        "model": {
+          "conditions": [
+            {
+              "evaluator": {
+                "params": [],
+                "type": "gt"
+              },
+              "operator": {
+                "type": "and"
+              },
+              "query": {
+                "params": [
+                  "B"
+                ]
+              },
+              "reducer": {
+                "params": [],
+                "type": "last"
+              },
+              "type": "query"
+            }
+          ],
+          "datasource": {
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "A",
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "reducer": "last",
+          "refId": "B",
+          "type": "reduce"
+        }
+      },
+      {
+        "refId": "C",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        },
+        "datasourceUid": "__expr__",
+        "model": {
+          "conditions": [
+            {
+              "evaluator": {
+                "params": [
+                  0
+                ],
+                "type": "gt"
+              },
+              "operator": {
+                "type": "and"
+              },
+              "query": {
+                "params": [
+                  "C"
+                ]
+              },
+              "reducer": {
+                "params": [],
+                "type": "last"
+              },
+              "type": "query"
+            }
+          ],
+          "datasource": {
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "B",
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "C",
+          "type": "threshold"
+        }
+      }
+    ],
+    "dashboardUid": "f05500d0-15ed-4719-b68d-fb898ca13cc8",
+    "panelId": 15,
+    "noDataState": "OK",
+    "execErrState": "Error",
+    "annotations": {
+      "__dashboardUid__": "f05500d0-15ed-4719-b68d-fb898ca13cc8",
+      "__panelId__": "15",
+      "summary": "A backup did not run in the last 24 hours."
+    },
+    "labels": {
+      "role": "sysadmin"
+    },
+    "isPaused": false
   }
 ]

modules/blocks/nginx.nix

@@ -1,10 +1,13 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  lib,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.nginx;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   fqdn = c: "${c.subdomain}.${c.domain}";
 
   vhostConfig = lib.types.submodule {
@@ -23,13 +26,14 @@ let
 
       ssl = lib.mkOption {
         description = "Path to SSL files";
-        type = lib.types.nullOr contracts.ssl.certs;
+        type = lib.types.nullOr shb.contracts.ssl.certs;
         default = null;
       };
 
       upstream = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.nullOr lib.types.str;
         description = "Upstream url to be protected.";
+        default = null;
         example = "http://127.0.0.1:1234";
       };
 
@@ -42,17 +46,58 @@ let
 
       autheliaRules = lib.mkOption {
         type = lib.types.listOf (lib.types.attrsOf lib.types.anything);
-        default = [];
+        default = [ ];
         description = "Authelia rule configuration";
-        example = lib.literalExpression ''[{
-        policy = "two_factor";
-        subject = ["group:service_user"];
-        }]'';
+        example = lib.literalExpression ''
+          [
+            # Protect /admin endpoint with 2FA
+            # and only allow access to admin users.
+            {
+              domain = "myapp.example.com";
+              policy = "two_factor";
+              subject = [ "group:service_admin" ];
+              resources = [
+                "^/admin"
+              ];
+            }
+            # Leave /api endpoint open - assumes an API key is used to protect it.
+            {
+              domain = "myapp.example.com";
+              policy = "bypass";
+              resources = [
+                "^/api"
+              ];
+            },
+            # Protect rest of app with 1FA
+            # and allow access to normal and admin users.
+            {
+              domain = "myapp.example.com";
+              policy = "one_factor";
+              subject = ["group:service_user"];
+            },
+          ]
+        '';
+      };
+
+      phpForwardAuth = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Authelia rule configuration";
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = "Extra config to add to the root / location. Strings separated by newlines.";
       };
     };
   };
 in
 {
+  imports = [
+    ./authelia.nix
+  ];
+
   options.shb.nginx = {
     accessLog = lib.mkOption {
       type = lib.types.bool;
@@ -71,41 +116,44 @@ in
     vhosts = lib.mkOption {
       description = "Endpoints to be protected by authelia.";
       type = lib.types.listOf vhostConfig;
-      default = [];
+      default = [ ];
     };
   };
 
   config = {
-    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
 
     services.nginx.enable = true;
     services.nginx.logError = lib.mkIf cfg.debugLog "stderr warn";
     services.nginx.appendHttpConfig = lib.mkIf cfg.accessLog ''
-        log_format apm
-          '{'
-          '"remote_addr":"$remote_addr",'
-          '"remote_user":"$remote_user",'
-          '"time_local":"$time_local",'
-          '"request":"$request",'
-          '"request_length":"$request_length",'
-          '"server_name":"$server_name",'
-          '"status":"$status",'
-          '"bytes_sent":"$bytes_sent",'
-          '"body_bytes_sent":"$body_bytes_sent",'
-          '"referrer":"$http_referrer",'
-          '"user_agent":"$http_user_agent",'
-          '"gzip_ration":"$gzip_ratio",'
-          '"post":"$request_body",'
-          '"upstream_addr":"$upstream_addr",'
-          '"upstream_status":"$upstream_status",'
-          '"request_time":"$request_time",'
-          '"upstream_response_time":"$upstream_response_time",'
-          '"upstream_connect_time":"$upstream_connect_time",'
-          '"upstream_header_time":"$upstream_header_time"'
-          '}';
+      log_format apm
+        '{'
+        '"remote_addr":"$remote_addr",'
+        '"remote_user":"$remote_user",'
+        '"time_local":"$time_local",'
+        '"request":"$request",'
+        '"request_length":"$request_length",'
+        '"server_name":"$server_name",'
+        '"status":"$status",'
+        '"bytes_sent":"$bytes_sent",'
+        '"body_bytes_sent":"$body_bytes_sent",'
+        '"referrer":"$http_referrer",'
+        '"user_agent":"$http_user_agent",'
+        '"gzip_ration":"$gzip_ratio",'
+        '"post":"$request_body",'
+        '"upstream_addr":"$upstream_addr",'
+        '"upstream_status":"$upstream_status",'
+        '"request_time":"$request_time",'
+        '"upstream_response_time":"$upstream_response_time",'
+        '"upstream_connect_time":"$upstream_connect_time",'
+        '"upstream_header_time":"$upstream_header_time"'
+        '}';
 
-        access_log syslog:server=unix:/dev/log apm;
-      '';
+      access_log syslog:server=unix:/dev/log apm;
+    '';
 
     services.nginx.virtualHosts =
       let
@@ -116,51 +164,58 @@ in
             sslCertificateKey = lib.mkIf (!(isNull c.ssl)) c.ssl.paths.key;
 
             # Taken from https://github.com/authelia/authelia/issues/178
-            locations."/".extraConfig = ''
-              add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
-              add_header X-Content-Type-Options nosniff;
-              add_header X-Frame-Options "SAMEORIGIN";
-              add_header X-XSS-Protection "1; mode=block";
-              add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
-              add_header X-Download-Options noopen;
-              add_header X-Permitted-Cross-Domain-Policies none;
+            locations."/".extraConfig =
+              lib.optionalString (c.upstream != null) ''
+                add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+                add_header X-Content-Type-Options nosniff;
+                add_header X-Frame-Options "SAMEORIGIN";
+                add_header X-XSS-Protection "1; mode=block";
+                add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
+                add_header X-Download-Options noopen;
+                add_header X-Permitted-Cross-Domain-Policies none;
 
-              proxy_set_header Host $host;
-              proxy_set_header X-Real-IP $remote_addr;
-              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-              proxy_set_header X-Forwarded-Proto $scheme;
-              proxy_http_version 1.1;
-              proxy_set_header Upgrade $http_upgrade;
-              proxy_set_header Connection "upgrade";
-              proxy_cache_bypass $http_upgrade;
+                proxy_set_header Host $host;
+                proxy_set_header X-Real-IP $remote_addr;
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header X-Forwarded-Proto $scheme;
+                proxy_http_version 1.1;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection "upgrade";
+                proxy_cache_bypass $http_upgrade;
 
-              proxy_pass ${c.upstream};
-            ''
-            + lib.optionalString (c.authEndpoint != null) ''
-              auth_request /authelia;
-              auth_request_set $user $upstream_http_remote_user;
-              auth_request_set $groups $upstream_http_remote_groups;
-              proxy_set_header X-Forwarded-User $user;
-              proxy_set_header X-Forwarded-Groups $groups;
-              # TODO: Are those needed?
-              # auth_request_set $name $upstream_http_remote_name;
-              # auth_request_set $email $upstream_http_remote_email;
-              # proxy_set_header Remote-Name $name;
-              # proxy_set_header Remote-Email $email;
-              # TODO: Would be nice to have this working, I think.
-              # set $new_cookie $http_cookie;
-              # if ($http_cookie ~ "(.*)(?:^|;)\s*example\.com\.session\.id=[^;]+(.*)") {
-              #     set $new_cookie $1$2;
-              # }
-              # proxy_set_header Cookie $new_cookie;
+                proxy_pass ${c.upstream};
+              ''
+              + c.extraConfig
+              + lib.optionalString (c.authEndpoint != null) ''
+                auth_request /authelia;
+                auth_request_set $user $upstream_http_remote_user;
+                auth_request_set $groups $upstream_http_remote_groups;
+                proxy_set_header X-Forwarded-User $user;
+                proxy_set_header X-Forwarded-Groups $groups;
+                # TODO: Are those needed?
+                # auth_request_set $name $upstream_http_remote_name;
+                # auth_request_set $email $upstream_http_remote_email;
+                # proxy_set_header Remote-Name $name;
+                # proxy_set_header Remote-Email $email;
+                # TODO: Would be nice to have this working, I think.
+                # set $new_cookie $http_cookie;
+                # if ($http_cookie ~ "(.*)(?:^|;)\s*example\.com\.session\.id=[^;]+(.*)") {
+                #     set $new_cookie $1$2;
+                # }
+                # proxy_set_header Cookie $new_cookie;
 
-              auth_request_set $redirect $scheme://$http_host$request_uri;
-              error_page 401 =302 ${c.authEndpoint}?rd=$redirect;
-              error_page 403 = ${c.authEndpoint}/error/403;
+                auth_request_set $redirect $scheme://$http_host$request_uri;
+                error_page 401 =302 ${c.authEndpoint}?rd=$redirect;
+                error_page 403 = ${c.authEndpoint}/error/403;
+              '';
+
+            locations."~ \\.php$".extraConfig = lib.mkIf (c.phpForwardAuth) ''
+              fastcgi_param HTTP_X_FORWARDED_USER   $user;
+              fastcgi_param HTTP_X_FORWARDED_GROUPS $groups;
             '';
 
             # Virtual endpoint created by nginx to forward auth requests.
-            locations."/authelia".extraConfig = lib.mkIf (!(isNull c.authEndpoint)) ''
+            locations."/authelia".extraConfig = lib.mkIf (c.authEndpoint != null) ''
               internal;
               proxy_pass ${c.authEndpoint}/api/verify;
 
@@ -182,13 +237,13 @@ in
           };
         };
       in
-        lib.mkMerge (map vhostCfg cfg.vhosts);
+      lib.mkMerge (map vhostCfg cfg.vhosts);
 
     shb.authelia.rules =
       let
         authConfig = c: map (r: r // { domain = fqdn c; }) c.autheliaRules;
       in
-        lib.flatten (map authConfig cfg.vhosts);
+      lib.flatten (map authConfig cfg.vhosts);
 
     security.acme.defaults.reloadServices = [
       "nginx.service"

modules/blocks/nginx/docs/default.md

@@ -0,0 +1,211 @@
+# Nginx Block {#blocks-nginx}
+
+Defined in [`/modules/blocks/nginx.nix`](@REPO@/modules/blocks/nginx.nix).
+
+This block sets up a [Nginx](https://nginx.org/) instance.
+
+It complements the upstream nixpkgs with some authentication and debugging improvements as shows in the Usage section.
+
+## Usage {#blocks-nginx-usage}
+
+### Access Logging {#blocks-nginx-usage-accesslog}
+
+JSON access logging is enabled with the [`shb.nginx.accessLog`](#blocks-nginx-options-shb.nginx.accessLog) option:
+
+```nix
+{
+  shb.nginx.accessLog = true;
+}
+```
+
+Looking at the systemd logs (`journalctl -fu nginx`) will show for example:
+
+```json
+nginx[969]: server nginx:
+  {
+    "remote_addr":"192.168.1.1",
+    "remote_user":"-",
+    "time_local":"29/Dec/2025:14:22:41 +0000",
+    "request":"POST /api/firstfactor HTTP/2.0",
+    "request_length":"264",
+    "server_name":"auth_example_com",
+    "status":"200",
+    "bytes_sent":"855",
+    "body_bytes_sent":"60",
+    "referrer":"-",
+    "user_agent":"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    "gzip_ration":"-",
+    "post":"{\x22username\x22:\x22charlie\x22,\x22password\x22:\x22CharliePassword\x22,\x22keepMeLoggedIn\x22:false,\x22targetURL\x22:\x22https://f.example.com/\x22,\x22requestMethod\x22:null}",
+    "upstream_addr":"127.0.0.1:9091",
+    "upstream_status":"200",
+    "request_time":"0.873",
+    "upstream_response_time":"0.873",
+    "upstream_connect_time":"0.001",
+    "upstream_header_time":"0.872"
+  }
+```
+
+This _will_ log the body of POST queries so it should only be enabled for debug logging.
+
+### Debug Logging {#blocks-nginx-usage-debuglog}
+
+Debug logging is enabled with the [`shb.nginx.debugLog`](#blocks-nginx-options-shb.nginx.debugLog) option:
+
+```nix
+{
+  shb.nginx.debugLog = true;
+}
+```
+
+If enabled, it sets:
+
+```
+error_log stderr warn;
+```
+
+### Virtual Host Upstream Proxy {#blocks-nginx-usage-upstream}
+
+Easy upstream proxy setup is done with the [`shb.nginx.vhosts.*.upstream`](#blocks-nginx-options-shb.nginx.vhosts._.upstream) option:
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      upstream = "http://127.0.0.1:9090";
+    }
+  ];
+}
+```
+
+This will set also a few headers.
+Some are shown here and others please see in the [nginx](@REPO@/modules/blocks/nginx.nix) module:
+
+- `Host` = `$host`;
+- `X-Real-IP` = `$remote_addr`;
+- `X-Forwarded-For` = `$proxy_add_x_forwarded_for`;
+- `X-Forwarded-Proto` = `$scheme`;
+
+### Virtual Host SSL Generator Contract Integration {#blocks-nginx-usage-ssl}
+
+This module integrates with the [SSL Generator Contract](./contracts-ssl.html)
+to setup HTTPs with the [`shb.nginx.vhosts.*.ssl`](#blocks-nginx-options-shb.nginx.vhosts._.ssl) option:
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      ssl = config.shb.certs.certs.letsencrypt.${domain};;
+    }
+  ];
+
+  shb.certs.certs.letsencrypt.${domain} = {
+    inherit domain;
+  };
+}
+```
+
+### Virtual Host SHB Forward Authentication {#blocks-nginx-usage-shbforwardauth}
+
+For services provided by SelfHostBlocks that do not handle [OIDC integration][OIDC],
+this block can provide [forward authentication][] which still allows the service
+to still be protected by an SSO server.
+
+[OIDC]: blocks-authelia.html#blocks-authelia-shb-oidc
+
+The user could still be required to authenticate to the service itself,
+although some services can automatically users authorized by Authelia.
+
+[forward authentication]: https://doc.traefik.io/traefik/middlewares/http/forwardauth/
+
+Integrating with this block is done with the following code:
+
+```nix
+shb.<services>.authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+```
+
+### Virtual Host Forward Authentication {#blocks-nginx-usage-forwardauth}
+
+Forward authentication is when Nginx talks with the SSO service directly
+and the user is authenticated before reaching the upstream application.
+
+The SSO service responds with the username, group and more information about the user.
+This is then forwarded to the upstream application by Nginx.
+
+Note that _every_ request is authenticated this way with the SSO server
+so it involves more hops than a direct [OIDC integration](blocks-authelia.html#blocks-authelia-shb-oidc).
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      authEndpoint = "authelia.example.com";
+      autheliaRules = [
+        [
+          # Protect /admin endpoint with 2FA
+          # and only allow access to admin users.
+          {
+            domain = "myapp.example.com";
+            policy = "two_factor";
+            subject = [ "group:service_admin" ];
+            resources = [
+              "^/admin"
+            ];
+          }
+          # Leave /api endpoint open - assumes an API key is used to protect it.
+          {
+            domain = "myapp.example.com";
+            policy = "bypass";
+            resources = [
+              "^/api"
+            ];
+          },
+          # Protect rest of app with 1FA
+          # and allow access to normal and admin users.
+          {
+            domain = "myapp.example.com";
+            policy = "one_factor";
+            subject = ["group:service_user"];
+          },
+        ]
+      ];
+    }
+  ];
+}
+```
+
+If PHP is used with fastCGI,
+extra headers must be added by enabling the [`shb.nginx.vhosts.*.phpForwardAuth`](#blocks-nginx-options-shb.nginx.vhosts._.phpForwardAuth) option.
+
+### Virtual Host Extra Config {#blocks-nginx-usage-extraconfig}
+
+To add extra configuration to a virtual host,
+use the [`shb.nginx.vhosts.*.extraConfig`](#blocks-nginx-options-shb.nginx.vhosts._.extraConfig) option.
+This can be used to add headers, for example:
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      extraConfig = ''
+        add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
+      '';
+    }
+  ];
+}
+```
+
+## Options Reference {#blocks-nginx-options}
+
+```{=include=} options
+id-prefix: blocks-nginx-options-
+list-id: selfhostblocks-block-nginx-options
+source: @OPTIONS_JSON@
+```

modules/blocks/postgresql.nix

@@ -1,9 +1,15 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  ...
+}:
 let
   cfg = config.shb.postgresql;
-  contracts = pkgs.callPackage ../contracts {};
 
-  upgrade-script = old: new:
+  upgrade-script =
+    old: new:
     let
       oldStr = builtins.toString old;
       newStr = builtins.toString new;
@@ -11,37 +17,41 @@ let
       oldPkg = pkgs.${"postgresql_${oldStr}"};
       newPkg = pkgs.${"postgresql_${newStr}"};
     in
-      pkgs.writeScriptBin "upgrade-pg-cluster-${oldStr}-${newStr}" ''
-        set -eux
-        # XXX it's perhaps advisable to stop all services that depend on postgresql
-        systemctl stop postgresql
+    pkgs.writeScriptBin "upgrade-pg-cluster-${oldStr}-${newStr}" ''
+      set -eux
+      # XXX it's perhaps advisable to stop all services that depend on postgresql
+      systemctl stop postgresql
 
-        export NEWDATA="/var/lib/postgresql/${newPkg.psqlSchema}"
-        export NEWBIN="${newPkg}/bin"
+      export NEWDATA="/var/lib/postgresql/${newPkg.psqlSchema}"
+      export NEWBIN="${newPkg}/bin"
 
-        export OLDDATA="/var/lib/postgresql/${oldPkg.psqlSchema}"
-        export OLDBIN="${oldPkg}/bin"
+      export OLDDATA="/var/lib/postgresql/${oldPkg.psqlSchema}"
+      export OLDBIN="${oldPkg}/bin"
 
-        install -d -m 0700 -o postgres -g postgres "$NEWDATA"
-        cd "$NEWDATA"
-        sudo -u postgres $NEWBIN/initdb -D "$NEWDATA"
+      install -d -m 0700 -o postgres -g postgres "$NEWDATA"
+      cd "$NEWDATA"
+      sudo -u postgres $NEWBIN/initdb -D "$NEWDATA"
 
-        sudo -u postgres $NEWBIN/pg_upgrade \
-          --old-datadir "$OLDDATA" --new-datadir "$NEWDATA" \
-          --old-bindir $OLDBIN --new-bindir $NEWBIN \
-          "$@"
-      '';
+      sudo -u postgres $NEWBIN/pg_upgrade \
+        --old-datadir "$OLDDATA" --new-datadir "$NEWDATA" \
+        --old-bindir $OLDBIN --new-bindir $NEWBIN \
+        "$@"
+    '';
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.postgresql = {
     debug = lib.mkOption {
       type = lib.types.bool;
       description = ''
-      Enable debugging options.
+        Enable debugging options.
 
-      Currently enables shared_preload_libraries = "auto_explain, pg_stat_statements"
+        Currently enables shared_preload_libraries = "auto_explain, pg_stat_statements"
 
-      See https://www.postgresql.org/docs/current/pgstatstatements.html'';
+        See https://www.postgresql.org/docs/current/pgstatstatements.html'';
       default = false;
     };
     enableTCPIP = lib.mkOption {
@@ -55,15 +65,15 @@ in
         Backup configuration.
       '';
 
-      default = {};
+      default = { };
       type = lib.types.submodule {
-        options = contracts.databasebackup.mkRequester {
+        options = shb.contracts.databasebackup.mkRequester {
           user = "postgres";
 
           backupName = "postgres.sql";
 
           backupCmd = ''
-            ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
+            ${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
           '';
 
           restoreCmd = ''
@@ -75,27 +85,29 @@ in
 
     ensures = lib.mkOption {
       description = "List of username, database and/or passwords that should be created.";
-      type = lib.types.listOf (lib.types.submodule {
-        options = {
-          username = lib.mkOption {
-            type = lib.types.str;
-            description = "Postgres user name.";
-          };
+      type = lib.types.listOf (
+        lib.types.submodule {
+          options = {
+            username = lib.mkOption {
+              type = lib.types.str;
+              description = "Postgres user name.";
+            };
 
-          database = lib.mkOption {
-            type = lib.types.str;
-            description = "Postgres database.";
-          };
+            database = lib.mkOption {
+              type = lib.types.str;
+              description = "Postgres database.";
+            };
 
-          passwordFile = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
-            description = "Optional password file for the postgres user. If not given, only peer auth is accepted for this user, otherwise password auth is allowed.";
-            default = null;
-            example = "/run/secrets/postgresql/password";
+            passwordFile = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = "Optional password file for the postgres user. If not given, only peer auth is accepted for this user, otherwise password auth is allowed.";
+              default = null;
+              example = "/run/secrets/postgresql/password";
+            };
           };
-        };
-      });
-      default = [];
+        }
+      );
+      default = [ ];
     };
   };
 
@@ -123,51 +135,61 @@ in
       dbConfig = ensureCfgs: {
         services.postgresql.enable = lib.mkDefault ((builtins.length ensureCfgs) > 0);
         services.postgresql.ensureDatabases = map ({ database, ... }: database) ensureCfgs;
-        services.postgresql.ensureUsers = map ({ username, database, ... }: {
-          name = username;
-          ensureDBOwnership = true;
-          ensureClauses.login = true;
-        }) ensureCfgs;
+        services.postgresql.ensureUsers = map (
+          { username, database, ... }:
+          {
+            name = username;
+            ensureDBOwnership = true;
+            ensureClauses.login = true;
+          }
+        ) ensureCfgs;
       };
 
       pwdConfig = ensureCfgs: {
-        systemd.services.postgresql.postStart =
+        systemd.services.postgresql-setup.script = lib.mkAfter (
           let
             prefix = ''
-            $PSQL -tA <<'EOF'
-              DO $$
-              DECLARE password TEXT;
-              BEGIN
+              psql -tA <<'EOF'
+                DO $$
+                DECLARE password TEXT;
+                BEGIN
             '';
             suffix = ''
-              END $$;
-            EOF
-            '';
-            exec = { username, passwordFile, ... }: ''
-            password := trim(both from replace(pg_read_file('${passwordFile}'), E'\n', '''));
-            EXECUTE format('ALTER ROLE ${username} WITH PASSWORD '''%s''';', password);
+                END $$;
+              EOF
             '';
+            exec =
+              { username, passwordFile, ... }:
+              ''
+                password := trim(both from replace(pg_read_file('${passwordFile}'), E'\n', '''));
+                EXECUTE format('ALTER ROLE "${username}" WITH PASSWORD '''%s''';', password);
+              '';
             cfgsWithPasswords = builtins.filter (cfg: cfg.passwordFile != null) ensureCfgs;
           in
-            if (builtins.length cfgsWithPasswords) == 0 then "" else
-              prefix + (lib.concatStrings (map exec cfgsWithPasswords)) + suffix;
+          if (builtins.length cfgsWithPasswords) == 0 then
+            ""
+          else
+            prefix + (lib.concatStrings (map exec cfgsWithPasswords)) + suffix
+        );
       };
 
-      debugConfig = enableDebug: lib.mkIf enableDebug {
-        services.postgresql.settings.shared_preload_libraries = "auto_explain, pg_stat_statements";
-      };
+      debugConfig =
+        enableDebug:
+        lib.mkIf enableDebug {
+          services.postgresql.settings.shared_preload_libraries = "auto_explain, pg_stat_statements";
+        };
     in
-      lib.mkMerge ([
-        commonConfig
-        (dbConfig cfg.ensures)
-        (pwdConfig cfg.ensures)
-        (lib.mkIf cfg.enableTCPIP tcpConfig)
-        (debugConfig cfg.debug)
-        {
-          environment.systemPackages = lib.mkIf config.services.postgresql.enable [
-            (upgrade-script 15 16)
-            (upgrade-script 16 17)
-          ];
-        }
-      ]);
+    lib.mkMerge ([
+      commonConfig
+      (dbConfig cfg.ensures)
+      (pwdConfig cfg.ensures)
+      (lib.mkIf cfg.enableTCPIP tcpConfig)
+      (debugConfig cfg.debug)
+      {
+        environment.systemPackages = lib.mkIf config.services.postgresql.enable [
+          (upgrade-script 15 16)
+          (upgrade-script 16 17)
+        ];
+      }
+    ]);
 }

modules/blocks/postgresql/docs/default.md

@@ -6,17 +6,46 @@ This block sets up a [PostgreSQL][] database.
 
 [postgresql]: https://www.postgresql.org/
 
-## Tests {#blocks-postgresql-tests}
+Compared to the upstream nixpkgs module, this module also sets up:
 
-Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
+- Enabling TCP/IP login and also accepting password authentication from localhost with [`shb.postgresql.enableTCPIP`](#blocks-postgresql-options-shb.postgresql.enableTCPIP).
+- Enhance the `ensure*` upstream option by setting up a database's password from a password file with [`shb.postgresql.ensures`](#blocks-postgresql-options-shb.postgresql.ensures).
+- Debug logging with `auto_explain` and `pg_stat_statements` with [`shb.postgresql.debug`](#blocks-postgresql-options-shb.postgresql.debug).
 
-## Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
+## Usage {#blocks-postgresql-usage}
+
+### Ensure User and Database {#blocks-postgresql-ensures}
+
+Ensure a database and user exists:
+
+```nix
+shb.postgresql.ensures = [
+  {
+    username = "firefly-iii";
+    database = "firefly-iii";
+  }
+];
+```
+
+Also set up the database password from a file path:
+
+```nix
+shb.postgresql.ensures = [
+  {
+    username = "firefly-iii";
+    database = "firefly-iii";
+    passwordFile = "/run/secrets/firefly-iii_db_password";
+  }
+];
+```
+
+### Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
 
 This block can be backed up using the [database backup](contracts-databasebackup.html) contract.
 
 Contract integration tests are defined in [`/test/contracts/databasebackup.nix`](@REPO@/test/contracts/databasebackup.nix).
 
-### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
+#### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
 
 ```nix
 {
@@ -30,6 +59,10 @@ Contract integration tests are defined in [`/test/contracts/databasebackup.nix`]
 }
 ```
 
+## Tests {#blocks-postgresql-tests}
+
+Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
+
 ## Options Reference {#blocks-postgresql-options}
 
 ```{=include=} options

modules/blocks/restic.nix

@@ -1,168 +1,234 @@
-{ config, pkgs, lib, utils, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  shb,
+  utils,
+  ...
+}:
 
 let
   cfg = config.shb.restic;
 
-  shblib = pkgs.callPackage ../../lib {};
-  contracts = pkgs.callPackage ../contracts {};
+  inherit (lib)
+    concatStringsSep
+    filterAttrs
+    flatten
+    literalExpression
+    optionals
+    listToAttrs
+    mapAttrsToList
+    mkEnableOption
+    mkOption
+    mkMerge
+    ;
+  inherit (lib)
+    hasPrefix
+    mkIf
+    nameValuePair
+    optionalAttrs
+    removePrefix
+    ;
+  inherit (lib.types)
+    attrsOf
+    enum
+    int
+    ints
+    oneOf
+    nonEmptyStr
+    nullOr
+    str
+    submodule
+    ;
 
-  inherit (lib) concatStringsSep filterAttrs flatten literalExpression optionals listToAttrs mapAttrsToList mkEnableOption mkOption mkMerge;
-  inherit (lib) generators hasPrefix mkIf nameValuePair optionalAttrs removePrefix;
-  inherit (lib.types) attrsOf enum int ints oneOf nonEmptyStr nullOr str submodule;
+  commonOptions =
+    {
+      name,
+      prefix,
+      config,
+      ...
+    }:
+    {
+      enable = mkEnableOption ''
+        SelfHostBlocks' Restic block
 
-  commonOptions = { name, prefix, config, ... }: {
-    enable = mkEnableOption ''
-      this backup intance.
+        A disabled instance will not backup data anymore
+        but still provides the helper tool to restore snapshots
+      '';
 
-      A disabled instance will not backup data anymore
-      but still provides the helper tool to restore snapshots
-    '';
-
-    passphrase = lib.mkOption {
-      description = "Encryption key for the backup repository.";
-      type = lib.types.submodule {
-        options = contracts.secret.mkRequester {
-          mode = "0400";
-          owner = config.request.user;
-          ownerText = "[shb.restic.${prefix}.<name>.request.user](#blocks-restic-options-shb.restic.${prefix}._name_.request.user)";
-          restartUnits = [ (fullName name config.settings.repository) ];
-          restartUnitsText = "[ [shb.restic.${prefix}.<name>.settings.repository](#blocks-restic-options-shb.restic.${prefix}._name_.settings.repository) ]";
+      passphrase = lib.mkOption {
+        description = "Encryption key for the backup repository.";
+        type = lib.types.submodule {
+          options = shb.contracts.secret.mkRequester {
+            mode = "0400";
+            owner = config.request.user;
+            ownerText = "[shb.restic.${prefix}.<name>.request.user](#blocks-restic-options-shb.restic.${prefix}._name_.request.user)";
+            restartUnits = [ "${fullName name config.settings.repository}.service" ];
+            restartUnitsText = "[ [shb.restic.${prefix}.<name>.settings.repository](#blocks-restic-options-shb.restic.${prefix}._name_.settings.repository) ]";
+          };
         };
       };
-    };
 
-    repository = mkOption {
-      description = "Repositories to back this instance to.";
-      type = submodule {
-        options = {
-          path = mkOption {
-            type = str;
-            description = "Repository location";
-          };
-
-          secrets = mkOption {
-            type = attrsOf shblib.secretFileType;
-            default = {};
-            description = ''
-              Secrets needed to access the repository where the backups will be stored.
-
-              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
-              and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
-
-            '';
-            example = literalExpression ''
-              {
-                AWS_ACCESS_KEY_ID.source = <path/to/secret>;
-                AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
-              }
-            '';
-          };
-
-          timerConfig = mkOption {
-            type = attrsOf utils.systemdUtils.unitOptions.unitOption;
-            default = {
-              OnCalendar = "daily";
-              Persistent = true;
+      repository = mkOption {
+        description = "Repositories to back this instance to.";
+        type = submodule {
+          options = {
+            path = mkOption {
+              type = str;
+              description = "Repository location";
             };
-            description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
-            example = {
-              OnCalendar = "00:05";
-              RandomizedDelaySec = "5h";
-              Persistent = true;
+
+            secrets = mkOption {
+              type = attrsOf shb.secretFileType;
+              default = { };
+              description = ''
+                Secrets needed to access the repository where the backups will be stored.
+
+                See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+                and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
+
+              '';
+              example = literalExpression ''
+                {
+                  AWS_ACCESS_KEY_ID.source = <path/to/secret>;
+                  AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
+                }
+              '';
+            };
+
+            timerConfig = mkOption {
+              type = attrsOf utils.systemdUtils.unitOptions.unitOption;
+              default = {
+                OnCalendar = "daily";
+                Persistent = true;
+              };
+              description = "When to run the backup. See {manpage}`systemd.timer(5)` for details.";
+              example = {
+                OnCalendar = "00:05";
+                RandomizedDelaySec = "5h";
+                Persistent = true;
+              };
             };
           };
         };
       };
-    };
 
-    retention = mkOption {
-      description = "For how long to keep backup files.";
-      type = attrsOf (oneOf [ int nonEmptyStr ]);
-      default = {
-        keep_within = "1d";
-        keep_hourly = 24;
-        keep_daily = 7;
-        keep_weekly = 4;
-        keep_monthly = 6;
+      retention = mkOption {
+        description = "For how long to keep backup files.";
+        type = attrsOf (oneOf [
+          int
+          nonEmptyStr
+        ]);
+        default = {
+          keep_within = "1d";
+          keep_hourly = 24;
+          keep_daily = 7;
+          keep_weekly = 4;
+          keep_monthly = 6;
+        };
+      };
+
+      limitUploadKiBs = mkOption {
+        type = nullOr int;
+        description = "Limit upload bandwidth to the given KiB/s amount.";
+        default = null;
+        example = 8000;
+      };
+
+      limitDownloadKiBs = mkOption {
+        type = nullOr int;
+        description = "Limit download bandwidth to the given KiB/s amount.";
+        default = null;
+        example = 8000;
       };
     };
 
-    limitUploadKiBs = mkOption {
-      type = nullOr int;
-      description = "Limit upload bandwidth to the given KiB/s amount.";
-      default = null;
-      example = 8000;
-    };
-
-    limitDownloadKiBs = mkOption {
-      type = nullOr int;
-      description = "Limit download bandwidth to the given KiB/s amount.";
-      default = null;
-      example = 8000;
-    };
-  };
-
-  repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (removePrefix "/" name);
+  repoSlugName = name: builtins.replaceStrings [ "/" ":" ] [ "_" "_" ] (removePrefix "/" name);
   fullName = name: repository: "restic-backups-${name}_${repoSlugName repository.path}";
 in
 {
+  imports = [
+    ../../lib/module.nix
+    ../blocks/monitoring.nix
+  ];
+
   options.shb.restic = {
+    enableDashboard = lib.mkEnableOption "the Backups SHB dashboard" // {
+      default = true;
+    };
+
     instances = mkOption {
-      description = "Files to backup following the [backup contract](./contracts-backup.html).";
-      default = {};
-      type = attrsOf (submodule ({ name, config, ... }: {
-        options = contracts.backup.mkProvider {
-          settings = mkOption {
-            description = ''
-              Settings specific to the Restic provider.
-            '';
+      description = "Files to backup following the [backup contract](./shb.contracts-backup.html).";
+      default = { };
+      type = attrsOf (
+        submodule (
+          { name, config, ... }:
+          {
+            options = shb.contracts.backup.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the Restic provider.
+                '';
 
-            type = submodule {
-              options = commonOptions { inherit name config; prefix = "instances"; };
+                type = submodule {
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "instances";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
             };
-          };
-
-          resultCfg = {
-            restoreScript = fullName name config.settings.repository;
-            restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
-
-            backupService = "${fullName name config.settings.repository}.service";
-            backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
-          };
-        };
-      }));
+          }
+        )
+      );
     };
 
     databases = mkOption {
-      description = "Databases to backup following the [database backup contract](./contracts-databasebackup.html).";
-      default = {};
-      type = attrsOf (submodule ({ name, config, ... }: {
-        options = contracts.databasebackup.mkProvider {
-          settings = mkOption {
-            description = ''
-              Settings specific to the Restic provider.
-            '';
+      description = "Databases to backup following the [database backup contract](./shb.contracts-databasebackup.html).";
+      default = { };
+      type = attrsOf (
+        submodule (
+          { name, config, ... }:
+          {
+            options = shb.contracts.databasebackup.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the Restic provider.
+                '';
 
-            type = submodule {
-              options = commonOptions { inherit name config; prefix = "databases"; };
+                type = submodule {
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "databases";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
             };
-          };
-
-          resultCfg = {
-            restoreScript = fullName name config.settings.repository;
-            restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
-
-            backupService = "${fullName name config.settings.repository}.service";
-            backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
-          };
-        };
-      }));
+          }
+        )
+      );
     };
 
     # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
     performance = mkOption {
       description = "Reduce performance impact of backup jobs.";
-      default = {};
+      default = { };
       type = submodule {
         options = {
           niceness = mkOption {
@@ -171,7 +237,11 @@ in
             default = 15;
           };
           ioSchedulingClass = mkOption {
-            type = enum [ "idle" "best-effort" "realtime" ];
+            type = enum [
+              "idle"
+              "best-effort"
+              "realtime"
+            ];
             description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
             default = "best-effort";
           };
@@ -185,23 +255,28 @@ in
     };
   };
 
-  config = mkIf (cfg.instances != {} || cfg.databases != {}) (
+  config = mkIf (cfg.instances != { } || cfg.databases != { }) (
     let
       enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances;
       enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases;
-    in mkMerge [
+    in
+    mkMerge [
       {
-        environment.systemPackages = optionals (enabledInstances != {} || enabledDatabases != {}) [ pkgs.restic ];
+        environment.systemPackages = optionals (enabledInstances != { } || enabledDatabases != { }) [
+          pkgs.restic
+        ];
       }
       {
         # Create repository if it is a local path.
         systemd.tmpfiles.rules =
           let
-            mkSettings = name: instance: optionals (hasPrefix "/" instance.settings.repository.path) [
-              "d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -"
-            ];
+            mkSettings =
+              name: instance:
+              optionals (hasPrefix "/" instance.settings.repository.path) [
+                "d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -"
+              ];
           in
-            flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
+          flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
       }
       {
         services.restic.backups =
@@ -220,8 +295,8 @@ in
 
                 inherit (instance.settings.repository) timerConfig;
 
-                pruneOpts = mapAttrsToList (name: value:
-                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+                pruneOpts = mapAttrsToList (
+                  name: value: "--${builtins.replaceStrings [ "_" ] [ "-" ] name} ${builtins.toString value}"
                 ) instance.settings.retention;
 
                 backupPrepareCommand = concatStringsSep "\n" instance.request.hooks.beforeBackup;
@@ -235,12 +310,13 @@ in
                   ++ (optionals (instance.settings.limitDownloadKiBs != null) [
                     "--limit-download=${toString instance.settings.limitDownloadKiBs}"
                   ]);
-              } // optionalAttrs (builtins.length instance.request.excludePatterns > 0) {
+              }
+              // optionalAttrs (builtins.length instance.request.excludePatterns > 0) {
                 exclude = instance.request.excludePatterns;
               };
             };
           in
-            mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
+          mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
       }
       {
         services.restic.backups =
@@ -259,8 +335,8 @@ in
 
                 inherit (instance.settings.repository) timerConfig;
 
-                pruneOpts = mapAttrsToList (name: value:
-                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+                pruneOpts = mapAttrsToList (
+                  name: value: "--${builtins.replaceStrings [ "_" ] [ "-" ] name} ${builtins.toString value}"
                 ) instance.settings.retention;
 
                 extraBackupArgs =
@@ -270,120 +346,139 @@ in
                   ++ (optionals (instance.settings.limitDownloadKiBs != null) [
                     "--limit-download=${toString instance.settings.limitDownloadKiBs}"
                   ])
-                  ++ 
-                  (let
-                    cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd;
-                  in
+                  ++ (
+                    let
+                      cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd;
+                    in
                     [
                       "--stdin-filename ${instance.request.backupName} --stdin-from-command -- ${cmd}/bin/dump.sh"
-                    ]);
+                    ]
+                  );
               };
             };
           in
-            mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
+          mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
       }
       {
         systemd.services =
           let
-            mkSettings = name: instance:
+            mkSettings =
+              name: instance:
               let
                 serviceName = fullName name instance.settings.repository;
               in
-                {
-                  ${serviceName} = mkMerge [
-                    {
-                      serviceConfig = {
-                        Nice = cfg.performance.niceness;
-                        IOSchedulingClass = cfg.performance.ioSchedulingClass;
-                        IOSchedulingPriority = cfg.performance.ioPriority;
-                        # BindReadOnlyPaths = instance.sourceDirectories;
-                      };
-                    }
-                    (optionalAttrs (instance.settings.repository.secrets != {})
-                      {
-                        serviceConfig.EnvironmentFile = [
-                          "/run/secrets_restic/${serviceName}"
-                        ];
-                        after = [ "${serviceName}-pre.service" ];
-                        requires = [ "${serviceName}-pre.service" ];
-                      })
-                  ];
+              {
+                ${serviceName} = mkMerge [
+                  {
+                    serviceConfig = {
+                      Nice = cfg.performance.niceness;
+                      IOSchedulingClass = cfg.performance.ioSchedulingClass;
+                      IOSchedulingPriority = cfg.performance.ioPriority;
+                      # BindReadOnlyPaths = instance.sourceDirectories;
+                    };
+                  }
+                  (optionalAttrs (instance.settings.repository.secrets != { }) {
+                    serviceConfig.EnvironmentFile = [
+                      "/run/secrets_restic/${serviceName}"
+                    ];
+                    after = [ "${serviceName}-pre.service" ];
+                    requires = [ "${serviceName}-pre.service" ];
+                  })
+                ];
 
-                  "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != {})
-                    (let
-                      script = shblib.genConfigOutOfBandSystemd {
-                        config = instance.settings.repository.secrets;
-                        configLocation = "/run/secrets_restic/${serviceName}";
-                        generator = name: v: pkgs.writeText "template" (generators.toINIWithGlobalSection {} { globalSection = v; });
-                        user = instance.request.user;
-                      };
-                    in
-                      {
-                        script = script.preStart;
-                        serviceConfig.Type = "oneshot";
-                        serviceConfig.LoadCredential = script.loadCredentials;
-                      });
-            };
+                "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != { }) (
+                  let
+                    script = shb.genConfigOutOfBandSystemd {
+                      config = instance.settings.repository.secrets;
+                      configLocation = "/run/secrets_restic/${serviceName}";
+                      generator = shb.toEnvVar;
+                      user = instance.request.user;
+                    };
+                  in
+                  {
+                    script = script.preStart;
+                    serviceConfig.Type = "oneshot";
+                    serviceConfig.LoadCredential = script.loadCredentials;
+                  }
+                );
+              };
           in
-            mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
+          mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
       }
       {
-        systemd.services = let
-          mkEnv = name: instance:
-            nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
-              enable = true;
-              wantedBy = [ "multi-user.target" ];
-              serviceConfig.Type = "oneshot";
-              script = (shblib.replaceSecrets {
-                userConfig = instance.settings.repository.secrets // {
-                  RESTIC_PASSWORD_FILE = toString instance.settings.passphrase.result.path;
-                  RESTIC_REPOSITORY = instance.settings.repository.path;
-                };
-                resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                generator = name: v: pkgs.writeText (fullName name instance.settings.repository) (generators.toINIWithGlobalSection {} { globalSection = v; });
-                user = instance.request.user;
-              });
-            };
-        in
+        systemd.services =
+          let
+            mkEnv =
+              name: instance:
+              nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
+                enable = true;
+                wantedBy = [ "multi-user.target" ];
+                # Purposely not a oneshot systemd service otherwise
+                # the service waits on the completion the backup before deactivating.
+                # This seems like a nice property at first but there is one annoying
+                # edge case when deploying. If a backup job somehow is started when
+                # the deploy happens, the deploy will wait on the service to finish
+                # before considering the deploy done. And worse, it will consider the
+                # deploy as failed if the backup fails, which is not what you want.
+                script = (
+                  shb.replaceSecrets {
+                    userConfig = instance.settings.repository.secrets // {
+                      RESTIC_PASSWORD_FILE = toString instance.settings.passphrase.result.path;
+                      RESTIC_REPOSITORY = instance.settings.repository.path;
+                    };
+                    resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
+                    generator = shb.toEnvVar;
+                    user = instance.request.user;
+                  }
+                );
+              };
+          in
           listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases)));
       }
       {
-        environment.systemPackages = let
-          mkResticBinary = name: instance:
-            pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
-              set -euo pipefail
-
-              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
-                       | xargs -d '\n')
-
-              if ! [ "$1" = "restore" ]; then
-                sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
-              else
-                shift
-                sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic restore $@ --target /"
-              fi
-              '';
-        in
+        environment.systemPackages =
+          let
+            mkResticBinary =
+              name: instance:
+              shb.contracts.backup.mkRestoreScript {
+                name = fullName name instance.settings.repository;
+                user = instance.request.user;
+                sudoPreserveEnvs = [
+                  "RESTIC_REPOSITORY"
+                  "RESTIC_PASSWORD_FILE"
+                ];
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
+                restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
+              };
+          in
           flatten (mapAttrsToList mkResticBinary cfg.instances);
       }
       {
-        environment.systemPackages = let
-          mkResticBinary = name: instance:
-            pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
-              set -euo pipefail
-
-              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
-                       | xargs -d '\n')
-
-              if ! [ "$1" = "restore" ]; then
-                sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
-              else
-                shift
-                sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupName} | ${instance.request.restoreCmd}"
-              fi
-              '';
-        in
+        environment.systemPackages =
+          let
+            mkResticBinary =
+              name: instance:
+              shb.contracts.backup.mkRestoreScript {
+                name = fullName name instance.settings.repository;
+                user = instance.request.user;
+                sudoPreserveEnvs = [
+                  "RESTIC_REPOSITORY"
+                  "RESTIC_PASSWORD_FILE"
+                ];
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
+                restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
+              };
+          in
           flatten (mapAttrsToList mkResticBinary cfg.databases);
       }
-    ]);
+
+      (lib.mkIf (cfg.enableDashboard && (cfg.instances != { } || cfg.databases != { })) {
+        shb.monitoring.dashboards = [
+          ./backup/dashboard/Backups.json
+        ];
+      })
+    ]
+  );
 }

modules/blocks/restic/docs/default.md

@@ -3,13 +3,10 @@
 Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
 
 This block sets up a backup job using [Restic][].
+It is heavily based on the nixpkgs Restic module.
 
 [restic]: https://restic.net/
 
-## Tests {#blocks-restic-tests}
-
-Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
-
 ## Provider Contracts {#blocks-restic-contract-provider}
 
 This block provides the following contracts:
@@ -25,9 +22,7 @@ This block provides the following contracts:
 [database backup contract tests]: @REPO@/test/contracts/databasebackup.nix
 
 As requested by those two contracts, when setting up a backup with Restic,
-a backup Systemd service and restore script are provided.
-The restore script has all the secrets needed to access the repo,
-the only requirement to run it is to be able to `sudo` in the expected user.
+a backup Systemd service and a [restore script](#blocks-restic-maintenance) are provided.
 
 ## Usage {#blocks-restic-usage}
 
@@ -56,7 +51,7 @@ shb.restic.instances."myservice" = {
   settings = {
     enable = true;
 
-    passphrase.result = shb.sops.secret."passphrase".result;
+    passphrase.result = config.shb.sops.secret."passphrase".result;
 
     repository = {
       path = "/srv/backups/myservice";
@@ -77,7 +72,7 @@ shb.restic.instances."myservice" = {
 };
 
 shb.sops.secret."passphrase".request =
-  shb.restic.instances."myservice".settings.passphrase.request;
+  config.shb.restic.instances."myservice".settings.passphrase.request;
 ```
 
 ### One folder backed up with contract {#blocks-restic-usage-provider-contract}
@@ -88,12 +83,12 @@ the snippet above becomes:
 
 ```nix
 shb.restic.instances."myservice" = {
-  request = config.myservice.backup;
+  request = config.myservice.backup.request;
 
   settings = {
     enable = true;
 
-    passphrase.result = shb.sops.secret."passphrase".result;
+    passphrase.result = config.shb.sops.secret."passphrase".result;
 
     repository = {
       path = "/srv/backups/myservice";
@@ -114,7 +109,7 @@ shb.restic.instances."myservice" = {
 };
 
 shb.sops.secret."passphrase".request =
-  shb.restic.instances."myservice".settings.passphrase.request;
+  config.shb.restic.instances."myservice".settings.passphrase.request;
 ```
 
 ### One folder backed up to S3 {#blocks-restic-usage-provider-remote}
@@ -124,7 +119,7 @@ Here we will only highlight the differences with the previous configuration.
 This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/).
 
 ```diff
-  shb.backup.instances.myservice = {
+  shb.test.backup.instances.myservice = {
 
     repository = {
 -     path = "/srv/pool1/backups/myfolder";
@@ -142,7 +137,7 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
   }
 ```
 
-## Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
+### Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
 
 The following snippet shows how to configure backup of any number of folders to 3 repositories,
 each happening at different times to avoid I/O contention.
@@ -217,28 +212,62 @@ backupcfg = repositories: name: sourceDirectories {
 Now, we can define multiple backup jobs to backup different folders:
 
 ```nix
-shb.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"];
-shb.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"];
+shb.test.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"];
+shb.test.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"];
 ```
 
 The difference between the above snippet and putting all the folders into one configuration (shown
 below) is the former splits the backups into sub-folders on the repositories.
 
 ```nix
-shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
+shb.test.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
 ```
 
-## Demo {#blocks-restic-demo}
-
-[WIP]
-
 ## Monitoring {#blocks-restic-monitoring}
 
-[WIP]
+A generic dashboard for all backup solutions is provided.
+See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backup) section in the monitoring chapter.
 
 ## Maintenance {#blocks-restic-maintenance}
 
-One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
+### Manual Backup {#blocks-restic-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'restic*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start restic<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-restic-maintenance-restore}
+
+One command-line helper is provided per backup instance and repository pair which allows to:
+
+- list snapshots: `<script> snapshots`
+- to restore a snapshot: `<script> restore <snapshot>`
+
+The restore script has all the secrets needed to access the repo,
+it will run `sudo` automatically
+and the user running it needs to have correct permissions for privilege escalation.
 
 In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
 
@@ -263,6 +292,10 @@ restic-myfolder1_srv_pool1_backups restore latest
 
 In case something bad happens with a backup, the [official documentation](https://restic.readthedocs.io/en/stable/077_troubleshooting.html) has a lot of tips.
 
+## Tests {#blocks-restic-tests}
+
+Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
+
 ## Options Reference {#blocks-restic-options}
 
 ```{=include=} options

modules/blocks/sanoid.nix

@@ -0,0 +1,164 @@
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  utils,
+  ...
+}:
+let
+  cfg = config.shb.sanoid;
+
+  restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore";
+  backupScriptBase = "sanoid";
+
+  restoreScript =
+    name:
+    pkgs.writers.writePython3Bin (restoreScriptName name)
+      {
+        flakeIgnore = [ "E501" ];
+      }
+      ''
+        import argparse
+        import subprocess
+        import sys
+
+
+        dataset = "${name}"
+
+
+        class ZFSError(Exception):
+            pass
+
+
+        def run_command(cmd: list[str]) -> str:
+            try:
+                result = subprocess.run(
+                    cmd,
+                    check=True,
+                    text=True,
+                    capture_output=True,
+                )
+                return result.stdout.strip()
+            except subprocess.CalledProcessError as e:
+                raise ZFSError(
+                    f"Command failed: {' '.join(cmd)}\n"
+                    f"Exit code: {e.returncode}\n"
+                    f"stderr: {e.stderr.strip()}"
+                ) from e
+
+
+        def list_snapshots() -> None:
+            """List all ZFS snapshots."""
+            output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset])
+            if not output:
+                return []
+            return output.splitlines()
+
+
+        def restore_snapshot(snapshot: str) -> None:
+            """Rollback to a given snapshot."""
+            if not snapshot:
+                raise ValueError("Snapshot name must not be empty")
+
+            print(f"Rolling back to snapshot: {snapshot}")
+            run_command(["zfs", "rollback", "-r", snapshot])
+            print("Rollback completed successfully.")
+
+
+        def build_parser() -> argparse.ArgumentParser:
+            parser = argparse.ArgumentParser(description=f"Restore script for {dataset}")
+
+            subparsers = parser.add_subparsers(dest="command", required=True)
+
+            subparsers.add_parser(
+                "snapshots",
+                help="List all ZFS snapshots",
+            )
+
+            restore_parser = subparsers.add_parser(
+                "restore",
+                help="Rollback to a specific snapshot",
+            )
+            restore_parser.add_argument(
+                "snapshot",
+                help="Snapshot name (e.g. pool/dataset@snapname)",
+            )
+
+            return parser
+
+
+        def main():
+            parser = build_parser()
+            args = parser.parse_args()
+
+            try:
+                if args.command == "snapshots":
+                    snapshots = list_snapshots()
+                    for s in snapshots:
+                        print(s)
+                elif args.command == "restore":
+                    restore_snapshot(args.snapshot)
+                else:
+                    parser.print_help()
+                    sys.exit(1)
+
+            except ZFSError as e:
+                print(f"ERROR: {e}", file=sys.stderr)
+                sys.exit(1)
+            except Exception as e:
+                print(f"Unexpected error: {e}", file=sys.stderr)
+                sys.exit(1)
+
+
+        if __name__ == "__main__":
+            main()
+      '';
+in
+{
+  imports = [
+    ../../lib/module.nix
+  ];
+
+  options.shb.sanoid.backup = lib.mkOption {
+    description = "Sanoid prodiver for file backup contract";
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, ... }:
+        {
+          options = shb.contracts.backup.mkProvider {
+            resultCfg = {
+              restoreScript = restoreScriptName name;
+              restoreScriptText = restoreScriptName "<name>";
+
+              backupService = "${backupScriptBase}.service";
+              backupServiceText = "${backupScriptBase}.service";
+            };
+
+            settings = lib.mkOption {
+              description = "Options passed to the `services.sanoid.datasets.<name>` option.";
+              default = { };
+              type = lib.types.attrsOf lib.types.anything;
+            };
+          };
+        }
+      )
+    );
+  };
+
+  config = lib.mkIf (cfg.backup != { }) {
+    services.sanoid.enable = true;
+
+    services.sanoid.datasets =
+      let
+        mkDataset = name: cfg': {
+          inherit name;
+          value = cfg'.settings;
+        };
+      in
+      lib.mapAttrs' mkDataset cfg.backup;
+
+    environment.systemPackages = map restoreScript (lib.attrNames cfg.backup);
+  };
+}

modules/blocks/sanoid/docs/default.md

@@ -0,0 +1,97 @@
+# Sanoid Block {#blocks-sanoid}
+
+Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix):
+
+```nix
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.sanoid
+  ];
+}
+```
+
+## Provider Contracts {#blocks-sanoid-contract-provider}
+
+This block provides the following contracts:
+
+- [dataset backup contract](contracts-datasetbackup.html) under the [`shb.sanoid.backup`][backup] option.
+  It is tested with the [generic contract tests][backup contract tests].
+
+[backup]: #blocks-sanoid-options-shb.sanoid.backup
+[backup contract tests]: @REPO@/test/contracts/backup.nix
+
+## Usage {#blocks-sanoid-usage}
+
+Sanoid uses templates to know when snapshots should be kept or pruned.
+
+### Default Template {#blocks-sanoid-usage-default-template}
+
+Backup a dataset using the default Sanoid template:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+  };
+}
+```
+
+This uses the dataset backup contract which is exposed through the ZFS module's [`shb.zfs.pools.<name>.datasets.<name>.datasetBackup`](blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup) option.
+
+### Custom Template {#blocks-sanoid-usage-custom-template}
+
+Create a custom template and use it:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  services.sanoid.templates."yearly" = {
+    hourly = 10;
+    daily = 3;
+    monthly = 3;
+    yearly = 2;
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+    template = "yearly";
+  };
+}
+```
+
+Note we use the upstream `services.sanoid.templates` option to define the templates.
+
+### Without contract {#blocks-sanoid-usage-without-contract}
+
+To backup a dataset that does not provide the dataset backup contract,
+we can just set the request manually:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request.dataset = "root/home";
+  };
+}
+```
+
+Note the attr name under the `shb.sanoid.backup` option does not
+set the dataset name.
+
+## Options Reference {#blocks-sanoid-options}
+
+```{=include=} options
+id-prefix: blocks-sanoid-options-
+list-id: selfhostblocks-blocks-sanoid-options
+source: @OPTIONS_JSON@
+```

modules/blocks/sops.nix

@@ -1,45 +1,59 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  shb,
+  ...
+}:
 let
   inherit (lib) mapAttrs mkOption;
   inherit (lib.types) attrsOf anything submodule;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   cfg = config.shb.sops;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.sops = {
     secret = mkOption {
       description = "Secret following the [secret contract](./contracts-secret.html).";
-      default = {};
-      type = attrsOf (submodule ({ name, options, ... }: {
-        options = contracts.secret.mkProvider {
-          settings = mkOption {
-            description = ''
-              Settings specific to the Sops provider.
+      default = { };
+      type = attrsOf (
+        submodule (
+          { name, options, ... }:
+          {
+            options = shb.contracts.secret.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the Sops provider.
 
-              This is a passthrough option to set [sops-nix options](https://github.com/Mic92/sops-nix/blob/master/modules/sops/default.nix).
+                  This is a passthrough option to set [sops-nix options](https://github.com/Mic92/sops-nix/blob/master/modules/sops/default.nix).
 
-              Note though that the `mode`, `owner`, `group`, and `restartUnits`
-              are managed by the [shb.sops.secret.<name>.request](#blocks-sops-options-shb.sops.secret._name_.request) option.
-            '';
+                  Note though that the `mode`, `owner`, `group`, and `restartUnits`
+                  are managed by the [shb.sops.secret.<name>.request](#blocks-sops-options-shb.sops.secret._name_.request) option.
+                '';
 
-            type = attrsOf anything;
-            default = {};
-          };
+                type = attrsOf anything;
+                default = { };
+              };
 
-          resultCfg = {
-            path = "/run/secrets/${name}";
-            pathText = "/run/secrets/<name>";
-          };
-        };
-      }));
+              resultCfg = {
+                path = "/run/secrets/${name}";
+                pathText = "/run/secrets/<name>";
+              };
+            };
+          }
+        )
+      );
     };
   };
 
   config = {
-    sops.secrets = let
-      mkSecret = n: secretCfg: secretCfg.request // secretCfg.settings;
-    in mapAttrs mkSecret cfg.secret;
+    sops.secrets =
+      let
+        mkSecret = n: secretCfg: secretCfg.request // secretCfg.settings;
+      in
+      mapAttrs mkSecret cfg.secret;
   };
 }

modules/blocks/sops/docs/default.md

@@ -5,7 +5,7 @@ Defined in [`/modules/blocks/sops.nix`](@REPO@/modules/blocks/sops.nix).
 This block sets up a [sops-nix][] secret.
 
 It is only a small layer on top of `sops-nix` options
-to adapt it to the [secret contract](./contract-secret.html).
+to adapt it to the [secret contract](./contracts-secret.html).
 
 [sops-nix]: https://github.com/Mic92/sops-nix
 
@@ -13,7 +13,7 @@ to adapt it to the [secret contract](./contract-secret.html).
 
 This block provides the following contracts:
 
-- [secret contract][] under the [`shb.sops.secrets`][secret] option.
+- [secret contract][] under the [`shb.sops.secret`][secret] option.
   It is not yet tested with [contract tests][secret contract tests] but it is used extensively on several machines.
 
 [secret]: #blocks-sops-options-shb.sops.secret
@@ -21,7 +21,7 @@ This block provides the following contracts:
 [secret contract tests]: @REPO@/test/contracts/secret.nix
 
 As requested by the contract, when asking for a secret with the `shb.sops` module,
-the path where the secret will be located can be found under the [`shb.sops.secrets.<name>.result`][result] option.
+the path where the secret will be located can be found under the [`shb.sops.secret.<name>.result`][result] option.
 
 [result]: #blocks-sops-options-shb.sops.secret._name_.result
 

modules/blocks/ssl/dashboard/SSL.json

@@ -0,0 +1,727 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 16,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "line+area"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              },
+              {
+                "color": "transparent",
+                "value": 604808
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": false
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "expr": "min by(exported_hostname, subject, path) (ssl_certificate_expiry_seconds{subject=~\"CN=$job\"})",
+          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Certificate Remaining Validity",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "fieldMinMax": false,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "dateTimeFromNow"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true,
+          "width": 300
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "systemd_timer_next_trigger_seconds{name=~\"acme-renew-$job.timer\"} * 1000",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Schedule",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "footer": {
+              "reducers": []
+            },
+            "inspect": false
+          },
+          "decimals": 0,
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "green",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "% failed"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percentunit"
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "color",
+                "value": {
+                  "mode": "continuous-GrYlRd"
+                }
+              },
+              {
+                "id": "max",
+                "value": 1
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "total"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "color",
+                "value": {
+                  "mode": "thresholds"
+                }
+              },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "red",
+                      "value": 0
+                    },
+                    {
+                      "color": "transparent",
+                      "value": 1
+                    }
+                  ]
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byType",
+              "options": "string"
+            },
+            "properties": [
+              {
+                "id": "custom.minWidth",
+                "value": 150
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byType",
+              "options": "number"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 100
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "cellHeight": "sm",
+        "enablePagination": true,
+        "frozenColumns": {},
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "increase(systemd_unit_state{name=~\"acme-(order-renew-)?[[job]].service\", state=\"activating\"}[7d])",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "increase(systemd_unit_state{name=~\"acme-(order-renew-)?[[job]].service\", state=\"failed\"}[7d])",
+          "hide": false,
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "B"
+        }
+      ],
+      "title": "Jobs in the Past Week",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {
+            "mode": "columns"
+          }
+        },
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "groupingToMatrix",
+          "options": {
+            "columnField": "state",
+            "rowField": "name",
+            "valueField": "Value"
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "total",
+            "binary": {
+              "left": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "activating"
+                }
+              },
+              "operator": "+",
+              "right": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "failed"
+                }
+              }
+            },
+            "mode": "binary",
+            "reduce": {
+              "include": [
+                "activating",
+                "failed"
+              ],
+              "reducer": "sum"
+            }
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "% failed",
+            "binary": {
+              "left": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "failed"
+                }
+              },
+              "operator": "/",
+              "right": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "total"
+                }
+              }
+            },
+            "mode": "binary",
+            "reduce": {
+              "reducer": "sum"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "total"
+              }
+            ]
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "failed"
+              }
+            ]
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "includeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "activating": "success",
+              "name\\state": "Job"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "description": "",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 7,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "If the log panel is empty, it may be because the amount of lines is too high. Try filtering a few jobs first.",
+        "mode": "markdown"
+      },
+      "pluginVersion": "12.2.0",
+      "title": "",
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "default": false,
+        "type": "loki",
+        "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 14,
+        "w": 12,
+        "x": 12,
+        "y": 10
+      },
+      "id": 8,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": false,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "{unit=~\"acme-(order-renew-)?($job).service\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Logs - $job",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "description": "The job duration is not accurate. Jobs taking less than 15s to run will sometimes appear as taking 100s.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineWidth": 0,
+            "spanNulls": false
+          },
+          "mappings": [
+            {
+              "options": {
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Running"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "#EAB839",
+                "value": 0
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "never",
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(label_replace(systemd_unit_state{name=~\"acme-(order-renew-)?$job.service\", state=\"activating\"}, \"name\", \"$1\", \"name\", \"acme-order-renew-(.*).service\") > 0 or on(name) label_replace(clamp(systemd_timer_last_trigger_seconds{name=~\"acme-renew-$job.timer\"} - (systemd_timer_last_trigger_seconds{name=~\"acme-renew-$job.timer\"} offset 100s) > 0, 0, 1), \"name\", \"$1\", \"name\", \"acme-renew-(.*).timer\")) by (name)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "key": "Q-e1d5c07a-8dcc-4f34-aa5c-cdebcbdda322-0",
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Job Runs",
+      "type": "state-timeline"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 42,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "definition": "label_values(systemd_unit_state{name=~\"acme-renew-.*.timer\"},name)",
+        "includeAll": true,
+        "label": "Job",
+        "multi": true,
+        "name": "job",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(systemd_unit_state{name=~\"acme-renew-.*.timer\"},name)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "/acme-renew-(?<value>.*).timer/",
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "SSL Certificates",
+  "uid": "ae818js0bvw8wb",
+  "version": 25
+}

modules/blocks/ssl/docs/default.md

@@ -62,6 +62,7 @@ We can ask Let's Encrypt to generate a certificate with:
 shb.certs.certs.letsencrypt."example.com" = {
   domain = "example.com";
   group = "nginx";
+  reloadServices = [ "nginx.service" ];
   dnsProvider = "linode";
   adminEmail = "admin@example.com";
   credentialsFile = /path/to/secret/file;
@@ -79,6 +80,13 @@ The credential file's content would be a key-value pair:
 LINODE_TOKEN=XYZ...
 ```
 
+If you use one subdomain per service,
+asking for certificates for a subdomain is done with:
+
+```nix
+shb.certs.certs.letsencrypt."example.com".extraDomains = [ "nextcloud.${domain}" ];
+```
+
 For other providers, see the [official instruction](https://go-acme.github.io/lego/dns/).
 
 ## Usage {#block-ssl-usage}
@@ -110,11 +118,35 @@ config.shb.certs.systemdService
 See also the [SSL certificate generator usage](contracts-ssl.html#ssl-contract-usage) for a more detailed usage
 example.
 
+## Monitoring {#blocks-ssl-monitoring}
+
+A dashboard for SSL certificates is provided.
+See [SSL Certificates Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-ssl) section in the monitoring chapter.
+
 ## Debug {#block-ssl-debug}
 
 Each CA and Cert is generated by a systemd service whose name can be seen in the `systemdService`
 option. You can then see the latest errors messages using `journalctl`.
 
+### Let's Encrypt debug {#blocks-ssl-debug-lets-encrypt}
+
+Since the SHB SSL block uses the [`security.acme`][] module under the hood,
+knowing how that one works can become required if something goes wrong.
+
+For each domain and subdomain, noted as `fqdn` hereunder,
+the following systemd timers and services are created:
+
+- `acme-renew-${fqdn}.timer` triggers the `acme-order-renew-${fqdn}.service` service every day.
+- `acme-${fqdn}.service` (re)generate the initial self-signed certificate,
+   only if the following job never succeeded at least once yet.
+- `acme-order-renew-${fqdn}.service` asks for a new certificate
+   only if the certificate will expire in the next 30 days.
+   Has logic to only renew if the list of domains has not changed.
+
+Also, a global service named `acme-setup.service` is created
+
+[`security.acme`]: https://nixos.org/manual/nixos/stable/#module-security-acme
+
 ## Tests {#block-ssl-tests}
 
 The self-signed implementation is tested in [`/tests/vm/ssl.nix`](@REPO@/tests/vm/ssl.nix).

modules/blocks/tinyproxy.nix

@@ -1,32 +1,43 @@
 # Inspired from https://github.com/NixOS/nixpkgs/pull/231152 but made it so we can have multiple instances.
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 with lib;
 
 let
   cfg = config.shb.tinyproxy;
 
-  mkValueStringTinyproxy = with lib; v:
-        if true  ==         v then "yes"
-        else if false ==    v then "no"
-        else generators.mkValueStringDefault {} v;
+  mkValueStringTinyproxy =
+    with lib;
+    v:
+    if true == v then
+      "yes"
+    else if false == v then
+      "no"
+    else
+      generators.mkValueStringDefault { } v;
 
-  mkKeyValueTinyproxy = {
-    mkValueString ? mkValueStringDefault {}
-  }: sep: k: v:
-    if null     ==  v then ""
-    else "${lib.strings.escape [sep] k}${sep}${mkValueString v}";
+  mkKeyValueTinyproxy =
+    {
+      mkValueString ? mkValueStringDefault { },
+    }:
+    sep: k: v:
+    if null == v then "" else "${lib.strings.escape [ sep ] k}${sep}${mkValueString v}";
 
-  settingsFormat = (pkgs.formats.keyValue {
+  settingsFormat = (
+    pkgs.formats.keyValue {
       mkKeyValue = mkKeyValueTinyproxy {
         mkValueString = mkValueStringTinyproxy;
       } " ";
-      listsAsDuplicateKeys= true;
-  });
+      listsAsDuplicateKeys = true;
+    }
+  );
 
   configFile = name: cfg: settingsFormat.generate "tinyproxy-${name}.conf" cfg.settings;
-
-  someEnabled = any (mapAttrsToList (name: c: c.enable) cfg);
 in
 {
   options =
@@ -35,115 +46,124 @@ in
         options = {
           enable = mkEnableOption "Tinyproxy daemon";
 
-          package = mkPackageOption pkgs "tinyproxy" {};
+          package = mkPackageOption pkgs "tinyproxy" { };
 
           dynamicBindFile = mkOption {
             description = ''
-            File holding the IP to bind to.
+              File holding the IP to bind to.
             '';
             default = "";
           };
 
           settings = mkOption {
             description = ''
-            Configuration for [tinyproxy](https://tinyproxy.github.io/).
+              Configuration for [tinyproxy](https://tinyproxy.github.io/).
             '';
             default = { };
-            example = literalExpression ''{
-            Port 8888;
-            Listen 127.0.0.1;
-            Timeout 600;
-            Allow 127.0.0.1;
-            Anonymous = ['"Host"' '"Authorization"'];
-            ReversePath = '"/example/" "http://www.example.com/"';
-            }'';
-            type = types.submodule ({name, ...}: {
-              freeformType = settingsFormat.type;
-              options = {
-                Listen = mkOption {
-                  type = types.str;
-                  default = "127.0.0.1";
-                  description = ''
-                  Specify which address to listen to.
-                  '';
+            example = literalExpression ''
+              {
+                          Port 8888;
+                          Listen 127.0.0.1;
+                          Timeout 600;
+                          Allow 127.0.0.1;
+                          Anonymous = ['"Host"' '"Authorization"'];
+                          ReversePath = '"/example/" "http://www.example.com/"';
+                          }'';
+            type = types.submodule (
+              { name, ... }:
+              {
+                freeformType = settingsFormat.type;
+                options = {
+                  Listen = mkOption {
+                    type = types.str;
+                    default = "127.0.0.1";
+                    description = ''
+                      Specify which address to listen to.
+                    '';
+                  };
+                  Port = mkOption {
+                    type = types.int;
+                    default = 8888;
+                    description = ''
+                      Specify which port to listen to.
+                    '';
+                  };
+                  Anonymous = mkOption {
+                    type = types.listOf types.str;
+                    default = [ ];
+                    description = ''
+                      If an `Anonymous` keyword is present, then anonymous proxying is enabled. The
+                      headers listed with `Anonymous` are allowed through, while all others are denied.
+                      If no Anonymous keyword is present, then all headers are allowed through. You must
+                      include quotes around the headers.
+                    '';
+                  };
+                  Filter = mkOption {
+                    type = types.nullOr types.path;
+                    default = null;
+                    description = ''
+                      Tinyproxy supports filtering of web sites based on URLs or domains. This option
+                      specifies the location of the file containing the filter rules, one rule per line.
+                    '';
+                  };
                 };
-                Port = mkOption {
-                  type = types.int;
-                  default = 8888;
-                  description = ''
-                  Specify which port to listen to.
-                  '';
-                };
-                Anonymous = mkOption {
-                  type = types.listOf types.str;
-                  default = [];
-                  description = ''
-                  If an `Anonymous` keyword is present, then anonymous proxying is enabled. The
-                  headers listed with `Anonymous` are allowed through, while all others are denied.
-                  If no Anonymous keyword is present, then all headers are allowed through. You must
-                  include quotes around the headers.
-                  '';
-                };
-                Filter = mkOption {
-                  type = types.nullOr types.path;
-                  default = null;
-                  description = ''
-                  Tinyproxy supports filtering of web sites based on URLs or domains. This option
-                  specifies the location of the file containing the filter rules, one rule per line.
-                  '';
-                };
-              };
-            });
+              }
+            );
           };
         };
       };
     in
-      {
-        shb.tinyproxy = mkOption {
-          description = "Tinyproxy instances.";
-          default = {};
-          type = types.attrsOf instanceOption;
-        };
+    {
+      shb.tinyproxy = mkOption {
+        description = "Tinyproxy instances.";
+        default = { };
+        type = types.attrsOf instanceOption;
       };
+    };
 
   config = {
     systemd.services =
       let
-        instanceConfig = name: c: mkIf c.enable {
-          "tinyproxy-${name}" = {
-            description = "TinyProxy daemon - instance ${name}";
-            after = [ "network.target" ];
-            wantedBy = [ "multi-user.target" ];
-            serviceConfig = {
-              User = "tinyproxy";
-              Group = "tinyproxy";
-              Type = "simple";
-              ExecStart = "${getExe c.package} -d -c /etc/tinyproxy/${name}.conf";
-              ExecReload = "${pkgs.coreutils}/bin/kill -SIGHUP $MAINPID";
-              KillSignal = "SIGINT";
-              TimeoutStopSec = "30s";
-              Restart = "on-failure";
-              RestartSec = "1s";
-              RestartSteps = "3";
-              RestartMaxDelaySec = "10s";
-              ConfigurationDirectory = "tinyproxy";
+        instanceConfig =
+          name: c:
+          mkIf c.enable {
+            "tinyproxy-${name}" = {
+              description = "TinyProxy daemon - instance ${name}";
+              after = [ "network.target" ];
+              wantedBy = [ "multi-user.target" ];
+              serviceConfig = {
+                User = "tinyproxy";
+                Group = "tinyproxy";
+                Type = "simple";
+                ExecStart = "${getExe c.package} -d -c /etc/tinyproxy/${name}.conf";
+                ExecReload = "${pkgs.coreutils}/bin/kill -SIGHUP $MAINPID";
+                KillSignal = "SIGINT";
+                TimeoutStopSec = "30s";
+                Restart = "on-failure";
+                RestartSec = "1s";
+                RestartSteps = "3";
+                RestartMaxDelaySec = "10s";
+                ConfigurationDirectory = "tinyproxy";
+              };
+              preStart = concatStringsSep "\n" (
+                [
+                  "cat ${configFile name c} > /etc/tinyproxy/${name}.conf"
+                ]
+                ++ optionals (c.dynamicBindFile != "") [
+                  "echo -n 'Bind ' >> /etc/tinyproxy/${name}.conf"
+                  "cat ${c.dynamicBindFile} >> /etc/tinyproxy/${name}.conf"
+                ]
+              );
             };
-            preStart = concatStringsSep "\n" ([
-              "cat ${configFile name c} > /etc/tinyproxy/${name}.conf"
-            ] ++ optionals (c.dynamicBindFile != "") [
-              "echo -n 'Bind ' >> /etc/tinyproxy/${name}.conf"
-              "cat ${c.dynamicBindFile} >> /etc/tinyproxy/${name}.conf"
-            ]);
           };
-        };
       in
-        mkMerge (mapAttrsToList instanceConfig cfg);
+      mkMerge (mapAttrsToList instanceConfig cfg);
 
     users.users.tinyproxy = {
       group = "tinyproxy";
       isSystemUser = true;
     };
-    users.groups.tinyproxy = {};
+    users.groups.tinyproxy = { };
   };
 
   meta.maintainers = with maintainers; [ tcheronneau ];

modules/blocks/vpn.nix

@@ -1,4 +1,9 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
 
 let
   cfg = config.shb.vpn;
@@ -6,199 +11,212 @@ let
   quoteEach = lib.concatMapStrings (x: ''"${x}"'');
 
   nordvpnConfig =
-    { name
-    , dev
-    , authFile
-    , remoteServerIP
-    , dependentServices ? []
-    }: ''
-    client
-    dev ${dev}
-    proto tcp
-    remote ${remoteServerIP} 443
-    resolv-retry infinite
-    remote-random
-    nobind
-    tun-mtu 1500
-    tun-mtu-extra 32
-    mssfix 1450
-    persist-key
-    persist-tun
-    ping 15
-    ping-restart 0
-    ping-timer-rem
-    reneg-sec 0
-    comp-lzo no
+    {
+      name,
+      dev,
+      authFile,
+      remoteServerIP,
+      dependentServices ? [ ],
+    }:
+    ''
+      client
+      dev ${dev}
+      proto tcp
+      remote ${remoteServerIP} 443
+      resolv-retry infinite
+      remote-random
+      nobind
+      tun-mtu 1500
+      tun-mtu-extra 32
+      mssfix 1450
+      persist-key
+      persist-tun
+      ping 15
+      ping-restart 0
+      ping-timer-rem
+      reneg-sec 0
+      comp-lzo no
 
-    status /tmp/openvpn/${name}.status
+      status /tmp/openvpn/${name}.status
 
-    remote-cert-tls server
+      remote-cert-tls server
 
-    auth-user-pass ${authFile}
-    verb 3
-    pull
-    fast-io
-    cipher AES-256-CBC
-    auth SHA512
+      auth-user-pass ${authFile}
+      verb 3
+      pull
+      fast-io
+      cipher AES-256-CBC
+      auth SHA512
 
-    script-security 2
-    route-noexec
-    route-up ${routeUp name dependentServices}/bin/routeUp.sh
-    down ${routeDown name dependentServices}/bin/routeDown.sh
+      script-security 2
+      route-noexec
+      route-up ${routeUp name dependentServices}/bin/routeUp.sh
+      down ${routeDown name dependentServices}/bin/routeDown.sh
 
-    <ca>
-    -----BEGIN CERTIFICATE-----
-    MIIFCjCCAvKgAwIBAgIBATANBgkqhkiG9w0BAQ0FADA5MQswCQYDVQQGEwJQQTEQ
-    MA4GA1UEChMHTm9yZFZQTjEYMBYGA1UEAxMPTm9yZFZQTiBSb290IENBMB4XDTE2
-    MDEwMTAwMDAwMFoXDTM1MTIzMTIzNTk1OVowOTELMAkGA1UEBhMCUEExEDAOBgNV
-    BAoTB05vcmRWUE4xGDAWBgNVBAMTD05vcmRWUE4gUm9vdCBDQTCCAiIwDQYJKoZI
-    hvcNAQEBBQADggIPADCCAgoCggIBAMkr/BYhyo0F2upsIMXwC6QvkZps3NN2/eQF
-    kfQIS1gql0aejsKsEnmY0Kaon8uZCTXPsRH1gQNgg5D2gixdd1mJUvV3dE3y9FJr
-    XMoDkXdCGBodvKJyU6lcfEVF6/UxHcbBguZK9UtRHS9eJYm3rpL/5huQMCppX7kU
-    eQ8dpCwd3iKITqwd1ZudDqsWaU0vqzC2H55IyaZ/5/TnCk31Q1UP6BksbbuRcwOV
-    skEDsm6YoWDnn/IIzGOYnFJRzQH5jTz3j1QBvRIuQuBuvUkfhx1FEwhwZigrcxXu
-    MP+QgM54kezgziJUaZcOM2zF3lvrwMvXDMfNeIoJABv9ljw969xQ8czQCU5lMVmA
-    37ltv5Ec9U5hZuwk/9QO1Z+d/r6Jx0mlurS8gnCAKJgwa3kyZw6e4FZ8mYL4vpRR
-    hPdvRTWCMJkeB4yBHyhxUmTRgJHm6YR3D6hcFAc9cQcTEl/I60tMdz33G6m0O42s
-    Qt/+AR3YCY/RusWVBJB/qNS94EtNtj8iaebCQW1jHAhvGmFILVR9lzD0EzWKHkvy
-    WEjmUVRgCDd6Ne3eFRNS73gdv/C3l5boYySeu4exkEYVxVRn8DhCxs0MnkMHWFK6
-    MyzXCCn+JnWFDYPfDKHvpff/kLDobtPBf+Lbch5wQy9quY27xaj0XwLyjOltpiST
-    LWae/Q4vAgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqG
-    SIb3DQEBDQUAA4ICAQC9fUL2sZPxIN2mD32VeNySTgZlCEdVmlq471o/bDMP4B8g
-    nQesFRtXY2ZCjs50Jm73B2LViL9qlREmI6vE5IC8IsRBJSV4ce1WYxyXro5rmVg/
-    k6a10rlsbK/eg//GHoJxDdXDOokLUSnxt7gk3QKpX6eCdh67p0PuWm/7WUJQxH2S
-    DxsT9vB/iZriTIEe/ILoOQF0Aqp7AgNCcLcLAmbxXQkXYCCSB35Vp06u+eTWjG0/
-    pyS5V14stGtw+fA0DJp5ZJV4eqJ5LqxMlYvEZ/qKTEdoCeaXv2QEmN6dVqjDoTAo
-    k0t5u4YRXzEVCfXAC3ocplNdtCA72wjFJcSbfif4BSC8bDACTXtnPC7nD0VndZLp
-    +RiNLeiENhk0oTC+UVdSc+n2nJOzkCK0vYu0Ads4JGIB7g8IB3z2t9ICmsWrgnhd
-    NdcOe15BincrGA8avQ1cWXsfIKEjbrnEuEk9b5jel6NfHtPKoHc9mDpRdNPISeVa
-    wDBM1mJChneHt59Nh8Gah74+TM1jBsw4fhJPvoc7Atcg740JErb904mZfkIEmojC
-    VPhBHVQ9LHBAdM8qFI2kRK0IynOmAZhexlP/aT/kpEsEPyaZQlnBn3An1CRz8h0S
-    PApL8PytggYKeQmRhl499+6jLxcZ2IegLfqq41dzIjwHwTMplg+1pKIOVojpWA==
-    -----END CERTIFICATE-----
-    </ca>
-    key-direction 1
-    <tls-auth>
-    #
-    # 2048 bit OpenVPN static key
-    #
-    -----BEGIN OpenVPN Static key V1-----
-    e685bdaf659a25a200e2b9e39e51ff03
-    0fc72cf1ce07232bd8b2be5e6c670143
-    f51e937e670eee09d4f2ea5a6e4e6996
-    5db852c275351b86fc4ca892d78ae002
-    d6f70d029bd79c4d1c26cf14e9588033
-    cf639f8a74809f29f72b9d58f9b8f5fe
-    fc7938eade40e9fed6cb92184abb2cc1
-    0eb1a296df243b251df0643d53724cdb
-    5a92a1d6cb817804c4a9319b57d53be5
-    80815bcfcb2df55018cc83fc43bc7ff8
-    2d51f9b88364776ee9d12fc85cc7ea5b
-    9741c4f598c485316db066d52db4540e
-    212e1518a9bd4828219e24b20d88f598
-    a196c9de96012090e333519ae18d3509
-    9427e7b372d348d352dc4c85e18cd4b9
-    3f8a56ddb2e64eb67adfc9b337157ff4
-    -----END OpenVPN Static key V1-----
-    </tls-auth>
+      <ca>
+      -----BEGIN CERTIFICATE-----
+      MIIFCjCCAvKgAwIBAgIBATANBgkqhkiG9w0BAQ0FADA5MQswCQYDVQQGEwJQQTEQ
+      MA4GA1UEChMHTm9yZFZQTjEYMBYGA1UEAxMPTm9yZFZQTiBSb290IENBMB4XDTE2
+      MDEwMTAwMDAwMFoXDTM1MTIzMTIzNTk1OVowOTELMAkGA1UEBhMCUEExEDAOBgNV
+      BAoTB05vcmRWUE4xGDAWBgNVBAMTD05vcmRWUE4gUm9vdCBDQTCCAiIwDQYJKoZI
+      hvcNAQEBBQADggIPADCCAgoCggIBAMkr/BYhyo0F2upsIMXwC6QvkZps3NN2/eQF
+      kfQIS1gql0aejsKsEnmY0Kaon8uZCTXPsRH1gQNgg5D2gixdd1mJUvV3dE3y9FJr
+      XMoDkXdCGBodvKJyU6lcfEVF6/UxHcbBguZK9UtRHS9eJYm3rpL/5huQMCppX7kU
+      eQ8dpCwd3iKITqwd1ZudDqsWaU0vqzC2H55IyaZ/5/TnCk31Q1UP6BksbbuRcwOV
+      skEDsm6YoWDnn/IIzGOYnFJRzQH5jTz3j1QBvRIuQuBuvUkfhx1FEwhwZigrcxXu
+      MP+QgM54kezgziJUaZcOM2zF3lvrwMvXDMfNeIoJABv9ljw969xQ8czQCU5lMVmA
+      37ltv5Ec9U5hZuwk/9QO1Z+d/r6Jx0mlurS8gnCAKJgwa3kyZw6e4FZ8mYL4vpRR
+      hPdvRTWCMJkeB4yBHyhxUmTRgJHm6YR3D6hcFAc9cQcTEl/I60tMdz33G6m0O42s
+      Qt/+AR3YCY/RusWVBJB/qNS94EtNtj8iaebCQW1jHAhvGmFILVR9lzD0EzWKHkvy
+      WEjmUVRgCDd6Ne3eFRNS73gdv/C3l5boYySeu4exkEYVxVRn8DhCxs0MnkMHWFK6
+      MyzXCCn+JnWFDYPfDKHvpff/kLDobtPBf+Lbch5wQy9quY27xaj0XwLyjOltpiST
+      LWae/Q4vAgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqG
+      SIb3DQEBDQUAA4ICAQC9fUL2sZPxIN2mD32VeNySTgZlCEdVmlq471o/bDMP4B8g
+      nQesFRtXY2ZCjs50Jm73B2LViL9qlREmI6vE5IC8IsRBJSV4ce1WYxyXro5rmVg/
+      k6a10rlsbK/eg//GHoJxDdXDOokLUSnxt7gk3QKpX6eCdh67p0PuWm/7WUJQxH2S
+      DxsT9vB/iZriTIEe/ILoOQF0Aqp7AgNCcLcLAmbxXQkXYCCSB35Vp06u+eTWjG0/
+      pyS5V14stGtw+fA0DJp5ZJV4eqJ5LqxMlYvEZ/qKTEdoCeaXv2QEmN6dVqjDoTAo
+      k0t5u4YRXzEVCfXAC3ocplNdtCA72wjFJcSbfif4BSC8bDACTXtnPC7nD0VndZLp
+      +RiNLeiENhk0oTC+UVdSc+n2nJOzkCK0vYu0Ads4JGIB7g8IB3z2t9ICmsWrgnhd
+      NdcOe15BincrGA8avQ1cWXsfIKEjbrnEuEk9b5jel6NfHtPKoHc9mDpRdNPISeVa
+      wDBM1mJChneHt59Nh8Gah74+TM1jBsw4fhJPvoc7Atcg740JErb904mZfkIEmojC
+      VPhBHVQ9LHBAdM8qFI2kRK0IynOmAZhexlP/aT/kpEsEPyaZQlnBn3An1CRz8h0S
+      PApL8PytggYKeQmRhl499+6jLxcZ2IegLfqq41dzIjwHwTMplg+1pKIOVojpWA==
+      -----END CERTIFICATE-----
+      </ca>
+      key-direction 1
+      <tls-auth>
+      #
+      # 2048 bit OpenVPN static key
+      #
+      -----BEGIN OpenVPN Static key V1-----
+      e685bdaf659a25a200e2b9e39e51ff03
+      0fc72cf1ce07232bd8b2be5e6c670143
+      f51e937e670eee09d4f2ea5a6e4e6996
+      5db852c275351b86fc4ca892d78ae002
+      d6f70d029bd79c4d1c26cf14e9588033
+      cf639f8a74809f29f72b9d58f9b8f5fe
+      fc7938eade40e9fed6cb92184abb2cc1
+      0eb1a296df243b251df0643d53724cdb
+      5a92a1d6cb817804c4a9319b57d53be5
+      80815bcfcb2df55018cc83fc43bc7ff8
+      2d51f9b88364776ee9d12fc85cc7ea5b
+      9741c4f598c485316db066d52db4540e
+      212e1518a9bd4828219e24b20d88f598
+      a196c9de96012090e333519ae18d3509
+      9427e7b372d348d352dc4c85e18cd4b9
+      3f8a56ddb2e64eb67adfc9b337157ff4
+      -----END OpenVPN Static key V1-----
+      </tls-auth>
     '';
 
-  routeUp = name: dependentServices: pkgs.writeShellApplication {
-    name = "routeUp.sh";
+  routeUp =
+    name: dependentServices:
+    pkgs.writeShellApplication {
+      name = "routeUp.sh";
 
-    runtimeInputs = [ pkgs.iproute2 pkgs.systemd pkgs.nettools ];
+      runtimeInputs = [
+        pkgs.iproute2
+        pkgs.systemd
+        pkgs.nettools
+      ];
 
-    text = ''
-    echo "Running route-up..."
+      text = ''
+        echo "Running route-up..."
 
-    echo "dev=''${dev:?}"
-    echo "ifconfig_local=''${ifconfig_local:?}"
-    echo "route_vpn_gateway=''${route_vpn_gateway:?}"
+        echo "dev=''${dev:?}"
+        echo "ifconfig_local=''${ifconfig_local:?}"
+        echo "route_vpn_gateway=''${route_vpn_gateway:?}"
 
-    set -x
+        set -x
 
-    ip rule
-    ip rule add from "''${ifconfig_local:?}/32" table ${name}
-    ip rule add to "''${route_vpn_gateway:?}/32" table ${name}
-    ip rule
+        ip rule
+        ip rule add from "''${ifconfig_local:?}/32" table ${name}
+        ip rule add to "''${route_vpn_gateway:?}/32" table ${name}
+        ip rule
 
-    ip route list table ${name} || :
-    retVal=$?
-    if [ $retVal -eq 2 ]; then
-      echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
-      exit 1
-    fi
-    ip route add default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name}
-    ip route flush cache
-    ip route list table ${name} || :
-    retVal=$?
-    if [ $retVal -eq 2 ]; then
-      echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
-      exit 1
-    fi
+        ip route list table ${name} || :
+        retVal=$?
+        if [ $retVal -eq 2 ]; then
+          echo "table is empty"
+        elif [ $retVal -ne 0 ]; then
+          exit 1
+        fi
+        ip route add default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name}
+        ip route flush cache
+        ip route list table ${name} || :
+        retVal=$?
+        if [ $retVal -eq 2 ]; then
+          echo "table is empty"
+        elif [ $retVal -ne 0 ]; then
+          exit 1
+        fi
 
-    echo "''${ifconfig_local:?}" > /run/openvpn/${name}/ifconfig_local
+        echo "''${ifconfig_local:?}" > /run/openvpn/${name}/ifconfig_local
 
-    dependencies=(${quoteEach dependentServices})
-    for i in "''${dependencies[@]}"; do
-        systemctl restart "$i" || :
-    done
+        dependencies=(${quoteEach dependentServices})
+        for i in "''${dependencies[@]}"; do
+            systemctl restart "$i" || :
+        done
 
-    echo "Running route-up DONE"
-    '';
-  };
+        echo "Running route-up DONE"
+      '';
+    };
 
-  routeDown = name: dependentServices: pkgs.writeShellApplication {
-    name = "routeDown.sh";
+  routeDown =
+    name: dependentServices:
+    pkgs.writeShellApplication {
+      name = "routeDown.sh";
 
-    runtimeInputs = [ pkgs.iproute2 pkgs.systemd pkgs.nettools pkgs.coreutils ];
+      runtimeInputs = [
+        pkgs.iproute2
+        pkgs.systemd
+        pkgs.nettools
+        pkgs.coreutils
+      ];
 
-    text = ''
-    echo "Running route-down..."
+      text = ''
+        echo "Running route-down..."
 
-    echo "dev=''${dev:?}"
-    echo "ifconfig_local=''${ifconfig_local:?}"
-    echo "route_vpn_gateway=''${route_vpn_gateway:?}"
+        echo "dev=''${dev:?}"
+        echo "ifconfig_local=''${ifconfig_local:?}"
+        echo "route_vpn_gateway=''${route_vpn_gateway:?}"
 
-    set -x
+        set -x
 
-    ip rule
-    ip rule del from "''${ifconfig_local:?}/32" table ${name}
-    ip rule del to "''${route_vpn_gateway:?}/32" table ${name}
-    ip rule
+        ip rule
+        ip rule del from "''${ifconfig_local:?}/32" table ${name}
+        ip rule del to "''${route_vpn_gateway:?}/32" table ${name}
+        ip rule
 
-    # This will probably fail because the dev is already gone.
-    ip route list table ${name} || :
-    retVal=$?
-    if [ $retVal -eq 2 ]; then
-      echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
-      exit 1
-    fi
-    ip route del default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name} || :
-    ip route flush cache
-    ip route list table ${name} || :
-    retVal=$?
-    if [ $retVal -eq 2 ]; then
-      echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
-      exit 1
-    fi
+        # This will probably fail because the dev is already gone.
+        ip route list table ${name} || :
+        retVal=$?
+        if [ $retVal -eq 2 ]; then
+          echo "table is empty"
+        elif [ $retVal -ne 0 ]; then
+          exit 1
+        fi
+        ip route del default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name} || :
+        ip route flush cache
+        ip route list table ${name} || :
+        retVal=$?
+        if [ $retVal -eq 2 ]; then
+          echo "table is empty"
+        elif [ $retVal -ne 0 ]; then
+          exit 1
+        fi
 
-    rm /run/openvpn/${name}/ifconfig_local
+        rm /run/openvpn/${name}/ifconfig_local
 
-    dependencies=(${quoteEach dependentServices})
-    for i in "''${dependencies[@]}"; do
-        systemctl stop "$i" || :
-    done
+        dependencies=(${quoteEach dependentServices})
+        for i in "''${dependencies[@]}"; do
+            systemctl stop "$i" || :
+        done
 
-    echo "Running route-down DONE"
-    '';
-  };
-
-  someEnabled = lib.any (lib.mapAttrsToList (name: c: c.enable) cfg);
+        echo "Running route-down DONE"
+      '';
+    };
 in
 {
   options =
@@ -207,7 +225,7 @@ in
         options = {
           enable = lib.mkEnableOption "OpenVPN config";
 
-          package = lib.mkPackageOption pkgs "openvpn" {};
+          package = lib.mkPackageOption pkgs "openvpn" { };
 
           provider = lib.mkOption {
             description = "VPN provider, if given uses ready-made configuration.";
@@ -245,68 +263,76 @@ in
         };
       };
     in
-      {
-        shb.vpn = lib.mkOption {
-          description = "OpenVPN instances.";
-          default = {};
-          type = lib.types.attrsOf instanceOption;
-        };
+    {
+      shb.vpn = lib.mkOption {
+        description = "OpenVPN instances.";
+        default = { };
+        type = lib.types.attrsOf instanceOption;
       };
+    };
 
   config = {
     services.openvpn.servers =
       let
-        instanceConfig = name: c: lib.mkIf c.enable {
-          ${name} = {
-            autoStart = true;
+        instanceConfig =
+          name: c:
+          lib.mkIf c.enable {
+            ${name} = {
+              autoStart = true;
 
-            up = "mkdir -p /run/openvpn/${name}";
+              up = "mkdir -p /run/openvpn/${name}";
 
-            config = nordvpnConfig {
-              inherit name;
-              inherit (c) dev remoteServerIP authFile;
-              dependentServices = lib.optional (c.proxyPort != null) "tinyproxy-${name}.service";
+              config = nordvpnConfig {
+                inherit name;
+                inherit (c) dev remoteServerIP authFile;
+                dependentServices = lib.optional (c.proxyPort != null) "tinyproxy-${name}.service";
+              };
             };
           };
-        };
       in
-        lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
+      lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
 
-    systemd.tmpfiles.rules = map (name:
-      "d /tmp/openvpn/${name}.status 0700 root root"
-    ) (lib.attrNames cfg);
+    systemd.tmpfiles.rules = map (name: "d /tmp/openvpn/${name}.status 0700 root root") (
+      lib.attrNames cfg
+    );
 
     networking.iproute2.enable = true;
-    networking.iproute2.rttablesExtraConfig =
-      lib.concatStringsSep "\n" (lib.mapAttrsToList (name: c: "${toString c.routingNumber} ${name}") cfg);
+    networking.iproute2.rttablesExtraConfig = lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (name: c: "${toString c.routingNumber} ${name}") cfg
+    );
 
     shb.tinyproxy =
       let
-        instanceConfig = name: c: lib.mkIf (c.enable && c.proxyPort != null) {
-          ${name} = {
-            enable = true;
-            # package = pkgs.tinyproxy.overrideAttrs (old: {
-            #   withDebug = false;
-            #   patches = old.patches ++ [
-            #     (pkgs.fetchpatch {
-            #       name = "";
-            #       url = "https://github.com/tinyproxy/tinyproxy/pull/494/commits/2532ba09896352b31f3538d7819daa1fc3f829f1.patch";
-            #       sha256 = "sha256-Q0MkHnttW8tH3+hoCt9ACjHjmmZQgF6pC/menIrU0Co=";
-            #     })
-            #   ];
-            # });
-            dynamicBindFile = "/run/openvpn/${name}/ifconfig_local";
-            settings = {
-              Port = c.proxyPort;
-              Listen = "127.0.0.1";
-              Syslog = "On";
-              LogLevel = "Info";
-              Allow = [ "127.0.0.1" "::1" ];
-              ViaProxyName = ''"tinyproxy"'';
+        instanceConfig =
+          name: c:
+          lib.mkIf (c.enable && c.proxyPort != null) {
+            ${name} = {
+              enable = true;
+              # package = pkgs.tinyproxy.overrideAttrs (old: {
+              #   withDebug = false;
+              #   patches = old.patches ++ [
+              #     (pkgs.fetchpatch {
+              #       name = "";
+              #       url = "https://github.com/tinyproxy/tinyproxy/pull/494/commits/2532ba09896352b31f3538d7819daa1fc3f829f1.patch";
+              #       sha256 = "sha256-Q0MkHnttW8tH3+hoCt9ACjHjmmZQgF6pC/menIrU0Co=";
+              #     })
+              #   ];
+              # });
+              dynamicBindFile = "/run/openvpn/${name}/ifconfig_local";
+              settings = {
+                Port = c.proxyPort;
+                Listen = "127.0.0.1";
+                Syslog = "On";
+                LogLevel = "Info";
+                Allow = [
+                  "127.0.0.1"
+                  "::1"
+                ];
+                ViaProxyName = ''"tinyproxy"'';
+              };
             };
           };
-        };
       in
-        lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
+      lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
   };
 }

modules/blocks/zfs.nix

@@ -1,73 +1,259 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  shb,
+  utils,
+  ...
+}:
 
 let
   cfg = config.shb.zfs;
+
+  datasets =
+    if cfg.snapshotBeforeActivation.datasets != null then
+      cfg.snapshotBeforeActivation.datasets
+    else
+      config.boot.zfs.extraPools;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.zfs = {
-    defaultPoolName = lib.mkOption {
-      type = lib.types.nullOr lib.types.str;
-      default = null;
-      description = "ZFS pool name datasets should be created on if no pool name is given in the dataset.";
+    pools = lib.mkOption {
+      description = ''
+        Attrset of ZFS pools under which datasets will be created.
+
+        The ZFS pools are not managed by this module, they should already exist.
+
+        Each pool named here will be added to the [`boot.zfs.extraPools`](https://search.nixos.org/options?channel=unstable&include_modular_service_options=0&include_nixos_options=1&query=boot.zfs.extrapools&show=option:boot.zfs.extraPools) option.
+      '';
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule {
+          options = {
+            datasets = lib.mkOption {
+              description = ''
+                ZFS Datasets.
+
+                Each entry in the attrset will be created and mounted in the given path.
+                The attrset name is the dataset name.
+
+                This block implements the following contracts:
+                  - mount
+              '';
+              default = { };
+              example = lib.literalExpression ''
+                shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
+              '';
+              type = lib.types.attrsOf (
+                lib.types.submodule (
+                  { config, name, ... }:
+                  {
+                    options = {
+                      enable = lib.mkEnableOption "shb.zfs.datasets";
+
+                      path = lib.mkOption {
+                        type = lib.types.str;
+                        description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
+                      };
+
+                      mode = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix mode to apply to the dataset root folder.";
+                        default = null;
+                        example = "ug=rwx,g+s";
+                      };
+
+                      owner = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix user to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      group = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix group to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      defaultACLs = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = ''
+                          If non null, default ACL to set on the dataset root folder.
+
+                          Executes "setfacl -d -m $acl $path"
+                        '';
+                        default = null;
+                        example = "g:syncthing:rwX";
+                      };
+
+                      after = lib.mkOption {
+                        type = lib.types.listOf lib.types.str;
+                        description = ''
+                          Order creating this dataset after the mentioned ones.
+                          This only works with datasets managed by this module.
+
+                          Use the name of the dataset without the pool name.
+                        '';
+                        default = [ ];
+                        example = lib.literalExpression ''
+                          [
+                            "backup"
+                          ]
+                        '';
+                      };
+
+                      backup = lib.mkOption {
+                        description = ''
+                          Backup contract consumer configuration.
+
+                          This contract will backup the files inside the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.backup.mkRequester {
+                            user = if config.owner == null then "root" else config.owner;
+                            sourceDirectories = [
+                              config.path
+                            ];
+                            sourceDirectoriesText = ''
+                              [
+                                shb.zfs.pools.<name>.datasets.<name>.path
+                              ]
+                            '';
+                          };
+                        };
+                      };
+
+                      datasetbackup = lib.mkOption {
+                        description = ''
+                          ZFS dataset backup contract configuration.
+
+                          This contract will take snaphots of the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.datasetbackup.mkRequester {
+                            dataset = name;
+                          };
+                        };
+                      };
+                    };
+                  }
+                )
+              );
+            };
+          };
+        }
+
+      );
     };
 
-    datasets = lib.mkOption {
-      description = ''
-        ZFS Datasets.
+    snapshotBeforeActivation = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        description = "Take a snapshot of all datasets before activation";
+        default = false;
+      };
 
-        Each entry in the attrset will be created and mounted in the given path.
-        The attrset name is the dataset name.
+      template = lib.mkOption {
+        type = lib.types.str;
+        description = "Bash command to generate the snapshot name. $1 is the path to the new generation.";
+        default = ''pre-$(date --utc '+%y%m%dT%H%M%S')-$(basename "$1")'';
+      };
 
-        This block implements the following contracts:
-          - mount
-      '';
-      default = {};
-      example = lib.literalExpression ''
-        shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
-      '';
-      type = lib.types.attrsOf (lib.types.submodule {
-        options = {
-          enable = lib.mkEnableOption "shb.zfs.datasets";
+      datasets = lib.mkOption {
+        type = lib.types.nullOr (lib.types.listOf lib.types.str);
+        description = ''
+          Defines all datasets to take a snapshot of.
 
-          poolName = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
-            default = null;
-            description = "ZFS pool name this dataset should be created on. Overrides the defaultPoolName.";
-          };
+          If set to `null`, the default, take the list of datasets from `config.boot.zfs.extraPools`.
 
-          path = lib.mkOption {
-            type = lib.types.str;
-            description = "Path this dataset should be mounted on.";
-          };
-        };
-      });
+          The snapshots are not recursive unless specificed in the `recursive` option.
+        '';
+      };
+
+      recursive = lib.mkOption {
+        type = lib.types.bool;
+        description = "If true, take snapshots recurisevly on the given datasets.";
+        default = false;
+      };
     };
   };
 
+  # The implementation is greatly inspired by https://discourse.nixos.org/t/configure-zfs-filesystems-after-install/48633/3
   config = {
-    assertions = [
-      {
-        assertion = lib.any (x: x.poolName == null) (lib.mapAttrsToList (n: v: v) cfg.datasets) -> cfg.defaultPoolName != null;
-        message = "Cannot have both datasets.poolName and defaultPoolName set to null";
-      }
-    ];
+    boot.zfs.extraPools = lib.uniqueStrings (builtins.attrNames cfg.pools);
 
-    system.activationScripts = lib.mapAttrs' (name: cfg':
+    systemd.services =
       let
-        dataset = (if cfg'.poolName != null then cfg'.poolName else cfg.defaultPoolName) + "/" + name;
-      in
-      lib.attrsets.nameValuePair "zfsCreate-${name}" {
-        text = ''
-          ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
-            || ${pkgs.zfs}/bin/zfs create \
-               -o mountpoint=none \
-               ${dataset} || :
+        mkPool =
+          poolName: poolCfg: lib.listToAttrs (lib.mapAttrsToList (mkDataset poolName) poolCfg.datasets);
 
-          [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
-            || ${pkgs.zfs}/bin/zfs set \
-               mountpoint=${cfg'.path} \
-               ${dataset}
-        '';
-      }) cfg.datasets;
+        mkDataset =
+          poolName: name: cfg':
+          let
+            dataset = poolName + "/" + name;
+          in
+          lib.attrsets.nameValuePair "zfs-create-${utils.escapeSystemdPath dataset}" {
+            # oneshot is used to make the systemd service wait on completion of the script.
+            serviceConfig.Type = "oneshot";
+            unitConfig.DefaultDependencies = false;
+            requiredBy = [ "local-fs.target" ];
+            before = [ "local-fs.target" ];
+            after = [
+              "zfs-import-${poolName}.service"
+              "zfs-mount.service"
+            ]
+            ++ map (n: "zfs-create-${poolName}-${n}.service") cfg'.after;
+
+            unitConfig.ConditionPathIsMountPoint = lib.mkIf (cfg'.path != "none") "!${cfg'.path}";
+
+            script = ''
+              ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
+                || ${pkgs.zfs}/bin/zfs create \
+                   -o mountpoint=none \
+                   ${dataset} || :
+
+              [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
+                || ${pkgs.zfs}/bin/zfs set \
+                   mountpoint="${cfg'.path}" \
+                   ${dataset}
+
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
+              chmod "${cfg'.mode}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
+              chown "${cfg'.owner}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
+              chown :"${cfg'.group}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
+              ${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
+            '';
+          };
+        mergeAttrs = lib.foldl lib.mergeAttrs { };
+      in
+      mergeAttrs (lib.mapAttrsToList mkPool cfg.pools);
+
+    system.preSwitchChecks."shb-zfs-snapshot" = lib.mkIf cfg.snapshotBeforeActivation.enable (
+      ''
+        name="${cfg.snapshotBeforeActivation.template}"
+      ''
+      + (
+        let
+          recursiveFlag = lib.optionalString cfg.snapshotBeforeActivation.recursive "-r";
+        in
+        lib.concatMapStringsSep "\n" (ds: ''
+          echo "Taking ZFS snapshot of ${ds}@$name"
+          zfs snapshot ${recursiveFlag} ${ds}@"$name"
+        '') (lib.uniqueStrings datasets)
+      )
+    );
   };
 }

modules/blocks/zfs/docs/default.md

@@ -0,0 +1,138 @@
+# ZFS Block {#blocks-zfs}
+
+Defined in [`/modules/blocks/zfs.nix`](@REPO@/modules/blocks/zfs.nix):
+
+```nix
+{
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.zfs
+  ];
+}
+```
+
+This block creates ZFS datasets, optionally mounts them and sets permissions on the mount point.
+It also enables taking snapshots on activation.
+
+## Features {#blocks-zfs-features}
+
+- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
+- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
+- Backup of the files in the dataset [`shb.zfs.<name>.backup`][backup] through the [backup contract](./contracts-backup.html).
+- Backup of the dataset itself [`shb.zfs.<name>.datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html).
+- Take a snapshot [before activating or deploying][activationsnapshot] allowing a safe rollback.
+
+[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup
+[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup
+[activationsnapshot]: #blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable
+
+## Usage {#blocks-zfs-usage}
+
+Create a dataset at `root/safe/users` mounted on `/var/lib/nixos`:
+
+```nix
+shb.zfs.pools.root.datasets."safe/users".path = "/var/lib/nixos";
+```
+
+Create a dataset at `backup/syncoid` but do not mount it:
+
+```nix
+shb.zfs.pools.backup.datasets."syncoid".path = "none";
+```
+
+Create a dataset at `root/syncthing` and set custom permissions and ACL.
+Permission and ACL are only enforced for the mount point.
+
+```nix
+shb.zfs.pools.root.datasets."syncthing" = {
+  path = "/srv/syncthing";
+
+  mode = "ug=rwx,g+s,o=";
+  owner = "syncthing";
+  group = "syncthing";
+  defaultACLs = "g:syncthing:rwX";
+};
+```
+
+### Backup dataset {#blocks-zfs-usage-backup-dataset}
+
+To backup the dataset directly, use the [dataset backup contract](contracts-datasetbackup.html).
+For example, with the sanoid module as the dataset backup contract provider:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+    template = "yearly";
+  };
+}
+```
+
+See the [Sanoid block](blocks-sanoid.html) for more examples.
+
+### Backup files {#blocks-zfs-usage-backup-files}
+
+To backup the files in the dataset, use the [file backup contract](contracts-backup.html).
+For example, with the restic module as the file backup contract provider:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.restic.instances."myservice" = {
+    request = shb.zfs.pools.root.datasets.home.backup.request;
+  };
+}
+```
+
+See the [Restic block](blocks-restic.html) for more examples.
+
+### Snapshot before activation {#blocks-zfs-usage-snapshot-before-activation}
+
+To take a snapshot of all datasets before activating a new configuration:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation.enable = true;
+}
+```
+
+This does not take a recursive snapshot by default, to do it recursively, do:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation = {
+    enable = true;
+    recursive = true;
+  };
+}
+```
+
+We did not specify datasets manually, which means the datasets list comes from `boot.zfs.extraPools`.
+To specify the datasets manually, you can use:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation = {
+    enable = true;
+    datasets = [ "root" "root/var" ];
+  };
+}
+```
+
+## Options Reference {#blocks-zfs-options}
+
+```{=include=} options
+id-prefix: blocks-zfs-options-
+list-id: selfhostblocks-block-zfs-options
+source: @OPTIONS_JSON@
+```

modules/contracts/backup.nix

@@ -1,24 +1,40 @@
-{ pkgs, lib, ... }:
+{
+  lib,
+  shb,
+  pkgs,
+  ...
+}:
 let
-  inherit (lib) concatStringsSep literalMD mkOption optionalAttrs optionalString;
-  inherit (lib.types) listOf nonEmptyListOf submodule str;
-
-  shblib = pkgs.callPackage ../../lib {};
-  inherit (shblib) anyNotNull;
+  inherit (lib)
+    concatStringsSep
+    literalMD
+    mkOption
+    optionalAttrs
+    optionalString
+    ;
+  inherit (lib.types)
+    listOf
+    nonEmptyListOf
+    submodule
+    str
+    ;
+  inherit (shb) anyNotNull;
 in
 {
   mkRequest =
-    { user ? "",
+    {
+      user ? "",
       userText ? null,
       sourceDirectories ? [ "/var/lib/example" ],
       sourceDirectoriesText ? null,
-      excludePatterns ? [],
+      excludePatterns ? [ ],
       excludePatternsText ? null,
-      beforeBackup ? [],
+      beforeBackup ? [ ],
       beforeBackupText ? null,
-      afterBackup ? [],
+      afterBackup ? [ ],
       afterBackupText ? null,
-    }: mkOption {
+    }:
+    mkOption {
       description = ''
         Request part of the backup contract.
 
@@ -33,72 +49,102 @@ in
         };
       };
 
-      defaultText = optionalString (anyNotNull [
-        userText
-        sourceDirectoriesText
-        excludePatternsText
-        beforeBackupText
-        afterBackupText
-      ]) (literalMD ''
-      {
-        user = ${if userText != null then userText else user};
-        sourceDirectories = ${if sourceDirectoriesText != null then sourceDirectoriesText else "[ " + concatStringsSep " " sourceDirectories + " ]"};
-        excludePatterns = ${if excludePatternsText != null then excludePatternsText else "[ " + concatStringsSep " " excludePatterns + " ]"};
-        hooks.beforeBackup = ${if beforeBackupText != null then beforeBackupText else "[ " + concatStringsSep " " beforeBackup + " ]"};
-        hooks.afterBackup = ${if afterBackupText != null then afterBackupText else "[ " + concatStringsSep " " afterBackup + " ]"};
-      };
-      '');
+      defaultText =
+        optionalString
+          (anyNotNull [
+            userText
+            sourceDirectoriesText
+            excludePatternsText
+            beforeBackupText
+            afterBackupText
+          ])
+          (literalMD ''
+            {
+              user = ${if userText != null then userText else user};
+              sourceDirectories = ${
+                if sourceDirectoriesText != null then
+                  sourceDirectoriesText
+                else
+                  "[ " + concatStringsSep " " sourceDirectories + " ]"
+              };
+              excludePatterns = ${
+                if excludePatternsText != null then
+                  excludePatternsText
+                else
+                  "[ " + concatStringsSep " " excludePatterns + " ]"
+              };
+              hooks.beforeBackup = ${
+                if beforeBackupText != null then
+                  beforeBackupText
+                else
+                  "[ " + concatStringsSep " " beforeBackup + " ]"
+              };
+              hooks.afterBackup = ${
+                if afterBackupText != null then afterBackupText else "[ " + concatStringsSep " " afterBackup + " ]"
+              };
+            };
+          '');
 
       type = submodule {
         options = {
-          user = mkOption {
-            description = ''
-              Unix user doing the backups.
-            '';
-            type = str;
-            example = "vaultwarden";
-            default = user;
-          } // optionalAttrs (userText != null) {
-            defaultText = literalMD userText;
-          };
+          user =
+            mkOption {
+              description = ''
+                Unix user doing the backups.
+              '';
+              type = str;
+              example = "vaultwarden";
+              default = user;
+            }
+            // optionalAttrs (userText != null) {
+              defaultText = literalMD userText;
+            };
 
-          sourceDirectories = mkOption {
-            description = "Directories to backup.";
-            type = nonEmptyListOf str;
-            example = "/var/lib/vaultwarden";
-            default = sourceDirectories;
-          } // optionalAttrs (sourceDirectoriesText != null) {
-            defaultText = literalMD sourceDirectoriesText;
-          };
+          sourceDirectories =
+            mkOption {
+              description = "Directories to backup.";
+              type = nonEmptyListOf str;
+              example = "/var/lib/vaultwarden";
+              default = sourceDirectories;
+            }
+            // optionalAttrs (sourceDirectoriesText != null) {
+              defaultText = literalMD sourceDirectoriesText;
+            };
 
-          excludePatterns = mkOption {
-            description = "File patterns to exclude.";
-            type = listOf str;
-            default = excludePatterns;
-          } // optionalAttrs (excludePatternsText != null) {
-            defaultText = literalMD excludePatternsText;
-          };
+          excludePatterns =
+            mkOption {
+              description = "File patterns to exclude.";
+              type = listOf str;
+              default = excludePatterns;
+            }
+            // optionalAttrs (excludePatternsText != null) {
+              defaultText = literalMD excludePatternsText;
+            };
 
           hooks = mkOption {
             description = "Hooks to run around the backup.";
-            default = {};
+            default = { };
             type = submodule {
               options = {
-                beforeBackup = mkOption {
-                  description = "Hooks to run before backup.";
-                  type = listOf str;
-                  default = beforeBackup;
-                } // optionalAttrs (beforeBackupText != null) {
-                  defaultText = literalMD beforeBackupText;
-                };
+                beforeBackup =
+                  mkOption {
+                    description = "Hooks to run before backup.";
+                    type = listOf str;
+                    default = beforeBackup;
+                  }
+                  // optionalAttrs (beforeBackupText != null) {
+                    defaultText = literalMD beforeBackupText;
+                  };
 
-                afterBackup = mkOption {
-                  description = "Hooks to run after backup.";
-                  type = listOf str;
-                  default = afterBackup;
-                } // optionalAttrs (afterBackupText != null) {
-                  defaultText = literalMD afterBackupText;
-                };
+                afterBackup =
+                  mkOption {
+                    description = "Hooks to run after backup.";
+                    type = listOf str;
+                    default = afterBackup;
+                  }
+                  // optionalAttrs (afterBackupText != null) {
+                    defaultText = literalMD afterBackupText;
+                  };
               };
             };
           };
@@ -106,70 +152,133 @@ in
       };
     };
 
-  mkResult = {
-    restoreScript ? "restore",
-    restoreScriptText ? null,
-    backupService ? "backup.service",
-    backupServiceText ? null,
-  }: mkOption {
-    description = ''
-      Result part of the backup contract.
-
-      Options set by the provider module that indicates the name of the backup and restor scripts.
-    '';
-    default = {
-      inherit restoreScript backupService;
-    };
-
-    defaultText = optionalString (anyNotNull [
-      restoreScriptText
-      backupServiceText
-    ]) (literalMD ''
+  mkResult =
     {
-      restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
-      backupService = ${if backupServiceText != null then backupServiceText else backupService};
-    }
-    '');
+      restoreScript ? "restore",
+      restoreScriptText ? null,
+      backupService ? "backup.service",
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the backup contract.
 
-    type = submodule {
-      options = {
-        restoreScript = mkOption {
-          description = ''
-            Name of script that can restore the database.
-            One can then list snapshots with:
+        Options set by the provider module that indicates the name of the backup and restore scripts.
+      '';
+      default = {
+        inherit restoreScript backupService;
+      };
 
-            ```bash
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
-            ```
+      defaultText =
+        optionalString
+          (anyNotNull [
+            restoreScriptText
+            backupServiceText
+          ])
+          (literalMD ''
+            {
+              restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
+              backupService = ${if backupServiceText != null then backupServiceText else backupService};
+            }
+          '');
 
-            And restore the database with:
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
 
-            ```bash
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
-            ```
-          '';
-          type = str;
-          default = restoreScript;
-        } // optionalAttrs (restoreScriptText != null) {
-          defaultText = literalMD restoreScriptText;
-        };
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                <snapshot 1> <metadata>
+                <snapshot 2> <metadata>
+                ```
 
-        backupService = mkOption {
-          description = ''
-            Name of service backing up the database.
+                And restore the database with:
 
-            This script can be ran manually to backup the database:
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
+                ```
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
 
-            ```bash
-            $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
-            ```
-          '';
-          type = str;
-          default = backupService;
-        } // optionalAttrs (backupServiceText != null) {
-          defaultText = literalMD backupServiceText;
+          backupService =
+            mkOption {
+              description = ''
+                Name of service backing up the database.
+
+                This script can be ran manually to backup the database:
+
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
         };
       };
     };
-  };
+
+  passthru.mkRestoreScript =
+    {
+      name,
+      user,
+      sudoPreserveEnvs ? [ ],
+      secretsFile,
+      restoreCmd,
+      listCmd,
+    }:
+    pkgs.writeShellApplication {
+      inherit name;
+
+      text = ''
+        usage() {
+          echo "$0 snapshots"
+          echo "$0 restore <snapshot>"
+        }
+
+        sudocmd() {
+          sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
+        }
+
+        loadenv() {
+          set -a
+          # shellcheck disable=SC1090
+          source <(sudocmd cat "${secretsFile}")
+          set +a
+        }
+
+        if [ "$1" = "restore" ]; then
+          shift
+          snapshot="$1"
+
+          loadenv
+
+          echo "Will restore snapshot '$snapshot'"
+
+          sudocmd sh -c "${restoreCmd}"
+        elif [ "$1" = "snapshots" ]; then
+          shift
+
+          loadenv
+
+          sudocmd sh -c "${listCmd}"
+        else
+          usage
+          exit 1
+        fi
+      '';
+    };
+
 }

modules/contracts/backup/docs/default.md

@@ -20,25 +20,22 @@ source: @OPTIONS_JSON@
 ## Usage {#contract-backup-usage}
 
 A service that can be backed up will provide a `backup` option.
-Such a service is a `requester` providing a `request` for a module `provider` of this contract. 
 
-What this option defines is, from the user perspective - that is _you_ - an implementation detail
-but it will at least define what directories to backup,
-the user to backup with
-and possibly hooks to run before or after the backup job runs.
-
-Here is an example module defining such a `backup` option:
+Here is an example module defining such a `backup` option,
+which defines what directories to backup (`sourceDirectories`)
+and the user to backup with (`user`).
 
 ```nix
 {
   options = {
     myservice.backup = mkOption {
-      type = contracts.backup.request;
-      default = {
-        user = "myservice";
-        sourceDirectories = [
-          "/var/lib/myservice"
-        ];
+      type = lib.types.submodule {
+        options = shb.contracts.backup.mkRequester {
+          user = "nextcloud";
+          sourceDirectories = [
+            "/var/lib/nextcloud"
+          ];
+        };
       };
     };
   };
@@ -48,13 +45,13 @@ Here is an example module defining such a `backup` option:
 Now, on the other side we have a service that uses this `backup` option and actually backs up files.
 This service is a `provider` of this contract and will provide a `result` option.
 
-Let's assume such a module is available under the `backupservice` option
-and that one can create multiple backup instances under `backupservice.instances`.
+Let's assume such a module is available under the `backupService` option
+and that one can create multiple backup instances under `backupService.instances`.
 Then, to actually backup the `myservice` service, one would write:
 
 ```nix
-backupservice.instances.myservice = {
-  request = myservice.backup;
+backupService.instances.myservice = {
+  request = myservice.backup.request;
   
   settings = {
     enable = true;
@@ -63,17 +60,17 @@ backupservice.instances.myservice = {
       path = "/srv/backup/myservice";
     };
 
-    # ... Other options specific to backupservice like scheduling.
+    # ... Other options specific to backupService like scheduling.
   };
 };
 ```
 
 It is advised to backup files to different location, to improve redundancy.
-Thanks to using contracts, this can be made easily either with the same `backupservice`:
+Thanks to using contracts, this can be made easily either with the same `backupService`:
 
 ```nix
-backupservice.instances.myservice_2 = {
-  request = myservice.backup;
+backupService.instances.myservice_2 = {
+  request = myservice.backup.request;
   
   settings = {
     enable = true;
@@ -85,12 +82,13 @@ backupservice.instances.myservice_2 = {
 };
 ```
 
-Or with another module `backupservice_2`!
+Or with another module `backupService_2`!
 
 ## Providers of the Backup Contract {#contract-backup-providers}
 
 - [Restic block](blocks-restic.html).
-- [Borgbackup block](blocks-borgbackup.html) [WIP].
+- [Borgbackup block](blocks-borgbackup.html).
+- [ZFS block](blocks-zfs.html).
 
 ## Requester Blocks and Services {#contract-backup-requesters}
 

modules/contracts/backup/dummyModule.nix

@@ -1,11 +1,13 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  contracts = pkgs.callPackage ../. {};
-
   inherit (lib) mkOption;
   inherit (lib.types) submodule;
 in
 {
+  imports = [
+    ../../../lib/module.nix
+  ];
+
   options.shb.contracts.backup = mkOption {
     description = ''
       Contract for backing up files
@@ -23,7 +25,7 @@ in
     '';
 
     type = submodule {
-      options = contracts.backup.contract;
+      options = shb.contracts.backup.contract;
     };
   };
 }

modules/contracts/backup/test.nix

@@ -1,120 +1,179 @@
-{ pkgs, lib, ... }:
+{
+  pkgs,
+  lib,
+  shb,
+}:
 let
-  testLib = pkgs.callPackage ../../../test/common.nix {};
-
-  inherit (lib) concatMapStringsSep getAttrFromPath mkIf optionalAttrs setAttrByPath;
+  inherit (lib)
+    concatMapStringsSep
+    getAttrFromPath
+    mkIf
+    optionalAttrs
+    setAttrByPath
+    ;
 in
-{ name,
+{
+  name,
   providerRoot,
-  modules ? [],
+  modules ? [ ],
   username ? "me",
   sourceDirectories ? [
     "/opt/files/A"
     "/opt/files/B"
   ],
-  settings, # { repository, config } -> attrset
-  extraConfig ? null, # { username, config } -> attrset
-}: pkgs.testers.runNixOSTest {
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
+  extraConfig ? null, # { filesRoot, username, config } -> attrset
+}:
+shb.test.runNixOSTest {
   inherit name;
 
-  nodes.machine = { config, ... }: {
-    imports = [ testLib.baseImports ] ++ modules;
+  nodes.machine =
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
 
-    config = lib.mkMerge [
-      (setAttrByPath providerRoot {
-        request = {
-          inherit sourceDirectories;
-          user = username;
-        };
-        settings = settings {
-          inherit config;
-          repository = "/opt/repos/${name}";
-        };
-      })
-      (mkIf (username != "root") {
-        users.users.${username} = {
-          isSystemUser = true;
-          extraGroups = [ "sudoers" ];
-          group = "root";
-        };
-      })
-      (optionalAttrs (extraConfig != null) (extraConfig { inherit username config; }))
-    ];
-  };
+      config = lib.mkMerge [
+        (setAttrByPath providerRoot {
+          request = {
+            inherit sourceDirectories;
+            user = username;
+          };
+          settings = settings {
+            inherit config;
+            filesRoot = "/opt/files";
+          };
+        })
+        (mkIf (username != "root") {
+          users.users.${username} = {
+            isSystemUser = true;
+            extraGroups = [ "sudoers" ];
+            group = "root";
+          };
+        })
+        (optionalAttrs (extraConfig != null) (extraConfig {
+          inherit username config;
+          filesRoot = "/opt/files";
+        }))
+      ];
+    };
 
   extraPythonPackages = p: [ p.dictdiffer ];
   skipTypeCheck = true;
 
-  testScript = { nodes, ... }: let
-    provider = (getAttrFromPath providerRoot nodes.machine).result;
-  in ''
-    from dictdiffer import diff
+  testScript =
+    { nodes, ... }:
+    let
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      from datetime import datetime, timedelta
+      from dictdiffer import diff
+      import re
 
-    username = "${username}"
-    sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
+      username = "${username}"
+      sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
 
-    def list_files(dir):
-        files_and_content = {}
+      def list_files(dir):
+          files_and_content = {}
 
-        files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
+          files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
 
-        for f in files:
-            content = machine.succeed(f"""cat {f}""").strip()
-            files_and_content[f] = content
+          for f in files:
+              content = machine.succeed(f"""cat {f}""").strip()
+              files_and_content[f] = content
 
-        return files_and_content
+          return files_and_content
 
-    def assert_files(dir, files):
-        result = list(diff(list_files(dir), files))
-        if len(result) > 0:
-            raise Exception("Unexpected files:", result)
+      def assert_files(dir, files):
+          result = list(diff(list_files(dir), files))
+          if len(result) > 0:
+              raise Exception("Unexpected files:", result)
 
-    with subtest("Create initial content"):
-        for path in sourceDirectories:
-            machine.succeed(f"""
-                mkdir -p {path}
-                echo repo_fileA_1 > {path}/fileA
-                echo repo_fileB_1 > {path}/fileB
+      with subtest("Create initial content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                  mkdir -p {path}
+                  echo repo_fileA_1 > {path}/fileA
+                  echo repo_fileB_1 > {path}/fileB
 
-                chown {username}: -R {path}
-                chmod go-rwx -R {path}
-            """)
-
-        for path in sourceDirectories:
-            assert_files(path, {
-                f'{path}/fileA': 'repo_fileA_1',
-                f'{path}/fileB': 'repo_fileB_1',
-            })
-
-    with subtest("First backup in repo"):
-        print(machine.succeed("systemctl cat ${provider.backupService}"))
-        machine.succeed("systemctl start ${provider.backupService}")
-
-    with subtest("New content"):
-        for path in sourceDirectories:
-            machine.succeed(f"""
-              echo repo_fileA_2 > {path}/fileA
-              echo repo_fileB_2 > {path}/fileB
+                  chown {username}: -R {path}
+                  chmod go-rwx -R {path}
               """)
 
-            assert_files(path, {
-                f'{path}/fileA': 'repo_fileA_2',
-                f'{path}/fileB': 'repo_fileB_2',
-            })
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
 
-    with subtest("Delete content"):
-        for path in sourceDirectories:
-            machine.succeed(f"""rm -r {path}/*""")
+      with subtest("Initial snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
 
-            assert_files(path, {})
+      with subtest("First backup in repo"):
+          print(machine.succeed("systemctl cat ${provider.backupService}"))
+          machine.succeed("systemctl start --wait ${provider.backupService}")
 
-    with subtest("Restore initial content from repo"):
-        machine.succeed("""${provider.restoreScript} restore latest""")
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
 
-        for path in sourceDirectories:
-            assert_files(path, {
-                f'{path}/fileA': 'repo_fileA_1',
-                f'{path}/fileB': 'repo_fileB_1',
-            })
+      # To accomodate for snapshot orchestrators which keep only a given amount
+      # of snapshots per unit of time, we set the time to now + 2h.
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
+      machine.succeed(f"timedatectl set-time '{new_date}'")
+
+      with subtest("New content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                echo repo_fileA_2 > {path}/fileA
+                echo repo_fileB_2 > {path}/fileB
+                """)
+
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("Delete content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""rm -r {path}/*""")
+
+              assert_files(path, {})
+
+      with subtest("Restore second backup"):
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
     '';
 }

modules/contracts/dashboard.nix

@@ -0,0 +1,77 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types)
+    nullOr
+    submodule
+    str
+    ;
+in
+{
+  mkRequest =
+    {
+      serviceName ? "",
+      externalUrl ? "",
+      externalUrlText ? null,
+      internalUrl ? null,
+      internalUrlText ? null,
+      apiKey ? null,
+    }:
+    mkOption {
+      description = ''
+        Request part of the dashboard contract.
+      '';
+      default = { };
+      type = submodule {
+        options = {
+          externalUrl =
+            mkOption {
+              description = ''
+                URL at which the service can be accessed.
+
+                This URL should go through the reverse proxy.
+              '';
+              type = str;
+              default = externalUrl;
+              example = "https://jellyfin.example.com";
+            }
+            // (lib.optionalAttrs (externalUrlText != null) {
+              defaultText = externalUrlText;
+            });
+
+          internalUrl =
+            mkOption {
+              description = ''
+                URL at which the service can be accessed directly.
+
+                This URL should bypass the reverse proxy.
+                It can be used for example to ping the service
+                and making sure it is up and running correctly.
+              '';
+              type = nullOr str;
+              default = internalUrl;
+              example = "http://127.0.0.1:8081";
+            }
+            // (lib.optionalAttrs (internalUrlText != null) {
+              defaultText = internalUrlText;
+            });
+        };
+      };
+    };
+
+  mkResult =
+    {
+    }:
+    mkOption {
+      description = ''
+        Result part of the dashboard contract.
+
+        No option is provided here.
+      '';
+      default = { };
+      type = submodule {
+        options = {
+        };
+      };
+    };
+}

modules/contracts/dashboard/docs/default.md

@@ -0,0 +1,65 @@
+# Dashboard Contract {#contract-dashboard}
+
+This NixOS contract is used for user-facing services
+that want to be displayed on a dashboard.
+
+It is a contract between a service that can be accessed through an URL
+and a service that wants to show a list of those services.
+
+## Providers {#contract-dashboard-providers}
+
+The providers of this contract in SHB are:
+
+<!-- Somehow generate this list -->
+
+- The homepage service under its [shb.homepage.servicesGroups.<name>.services.<name>.dashboard](#services-homepage-options-shb.homepage.servicesGroups._name_.services._name_.dashboard) option.
+
+## Usage {#contracts-dashboard-usage}
+
+A service that can be shown on a dashboard will provide a `dashboard` option.
+
+Here is an example module defining such a requester option for this dashboard contract:
+
+```nix
+{
+  options = {
+    myservice.dashboard = lib.mkOption {
+      description = ''
+        Dashboard contract consumer
+      '';
+      default = { };
+      type = lib.types.submodule {
+        options = shb.contracts.dashboard.mkRequester {
+          externalUrl = "https://${config.myservice.subdomain}.${config.myservice.domain}";
+          internalUrl = "http://127.0.0.1:${config.myservice.port}";
+        };
+      };
+    };
+  };
+};
+```
+
+Then, plug both consumer and provider together in the `config`:
+
+```nix
+{
+  config = {
+    <provider-module> = {
+      dashboard.request = config.myservice.dashboard.request;
+    };
+  };
+}
+```
+
+And that's it for the contract part.
+For more specific details on each provider, go to their respective manual pages.
+
+## Contract Reference {#contract-dashboard-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-dashboard-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```

modules/contracts/dashboard/dummyModule.nix

@@ -0,0 +1,30 @@
+{ lib, shb, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) submodule;
+in
+{
+  imports = [
+    ../../../lib/module.nix
+  ];
+
+  options.shb.contracts.dashboard = mkOption {
+    description = ''
+      Contract for user-facing services that want to 
+      be displayed on a dashboard.
+
+      The requester communicates to the provider
+      how to access the service
+      through the `request` options.
+
+      The provider reads from the `request` options
+      and configures what is necessary on its side
+      to show the service and check its availability.
+      It does not communicate back to the requester.
+    '';
+
+    type = submodule {
+      options = shb.contracts.dashboard.contract;
+    };
+  };
+}

modules/contracts/databasebackup.nix

@@ -1,14 +1,19 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  inherit (lib) mkOption literalExpression literalMD optionalAttrs optionalString;
+  inherit (lib)
+    mkOption
+    literalExpression
+    literalMD
+    optionalAttrs
+    optionalString
+    ;
   inherit (lib.types) submodule str;
-
-  shblib = pkgs.callPackage ../../lib {};
-  inherit (shblib) anyNotNull;
+  inherit (shb) anyNotNull;
 in
 {
   mkRequest =
-    { user ? "root",
+    {
+      user ? "root",
       userText ? null,
       backupName ? "dump",
       backupNameText ? null,
@@ -16,146 +21,171 @@ in
       backupCmdText ? null,
       restoreCmd ? "",
       restoreCmdText ? null,
-    }: mkOption {
+    }:
+    mkOption {
       description = ''
-        Request part of the backup contract.
+        Request part of the database backup contract.
 
         Options set by the requester module
         enforcing how to backup files.
       '';
 
       default = {
-        inherit user backupName backupCmd restoreCmd;
+        inherit
+          user
+          backupName
+          backupCmd
+          restoreCmd
+          ;
       };
 
-      defaultText = optionalString (anyNotNull [
-        userText
-        backupNameText
-        backupCmdText
-        restoreCmdText
-      ]) (literalMD ''
-      {
-        user = ${if userText != null then userText else user};
-        backupName = ${if backupNameText != null then backupNameText else backupName};
-        backupCmd = ${if backupCmdText != null then backupCmdText else backupCmd};
-        restoreCmd = ${if restoreCmdText != null then restoreCmdText else restoreCmd};
-      }
-      '');
+      defaultText =
+        optionalString
+          (anyNotNull [
+            userText
+            backupNameText
+            backupCmdText
+            restoreCmdText
+          ])
+          (literalMD ''
+            {
+              user = ${if userText != null then userText else user};
+              backupName = ${if backupNameText != null then backupNameText else backupName};
+              backupCmd = ${if backupCmdText != null then backupCmdText else backupCmd};
+              restoreCmd = ${if restoreCmdText != null then restoreCmdText else restoreCmd};
+            }
+          '');
 
       type = submodule {
         options = {
-          user = mkOption {
-            description = ''
-              Unix user doing the backups.
+          user =
+            mkOption {
+              description = ''
+                Unix user doing the backups.
 
-              This should be an admin user having access to all databases.
-            '';
-            type = str;
-            example = "postgres";
-            default = user;
-          } // optionalAttrs (userText != null) {
-            defaultText = literalMD userText;
-          };
+                This should be an admin user having access to all databases.
+              '';
+              type = str;
+              example = "postgres";
+              default = user;
+            }
+            // optionalAttrs (userText != null) {
+              defaultText = literalMD userText;
+            };
 
-          backupName = mkOption {
-            description = "Name of the backup in the repository.";
-            type = str;
-            example = "postgresql.sql";
-            default = backupName;
-          } // optionalAttrs (backupNameText != null) {
-            defaultText = literalMD backupNameText;
-          };
+          backupName =
+            mkOption {
+              description = "Name of the backup in the repository.";
+              type = str;
+              example = "postgresql.sql";
+              default = backupName;
+            }
+            // optionalAttrs (backupNameText != null) {
+              defaultText = literalMD backupNameText;
+            };
 
-          backupCmd = mkOption {
-            description = "Command that produces the database dump on stdout.";
-            type = str;
-            example = literalExpression ''
-              ''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable
-            '';
-            default = backupCmd;
-          } // optionalAttrs (backupCmdText != null) {
-            defaultText = literalMD backupCmdText;
-          };
+          backupCmd =
+            mkOption {
+              description = "Command that produces the database dump on stdout.";
+              type = str;
+              example = literalExpression ''
+                ''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable
+              '';
+              default = backupCmd;
+            }
+            // optionalAttrs (backupCmdText != null) {
+              defaultText = literalMD backupCmdText;
+            };
 
-          restoreCmd = mkOption {
-            description = "Command that reads the database dump on stdin and restores the database.";
-            type = str;
-            example = literalExpression ''
-              ''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres
-            '';
-            default = restoreCmd;
-          } // optionalAttrs (restoreCmdText != null) {
-            defaultText = literalMD restoreCmdText;
-          };
+          restoreCmd =
+            mkOption {
+              description = "Command that reads the database dump on stdin and restores the database.";
+              type = str;
+              example = literalExpression ''
+                ''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres
+              '';
+              default = restoreCmd;
+            }
+            // optionalAttrs (restoreCmdText != null) {
+              defaultText = literalMD restoreCmdText;
+            };
         };
       };
     };
 
-
-  mkResult = {
-    restoreScript ? "restore",
-    restoreScriptText ? null,
-    backupService ? "backup.service",
-    backupServiceText ? null,
-  }: mkOption {
-    description = ''
-      Result part of the backup contract.
-
-      Options set by the provider module that indicates the name of the backup and restor scripts.
-    '';
-    default = {
-      inherit restoreScript backupService;
-    };
-
-    defaultText = optionalString (anyNotNull [
-      restoreScriptText
-      backupServiceText
-    ]) (literalMD ''
+  mkResult =
     {
-      restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
-      backupService = ${if backupServiceText != null then backupServiceText else backupService};
-    }
-    '');
+      restoreScript ? "restore",
+      restoreScriptText ? null,
+      backupService ? "backup.service",
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the database backup contract.
 
-    type = submodule {
-      options = {
-        restoreScript = mkOption {
-          description = ''
-            Name of script that can restore the database.
-            One can then list snapshots with:
+        Options set by the provider module that indicates the name of the backup and restore scripts.
+      '';
+      default = {
+        inherit restoreScript backupService;
+      };
 
-            ```bash
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
-            ```
+      defaultText =
+        optionalString
+          (anyNotNull [
+            restoreScriptText
+            backupServiceText
+          ])
+          (literalMD ''
+            {
+              restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
+              backupService = ${if backupServiceText != null then backupServiceText else backupService};
+            }
+          '');
 
-            And restore the database with:
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
 
-            ```bash
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
-            ```
-          '';
-          type = str;
-          default = restoreScript;
-        } // optionalAttrs (restoreScriptText != null) {
-          defaultText = literalMD restoreScriptText;
-        };
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                ```
 
-        backupService = mkOption {
-          description = ''
-            Name of service backing up the database.
+                And restore the database with:
 
-            This script can be ran manually to backup the database:
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
+                ```
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
 
-            ```bash
-            $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
-            ```
-          '';
-          type = str;
-          default = backupService;
-        } // optionalAttrs (backupServiceText != null) {
-          defaultText = literalMD backupServiceText;
+          backupService =
+            mkOption {
+              description = ''
+                Name of service backing up the database.
+
+                This script can be ran manually to backup the database:
+
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
         };
       };
     };
-  };
 }

modules/contracts/databasebackup/docs/default.md

@@ -20,21 +20,21 @@ source: @OPTIONS_JSON@
 ## Usage {#contract-databasebackup-usage}
 
 A database that can be backed up will provide a `databasebackup` option.
-Such a service is a `requester` providing a `request` for a module `provider` of this contract. 
 
 What this option defines is, from the user perspective - that is _you_ - an implementation detail
 but it will at least define how to create a database dump,
 the user to backup with
 and how to restore from a database dump.
 
-Here is an example module defining such a `databasebackup` option:
+Here is an example module defining such a `databasebackup` option,
+which defines the user to backup with (`user`)
+and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
 
 ```nix
 {
   options = {
     myservice.databasebackup = mkOption {
-      type = contracts.databasebackup.request;
-      default = {
+      type = shb.contracts.databasebackup.mkRequester = {
         user = "myservice";
         backupCmd = ''
           ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
@@ -48,16 +48,16 @@ Here is an example module defining such a `databasebackup` option:
 };
 ```
 
-Now, on the other side we have a service that uses this `backup` option and actually backs up files.
+Now, on the other side we have a service that uses this `databasebackup` option and actually backs up files.
 This service is a `provider` of this contract and will provide a `result` option.
 
-Let's assume such a module is available under the `databasebackupservice` option
-and that one can create multiple backup instances under `databasebackupservice.instances`.
+Let's assume such a module is available under the `databaseBackupService` option
+and that one can create multiple backup instances under `databaseBackupService.instances`.
 Then, to actually backup the `myservice` service, one would write:
 
 ```nix
-databasebackupservice.instances.myservice = {
-  request = myservice.databasebackup;
+databaseBackupService.instances.myservice = {
+  request = myservice.databasebackup.request;
   
   settings = {
     enable = true;
@@ -72,11 +72,11 @@ databasebackupservice.instances.myservice = {
 ```
 
 It is advised to backup files to different location, to improve redundancy.
-Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
+Thanks to using contracts, this can be made easily either with the same `databaseBackupService`:
 
 ```nix
-databasebackupservice.instances.myservice_2 = {
-  request = myservice.backup;
+databaseBackupService.instances.myservice_2 = {
+  request = myservice.databasebackup.request;
   
   settings = {
     enable = true;
@@ -88,12 +88,12 @@ databasebackupservice.instances.myservice_2 = {
 };
 ```
 
-Or with another module `databasebackupservice_2`!
+Or with another module `databaseBackupService_2`!
 
 ## Providers of the Database Backup Contract {#contract-databasebackup-providers}
 
 - [Restic block](blocks-restic.html).
-- [Borgbackup block](blocks-borgbackup.html) [WIP].
+- [Borgbackup block](blocks-borgbackup.html).
 
 ## Requester Blocks and Services {#contract-databasebackup-requesters}
 

modules/contracts/databasebackup/dummyModule.nix

@@ -1,11 +1,13 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  contracts = pkgs.callPackage ../. {};
-
   inherit (lib) mkOption;
   inherit (lib.types) submodule;
 in
 {
+  imports = [
+    ../../../lib/module.nix
+  ];
+
   options.shb.contracts.databasebackup = mkOption {
     description = ''
       Contract for database backup between a requester module
@@ -23,7 +25,7 @@ in
     '';
 
     type = submodule {
-      options = contracts.databasebackup.contract;
+      options = shb.contracts.databasebackup.contract;
     };
   };
 }

modules/contracts/databasebackup/test.nix

@@ -1,86 +1,142 @@
-{ pkgs, lib, ... }:
+{
+  pkgs,
+  lib,
+  shb,
+}:
 let
-  testLib = pkgs.callPackage ../../../test/common.nix {};
-
-  inherit (lib) getAttrFromPath mkIf optionalAttrs setAttrByPath;
+  inherit (lib)
+    getAttrFromPath
+    mkIf
+    optionalAttrs
+    setAttrByPath
+    ;
 in
-{ name,
+{
+  name,
   requesterRoot,
   providerRoot,
   extraConfig ? null, # { config, database } -> attrset
-  modules ? [],
+  modules ? [ ],
   database ? "me",
   settings, # { repository, config } -> attrset
-}: pkgs.testers.runNixOSTest {
+}:
+shb.test.runNixOSTest {
   inherit name;
 
-  nodes.machine = { config, ... }: {
-    imports = [ testLib.baseImports ] ++ modules;
-    config = lib.mkMerge [
-      (setAttrByPath providerRoot {
-        request = (getAttrFromPath requesterRoot config).request;
-        settings = settings {
-          inherit config;
-          repository = "/opt/repos/database";
-        };
-      })
-      (mkIf (database != "root") {
-        users.users.${database} = {
-          isSystemUser = true;
-          extraGroups = [ "sudoers" ];
-          group = "root";
-        };
-      })
-      (optionalAttrs (extraConfig != null) (extraConfig { inherit config database; }))
-    ];
-  };
+  nodes.machine =
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
+      config = lib.mkMerge [
+        (setAttrByPath providerRoot {
+          request = (getAttrFromPath requesterRoot config).request;
+          settings = settings {
+            inherit config;
+            repository = "/opt/repos/database";
+          };
+        })
+        (mkIf (database != "root") {
+          users.users.${database} = {
+            isSystemUser = true;
+            extraGroups = [ "sudoers" ];
+            group = "root";
+          };
+        })
+        (optionalAttrs (extraConfig != null) (extraConfig {
+          inherit config database;
+        }))
+      ];
+    };
 
-  testScript = { nodes, ... }: let
-    provider = getAttrFromPath providerRoot nodes.machine;
-  in ''
-    import csv
+  testScript =
+    { nodes, ... }:
+    let
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      import csv
+      import re
 
-    start_all()
-    machine.wait_for_unit("postgresql.service")
-    machine.wait_for_open_port(5432)
+      start_all()
+      machine.wait_for_unit("postgresql.service")
+      machine.wait_for_unit("postgresql-setup.service")
+      machine.wait_for_open_port(5432)
 
-    def peer_cmd(cmd, db="me"):
-        return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
+      def peer_cmd(cmd, db="${database}"):
+          return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
 
-    def query(query):
-        res = machine.succeed(peer_cmd(query))
-        return list(dict(l) for l in csv.DictReader(res.splitlines()))
+      def query(query):
+          res = machine.succeed(peer_cmd(query))
+          return list(dict(l) for l in csv.DictReader(res.splitlines()))
 
-    def cmp_tables(a, b):
-        for i in range(max(len(a), len(b))):
-            diff = set(a[i]) ^ set(b[i])
-            if len(diff) > 0:
-                raise Exception(i, diff)
+      def cmp_tables(a, b):
+          for i in range(max(len(a), len(b))):
+              diff = set(a[i]) ^ set(b[i])
+              if len(diff) > 0:
+                  raise Exception(i, diff)
 
-    table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+      with subtest("create fixture"):
+          machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
 
-    with subtest("create fixture"):
-        machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
-        machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
+          cmp_tables(res, table)
 
-        res = query("SELECT * FROM test")
-        cmp_tables(res, table)
+      with subtest("Initial snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 0:
+              raise Exception(f"Unexpected snapshots:\n{out}")
 
-    with subtest("backup"):
-        print(machine.succeed("systemctl cat ${provider.result.backupService}"))
-        print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
-        machine.succeed("systemctl start ${provider.result.backupService}")
+      with subtest("backup"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
 
-    with subtest("drop database"):
-        machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
-        machine.fail(peer_cmd("SELECT * FROM test"))
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+              raise Exception(f"Unexpected snapshots:\n{out}")
 
-    with subtest("restore"):
-        print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
-        machine.succeed("${provider.result.restoreScript} restore latest ")
+      with subtest("New content"):
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
 
-    with subtest("check restoration"):
-        res = query("SELECT * FROM test")
-        cmp_tables(res, table)
-  '';
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+          cmp_tables(res, table)
+
+      with subtest("backup"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("Two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("drop database"):
+          machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
+          machine.fail(peer_cmd("SELECT * FROM test"))
+
+      with subtest("restore second snapshot"):
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+          cmp_tables(res, table)
+
+      with subtest("restore first snapshot"):
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
+          cmp_tables(res, table)
+    '';
 }

modules/contracts/datasetbackup.nix

@@ -0,0 +1,113 @@
+{ lib, ... }:
+let
+  inherit (lib)
+    literalMD
+    mkOption
+    optionalAttrs
+    ;
+  inherit (lib.types)
+    submodule
+    str
+    ;
+in
+{
+  mkRequest =
+    {
+      dataset ? "",
+      datasetText ? null,
+    }:
+    mkOption {
+      description = ''
+        Request part of the backup contract.
+
+        Options set by the requester module
+        enforcing how to backup files.
+      '';
+
+      default = { };
+
+      type = submodule {
+        options = {
+          dataset =
+            mkOption {
+              description = ''
+                Dataset to backup, including the pool name.
+              '';
+              type = str;
+              example = "root/home";
+              default = dataset;
+            }
+            // optionalAttrs (datasetText != null) {
+              defaultText = literalMD datasetText;
+            };
+        };
+      };
+    };
+
+  mkResult =
+    {
+      restoreScript ? "restore",
+      restoreScriptText ? null,
+      backupService ? "backup.service",
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the backup contract.
+
+        Options set by the provider module that indicates the name of the backup and restore scripts.
+      '';
+      default = { };
+
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
+
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                <snapshot 1> <metadata>
+                <snapshot 2> <metadata>
+                ```
+
+                And restore the database with:
+
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
+                ```
+
+                It is not garanteed to be able to restore back to a snapshot in the future.
+                With the above example, it may not be possible to restore `<snapshot 2>`
+                after having restored `<snapshot 1>`.
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
+
+          backupService =
+            mkOption {
+              description = ''
+                Name of service backing up the database.
+
+                This script can be ran manually to backup the database:
+
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
+        };
+      };
+    };
+}

modules/contracts/datasetbackup/docs/default.md

@@ -0,0 +1,85 @@
+# Dataset Backup Contract {#contract-dataset-backup}
+
+This NixOS contract represents a backup job
+that will backup one ZFS dataset.
+
+It is a contract between a service that manages ZFS datasets
+and a service that backs up ZFS datasets.
+
+## Contract Reference {#contract-datatsetbackup-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-datasetbackup-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```
+
+## Usage {#contract-datasetbackup-usage}
+
+A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option.
+
+Here is an example module defining such a `backup` option,
+which defines what directories to backup (`sourceDirectories`)
+and the user to backup with (`user`).
+
+```nix
+{
+  options = {
+    myservice.datasetbackup = mkOption {
+      type = lib.types.submodule {
+        options = shb.contracts.datasetbackup.mkRequester {
+          dataset = "root/test";
+        };
+      };
+    };
+  };
+};
+```
+
+Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset.
+This service is a `provider` of this contract and will provide a `result` option.
+
+Let's assume such a module is available under the `backupService` option
+and that one can create multiple backup instances under `backupService.instances`.
+Then, to actually backup the `myservice` service, one would write:
+
+```nix
+backupService.instances.myservice = {
+  request = myservice.datasetbackup.request;
+  
+  settings = {
+    enable = true;
+
+    targetDataset = "backup/myservice";
+
+    # ... Other options specific to backupService like scheduling.
+  };
+};
+```
+
+It is advised to backup files to different location, to improve redundancy.
+Thanks to using contracts, this can be made easily either with the same `backupService`:
+
+```nix
+backupService.instances.myservice_2 = {
+  request = myservice.datasetbackup.request;
+  
+  settings = {
+    enable = true;
+  
+    targetDataset = "backup2/myservice";
+  };
+};
+```
+
+Or with another module `backupService_2`!
+
+## Providers of the Backup Contract {#contract-datasetbackup-providers}
+
+- [Sanoid block](blocks-sanoid.html).
+
+## Requester Blocks and Services {#contract-datasetbackup-requesters}
+
+- [ZFS](blocks-zfs.html#blocks-zfs-backup).

modules/contracts/datasetbackup/dummyModule.nix

@@ -0,0 +1,30 @@
+{ lib, shb, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) submodule;
+in
+{
+  imports = [
+    ../../../lib/module.nix
+  ];
+
+  options.shb.contracts.datasetbackup = mkOption {
+    description = ''
+      Contract for backing up ZFS datasets.
+
+      The requester communicates to the provider
+      the dataset to backup
+      through the `request` options.
+
+      The provider reads from the `request` options
+      and backs up the requested dataset.
+      It communicates to the requester what script is used
+      to backup and restore the files
+      through the `result` options.
+    '';
+
+    type = submodule {
+      options = shb.contracts.datasetbackup.contract;
+    };
+  };
+}

modules/contracts/datasetbackup/test.nix

@@ -0,0 +1,164 @@
+{
+  pkgs,
+  lib,
+  shb,
+}:
+let
+  inherit (lib)
+    concatMapStringsSep
+    getAttrFromPath
+    setAttrByPath
+    ;
+in
+{
+  name,
+  providerRoot,
+  modules ? [ ],
+  dataset ? "root/test",
+  sourceDirectories ? [
+    "/opt/files/A"
+    "/opt/files/B"
+  ],
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
+  extraConfig ? null, # { filesRoot, config } -> attrset
+}:
+shb.test.runNixOSTest {
+  inherit name;
+
+  nodes.machine =
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
+
+      config = lib.mkMerge [
+        (setAttrByPath providerRoot {
+          request = {
+            inherit dataset;
+          };
+          settings = settings {
+            inherit config;
+            filesRoot = "/opt/files";
+          };
+        })
+        (lib.optionalAttrs (extraConfig != null) (extraConfig {
+          inherit config;
+          filesRoot = "/opt/files";
+        }))
+      ];
+    };
+
+  extraPythonPackages = p: [ p.dictdiffer ];
+  skipTypeCheck = true;
+
+  testScript =
+    { nodes, ... }:
+    let
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      from datetime import datetime, timedelta
+      from dictdiffer import diff
+      import re
+
+      sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
+
+      def list_files(dir):
+          files_and_content = {}
+
+          files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
+
+          for f in files:
+              content = machine.succeed(f"""cat {f}""").strip()
+              files_and_content[f] = content
+
+          return files_and_content
+
+      def assert_files(dir, files):
+          result = list(diff(list_files(dir), files))
+          if len(result) > 0:
+              raise Exception("Unexpected files:", result)
+
+      with subtest("Create initial content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                  mkdir -p {path}
+                  echo repo_fileA_1 > {path}/fileA
+                  echo repo_fileB_1 > {path}/fileB
+              """)
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
+
+      with subtest("Initial snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("First backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      # To accomodate for snapshot orchestrators which keep only a given amount
+      # of snapshots per unit of time, we set the time to now + 2h.
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
+      machine.succeed(f"timedatectl set-time '{new_date}'")
+
+      with subtest("New content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                echo repo_fileA_2 > {path}/fileA
+                echo repo_fileB_2 > {path}/fileB
+                """)
+
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("Delete content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""rm -r {path}/*""")
+
+              assert_files(path, {})
+
+      with subtest("Restore second backup"):
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
+    '';
+}