open-webui: more consistent example

Open WebUI 0.9.6 still initializes its embedding function during startup, even when OFFLINE_MODE=true and BYPASS_EMBEDDING_AND_RETRIEVAL=true. With OFFLINE_MODE=true and no cached SentenceTransformer model, startup failed with:

ValueError: No embedding model is loaded. Set RAG_EMBEDDING_MODEL to a valid SentenceTransformer model name, or configure an external RAG_EMBEDDING_ENGINE (ollama, openai, azure_openai).

Configure the tests to use a dummy Ollama embedding endpoint so startup does not try to load or download a local embedding model. The endpoint is not meant to be contacted by these tests; it only keeps startup on the external embedding configuration path.

.github/workflows/auto-merge.yaml

@@ -23,7 +23,7 @@ on:
         required: false
 
 jobs:
-  automerge:
+  automerge-rebase:
     runs-on: ubuntu-latest
     steps:
       - uses: reitermarkus/automerge@v2
@@ -35,3 +35,16 @@ jobs:
           pull-request: ${{ github.event.inputs.pull-request }}
           review: ${{ github.event.inputs.review }}
           dry-run: false
+
+  automerge-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: reitermarkus/automerge@v2
+        with:
+          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          merge-method: merge
+          do-not-merge-labels: never-merge
+          required-labels: automerge-merge
+          pull-request: ${{ github.event.inputs.pull-request }}
+          review: ${{ github.event.inputs.review }}
+          dry-run: false

.github/workflows/build.yaml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - uses: tj-actions/changed-files@v47
         id: filter
@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Install Nix
         uses: cachix/install-nix-action@v31
         with:
@@ -61,13 +61,15 @@ jobs:
             keep-outputs = true
             keep-failed = true
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
       - name: Generate Matrix
         id: generate-matrix
         run: |
+          set -euox pipefail
+
           nix flake show --allow-import-from-derivation --json \
               | jq -c '.["checks"]["x86_64-linux"] | keys' > .output
 
@@ -81,7 +83,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Install Nix
         uses: cachix/install-nix-action@v31
         with:
@@ -91,7 +93,7 @@ jobs:
             keep-outputs = true
             keep-failed = true
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -112,7 +114,7 @@ jobs:
         check: ${{ fromJson(needs.build-matrix.outputs.check) }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Install Nix
         uses: cachix/install-nix-action@v31
         with:
@@ -122,7 +124,7 @@ jobs:
             keep-outputs = true
             keep-failed = true
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -131,7 +133,7 @@ jobs:
           echo "resultPath=$(nix eval .#checks.x86_64-linux.${{ matrix.check }} --raw)" >> $GITHUB_ENV
           nix build --print-build-logs --show-trace .#checks.x86_64-linux.${{ matrix.check }}
       - name: Upload Build Result
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: always() && startsWith(matrix.check, 'vm_')
         with:
           name: ${{ matrix.check }}

.github/workflows/demo.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - uses: tj-actions/changed-files@v47
         id: filter
@@ -40,6 +40,7 @@ jobs:
     needs: [ "path-filter" ]
     if: needs.path-filter.outputs.changed == 'true'
     strategy:
+      fail-fast: false
       matrix:
         demo:
           - name:  homeassistant
@@ -54,11 +55,18 @@ jobs:
           - name:  nextcloud
             flake: sso
 
+          - name:  minimal
+            flake: minimal
+          - name:  minimal
+            flake: lowlevel
+          - name:  minimal
+            flake: sops
+
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Install Nix
         uses: cachix/install-nix-action@v31
@@ -69,7 +77,7 @@ jobs:
             keep-outputs = true
             keep-failed = true
 
-      - uses: cachix/cachix-action@v16
+      - uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -78,6 +86,11 @@ jobs:
         run: |
           cd demo/${{ matrix.demo.name }}
           nix flake update --override-input selfhostblocks ../.. selfhostblocks
+          nix \
+            --print-build-logs \
+            --option keep-going true \
+            --show-trace \
+            build .#nixosConfigurations.${{ matrix.demo.flake }}.config.system.build.toplevel
           nix \
             --print-build-logs \
             --option keep-going true \

.github/workflows/format.yaml

@@ -0,0 +1,24 @@
+name: "format"
+on:
+  pull_request:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Caching
+        uses: cachix/cachix-action@v17
+        with:
+          name: selfhostblocks
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+      - name: Check Formatting
+        run: |
+          find . -name '*.nix' | nix fmt -- --ci

.github/workflows/lock-update.yaml

@@ -1,23 +1,43 @@
+# See shb.skarabox.com/misc.html
 name: Update Flake Lock
 
 on:
-  workflow_dispatch:
   schedule:
-    - cron: '0 0 * * *' # runs daily at 00:00
+    - cron: '0 */3 * * *' # runs every 3 hours at :00
+
+  workflow_dispatch:
+    inputs:
+      bisect_future:
+        description: "Bisect in the future instead of the past"
+        required: false
+        default: "false"
 
 jobs:
   lockfile:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Install Nix
         uses: cachix/install-nix-action@v31
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Update flake.lock
+      - name: Cache nixpkgs mirror
-        uses: DeterminateSystems/update-flake-lock@main
+        uses: actions/cache@v5
         with:
-          token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          path: .cache/nixpkgs.git
-          pr-labels: |
+          key: nixpkgs-mirror-v1
-            automerge
+      - name: Update flake.lock
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
+          BISECT_FUTURE: ${{ inputs.bisect_future }}
+        run: |
+          git config user.email "update-flake-lock-pr@selfhostblocks.com"
+          git config user.name "Update Flake Lock PR"
+          if [ "$BISECT_FUTURE" = false ]; then
+            BISECT_FUTURE=
+          elif [ -n "$BISECT_FUTURE" ]; then
+            BISECT_FUTURE=future
+          fi
+          nix run .#update-flake-lock-pr $BISECT_FUTURE
+

.github/workflows/pages.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Install Nix
         uses: cachix/install-nix-action@v31
@@ -37,7 +37,7 @@ jobs:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Setup Caching
-        uses: cachix/cachix-action@v16
+        uses: cachix/cachix-action@v17
         with:
           name: selfhostblocks
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@@ -59,13 +59,13 @@ jobs:
             public
 
       - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@v6
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v4
+        uses: actions/upload-pages-artifact@v5
         with:
           path: ./public
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v5

.github/workflows/update-flake-lock-pr.nix

@@ -0,0 +1,223 @@
+{
+  gh,
+  writeShellApplication,
+}:
+writeShellApplication {
+  name = "update-flake-lock-pr";
+  runtimeInputs = [
+    gh
+  ];
+  text = ''
+    branch=ci/nixpkgs-update
+    if [ "''${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
+      branch=ci/nixpkgs-update-auto
+    fi
+
+    goto_future=no
+    if [ -n "''${1:-}" ]; then
+      goto_future=yes
+    fi
+
+    main () {
+      PR=$(get_pr)
+
+      if [ -z "$PR" ]; then
+        create_pr
+        exit 0
+      fi
+
+      case "$(get_pr_check_status "$PR")" in
+        pending)
+          echo "Checks are running on PR $PR, nothing to do yet."
+          exit 0
+          ;;
+
+        succeeded)
+          echo "Checks succeeded, nothing to do."
+          exit 0
+          ;;
+
+        failing)
+          echo "Checks failed, updating PR to midpoint commit."
+
+          local CURRENT_COMMIT
+          local LAST_WORKING_COMMIT
+          local FAILING_COMMIT
+          local NEW_COMMIT
+
+          CURRENT_COMMIT="$(get_main_branch_commit)"
+          if [ "$goto_future" = "no" ]; then
+            LAST_WORKING_COMMIT="$CURRENT_COMMIT"
+            FAILING_COMMIT="$(get_pr_commit "$PR")"
+          else
+            LAST_WORKING_COMMIT="$(get_pr_commit "$PR")"
+            FAILING_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+          NEW_COMMIT="$(midpoint_commit "$LAST_WORKING_COMMIT" "$FAILING_COMMIT")"
+
+          if [ "$NEW_COMMIT" = "$LAST_WORKING_COMMIT" ] || [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            NEW_COMMIT="$(get_latest_nixpkgs_commit)"
+          fi
+
+          if [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
+            echo "Latest nixpkgs commit is still same failing commit, will retry later."
+            exit 0
+          fi
+
+          update_pr "$PR" "$CURRENT_COMMIT" "$NEW_COMMIT"
+          exit 0
+          ;;
+      esac
+    }
+
+    get_main_branch_commit() {
+      git fetch origin main
+
+      git show origin/main:flake.lock \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    midpoint_commit() {
+      local good="$1"
+      local bad="$2"
+
+      mkdir -p .cache
+      if [ ! -d .cache/nixpkgs.git ]; then
+        git clone --mirror https://github.com/NixOS/nixpkgs.git .cache/nixpkgs.git
+      else
+        git --git-dir=.cache/nixpkgs.git fetch origin
+      fi
+
+      local commits=()
+
+      mapfile -t commits < <(
+        git --git-dir=.cache/nixpkgs.git \
+          rev-list \
+          --reverse \
+          --ancestry-path \
+          "''${good}..''${bad}"
+      )
+
+      local count="''${#commits[@]}"
+
+      if [ "$count" -eq 0 ]; then
+        echo "$bad"
+        return
+      fi
+
+      local midpoint_index=$((count / 2))
+
+      echo "''${commits[$midpoint_index]}"
+    }
+
+    get_latest_nixpkgs_commit () {
+      git ls-remote https://github.com/NixOS/nixpkgs nixos-unstable | cut -f1
+    }
+
+    get_pr () {
+      gh pr list \
+        --head "$branch" \
+        --state open \
+        --json number \
+        --jq '.[0].number // empty'
+    }
+
+    create_pr() {
+      local test_commit
+
+      test_commit="$(get_latest_nixpkgs_commit)"
+      echo "Creating PR on nixpkgs' latest commit $test_commit"
+
+      git checkout -B "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push -u origin "$branch" --force
+
+      current_commit="$(get_main_branch_commit)"
+      gh pr create \
+        --title "update nixpkgs to $test_commit" \
+        --body "$(printf "%s\n\n%s" "Automated nixpkgs update. Latest tries:" " - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit")" \
+        --label automerge-merge
+    }
+
+    append_pr_body() {
+      local pr="$1"
+      local text="$2"
+
+      local current_body
+
+      current_body="$(gh pr view "$pr" --json body --jq .body)"
+
+      gh pr edit "$pr" \
+        --body "$(printf '%s%b' "$current_body" "$text")"
+    }
+
+    update_pr() {
+      local pr="$1"
+      local current_commit="$2"
+      local test_commit="$3"
+
+      echo "Updating PR to nixpkgs' commit $test_commit"
+
+      git checkout "$branch"
+
+      nix flake lock \
+        --override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
+
+      git add flake.lock
+
+      git commit -m "update nixpkgs to $test_commit"
+
+      git push --force-with-lease origin "$branch"
+
+      local future_text
+      if [ "$goto_future" = "yes" ]; then
+        future_text="bisected in the future"
+      else
+        future_text="bisected in the past"
+      fi
+
+      gh pr edit "$pr" \
+        --title "update nixpkgs to $test_commit"
+
+      append_pr_body "$pr" \
+        " ❌\n - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit ($future_text)"
+    }
+
+    get_pr_check_status() {
+      local pr="$1"
+
+      local status
+      status="$(gh pr checks "$pr" --json state --jq '.[].state' || true)"
+
+      if echo "$status" | grep -qE 'PENDING|QUEUED|IN_PROGRESS'; then
+        echo "pending"
+        return
+      fi
+
+      if echo "$status" | grep -qE 'FAILURE|TIMED_OUT|CANCELLED'; then
+        echo "failing"
+        return
+      fi
+
+      echo "success"
+    }
+
+    get_pr_commit() {
+      local pr="$1"
+
+      git fetch origin "$branch"
+
+      git show "origin/$branch:flake.lock" \
+        | jq -r '.nodes.nixpkgs.locked.rev'
+    }
+
+    main
+  '';
+}

.github/workflows/version.yaml

@@ -11,7 +11,7 @@ jobs:
   create-tag:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 1
       - name: Get version

CHANGELOG.md

@@ -16,6 +16,256 @@ Template:
 
 # Upcoming Release
 
+# v0.9.0
+
+Commits: https://github.com/ibizaman/selfhostblocks/compare/v0.8.0...v0.9.0
+
+## Breaking Changes
+
+- Updated simple-nixos-mailserver. Update all options following this pattern:
+
+  ```diff
+  - mailserver.mailboxes.<name>.specialUse = "value"
+  + mailserver.mailboxes.<name>.special_use = "\\value"
+  ```
+
+  Note that simple-nixos-mailserver wanted to upgrade the location of the storage path
+  to include the ldap UID instead of the email but I kept the email address.
+  This means there is no migration to do.
+- Update nixpkgs [from 6201e2 to abd1ea](https://github.com/ibizaman/selfhostblocks/compare/6201e203d09599479a3b3450ed24fa81537ebc4e...abd1ea17fdbedd48cbca770847b39e3a7a09ab5b).
+
+## New Features
+
+- Add `shb.zfs.snapshotBeforeActivation` option to take a ZFS snapshot of given datasets before activation.
+  See [the manual](https://shb.skarabox.com/blocks-zfs.html) for more details.
+- Add [sanoid block](https://shb.skarabox.com/blocks-sanoid.html)
+- Add option to take snapshot before activation with ZFS block.
+
+## Fixes
+
+- Fix oidc_login build for Nextcloud
+- Fix mailserver build
+
+## Other Changes
+
+- Harden lldap configuration
+- Convert ZFS block to system and [add documentation](https://shb.skarabox.com/blocks-zfs.html)
+- Switch monitoring log fetching from deprecated promtail to fluent-bit
+- Add test to catch drift for Let's Encrypt DNS provider
+
+# [BROKEN] v0.8.0
+
+## Breaking Changes
+
+- Bump of Nextcloud version to 32 and 33 because of nixpkgs bump. All provided apps are verified compatible with Nextcloud 33 thanks to new tests.
+
+## New Features
+
+- Added Immich Public Proxy service
+- Add homepage service with dashboard contract implemented by all services
+- Add scrutiny service.
+- ZFS module now supports setting permissions
+- Add landing page for mailserver and dashboard contract integration
+
+## Bug Fixes
+
+- Use configurable dataDir in arr stack
+- Forgejo ensures ldap is setup when sso is configured
+- Add nixpkgs patches on aarch64-linux too
+- Self-signed certs are now idempotent
+- Prometheus scrapes metrics at 15s interval instead of 1m
+
+## Other Changes
+
+- Arr stack declares ldap groups, declare ApiKeys and bypasses auth for readarr when sso is enabled
+- Forgejo declares ldap group
+
+# v0.7.3
+
+## New Features
+
+- Add [mailserver module](https://shb.skarabox.com/services-mailserver.html) integrating with [Simple NixOS Mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver) and allowing full backup of an email provider.
+- Bump nixpkgs from https://github.com/NixOS/nixpkgs/commit/5e2a59a5b1a82f89f2c7e598302a9cacebb72a67 to https://github.com/NixOS/nixpkgs/commit/bfc1b8a4574108ceef22f02bafcf6611380c100d. [Full diff](https://github.com/nixos/nixpkgs/compare/5e2a59a5b1a82f89f2c7e598302a9cacebb72a67...bfc1b8a4574108ceef22f02bafcf6611380c100d).
+  On top of minor changes, the most notable one was:
+  - Updated Jellyfin LDAP and SSO plugins and configuration. @Codys-Wright
+
+## Bug Fixes
+
+- Fix Restic and Authelia modules referencing systemd services without the `.service` suffix and leading to 
+
+# v0.7.2
+
+## New Features
+
+- Forgejo uses secrets contract for smtp password.
+- Add [Firefly-iii](https://shb.skarabox.com/services-firefly-iii.html) service.
+- Jellyfin can [install plugins declaratively](https://shb.skarabox.com/services-jellyfin.html#services-jellyfin-options-shb.jellyfin.plugins).
+  (Support is quite crude and WIP).
+- Jellyfin configures LDAP and SSO fully declaratively, including installing necessary plugins.
+- Nextcloud 32 is fully supported thanks to tests for version 31 and 32.
+
+## Fixes
+
+- Revert Authelia to continue using dots in systemd service names.
+  This caused issue with nginx name resolution.
+
+## Other Changes
+
+- Authelia uses non deprecated `smtp.address` option.
+- Add documentation for Nginx block
+- Now a user which is only member of the admin LDAP group of a service can login.
+  Before, some services required a user to be member of both the user and admin LDAP group.
+  This is ensured by regression tests going forward.
+
+# v0.7.1
+
+## New Features
+
+- Add a Grafana dashboard showing SSL certificate renewal jobs
+
+## Fixes
+
+- Fix let's encrypt certificate renewal jobs by removing duplicated domain name.
+  Also adds an assertion to catch these kinds of errors.
+
+## Other Changes
+
+- Reduce number of late SSL renewal alert by merging all metrics corresponding to one CN.
+
+# v0.7.0
+
+## Breaking Changes
+
+- Fix pkgs overrides not being passed to users of SelfHostBlocks.
+  This will require to update your flake to follow the example in the [Usage](https://shb.skarabox.com/usage.html) section.
+
+## New Features
+
+- Add a Grafana dashboard showing stats on backup jobs
+  and also an alert if a backup job did not run in the last 24 hours or never succeeded in the last 24 hours.
+- Add SSO integration in Grafana.
+- Add Paperless service.
+
+## Fixes
+
+- Allow to upload big files in Immich.
+- Only enable php-fpm Prometheus exporter if Nextcloud is enabled.
+
+## Other Changes
+
+- Add recipe to setup DNS server with DNSSEC.
+
+# v0.6.1
+
+## New Features
+
+- Implement backup and databasebackup contracts with BorgBackup block.
+
+## Fixes
+
+- Add back erroneously removed Prometheus collectors.
+
+# v0.6.0
+
+## Breaking Changes
+
+- Removed Nextcloud 30, update to Nextcloud 31 then after to 32.
+- Removed the `sops` module in the `default` NixOS module. Removed the `all` NixOS module.
+
+## New Features
+
+- Meilisearch configured with production environment and master key.
+
+## Other Changes
+
+- Only import hardcodedsecret module in tests.
+- Better usage section in manual.
+- Added new demo for minimal SelfHostBlocks setup, which is tested in CI.
+- Format all files in repo and make sure they are formatted in CI.
+
+# v0.5.1
+
+## New Features
+
+- Added Karakeep service with SSO integration.
+- Add SelfHostBlocks' `lib` into `pkgs.lib.shb`. Integrates with [Skarabox](https://github.com/ibizaman/skarabox/blob/631ff5af0b5c850bb63a3b3df451df9707c0af4e/template/flake.nix#L42-L43) too.
+
+## Other Changes
+
+- Moved implementation guide under contributing section.
+
+# v0.5.0
+
+## Breaking Changes
+
+- Modules in the `nixosModules` output field do not anymore have the `system` in their path.
+  `selfhostblocks.nixosModules.x86_64-linux.home-assistant` becomes `selfhostblocks.nixosModules.home-assistant`
+  like it always should have been.
+
+## Fixes
+
+- Added test case making sure a user belonging to a not authorized LDAP group cannot login.
+  Fixed Open WebUI module.
+- Now importing a single module, like `selfhostblocks.nixosModules.home-assistant`, will
+  import all needed block modules at the same time.
+
+## Other Changes
+
+- Nextcloud module can now setup SSO integration without setting up LDAP integration.
+
+# v0.4.4
+
+## New Features
+
+- Added Pinchflat service with SSO integration. Declarative user creation only supported through SSO integration.
+- Added Immich service with SSO integration.
+- Added Open WebUI service with SSO integration.
+
+# v0.4.3
+
+## New Features
+
+- Allow user to change their SSO password in Authelia.
+- Make Audiobookshelf SSO integration respect admin users.
+
+## Fixes
+
+- Fix permission on Nextcloud systemd service.
+- Delete Forgejo backups correctly to avoid them piling up.
+
+## Other Changes
+
+- Add recipes section to the documentation.
+
+# v0.4.2
+
+## New Features
+
+- The LLDAP and Authelia modules gain a debug mode where a mitmdump instance is added so all traffic is printed.
+
+## Fixes
+
+- By default, LLDAP module only enforces groups declaratively. Users that are not defined declaratively
+  are not anymore deleted by inadvertence.
+- SSO integration with most services got fixed. A recent incompatible change in upstream Authelia broke most of them.
+- Fixed PostgreSQL and Home Assistant modules after nixpkgs updates.
+- Fixed Nextcloud module SSO integration with Authelia.
+- Make Nextcloud SSO integration respect admin users.
+
+# v0.4.1
+
+## New Features
+
+- LLDAP now manages users, groups, user attributes and group attributes declaratively.
+- Individual modules are exposed in the flake output for each block and service.
+- A mitmdump block is added that can be placed between two services and print all requests and responses.
+- The SSO setup for Audiobookshelf is now a bit more declarative.
+
+## Other Changes
+
+- Forgejo got a new playwright test to check the LDAP integration.
+- Some renaming options have been added retroactively for jellyfin and forgejo.
+
 # v0.4.0
 
 ## Breaking Changes

README.md

@@ -90,18 +90,33 @@ that fit together to build their server.
 > production server, this is really just a one person effort for now and there are most certainly
 > bugs that I didn't discover yet.
 
-### At a Glance
+To get started using SelfHostBlocks, the following snippet is enough:
 
-Head over to the [recipes section](https://shb.skarabox.com/recipes.html) of the manual to see how SelfHostBlocks can help you quickly setup common use cases.
+```nix
+{
+  inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks";
 
-The [services section](https://shb.skarabox.com/services.html) lists all integrated services you can quickly spin up using SelfHostBlocks.
+  outputs = { selfhostblocks, ... }: let
+    system = "x86_64-linux";
+    nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
+  in
+    nixosConfigurations = {
+      myserver = nixpkgs'.nixosSystem {
+        inherit system;
+        modules = [
+          selfhostblocks.nixosModules.default
+          ./configuration.nix
+        ];
+      };
+    };
+}
+```
 
-### Existing Installation
+SelfHostBlocks provides its own patched nixpkgs, so you are required to use it
-
+otherwise evaluation can quickly break.
-To get started using SelfHostBlocks,
+[The usage section](https://shb.skarabox.com/usage.html) of the manual has
-follow [the usage section](https://shb.skarabox.com/usage.html) of the manual.
+more details and goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
-It goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
+and also how to handle secrets management with [SOPS][].
-and also goes over secrets management with [SOPS][].
 
 [Colmena]: https://shb.skarabox.com/usage.html#usage-example-colmena
 [nixos-rebuild]: https://shb.skarabox.com/usage.html#usage-example-nixosrebuild
@@ -109,8 +124,9 @@ and also goes over secrets management with [SOPS][].
 [SOPS]: https://shb.skarabox.com/usage.html#usage-secrets
 
 Then, to actually configure services, you can choose which one interests you in
-[the services section](https://shb.skarabox.com/services.html) of the manual.
+the [services section](https://shb.skarabox.com/services.html) of the manual.
-Not all services have a corresponding manual page yet.
+
+The [recipes section](https://shb.skarabox.com/recipes.html) of the manual shows some other common use cases.
 
 Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org)
 for any remaining question, or just to say hi :)
@@ -137,7 +153,7 @@ SelfHostBlocks provides building blocks that take care of common self-hosting ne
 - Backup for all services.
 - Automatic creation of ZFS datasets per service.
 - LDAP and SSO integration for most services.
-- Monitoring with Grafana and Prometheus stack with provided dashboards.
+- Monitoring with Grafana and Prometheus stack with provided dashboards and integration with Scrutiny.
 - Automatic reverse proxy and certificate management for HTTPS.
 - VPN and proxy tunneling services.
 
@@ -158,11 +174,15 @@ Also, the stack fits together nicely thanks to [contracts](#contracts).
 - Nextcloud
 - Audiobookshelf
 - Deluge + *arr stack
+- Simple NixOS Mailserver
+- Firefly-iii
 - Forgejo
 - Grocy
 - Hledger
 - Home-Assistant
 - Jellyfin
+- Karakeep
+- Open WebUI
 - Pinchflat
 - Vaultwarden
 
@@ -188,7 +208,7 @@ which altogether provides a solid foundation for self-hosting services:
 - BorgBackup
 - Davfs
 - LDAP
-- Monitoring (Grafana - Prometheus - Loki stack)
+- Monitoring (Grafana - Prometheus - Loki stack + Scrutiny)
 - Nginx
 - PostgreSQL
 - Restic
@@ -230,14 +250,14 @@ shb.nextcloud = {
     host = "127.0.0.1";
     port = config.shb.lldap.ldapPort;
     dcdomain = config.shb.lldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
   apps.sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
+    secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
+    secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
   };
 };
 ```
@@ -255,15 +275,15 @@ shb.forgejo = {
     host = "127.0.0.1";
     port = config.shb.lldap.ldapPort;
     dcdomain = config.shb.lldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
 
   sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."forgejo/sso/secret".result;
+    secret.result = config.shb.sops.secret."forgejo/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."forgejo/sso/secretForAuthelia".result;
+    secretForAuthelia.result = config.shb.sops.secret."forgejo/sso/secretForAuthelia".result;
   };
 };
 ```
@@ -271,7 +291,7 @@ shb.forgejo = {
 As you can see, they are pretty similar!
 This makes setting up a new service pretty easy and intuitive.
 
-SelfHostBlocks provides an ever growing list of [services](#provided-services)
+SelfHostBlocks provides an ever growing list of [services](#services)
 that are configured in the same way.
 
 ### Contracts

VERSION

@@ -1 +1 @@
-0.4.0
+0.9.0

demo/homeassistant/configuration.nix

@@ -5,17 +5,17 @@ let
   targetPort = 2222;
 in
 {
-  imports =
+  imports = [
-    [ # Include the results of the hardware scan.
+    # Include the results of the hardware scan.
-      ./hardware-configuration.nix
+    ./hardware-configuration.nix
-    ];
+  ];
 
   boot.loader.grub.enable = true;
   boot.kernelModules = [ "kvm-intel" ];
   system.stateVersion = "22.11";
 
   # Options above are generate by running nixos-generate-config on the VM.
-  
+
   # Needed otherwise deploy will say system won't be able to boot.
   boot.loader.grub.device = "/dev/vdb";
   # Needed to avoid getting into not available disk space in /boot.
@@ -26,7 +26,10 @@ in
 
   # Options above are needed to deploy in a VM.
 
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
 
   # We need to create the user we will deploy with.
   users.users.${targetUser} = {
@@ -41,9 +44,11 @@ in
 
   # The user we're deploying with must be able to run sudo without password.
   security.sudo.extraRules = [
-    { users = [ targetUser ];
+    {
+      users = [ targetUser ];
       commands = [
-        { command = "ALL";
+        {
+          command = "ALL";
           options = [ "NOPASSWD" ];
         }
       ];

demo/homeassistant/flake.nix

@@ -6,148 +6,160 @@
     sops-nix.url = "github:Mic92/sops-nix";
   };
 
-  outputs = inputs@{ self, selfhostblocks, sops-nix }:
+  outputs =
+    inputs@{
+      self,
+      selfhostblocks,
+      sops-nix,
+    }:
     let
       system = "x86_64-linux";
       nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
-      nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
 
-      basic = { config, ...  }: {
+      basic =
-        imports = [
+        { config, ... }:
-          ./configuration.nix
+        {
-          selfhostblocks.nixosModules.authelia
+          imports = [
-          selfhostblocks.nixosModules.home-assistant
+            ./configuration.nix
-          selfhostblocks.nixosModules.sops
+            selfhostblocks.nixosModules.authelia
-          selfhostblocks.nixosModules.ssl
+            selfhostblocks.nixosModules.home-assistant
-          sops-nix.nixosModules.default
+            selfhostblocks.nixosModules.sops
-        ];
+            selfhostblocks.nixosModules.ssl
+            sops-nix.nixosModules.default
+          ];
 
-        sops.defaultSopsFile = ./secrets.yaml;
+          sops.defaultSopsFile = ./secrets.yaml;
 
-        shb.home-assistant = {
+          shb.home-assistant = {
-          enable = true;
+            enable = true;
-          domain = "example.com";
+            domain = "example.com";
-          subdomain = "ha";
+            subdomain = "ha";
-          config = {
+            config = {
-            name = "SHB Home Assistant";
+              name = "SHB Home Assistant";
-            country.source = config.shb.sops.secret."home-assistant/country".result.path;
+              country.source = config.shb.sops.secret."home-assistant/country".result.path;
-            latitude.source = config.shb.sops.secret."home-assistant/latitude".result.path;
+              latitude.source = config.shb.sops.secret."home-assistant/latitude".result.path;
-            longitude.source = config.shb.sops.secret."home-assistant/longitude".result.path;
+              longitude.source = config.shb.sops.secret."home-assistant/longitude".result.path;
-            time_zone.source = config.shb.sops.secret."home-assistant/time_zone".result.path;
+              time_zone.source = config.shb.sops.secret."home-assistant/time_zone".result.path;
-            unit_system = "metric";
+              unit_system = "metric";
+            };
+          };
+          shb.sops.secret."home-assistant/country".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+          shb.sops.secret."home-assistant/latitude".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+          shb.sops.secret."home-assistant/longitude".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+          shb.sops.secret."home-assistant/time_zone".request = {
+            mode = "0440";
+            owner = "hass";
+            group = "hass";
+            restartUnits = [ "home-assistant.service" ];
+          };
+
+          nixpkgs.config.permittedInsecurePackages = [
+            "openssl-1.1.1w"
+          ];
+        };
+
+      ldap =
+        { config, ... }:
+        {
+          shb.lldap = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "ldap";
+            ldapPort = 3890;
+            webUIListenPort = 17170;
+            dcdomain = "dc=example,dc=com";
+            ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
+            jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
+          };
+          shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
+          shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
+
+          shb.home-assistant.ldap = {
+            enable = true;
+            host = "127.0.0.1";
+            port = config.shb.lldap.webUIListenPort;
+            userGroup = "homeassistant_user";
           };
         };
-        shb.sops.secret."home-assistant/country".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-        shb.sops.secret."home-assistant/latitude".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-        shb.sops.secret."home-assistant/longitude".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-        shb.sops.secret."home-assistant/time_zone".request = {
-          mode = "0440";
-          owner = "hass";
-          group = "hass";
-          restartUnits = [ "home-assistant.service" ];
-        };
-
-        nixpkgs.config.permittedInsecurePackages = [
-          "openssl-1.1.1w"
-        ];
-      };
-
-      ldap = { config, ...  }: {
-        shb.lldap = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "ldap";
-          ldapPort = 3890;
-          webUIListenPort = 17170;
-          dcdomain = "dc=example,dc=com";
-          ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
-          jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
-        };
-        shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
-        shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
-
-        shb.home-assistant.ldap = {
-          enable = true;
-          host = "127.0.0.1";
-          port = config.shb.lldap.webUIListenPort;
-          userGroup = "homeassistant_user";
-        };
-      };
 
       sopsConfig = {
         sops.age.keyFile = "/etc/sops/my_key";
         environment.etc."sops/my_key".source = ./keys.txt;
       };
     in
-      {
+    {
-        nixosConfigurations = {
+      nixosConfigurations = {
-          basic = nixosSystem' {
+        basic = nixpkgs'.nixosSystem {
-            system = "x86_64-linux";
+          system = "x86_64-linux";
-            modules = [
+          modules = [
-              basic
+            basic
-              sopsConfig
+            sopsConfig
-            ];
+          ];
-          };
-
-          ldap = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              basic
-              ldap
-              sopsConfig
-            ];
-          };
         };
 
-        colmena = {
+        ldap = nixpkgs'.nixosSystem {
-          meta = {
+          system = "x86_64-linux";
-            nixpkgs = import nixpkgs' {
+          modules = [
-              system = "x86_64-linux";
+            basic
-            };
+            ldap
-            specialArgs = inputs;
+            sopsConfig
-          };
+          ];
-
-          basic = { config, ... }: {
-            imports = [
-              basic
-            ];
-
-            # Used by colmena to know which target host to deploy to.
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
-
-          ldap = { config, ... }: {
-            imports = [
-              basic
-              ldap
-            ];
-
-            # Used by colmena to know which target host to deploy to.
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
         };
       };
+
+      colmena = {
+        meta = {
+          nixpkgs = import nixpkgs' {
+            system = "x86_64-linux";
+          };
+          specialArgs = inputs;
+        };
+
+        basic =
+          { config, ... }:
+          {
+            imports = [
+              basic
+            ];
+
+            # Used by colmena to know which target host to deploy to.
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+
+        ldap =
+          { config, ... }:
+          {
+            imports = [
+              basic
+              ldap
+            ];
+
+            # Used by colmena to know which target host to deploy to.
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+      };
+    };
 }

demo/homeassistant/hardware-configuration.nix

@@ -3,52 +3,65 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
 
 {
-  imports =
+  imports = [
-    [ (modulesPath + "/profiles/qemu-guest.nix")
+    (modulesPath + "/profiles/qemu-guest.nix")
-    ];
+  ];
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "floppy" "sr_mod" "virtio_blk" ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "floppy"
+    "sr_mod"
+    "virtio_blk"
+  ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
 
-  fileSystems."/" =
+  fileSystems."/" = {
-    { device = "/dev/vda";
+    device = "/dev/vda";
-      fsType = "ext4";
+    fsType = "ext4";
-    };
+  };
 
-  fileSystems."/nix/.ro-store" =
+  fileSystems."/nix/.ro-store" = {
-    { device = "nix-store";
+    device = "nix-store";
-      fsType = "9p";
+    fsType = "9p";
-    };
+  };
 
-  fileSystems."/nix/.rw-store" =
+  fileSystems."/nix/.rw-store" = {
-    { device = "tmpfs";
+    device = "tmpfs";
-      fsType = "tmpfs";
+    fsType = "tmpfs";
-    };
+  };
 
-  fileSystems."/tmp/shared" =
+  fileSystems."/tmp/shared" = {
-    { device = "shared";
+    device = "shared";
-      fsType = "9p";
+    fsType = "9p";
-    };
+  };
 
-  fileSystems."/tmp/xchg" =
+  fileSystems."/tmp/xchg" = {
-    { device = "xchg";
+    device = "xchg";
-      fsType = "9p";
+    fsType = "9p";
-    };
+  };
 
-  fileSystems."/nix/store" =
+  fileSystems."/nix/store" = {
-    { device = "overlay";
+    device = "overlay";
-      fsType = "overlay";
+    fsType = "overlay";
-    };
+  };
 
-  fileSystems."/boot" =
+  fileSystems."/boot" = {
-    { device = "/dev/vdb2";
+    device = "/dev/vdb2";
-      fsType = "vfat";
+    fsType = "vfat";
-    };
+  };
 
   swapDevices = [ ];
 

demo/minimal/flake.nix

@@ -0,0 +1,223 @@
+{
+  description = "Minimal example to setup SelfHostBlocks";
+
+  inputs = {
+    selfhostblocks.url = "github:ibizaman/selfhostblocks";
+
+    sops-nix = {
+      url = "github:Mic92/sops-nix";
+    };
+  };
+
+  outputs =
+    {
+      self,
+      selfhostblocks,
+      sops-nix,
+    }:
+    {
+      nixosConfigurations =
+        let
+          system = "x86_64-linux";
+          nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
+
+          # This module makes the assertions happy and the build succeed.
+          # This is of course wrong and will not work on any real system.
+          filesystemModule = {
+            fileSystems."/" = {
+              device = "/dev/null";
+              fsType = "none";
+            };
+            boot.loader.grub.devices = [ "/dev/null" ];
+          };
+        in
+        {
+          # Test with:
+          #   nix build .#nixosConfigurations.minimal.config.system.build.toplevel
+          minimal = nixpkgs'.nixosSystem {
+            inherit system;
+            modules = [
+              selfhostblocks.nixosModules.default
+              filesystemModule
+              # This modules showcases the use of SHB's lib.
+              (
+                {
+                  config,
+                  lib,
+                  shb,
+                  ...
+                }:
+                {
+                  options.myOption = lib.mkOption {
+                    # Using provided nixosSystem directly.
+                    # SHB's lib is available under `shb` thanks to the overlay.
+                    type = shb.secretFileType;
+                  };
+                  config = {
+                    myOption.source = "/a/path";
+                    # Use the option.
+                    environment.etc.myOption.text = config.myOption.source;
+                  };
+                }
+              )
+            ];
+          };
+
+          # Test with:
+          #   nix build .#nixosConfigurations.sops.config.system.build.toplevel
+          #   nix eval .#nixosConfigurations.sops.config.myOption
+          sops = nixpkgs'.nixosSystem {
+            inherit system;
+            modules = [
+              selfhostblocks.nixosModules.default
+              selfhostblocks.nixosModules.sops
+              sops-nix.nixosModules.default
+              filesystemModule
+              # This modules showcases the use of SHB's lib.
+              (
+                {
+                  config,
+                  lib,
+                  shb,
+                  ...
+                }:
+                {
+                  options.myOption = lib.mkOption {
+                    # Using provided nixosSystem directly.
+                    # SHB's lib is available under `shb` thanks to the overlay.
+                    type = shb.secretFileType;
+                  };
+                  config = {
+                    myOption.source = "/a/path";
+                    # Use the option.
+                    environment.etc.myOption.text = config.myOption.source;
+                  };
+                }
+              )
+            ];
+          };
+
+          # This example shows how to import the nixosSystem patches to nixpkgs manually.
+          #
+          # Test with:
+          #   nix build .#nixosConfigurations.lowlevel.config.system.build.toplevel
+          #   nix eval .#nixosConfigurations.lowlevel.config.myOption
+          lowlevel =
+            let
+              # We must import nixosSystem directly from the patched nixpkgs
+              # otherwise we do not get the patches.
+              nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
+            in
+            nixosSystem' {
+              inherit system;
+              modules = [
+                selfhostblocks.nixosModules.default
+                filesystemModule
+                # This modules showcases the use of SHB's lib.
+                (
+                  {
+                    config,
+                    lib,
+                    shb,
+                    ...
+                  }:
+                  {
+                    options.myOption = lib.mkOption {
+                      # Using provided nixosSystem directly.
+                      # SHB's lib is available under `shb` thanks to the overlay.
+                      type = shb.secretFileType;
+                    };
+                    config = {
+                      myOption.source = "/a/path";
+                      # Use the option.
+                      environment.etc.myOption.text = config.myOption.source;
+                    };
+                  }
+                )
+              ];
+            };
+
+          # This example shows how to apply patches to nixpkgs manually.
+          #
+          # Test with:
+          #   nix build .#nixosConfigurations.manual.config.system.build.toplevel
+          #   nix eval .#nixosConfigurations.manual.config.myOption
+          manual =
+            let
+              pkgs = import selfhostblocks.inputs.nixpkgs {
+                inherit system;
+              };
+              nixpkgs' = pkgs.applyPatches {
+                name = "nixpkgs-patched";
+                src = selfhostblocks.inputs.nixpkgs;
+                patches = selfhostblocks.lib.${system}.patches;
+              };
+              # We must import nixosSystem directly from the patched nixpkgs
+              # otherwise we do not get the patches.
+              nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
+            in
+            nixosSystem' {
+              inherit system;
+              modules = [
+                selfhostblocks.nixosModules.default
+                filesystemModule
+                # This modules showcases the use of SHB's lib.
+                (
+                  {
+                    config,
+                    lib,
+                    shb,
+                    ...
+                  }:
+                  {
+                    options.myOption = lib.mkOption {
+                      # Using provided nixosSystem directly.
+                      # SHB's lib is available under `shb` thanks to the overlay.
+                      type = shb.secretFileType;
+                    };
+                    config = {
+                      myOption.source = "/a/path";
+                      # Use the option.
+                      environment.etc.myOption.text = config.myOption.source;
+                    };
+                  }
+                )
+              ];
+            };
+
+          # Test with:
+          #   nix build .#nixosConfigurations.contractsDirect.config.system.build.toplevel
+          contractsDirect =
+            let
+              nixosSystem' = import "${selfhostblocks.inputs.nixpkgs}/nixos/lib/eval-config.nix";
+            in
+            nixosSystem' {
+              inherit system;
+              modules = [
+                filesystemModule
+                (import "${selfhostblocks}/lib/module.nix")
+                (
+                  {
+                    config,
+                    lib,
+                    shb,
+                    ...
+                  }:
+                  {
+                    options.myOption = lib.mkOption {
+                      # Using provided nixosSystem directly.
+                      # SHB's lib is available under `shb` thanks to the overlay.
+                      type = shb.secretFileType;
+                    };
+                    config = {
+                      myOption.source = "/a/path";
+                      # Use the option.
+                      environment.etc.myOption.text = config.myOption.source;
+                    };
+                  }
+                )
+              ];
+            };
+        };
+    };
+}

demo/nextcloud/README.md

@@ -187,8 +187,9 @@ This section corresponds to the `sso` section of the [Nextcloud
 manual](services-nextcloud.html#services-nextcloudserver-usage-oidc).
 ::::
 
-At this point, it is assumed you already deployed the `sso` demo. There is no host to add to
+At this point, it is assumed you already deployed the `sso` demo. This time, we cannot simply edit local
-`/etc/hosts` here. Instead, there is a `dnsmasq` server running in the VM and you must create a
+`/etc/hosts`, because Nextcloud SSO addon must be able to connect to Authelia by domain name
+(`auth.example.com`). Instead, there is a `dnsmasq` server running in the VM and you must create a
 SOCKS proxy to connect to it like so:
 
 ```bash

demo/nextcloud/configuration.nix

@@ -5,17 +5,17 @@ let
   targetPort = 2222;
 in
 {
-  imports =
+  imports = [
-    [ # Include the results of the hardware scan.
+    # Include the results of the hardware scan.
-      ./hardware-configuration.nix
+    ./hardware-configuration.nix
-    ];
+  ];
 
   boot.loader.grub.enable = true;
   boot.kernelModules = [ "kvm-intel" ];
   system.stateVersion = "22.11";
 
   # Options above are generate by running nixos-generate-config on the VM.
-  
+
   # Needed otherwise deploy will say system won't be able to boot.
   boot.loader.grub.device = "/dev/vdb";
   # Needed to avoid getting into not available disk space in /boot.
@@ -27,7 +27,10 @@ in
 
   # Options above are needed to deploy in a VM.
 
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
 
   # We need to create the user we will deploy with.
   users.users.${targetUser} = {
@@ -42,9 +45,11 @@ in
 
   # The user we're deploying with must be able to run sudo without password.
   security.sudo.extraRules = [
-    { users = [ targetUser ];
+    {
+      users = [ targetUser ];
       commands = [
-        { command = "ALL";
+        {
+          command = "ALL";
           options = [ "NOPASSWD" ];
         }
       ];

demo/nextcloud/flake.nix

@@ -1,241 +1,260 @@
 {
-  description = "Home Assistant example for Self Host Blocks";
+  description = "Nextcloud example for Self Host Blocks";
 
   inputs = {
     selfhostblocks.url = "github:ibizaman/selfhostblocks";
     sops-nix.url = "github:Mic92/sops-nix";
   };
 
-  outputs = inputs@{ self, selfhostblocks, sops-nix }:
+  outputs =
+    inputs@{
+      self,
+      selfhostblocks,
+      sops-nix,
+    }:
     let
       system = "x86_64-linux";
       nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
-      nixosSystem' = import "${nixpkgs'}/nixos/lib/eval-config.nix";
 
-      basic = { config, ... }: {
+      basic =
-        imports = [
+        { config, ... }:
-          ./configuration.nix
+        {
-          selfhostblocks.nixosModules.authelia
+          imports = [
-          selfhostblocks.nixosModules.nextcloud-server
+            ./configuration.nix
-          selfhostblocks.nixosModules.nginx
+            selfhostblocks.nixosModules.authelia
-          selfhostblocks.nixosModules.sops
+            selfhostblocks.nixosModules.nextcloud-server
-          selfhostblocks.nixosModules.ssl
+            selfhostblocks.nixosModules.nginx
-          sops-nix.nixosModules.default
+            selfhostblocks.nixosModules.sops
-        ];
+            selfhostblocks.nixosModules.ssl
+            sops-nix.nixosModules.default
+          ];
 
-        sops.defaultSopsFile = ./secrets.yaml;
+          sops.defaultSopsFile = ./secrets.yaml;
 
-        shb.nextcloud = {
+          shb.nextcloud = {
-          enable = true;
+            enable = true;
-          domain = "example.com";
+            domain = "example.com";
-          subdomain = "n";
+            subdomain = "n";
-          dataDir = "/var/lib/nextcloud";
+            dataDir = "/var/lib/nextcloud";
-          tracing = null;
+            tracing = null;
-          defaultPhoneRegion = "US";
+            defaultPhoneRegion = "US";
 
-          # This option is only needed because we do not access Nextcloud at the default port in the VM.
+            # This option is only needed because we do not access Nextcloud at the default port in the VM.
-          port = 8080;
+            port = 8080;
 
-          adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result;
+            adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result;
 
-          apps = {
+            apps = {
-            previewgenerator.enable = true;
+              previewgenerator.enable = true;
-          };
-        };
-        shb.sops.secret."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
-
-        # Set to true for more debug info with `journalctl -f -u nginx`.
-        shb.nginx.accessLog = true;
-        shb.nginx.debugLog = false;
-      };
-
-      ldap = { config, ... }: {
-        shb.lldap = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "ldap";
-          ldapPort = 3890;
-          webUIListenPort = 17170;
-          dcdomain = "dc=example,dc=com";
-          ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
-          jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
-        };
-        shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
-        shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
-
-        shb.nextcloud.apps.ldap = {
-          enable = true;
-          host = "127.0.0.1";
-          port = config.shb.lldap.ldapPort;
-          dcdomain = config.shb.lldap.dcdomain;
-          adminName = "admin";
-          adminPassword.result = config.shb.sops.secret."nextcloud/ldap_admin_password".result;
-          userGroup = "nextcloud_user";
-        };
-        shb.sops.secret."nextcloud/ldap_admin_password" = {
-          request = config.shb.nextcloud.apps.ldap.adminPassword.request;
-          settings.key = "lldap/user_password";
-        };
-      };
-
-      sso = { config, lib, ... }: {
-        shb.certs = {
-          cas.selfsigned.myca = {
-            name = "My CA";
-          };
-          certs.selfsigned = {
-            n = {
-              ca = config.shb.certs.cas.selfsigned.myca;
-              domain = "*.example.com";
-              group = "nginx";
             };
           };
-        };
+          shb.sops.secret."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
-        shb.nextcloud = {
-          port = lib.mkForce null;
-          ssl = config.shb.certs.certs.selfsigned.n;
-        };
-        shb.lldap.ssl = config.shb.certs.certs.selfsigned.n;
 
-        services.dnsmasq = {
+          # Set to true for more debug info with `journalctl -f -u nginx`.
-          enable = true;
+          shb.nginx.accessLog = true;
-          settings = {
+          shb.nginx.debugLog = false;
-            domain-needed = true;
+        };
-            # no-resolv = true;
+
-            bogus-priv = true;
+      ldap =
-            address =
+        { config, ... }:
-              map (hostname: "/${hostname}/127.0.0.1") [
+        {
+          shb.lldap = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "ldap";
+            ldapPort = 3890;
+            webUIListenPort = 17170;
+            dcdomain = "dc=example,dc=com";
+            ldapUserPassword.result = config.shb.sops.secret."lldap/user_password".result;
+            jwtSecret.result = config.shb.sops.secret."lldap/jwt_secret".result;
+          };
+          shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPassword.request;
+          shb.sops.secret."lldap/jwt_secret".request = config.shb.lldap.jwtSecret.request;
+
+          shb.nextcloud.apps.ldap = {
+            enable = true;
+            host = "127.0.0.1";
+            port = config.shb.lldap.ldapPort;
+            dcdomain = config.shb.lldap.dcdomain;
+            adminName = "admin";
+            adminPassword.result = config.shb.sops.secret."nextcloud/ldap_admin_password".result;
+            userGroup = "nextcloud_user";
+          };
+          shb.sops.secret."nextcloud/ldap_admin_password" = {
+            request = config.shb.nextcloud.apps.ldap.adminPassword.request;
+            settings.key = "lldap/user_password";
+          };
+        };
+
+      sso =
+        { config, lib, ... }:
+        {
+          shb.certs = {
+            cas.selfsigned.myca = {
+              name = "My CA";
+            };
+            certs.selfsigned = {
+              n = {
+                ca = config.shb.certs.cas.selfsigned.myca;
+                domain = "*.example.com";
+                group = "nginx";
+              };
+            };
+          };
+          shb.nextcloud = {
+            port = lib.mkForce null;
+            ssl = config.shb.certs.certs.selfsigned.n;
+          };
+          shb.lldap.ssl = config.shb.certs.certs.selfsigned.n;
+
+          services.dnsmasq = {
+            enable = true;
+            settings = {
+              domain-needed = true;
+              # no-resolv = true;
+              bogus-priv = true;
+              address = map (hostname: "/${hostname}/127.0.0.1") [
                 "example.com"
                 "n.example.com"
                 "ldap.example.com"
                 "auth.example.com"
               ];
+            };
+          };
+
+          shb.authelia = {
+            enable = true;
+            domain = "example.com";
+            subdomain = "auth";
+            ssl = config.shb.certs.certs.selfsigned.n;
+            ldapPort = config.shb.lldap.ldapPort;
+            ldapHostname = "127.0.0.1";
+            dcdomain = config.shb.lldap.dcdomain;
+
+            secrets = {
+              jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
+              ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
+              sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
+              storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
+              identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
+              identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
+            };
+          };
+          shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
+          shb.sops.secret."authelia/ldap_admin_password" = {
+            request = config.shb.authelia.secrets.ldapAdminPassword.request;
+            settings.key = "lldap/user_password";
+          };
+          shb.sops.secret."authelia/session_secret".request =
+            config.shb.authelia.secrets.sessionSecret.request;
+          shb.sops.secret."authelia/storage_encryption_key".request =
+            config.shb.authelia.secrets.storageEncryptionKey.request;
+          shb.sops.secret."authelia/hmac_secret".request =
+            config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
+          shb.sops.secret."authelia/private_key".request =
+            config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
+
+          shb.nextcloud.apps.sso = {
+            enable = true;
+            endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+            clientID = "nextcloud";
+            fallbackDefaultAuth = true;
+
+            secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
+            secretForAuthelia.result = config.shb.sops.secret."authelia/nextcloud_sso_secret".result;
+          };
+          shb.sops.secret."nextcloud/sso/secret".request = config.shb.nextcloud.apps.sso.secret.request;
+          shb.sops.secret."authelia/nextcloud_sso_secret" = {
+            request = config.shb.nextcloud.apps.sso.secretForAuthelia.request;
+            settings.key = "nextcloud/sso/secret";
           };
         };
 
-        shb.authelia = {
-          enable = true;
-          domain = "example.com";
-          subdomain = "auth";
-          ssl = config.shb.certs.certs.selfsigned.n;
-          ldapPort = config.shb.lldap.ldapPort;
-          ldapHostname = "127.0.0.1";
-          dcdomain = config.shb.lldap.dcdomain;
-
-          secrets = {
-            jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
-            ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
-            sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
-            storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
-            identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
-            identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
-          };
-        };
-        shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
-        shb.sops.secret."authelia/ldap_admin_password" = {
-          request = config.shb.authelia.secrets.ldapAdminPassword.request;
-          settings.key = "lldap/user_password";
-        };
-        shb.sops.secret."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
-        shb.sops.secret."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
-        shb.sops.secret."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
-        shb.sops.secret."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
-
-        shb.nextcloud.apps.sso = {
-          enable = true;
-          endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
-          clientID = "nextcloud";
-          fallbackDefaultAuth = true;
-
-          secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
-          secretForAuthelia.result = config.shb.sops.secret."authelia/nextcloud_sso_secret".result;
-        };
-        shb.sops.secret."nextcloud/sso/secret".request = config.shb.nextcloud.apps.sso.secret.request;
-        shb.sops.secret."authelia/nextcloud_sso_secret" = {
-          request = config.shb.nextcloud.apps.sso.secretForAuthelia.request;
-          settings.key = "nextcloud/sso/secret";
-        };
-      };
-
       sopsConfig = {
         sops.age.keyFile = "/etc/sops/my_key";
         environment.etc."sops/my_key".source = ./keys.txt;
       };
     in
-      {
+    {
-        nixosConfigurations = {
+      nixosConfigurations = {
-          basic = nixosSystem' {
+        basic = nixpkgs'.nixosSystem {
-            system = "x86_64-linux";
+          system = "x86_64-linux";
-            modules = [
+          modules = [
-              sopsConfig
+            sopsConfig
-              basic
+            basic
-            ];
+          ];
-          };
-          ldap = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              sopsConfig
-              basic
-              ldap
-            ];
-          };
-          sso = nixosSystem' {
-            system = "x86_64-linux";
-            modules = [
-              sopsConfig
-              basic
-              ldap
-              sso
-            ];
-          };
         };
-
+        ldap = nixpkgs'.nixosSystem {
-        colmena = {
+          system = "x86_64-linux";
-          meta = {
+          modules = [
-            nixpkgs = import nixpkgs' {
+            sopsConfig
-              system = "x86_64-linux";
+            basic
-            };
+            ldap
-            specialArgs = inputs;
+          ];
-          };
+        };
-
+        sso = nixpkgs'.nixosSystem {
-          basic = { config, ... }: {
+          system = "x86_64-linux";
-            imports = [
+          modules = [
-              basic
+            sopsConfig
-            ];
+            basic
-
+            ldap
-            deployment = {
+            sso
-              targetHost = "example";
+          ];
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
-
-          ldap = { config, ... }: {
-            imports = [
-              basic
-              ldap
-            ];
-
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
-
-          sso = { config, ... }: {
-            imports = [
-              basic
-              ldap
-              sso
-            ];
-
-            deployment = {
-              targetHost = "example";
-              targetUser = "nixos";
-              targetPort = 2222;
-            };
-          };
         };
       };
+
+      colmena = {
+        meta = {
+          nixpkgs = import nixpkgs' {
+            system = "x86_64-linux";
+          };
+          specialArgs = inputs;
+        };
+
+        basic =
+          { config, ... }:
+          {
+            imports = [
+              basic
+            ];
+
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+
+        ldap =
+          { config, ... }:
+          {
+            imports = [
+              basic
+              ldap
+            ];
+
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+
+        sso =
+          { config, ... }:
+          {
+            imports = [
+              basic
+              ldap
+              sso
+            ];
+
+            deployment = {
+              targetHost = "example";
+              targetUser = "nixos";
+              targetPort = 2222;
+            };
+          };
+      };
+    };
 }

demo/nextcloud/hardware-configuration.nix

@@ -3,52 +3,65 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
 
 {
-  imports =
+  imports = [
-    [ (modulesPath + "/profiles/qemu-guest.nix")
+    (modulesPath + "/profiles/qemu-guest.nix")
-    ];
+  ];
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "floppy" "sr_mod" "virtio_blk" ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "floppy"
+    "sr_mod"
+    "virtio_blk"
+  ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
 
-  fileSystems."/" =
+  fileSystems."/" = {
-    { device = "/dev/vda";
+    device = "/dev/vda";
-      fsType = "ext4";
+    fsType = "ext4";
-    };
+  };
 
-  fileSystems."/nix/.ro-store" =
+  fileSystems."/nix/.ro-store" = {
-    { device = "nix-store";
+    device = "nix-store";
-      fsType = "9p";
+    fsType = "9p";
-    };
+  };
 
-  fileSystems."/nix/.rw-store" =
+  fileSystems."/nix/.rw-store" = {
-    { device = "tmpfs";
+    device = "tmpfs";
-      fsType = "tmpfs";
+    fsType = "tmpfs";
-    };
+  };
 
-  fileSystems."/tmp/shared" =
+  fileSystems."/tmp/shared" = {
-    { device = "shared";
+    device = "shared";
-      fsType = "9p";
+    fsType = "9p";
-    };
+  };
 
-  fileSystems."/tmp/xchg" =
+  fileSystems."/tmp/xchg" = {
-    { device = "xchg";
+    device = "xchg";
-      fsType = "9p";
+    fsType = "9p";
-    };
+  };
 
-  fileSystems."/nix/store" =
+  fileSystems."/nix/store" = {
-    { device = "overlay";
+    device = "overlay";
-      fsType = "overlay";
+    fsType = "overlay";
-    };
+  };
 
-  fileSystems."/boot" =
+  fileSystems."/boot" = {
-    { device = "/dev/vdb2";
+    device = "/dev/vdb2";
-      fsType = "vfat";
+    fsType = "vfat";
-    };
+  };
 
   swapDevices = [ ];
 

docs/blocks.md

@@ -5,8 +5,13 @@ Blocks help you self-host apps or services. They implement a specific function l
 access through a subdomain. Each block is designed to be usable on its own and to fit nicely with
 others.
 
-Not all blocks are documented yet.
+All blocks are implemented under the blocks folder [in the repository](@REPO@/modules/blocks).
-You can find all available blocks [in the repository](@REPO@/modules/blocks).
+
+All services in SHB document how to setup the various blocks provided here.
+For custom services or those not provided by SHB,
+the [Expose a service Recipe](recipes-exposeService.html) explains how to use the blocks here.
+
+## Authentication {#blocks-category-authentication}
 
 ```{=include=} chapters html:into-file=//blocks-authelia.html
 modules/blocks/authelia/docs/default.md
@@ -16,26 +21,54 @@ modules/blocks/authelia/docs/default.md
 modules/blocks/lldap/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//blocks-mitmdump.html
+## Backup {#blocks-category-backup}
-modules/blocks/mitmdump/docs/default.md
-```
 
-```{=include=} chapters html:into-file=//blocks-monitoring.html
+```{=include=} chapters html:into-file=//blocks-borgbackup.html
-modules/blocks/monitoring/docs/default.md
+modules/blocks/borgbackup/docs/default.md
-```
-
-```{=include=} chapters html:into-file=//blocks-postgresql.html
-modules/blocks/postgresql/docs/default.md
-```
-
-```{=include=} chapters html:into-file=//blocks-sops.html
-modules/blocks/sops/docs/default.md
-```
-
-```{=include=} chapters html:into-file=//blocks-ssl.html
-modules/blocks/ssl/docs/default.md
 ```
 
 ```{=include=} chapters html:into-file=//blocks-restic.html
 modules/blocks/restic/docs/default.md
 ```
+
+```{=include=} chapters html:into-file=//blocks-sanoid.html
+modules/blocks/sanoid/docs/default.md
+```
+
+## Database {#blocks-category-database}
+
+```{=include=} chapters html:into-file=//blocks-postgresql.html
+modules/blocks/postgresql/docs/default.md
+```
+
+## Secrets {#blocks-category-secrets}
+
+```{=include=} chapters html:into-file=//blocks-sops.html
+modules/blocks/sops/docs/default.md
+```
+
+## Filesystem {#blocks-category-filesystem}
+
+```{=include=} chapters html:into-file=//blocks-zfs.html
+modules/blocks/zfs/docs/default.md
+```
+
+## Network {#blocks-category-network}
+
+```{=include=} chapters html:into-file=//blocks-ssl.html
+modules/blocks/ssl/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//blocks-nginx.html
+modules/blocks/nginx/docs/default.md
+```
+
+## Introspection {#blocks-category-introspection}
+
+```{=include=} chapters html:into-file=//blocks-monitoring.html
+modules/blocks/monitoring/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//blocks-mitmdump.html
+modules/blocks/mitmdump/docs/default.md
+```

docs/contracts.md

@@ -37,14 +37,21 @@ Provided contracts are:
 - [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
   Two providers are implemented: self-signed and Let's Encrypt.
 - [Backup contract][] to backup directories.
-  One provider is implemented: [Restic][].
+  Two providers are implemented: [BorgBackup][] and [Restic][].
 - [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
-  One provider is implemented: [Restic][].
+  Two providers are implemented: [BorgBackup][] and [Restic][].
-- [Secret contract](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
+- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets.
+  One provider is implemented: [Sanoid][]
+- [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
   One provider is implemented: [SOPS][].
+- [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard.
+  One provider is implemented: [Homepage][].
 
 [backup contract]: contracts-backup.html
+[borgbackup]: blocks-borgbackup.html
+[homepage]: services-homepage.html
 [restic]: blocks-restic.html
+[sanoid]: blocks-sanoid.html
 [sops]: blocks-sops.html
 
 ```{=include=} chapters html:into-file=//contracts-ssl.html
@@ -59,10 +66,18 @@ modules/contracts/backup/docs/default.md
 modules/contracts/databasebackup/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//contracts-datasetbackup.html
+modules/contracts/datasetbackup/docs/default.md
+```
+
 ```{=include=} chapters html:into-file=//contracts-secret.html
 modules/contracts/secret/docs/default.md
 ```
 
+```{=include=} chapters html:into-file=//contracts-dashboard.html
+modules/contracts/dashboard/docs/default.md
+```
+
 ## Problem Statement {#contracts-why}
 
 Currently in nixpkgs, every module accessing a shared resource
@@ -445,7 +460,7 @@ A simplified test for a secret contract would look like the following.
 First, there is the generic test:
 
 ```nix
-{ pkgs, lib, ... }:
+{ pkgs, lib, shb, ... }:
 let
   inherit (lib) getAttrFromPath setAttrByPath;
 in
@@ -455,7 +470,7 @@ in
     modules ? [],
     owner ? "root",
     content ? "secretPasswordA",
-  }: pkgs.testers.runNixOSTest {
+  }: shb.test.runNixOSTest {
     inherit name;
     
     nodes.machine = { config, ... }: {

docs/contributing.md

@@ -53,15 +53,7 @@ $ nix build .#checks.${system}.modules
 $ nix build .#checks.${system}.vm_postgresql_peerAuth
 ```
 
-Run one VM test interactively:
+### Playwright Tests {#contributing-playwright-tests}
-
-```bash
-$ nix run .#checks.${system}.vm_postgresql_peerAuth.driverInteractive
-```
-
-When you get to the shell, run either `start_all()` or `test_script()`. The former just starts all
-the VMs and service, then you can introspect. The latter also starts the VMs if they are not yet and
-then will run the test script.
 
 If the test includes playwright tests, you can see the playwright trace with:
 
@@ -69,6 +61,60 @@ If the test includes playwright tests, you can see the playwright trace with:
 $ nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
 ```
 
+If the test fails, you won't have the output directory available.
+Instead, run the test with `--keep-failed` and at the end you will see a line saying
+
+```
+Keeping build directory '/nix/var/nix/builds/nix-31776-4270051001/build'
+```
+
+Copy the path and run:
+
+```bash
+sudo cp -r /nix/var/nix/builds/nix-31776-4270051001/build/shared-xchg/trace trace && sudo chown -R $USER: trace
+```
+
+Now, open that file with playwright:
+
+```bash
+nix run .#playwright -- show-trace trace/0.zip
+```
+
+### Debug Tests {#contributing-debug-tests}
+
+Run the test in driver interactive mode:
+
+```bash
+$ nix run .#checks.${system}.vm_postgresql_peerAuth.driverInteractive
+```
+
+When you get to the shell, start the server and/or client with one of the following commands:
+
+```bash
+server.start()
+client.start()
+start_all()
+```
+
+To run the test from the shell, use `test_script()`.
+Note that if the test script ends in error,
+the shell will exit and you will need to restart the VMs.
+
+After the shell started, you will see lines like so:
+
+```
+SSH backdoor enabled, the machines can be accessed like this:
+Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer).
+    client:  ssh -o User=root vsock/3
+    server:  ssh -o User=root vsock/4
+```
+
+With the following command, you can directly access the server's nginx instance with your browser at `http://localhost:8000`:
+
+```bash
+ssh-keygen -R vsock/4; ssh -o User=root -L 8000:localhost:80 vsock/4
+```
+
 ## Upload test results to CI {#contributing-upload}
 
 Github actions do now have hardware acceleration, so running them there is not slow anymore. If
@@ -80,6 +126,18 @@ After running the `nix-fast-build` command from the previous section, run:
 $ find . -type l -name "result-vm_*" | xargs readlink | nix run nixpkgs#cachix -- push selfhostblocks
 ```
 
+## Upload package to CI {#contributing-upload-package}
+
+In the rare case where a package must be built but cannot in CI,
+for example because of not enough memory,
+you can push the package directly to the cache with:
+
+```bash
+nix build .#checks.x86_64-linux.vm_karakeep_backup.nodes.server.services.karakeep.package
+readlink result | nix run nixpkgs#cachix -- push selfhostblocks
+
+```
+
 ## Deploy using colmena {#contributing-deploy-colmena}
 
 ```bash
@@ -164,6 +222,12 @@ $ nix run nixpkgs#nvd -- diff \
 $ nix run nixpkgs#openssl -- rand -hex 64
 ```
 
+## Write code {#contributing-code}
+
+```{=include=} chapters html:into-file=//service-implementation-guide.html
+service-implementation-guide.md
+```
+
 ## Links that helped {#contributing-links}
 
 While creating NixOS tests:

docs/default.nix

@@ -1,40 +1,49 @@
 # Taken nearly verbatim from https://github.com/nix-community/home-manager/pull/4673
 # Read these docs online at https://shb.skarabox.com.
-{ pkgs
+{
-, buildPackages
+  pkgs,
-, lib
+  buildPackages,
-, nmdsrc
+  lib,
-, stdenv
+  nmdsrc,
-, documentation-highlighter
+  stdenv,
-, nixos-render-docs
+  documentation-highlighter,
+  nixos-render-docs,
 
-, release
+  release,
-, allModules
+  allModules,
 
-, version ? builtins.readFile ../VERSION
+  version ? builtins.readFile ../VERSION,
-, substituteVersionIn
+  substituteVersionIn,
 
-, modules
+  modules,
 }:
 
 let
   shbPath = toString ./..;
 
-  gitHubDeclaration = user: repo: subpath:
+  gitHubDeclaration =
-    let urlRef = "main";
+    user: repo: subpath:
-        end = if subpath == "" then "" else "/" + subpath;
+    let
-    in {
+      urlRef = "main";
+      end = if subpath == "" then "" else "/" + subpath;
+    in
+    {
       url = "https://github.com/${user}/${repo}/blob/${urlRef}${end}";
       name = "<${repo}${end}>";
     };
 
   ghRoot = (gitHubDeclaration "ibizaman" "selfhostblocks" "").url;
 
-  buildOptionsDocs = { modules, filterOptionPath ? null }: args:
+  buildOptionsDocs =
+    {
+      modules,
+      filterOptionPath ? null,
+    }:
+    args:
     let
       config = {
         _module.check = false;
-        _module.args = {};
+        _module.args = { };
         system.stateVersion = "22.11";
       };
 
@@ -52,41 +61,56 @@ let
       };
 
       options = lib.setAttrByPath filterOptionPath (lib.getAttrFromPath filterOptionPath eval.options);
-    in buildPackages.nixosOptionsDoc ({
+    in
-      inherit options;
+    buildPackages.nixosOptionsDoc (
+      {
+        inherit options;
 
-      transformOptions = opt:
+        transformOptions =
-        opt // {
+          opt:
-          # Clean up declaration sites to not refer to the Home Manager
+          opt
-          # source tree.
+          // {
-          declarations = map (decl:
+            # Clean up declaration sites to not refer to the Home Manager
-            gitHubDeclaration "ibizaman" "selfhostblocks"
+            # source tree.
-              (lib.removePrefix "/" (lib.removePrefix shbPath (toString decl)))) opt.declarations;
+            declarations = map (
-        };
+              decl:
-    } // builtins.removeAttrs args [ "includeModuleSystemOptions" ]);
+              gitHubDeclaration "ibizaman" "selfhostblocks" (
+                lib.removePrefix "/" (lib.removePrefix shbPath (toString decl))
+              )
+            ) opt.declarations;
+          };
+      }
+      // builtins.removeAttrs args [ "includeModuleSystemOptions" ]
+    );
 
   scrubbedModule = {
     _module.args.pkgs = lib.mkForce (nmd.scrubDerivations "pkgs" pkgs);
     _module.check = false;
   };
 
-  allOptionsDocs = paths: (buildOptionsDocs
+  allOptionsDocs =
-    {
+    paths:
-      modules = paths ++ allModules ++ [ scrubbedModule ];
+    (buildOptionsDocs
-      filterOptionPath = [ "shb" ];
+      {
-    }
+        modules = paths ++ allModules ++ [ scrubbedModule ];
-    {
+        filterOptionPath = [ "shb" ];
-      variablelistId = "selfhostblocks-options";
+      }
-    }).optionsJSON;
+      {
+        variablelistId = "selfhostblocks-options";
+      }
+    ).optionsJSON;
 
-  individualModuleOptionsDocs = filterOptionPath: paths: (buildOptionsDocs
+  individualModuleOptionsDocs =
-    {
+    filterOptionPath: paths:
-      modules = paths ++ [ scrubbedModule ];
+    (buildOptionsDocs
-      inherit filterOptionPath;
+      {
-    }
+        modules = paths ++ [ scrubbedModule ];
-    {
+        inherit filterOptionPath;
-      variablelistId = "selfhostblocks-options";
+      }
-    }).optionsJSON;
+      {
+        variablelistId = "selfhostblocks-options";
+      }
+    ).optionsJSON;
 
   nmd = import nmdsrc {
     inherit lib;
@@ -94,15 +118,15 @@ let
     # `nmd` uses to work around the broken stylesheets in
     # `docbook-xsl-ns`, so we restore the patched version here.
     pkgs = pkgs // {
-      docbook-xsl-ns =
+      docbook-xsl-ns = pkgs.docbook-xsl-ns.override { withManOptDedupPatch = true; };
-        pkgs.docbook-xsl-ns.override { withManOptDedupPatch = true; };
     };
   };
 
   outputPath = "share/doc/selfhostblocks";
 
-  manpage-urls = pkgs.writeText "manpage-urls.json" ''{}'';
+  manpage-urls = pkgs.writeText "manpage-urls.json" "{}";
-in stdenv.mkDerivation {
+in
+stdenv.mkDerivation {
   name = "self-host-blocks-manual";
 
   nativeBuildInputs = [ nixos-render-docs ];
@@ -135,28 +159,41 @@ in stdenv.mkDerivation {
       ${nmdsrc}/static/highlightjs/highlight.load.js
 
   ''
-  + lib.concatStringsSep "\n" (map (m: ''
+  + lib.concatStringsSep "\n" (
-    substituteInPlace ${m} --replace '@VERSION@' ${version}
+    map (m: ''
-  '') substituteVersionIn)
+      substituteInPlace ${m} --replace '@VERSION@' ${version}
+    '') substituteVersionIn
+  )
   + ''
     substituteInPlace ./options.md \
       --replace \
         '@OPTIONS_JSON@' \
-        ${allOptionsDocs [
+        ${
-          (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
+          allOptionsDocs [
-        ]}/share/doc/nixos/options.json
+            (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
+          ]
+        }/share/doc/nixos/options.json
   ''
-  + lib.concatStringsSep "\n" (lib.mapAttrsToList (name: cfg':
+  + lib.concatStringsSep "\n" (
-    let
+    lib.mapAttrsToList (
-      cfg = if builtins.isAttrs cfg' then cfg' else { module = cfg'; };
+      name: cfg':
-      module = if builtins.isList cfg.module then cfg.module else [ cfg.module ];
+      let
-      optionRoot = cfg.optionRoot or [ "shb" (lib.last (lib.splitString "/" name)) ];
+        cfg = if builtins.isAttrs cfg' then cfg' else { module = cfg'; };
-    in ''
+        module = if builtins.isList cfg.module then cfg.module else [ cfg.module ];
-    substituteInPlace ./modules/${name}/docs/default.md \
+        optionRoot =
-      --replace-fail \
+          cfg.optionRoot or [
-        '@OPTIONS_JSON@' \
+            "shb"
-        ${individualModuleOptionsDocs optionRoot module}/share/doc/nixos/options.json
+            (lib.last (lib.splitString "/" name))
-  '') modules)
+          ];
+      in
+      ''
+        substituteInPlace ./modules/${name}/docs/default.md \
+          --replace-fail \
+            '@OPTIONS_JSON@' \
+            ${individualModuleOptionsDocs optionRoot module}/share/doc/nixos/options.json
+      ''
+    ) modules
+  )
   + ''
     find . -name "*.md" -print0 | \
       while IFS= read -r -d ''' f; do

docs/manual.md

@@ -28,6 +28,10 @@ blocks.md
 recipes.md
 ```
 
+```{=include=} chapters html:into-file=//misc.html
+misc.md
+```
+
 ```{=include=} chapters html:into-file=//demos.html
 demos.md
 ```
@@ -36,10 +40,6 @@ demos.md
 contributing.md
 ```
 
-```{=include=} chapters html:into-file=//service-implementation-guide.html
-service-implementation-guide.md
-```
-
 ```{=include=} appendix html:into-file=//options.html
 options.md
 ```

docs/misc.md

@@ -0,0 +1,39 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Miscellaneous {#misc}
+
+Here goes meta-discussions around the repository and other topics not related directly to the usage of SHB.
+
+## Lock file update {#misc-lock-file-update}
+
+*SHB has an unusual strategy to keep up with its flake inputs. This section explains why.*
+
+SHB only depends on `nixpkgs` as far as flake inputs goes.
+There are others but they only matter for tasks around taking care of the repository itself.
+So only the `nixpkgs` input is important for downstream consumers of SHB.
+
+To keep up to date with nixpkgs unstable,
+initially a job was updating the nixpkgs input by simply updating the lock file and creating a PR.
+Now, this job failed most of the time because the tip of unstable often breaks modules and packages, resulting in failing SHB tests.
+To fix this, we usually needed to choose a commit around 2 weeks prior to the latest unstable commit.
+This usually did not lead to successful tests but the reason was then not
+because of failing packages but because nixpkgs modules evolved and SHB needs to adapt to that.
+The net effect was SHB was lagging too far behind unstable and updating it was becoming annoying.
+
+The new strategy now is as follows. On every job run:
+
+- If there is no PR, update nixpkgs input to latest unstable commit and create a PR which auto-merges when tests are successful.
+- If there is a PR:
+  - If the checks succeeded, do nothing because the PR will auto-merge.
+  - If the checks are pending, do nothing yet.
+  - If the checks failed, bisect and update nixpkgs input to between the commit on main and the commit in the PR.
+
+The effect here is as long as the tests are failing, we'll try a commit further in the past from nixpkgs unstable
+and closer to the tip of main branch.
+
+This could be made smarter by distinguishing between types of failures.
+We will see if this is needed.
+
+One can also run the bisect manually with `nix run .#update-flake-lock-pr`.
+This will create a PR on a different branch than the one used for the automated workflow.
+It accepts also one argument `future` which instead of trying a commit in between the tip of SHB main and the PR's failing one,
+it tries a commit between the PR's failing one and the nixpkgs unstable latest commit.

docs/preface.md

@@ -10,22 +10,203 @@ Feel free to join the dedicated Matrix room
 [matrix.org#selfhostblocks](https://matrix.to/#/#selfhostblocks:matrix.org).
 :::
 
-Self Host Blocks intends to help you self host any service you would like
+SelfHostBlocks is:
-with best practices out of the box.
 
-Compared to the stock nixpkgs experience, Self Host Blocks provides
+- Your escape from the cloud, for privacy and data sovereignty enthusiast. [Why?](#preface-why-self-hosting)
-an unified interface to setup common dependencies, called blocks
+- A groupware to self-host [all your data](#preface-services): documents, pictures, calendars, contacts, etc.
-in this project:
+- An opinionated NixOS server management OS for a [safe self-hosting experience](#preface-features).
+- A NixOS distribution making sure all services build and work correctly thanks to NixOS VM tests.
+- A collection of NixOS modules standardizing options so configuring services [look the same](#preface-unified-interfaces).
+- A testing ground for [contracts](#preface-contracts) which intents to make nixpkgs modules more modular.
+- [Upstreaming][] as much as possible.
 
-- reverse proxy
+[upstreaming]: https://github.com/pulls?page=1&q=created%3A%3E2023-06-01+is%3Apr+author%3Aibizaman+archived%3Afalse+-repo%3Aibizaman%2Fselfhostblocks+-repo%3Aibizaman%2Fskarabox
-- TLS certificate management
+
-- serving service under subdomain
+## Why Self-Hosting {#preface-why-self-hosting}
-- backup
+
+It is obvious by now that
+a deep dependency on proprietary service providers - "the cloud" -
+is a significant liability.
+One aspect often talked about is privacy
+which is inherently not guaranteed when using a proprietary service
+and is a valid concern.
+A more punishing issue is having your account closed or locked
+without prior warning
+When that happens,
+you get an instantaneous sinking feeling in your stomach
+at the realization you lost access to your data,
+possibly without recourse.
+
+Hosting services yourself is the obvious alternative
+to alleviate those concerns
+but it tends to require a lot of technical skills and time.
+SelfHostBlocks (together with its sibling project [Skarabox][])
+aims to lower the bar to self-hosting,
+and provides an opinionated server management system based on NixOS modules
+embedding best practices.
+Contrary to other server management projects,
+its main focus is ease of long term maintenance
+before ease of installation.
+To achieve this, it provides building blocks to setup services.
+Some are already provided out of the box,
+and customizing or adding additional ones is done easily.
+
+The building blocks fit nicely together thanks to [contracts](#contracts)
+which SelfHostBlocks sets out to introduce into nixpkgs.
+This will increase modularity, code reuse
+and empower end users to assemble components
+that fit together to build their server.
+
+## Usage {#preface-usage}
+
+> **Caution:** You should know that although I am using everything in this repo for my personal
+> production server, this is really just a one person effort for now and there are most certainly
+> bugs that I didn't discover yet.
+
+To get started using SelfHostBlocks, the following snippet is enough:
+
+```nix
+{
+  inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks";
+
+  outputs = { selfhostblocks, ... }: let
+    system = "x86_64-linux";
+    nixpkgs' = selfhostblocks.lib.${system}.patchedNixpkgs;
+  in
+    nixosConfigurations = {
+      myserver = nixpkgs'.nixosSystem {
+        inherit system;
+        modules = [
+          selfhostblocks.nixosModules.default
+          ./configuration.nix
+        ];
+      };
+    };
+}
+```
+
+SelfHostBlocks provides its own patched nixpkgs, so you are required to use it
+otherwise evaluation can quickly break.
+[The usage section](https://shb.skarabox.com/usage.html) of the manual has
+more details and goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
+and also how to handle secrets management with [SOPS][].
+
+[Colmena]: https://shb.skarabox.com/usage.html#usage-example-colmena
+[nixos-rebuild]: https://shb.skarabox.com/usage.html#usage-example-nixosrebuild
+[deploy-rs]: https://shb.skarabox.com/usage.html#usage-example-deployrs
+[SOPS]: https://shb.skarabox.com/usage.html#usage-secrets
+
+Then, to actually configure services, you can choose which one interests you in
+the [services section](https://shb.skarabox.com/services.html) of the manual.
+
+The [recipes section](https://shb.skarabox.com/recipes.html) of the manual shows some other common use cases.
+
+Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org)
+for any remaining question, or just to say hi :)
+
+### Installation From Scratch {#preface-usage-installation-from-scratch}
+
+I do recommend for this my sibling project [Skarabox][]
+which bootstraps a new server and sets up a few tools:
+
+- Create a bootable ISO, installable on an USB key.
+- Handles one or two (in raid 1) SSDs for root partition.
+- Handles two (in raid 1) or more hard drives for data partition.
+- [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) to install NixOS headlessly.
+- [disko](https://github.com/nix-community/disko) to format the drives using native ZFS encryption with remote unlocking through ssh.
+- [sops-nix](https://github.com/Mic92/sops-nix) to handle secrets.
+- [deploy-rs](https://github.com/serokell/deploy-rs) to deploy updates.
+
+[Skarabox]:  https://github.com/ibizaman/skarabox
+
+## Features {#preface-features}
+
+SelfHostBlocks provides building blocks that take care of common self-hosting needs:
+
+- Backup for all services.
+- Automatic creation of ZFS datasets per service.
+- LDAP and SSO integration for most services.
+- Monitoring with Grafana and Prometheus stack with provided dashboards.
+- Automatic reverse proxy and certificate management for HTTPS.
+- VPN and proxy tunneling services.
+
+Great care is taken to make the proposed stack robust.
+This translates into a test suite comprised of automated NixOS VM tests
+which includes playwright tests to verify some important workflow
+like logging in.
+
+This test suite also serves as a guaranty that all services provided by SelfHostBlocks
+all evaluate, build and work correctly together. It works similarly as a distribution but here it's all [automated](#preface-updates).
+
+Also, the stack fits together nicely thanks to [contracts](#preface-contracts).
+
+### Services {#preface-services}
+
+[Provided services](https://shb.skarabox.com/services.html) are:
+
+- Nextcloud
+- Audiobookshelf
+- Deluge + *arr stack
+- Forgejo
+- Grocy
+- Hledger
+- Home-Assistant
+- Jellyfin
+- Karakeep
+- Open WebUI
+- Pinchflat
+- Vaultwarden
+
+Like explained above, those services all benefit from
+out of the box backup,
+LDAP and SSO integration,
+monitoring with Grafana,
+reverse proxy and certificate management
+and VPN integration for the *arr suite.
+
+Some services do not have an entry yet in the manual.
+To know options for those, the only way for now
+is to go to the [All Options][] section of the manual.
+
+[All Options]: https://shb.skarabox.com/options.html
+
+### Blocks {#preface-blocks}
+
+The services above rely on the following [common blocks][]
+which altogether provides a solid foundation for self-hosting services:
+
+- Authelia
+- BorgBackup
+- Davfs
 - LDAP
-- SSO.
+- Monitoring (Grafana - Prometheus - Loki stack)
+- Nginx
+- PostgreSQL
+- Restic
+- Sops
+- SSL
+- Tinyproxy
+- VPN
+- ZFS
+
+Those blocks can be used with services
+not provided by SelfHostBlocks as shown [in the manual][common blocks].
+
+[common blocks]: https://shb.skarabox.com/blocks.html
+
+The manual also provides documentation for each individual blocks.
+
+### Unified Interfaces {#preface-unified-interfaces}
+
+Thanks to the blocks,
+SelfHostBlocks provides an unified configuration interface
+for the services it provides.
 
 Compare the configuration for Nextcloud and Forgejo.
-The following snippets focus on similitudes and assume the relevant blocks are configured off-screen.
+The following snippets focus on similitudes and assume the relevant blocks - like secrets - are configured off-screen.
+It also does not show specific options for each service.
+These are still complete snippets that configure HTTPS,
+subdomain serving the service, LDAP and SSO integration.
 
 ```nix
 shb.nextcloud = {
@@ -40,14 +221,14 @@ shb.nextcloud = {
     host = "127.0.0.1";
     port = config.shb.lldap.ldapPort;
     dcdomain = config.shb.lldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
   apps.sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
+    secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
+    secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
   };
 };
 ```
@@ -65,56 +246,160 @@ shb.forgejo = {
     host = "127.0.0.1";
     port = config.shb.lldap.ldapPort;
     dcdomain = config.shb.lldap.dcdomain;
-    adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
+    adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
   };
 
   sso = {
     enable = true;
     endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-    secret.result = config.shb.sops.secrets."forgejo/sso/secret".result;
+    secret.result = config.shb.sops.secret."forgejo/sso/secret".result;
-    secretForAuthelia.result = config.shb.sops.secrets."forgejo/sso/secretForAuthelia".result;
+    secretForAuthelia.result = config.shb.sops.secret."forgejo/sso/secretForAuthelia".result;
   };
 };
 ```
 
-SHB facilitates testing NixOS and slowly switching an existing installation to NixOS.
+As you can see, they are pretty similar!
+This makes setting up a new service pretty easy and intuitive.
 
-To achieve this, SHB pioneers [contracts][]
+SelfHostBlocks provides an ever growing list of [services](#preface-services)
-which allows you, the final user, to be more in control of which piece go where.
+that are configured in the same way.
+
+### Contracts {#preface-contracts}
+
+To make building blocks that fit nicely together,
+SelfHostBlocks pioneers [contracts][] which allows you, the final user,
+to be more in control of which piece goes where.
 This lets you choose, for example,
 any reverse proxy you want or any database you want,
 without requiring work from maintainers of the services you want to self host.
 
-[contracts]: contracts.html
+An [RFC][] exists to upstream this concept into `nixpkgs`.
+The [manual][contracts] also provides an explanation of the why and how of contracts.
 
-To achieve this, Self Host Blocks provides building blocks
+Also, two videos exist of me presenting the topic,
-which each provide part of what a self hosted app should do (SSO, HTTPS, etc.).
+the first at [NixCon North America in spring of 2024][NixConNA2024]
-It also provides some services that are already integrated with all those building blocks.
+and the second at [NixCon in Berlin in fall of 2024][NixConBerlin2024].
 
-Self Host Blocks uses the full power of NixOS modules to achieve these goals.
+[contracts]: https://shb.skarabox.com/contracts.html
-Blocks and service are both NixOS modules.
+[RFC]: https://github.com/NixOS/rfcs/pull/189
+[NixConNA2024]: https://www.youtube.com/watch?v=lw7PgphB9qM
+[NixConBerlin2024]: https://www.youtube.com/watch?v=CP0hR6w1csc
 
-## Next Steps {#next-steps}
+### Interfacing With Other OSes {#preface-interface}
 
-To get started using SelfHostBlocks,
+Thanks to [contracts](#contracts), one can interface NixOS
-follow [the usage section](https://shb.skarabox.com/usage.html) of the manual.
+with systems on other OSes.
-It goes over how to deploy with [Colmena][], [nixos-rebuild][] and [deploy-rs][]
+The [RFC][] explains how that works.
-and also goes over secrets management with [SOPS][].
 
-[Colmena]: https://shb.skarabox.com/usage.html#usage-example-colmena
+### Sitting on the Shoulders of a Giant {#preface-giants}
-[nixos-rebuild]: https://shb.skarabox.com/usage.html#usage-example-nixosrebuild
-[deploy-rs]: https://shb.skarabox.com/usage.html#usage-example-deployrs
-[SOPS]: https://shb.skarabox.com/usage.html#usage-secrets
 
-Then, depending on what you want to do:
+By using SelfHostBlocks, you get all the benefits of NixOS
+which are, for self hosted applications specifically:
 
-- You are new to self hosting and want pre-configured services to deploy easily.
+- declarative configuration;
-  Look at the [services section](services.html).
+- atomic configuration rollbacks;
-- You are a seasoned self-hoster but want to enhance some services you deploy already.
+- real programming language to define configurations;
-  Go to the [blocks section](blocks.html) and the [recipes section](recipes.html).
+- create your own higher level abstractions on top of SelfHostBlocks;
-- You are a user of Self Host Blocks but would like to use your own implementation for a block.
+- integration with the rest of nixpkgs;
-  Go to the [contracts section](https://shb.skarabox.com/contracts.html).
+- much fewer "works on my machine" type of issues.
 
-Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org)
+### Automatic Updates {#preface-updates}
-for any remaining question, or just to say hi :)
+
+SelfHostBlocks follows nixpkgs unstable branch closely.
+There is a GitHub action running every couple of days that updates
+the `nixpkgs` input in the root `flakes.nix`,
+runs the tests and merges the PR automatically
+if the tests pass.
+
+A release is then made every few commits,
+whenever deemed sensible.
+On your side, to update I recommend pinning to a release
+with the following command,
+replacing the RELEASE with the one you want:
+
+```bash
+RELEASE=0.2.4
+nix flake update \
+  --override-input selfhostblocks github:ibizaman/selfhostblocks/$RELEASE \
+  selfhostblocks
+```
+
+### Demos {#preface-demos}
+
+Demos that start and deploy a service
+on a Virtual Machine on your computer are located
+under the [demo](./demo/) folder.
+
+These show the onboarding experience you would get
+if you deployed one of the services on your own server.
+
+## Roadmap {#preface-roadmap}
+
+Currently, the Nextcloud and Vaultwarden services
+and the SSL and backup blocks
+are the most advanced and most documented.
+
+Documenting all services and blocks will be done
+as I make all blocks and services use the contracts.
+
+Upstreaming changes is also on the roadmap.
+
+Check the [issues][] and the [milestones]() to see planned work.
+Feel free to add more or to contribute!
+
+[issues]: (https://github.com/ibizaman/selfhostblocks/issues)
+[milestones]: https://github.com/ibizaman/selfhostblocks/milestones
+
+All blocks and services have NixOS tests.
+Also, I am personally using all the blocks and services in this project, so they do work to some extent.
+
+## Community {#preface-community}
+
+This project has been the main focus
+of my (non work) life for the past 3 year now
+and I intend to continue working on this for a long time.
+
+All issues and PRs are welcome:
+
+- Use this project. Something does not make sense? Something's not working?
+- Documentation. Something is not clear?
+- New services. Have one of your preferred service not integrated yet?
+- Better patterns. See something weird in the code?
+
+For PRs, if they are substantial changes, please open an issue to
+discuss the details first. More details in [the contributing section](https://shb.skarabox.com/contributing.html)
+of the manual.
+
+Issues that are being worked on are labeled with the [in progress][] label.
+Before starting work on those, you might want to talk about it in the issue tracker
+or in the [matrix][] channel.
+
+The prioritized issues are those belonging to the [next milestone][milestone].
+Those issues are not set in stone and I'd be very happy to solve
+an issue an user has before scratching my own itch.
+
+[in progress]: https://github.com/ibizaman/selfhostblocks/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22in%20progress%22
+[matrix]: https://matrix.to/#/%23selfhostblocks%3Amatrix.org
+[milestone]: https://github.com/ibizaman/selfhostblocks/milestones
+
+One aspect that's close to my heart is I intent to make SelfHostBlocks the lightest layer on top of nixpkgs as
+possible. I want to upstream as much as possible. I will still take some time to experiment here but
+when I'm satisfied with how things look, I'll upstream changes.
+
+## Funding {#preface-funding}
+
+I was lucky to [obtain a grant][nlnet] from NlNet which is an European fund,
+under [NGI Zero Core][NGI0],
+to work on this project.
+This also funds the contracts RFC.
+
+Go apply for a grant too!
+
+[nlnet]: https://nlnet.nl/project/SelfHostBlocks
+[NGI0]: https://nlnet.nl/core/
+
+## License {#preface-license}
+
+I'm following the [Nextcloud](https://github.com/nextcloud/server) license which is AGPLv3.
+See [this article](https://www.fsf.org/bulletin/2021/fall/the-fundamentals-of-the-agplv3) from the FSF that explains what this license adds to the GPL one.

docs/recipes.md

@@ -3,6 +3,10 @@
 
 This section of the manual gives you easy to follow recipes for common use cases. 
 
+```{=include=} chapters html:into-file=//recipes-dnsServer.html
+recipes/dnsServer.md
+```
+
 ```{=include=} chapters html:into-file=//recipes-exposeService.html
 recipes/exposeService.md
 ```

docs/recipes/dnsServer.md

@@ -0,0 +1,186 @@
+<!-- Read these docs at https://shb.skarabox.com -->
+# Self-Host a DNS server {#recipes-dnsServer}
+
+This recipe will show how to setup [dnsmasq][] as a local DNS server
+that forwards all queries to your own domain `example.com` to a local IP - your server running SelfHostBlocks for example.
+
+[dnsmasq]: https://dnsmasq.org/doc.html
+
+Other DNS queries will be forwarded to an external DNS server
+using [DNSSEC][] to encrypt your queries.
+
+[DNSSEC]: https://en.wikipedia.org/wiki/Domain_Name_System_Security_Extensions
+
+For this to work, you must configure the DHCP server of your network
+to set the DNS server to the IP of the host where the DNS server is running.
+Usually, your ISP's router can do this but probably easier is to disable completely that DHCP server
+and also self-host the DHCP server.
+This recipe shows how to do that too.
+
+## Why {#recipes-dnsServer-why}
+
+_You want to hide your DNS queries from your ISP or other prying eyes._
+
+Even if you use HTTPS to access an URL,
+DNS queries are by default made in plain text.
+Crazy, right?
+So, even if the actual communication is encrypted,
+everyone can see which site you're trying to access.
+Using DNSSEC means encrypting the traffic to your preferred external DNS server.
+Of course, that server will see what domain names you're trying to resolve,
+but at least intermediary hops will not be able to anymore.
+
+_You want more control on which DNS queries can be made._
+
+Self-hosting your own DNS server means you can block some domains or subdomains.
+This is done in practice by instructing your DNS server
+to fail resolving some domains or subdomains.
+Want to block Facebook for every host in the house?
+That's the way to go.
+
+Some routers allow this level of fine-tuning but if not,
+self-hosting your own DNS server is the way to go.
+
+## Drawbacks {#recipes-dnsServer-drawbacks}
+
+Although it has some nice advantages,
+self-hosting your own DNS server has one major drawback:
+if it goes down, the whole household will be impacted.
+By experience, it takes up to 5 minutes for others to notice something is wrong with internet.
+
+So be wary when you deploy a new config.
+
+## Recipe {#recipes-dnsServer-recipe}
+
+The following snippet:
+
+- Opens UDP port 53 in the firewall which is the ubiquitous (and hardcoded, crazy I know) port for DNS queries.
+- Disables the default DNS resolver.
+- Sets up dnsmasq as the DNS server.
+- Optionally sets up dnsmasq as the DHCP server.
+- Answers all DNS requests to your domain with the internal IP of the server.
+- Forwards all other DNS requests to an external DNS server using DNSSEC.
+  This is done using [stubby][].
+
+[stubby]: https://dnsprivacy.org/dns_privacy_daemon_-_stubby/
+
+For more information about options, read the dnsmasq [manual][].
+
+[manual]: https://dnsmasq.org/docs/dnsmasq-man.html
+
+```nix
+let
+  # Replace these values with what matches your network.
+  domain = "example.com";
+  serverIP = "192.168.1.30";
+
+  # This port is used internally for dnsmasq to talk to stubby on the loopback interface.
+  # Only change this if that port is already taken.
+  stubbyPort = 53000;
+in
+{
+  networking.firewall.allowedUDPPorts = [ 53 ];
+
+  services.resolved.enable = false;
+  services.dnsmasq = {
+    enable = true;
+    settings = {
+      inherit domain;
+
+      # Redirect queries to the stubby instance.
+      server = [
+        "127.0.0.1#${stubbyPort}"
+        "::1#${stubbyPort}"
+      ];
+      # We do trust our own instance of stubby
+      # so we can proxy DNSSEC stuff.
+      # I'm not sure how useful this is.
+      proxy-dnssec = true;
+
+      # Log all queries.
+      # This produces a lot of log lines
+      # and looking at those can be scary!
+      log-queries = true;
+
+      # Do not look at /etc/resolv.conf
+      no-resolv = true;
+
+      # Do not forward externally reverse DNS lookups for internal IPs.
+      bogus-priv = true;
+
+      address = [
+        "/.${domain}/${serverIP}"
+        # You can redirect anything anywhere too.
+        "/pikvm.${domain}/192.168.1.31"
+      ];
+    };
+  };
+
+  services.stubby = {
+    enable = true;
+    # It's a bit weird but default values comes from the examples settings hosted at
+    # https://github.com/getdnsapi/stubby/blob/develop/stubby.yml.example
+    settings = pkgs.stubby.passthru.settingsExample // {
+      listen_addresses = [
+        "127.0.0.1@${stubbyPort}"
+        "0::1@${stubbyPort}"
+      ];
+
+      # For more example of good DNS resolvers,
+      # head to https://dnsprivacy.org/public_resolvers/
+      #
+      # The digest comes from https://nixos.wiki/wiki/Encrypted_DNS#Stubby
+      upstream_recursive_servers = [
+        {
+          address_data = "9.9.9.9";
+          tls_auth_name = "dns.quad9.net";
+          tls_pubkey_pinset = [
+            {
+              digest = "sha256";
+              value = "i2kObfz0qIKCGNWt7MjBUeSrh0Dyjb0/zWINImZES+I=";
+            }
+          ];
+        }
+        {
+          address_data = "149.112.112.112";
+          tls_auth_name = "dns.quad9.net";
+          tls_pubkey_pinset = [
+            {
+              digest = "sha256";
+              value = "i2kObfz0qIKCGNWt7MjBUeSrh0Dyjb0/zWINImZES+I=";
+            }
+          ];
+        }
+      ];
+    };
+  };
+}
+```
+
+Optionally, to use dnsmasq as the DHCP server too,
+use the following snippet:
+
+```nix
+services.dnsmasq = {
+  settings = {
+    # When switching DNS server, accept old leases from previous server.
+    dhcp-authoritative = true;
+
+    # Adapt to your needs
+    # <ip-from>,<ip-to>,<mask>,<lease-ttl>
+    dhcp-range = "192.168.1.101,192.168.1.150,255.255.255.0,6h";
+
+    # Static DNS leases if needed.
+    # Choose an IP outside of the DHCP range
+    # <mac-address>,<DNS name>,<ip>,<lease-ttl>
+    dhcp-host = [
+      "12:34:56:78:9a:bc,server,192.168.1.50,infinite"
+    ];
+
+    # Set default route to the router that can acccess the internet.
+    dhcp-option = [
+      "3,192.168.1.1"
+    ];
+  };
+};
+```

docs/recipes/exposeService.md

@@ -20,6 +20,7 @@ let
   fqdn = "${subdomain}.${domain}";
   listenPort = 9000;
   dataDir = "/var/lib/awesome";
+  ldapGroup = "awesome_user";
 in
 ```
 
@@ -75,11 +76,11 @@ shb.nginx.vhosts = [
   {
     inherit subdomain domain;
     ssl = config.shb.certs.certs.letsencrypt.${domain};
-    upstream = "http://127.0.0.1:${toString config.services.calibre-web.listen.port}";
+    upstream = "http://127.0.0.1:${toString listenPort}";
     authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
     autheliaRules = [{
       policy = "one_factor";
-      subject = [ "group:${config.shb.lldap.ensureGroups.calibre_user.name}" ];
+      subject = [ "group:${ldapGroup}" ];
     }];
   }
 ];
@@ -120,10 +121,36 @@ shb.restic.instances.awesome = {
   request.sourceDirectories = [ dataDir ];
   settings.enable = true;
   settings.passphrase.result = config.shb.sops.secret.awesome.result;
-  settings.repository.path = "/srv/backup/awesome";
+  settings.repository.path = config.services.awesome.dataDir;
 };
 
 shb.sops.secret."awesome" = {
   request = config.shb.restic.instances.awesome.settings.passphrase.request;
 };
 ```
+
+## Impermanence {#recipes-exposeService-impermanence}
+
+To save the data folder in an impermanence setup, add:
+
+```nix
+{
+  shb.zfs.datasets."safe/awesome".path = config.services.awesome.dataDir;
+}
+```
+
+## Application Dashboard {#recipes-exposeService-applicationdashboard}
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.MyServices.services.Awesome = {
+    sortOrder = 1;
+    dashboard.request = {
+      externalUrl = "https://${fqdn}";
+      internalUrl = "http://127.0.0.1:${toString listenPort}";
+    };
+  };
+}
+```

docs/service-implementation-guide.md

@@ -71,7 +71,6 @@ Location: `modules/services/servicename.nix`
 let
   cfg = config.shb.servicename;
   contracts = pkgs.callPackage ../contracts {};
-  shblib = pkgs.callPackage ../../lib {};
   fqdn = "${cfg.subdomain}.${cfg.domain}";
   
   # Choose appropriate format based on service config
@@ -296,12 +295,12 @@ let
   
 in {
   # Test variants (all 6 required)
-  basic = pkgs.testers.runNixOSTest { ... };
+  basic = lib.shb.test.runNixOSTest { ... };
-  backup = pkgs.testers.runNixOSTest { ... };
+  backup = lib.shb.test.runNixOSTest { ... };
-  https = pkgs.testers.runNixOSTest { ... };
+  https = lib.shb.test.runNixOSTest { ... };
-  ldap = pkgs.testers.runNixOSTest { ... };
+  ldap = lib.shb.test.runNixOSTest { ... };
-  monitoring = pkgs.testers.runNixOSTest { ... };
+  monitoring = lib.shb.test.runNixOSTest { ... };
-  sso = pkgs.testers.runNixOSTest { ... };
+  sso = lib.shb.test.runNixOSTest { ... };
 }
 ```
 
@@ -462,6 +461,13 @@ nix flake check
 nix build .#manualHtml
 ```
 
+To continuously rebuild the documentation of file change, run the following command.
+To exit, you'll need to do Ctrl-C twice in a row.
+
+```bash
+nix run .#manualHtml-watch
+```
+
 ### Iterative Development Approach {#iterative-development-approach}
 
 1. **Start with basic functionality** - get core service working

docs/services.md

@@ -1,8 +1,10 @@
 <!-- Read these docs at https://shb.skarabox.com -->
 # Services {#services}
 
-Services are usually web applications that SHB help you self-host. Configuration of those is
+Services are usually web applications that SHB help you self-host some of your data.
-purposely made opinionated and require as few nix options as can make sense. That is possible thanks to the extensive use of blocks provided by SHB.
+Configuration of those is purposely made more opinionated than the upstream nixpkgs modules
+in exchange for an uniformized configuration experience.
+That is possible thanks to the extensive use of blocks provided by SHB.
 
 ::: {.note}
 Not all services are yet documented. You can find all available services [in the repository](@REPO@/modules/services).
@@ -11,16 +13,20 @@ Not all services are yet documented. You can find all available services [in the
 The following table summarizes for each documented service what features it provides. More
 information is provided in the respective manual sections.
 
-| Service              | Backup | Reverse Proxy | SSO | LDAP  | Monitoring | Profiling |
+| Service                     | Backup | Reverse Proxy | SSO | LDAP  | Monitoring | Profiling |
-|----------------------|--------|---------------|-----|-------|------------|-----------|
+|-----------------------------|--------|---------------|-----|-------|------------|-----------|
-| [*Arr][]             | Y (1)  | Y             | Y   | Y (4) | Y (2)      | N         |
+| [*Arr][]                    | Y (1)  | Y             | Y   | Y (4) | Y (2)      | N         |
-| [Forgejo][]          | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Firefly-iii][]             | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
-| [Jellyfin][]         | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Forgejo][]                 | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
-| [Home-Assistant][]   | Y (1)  | Y             | N   | Y     | Y (2)      | N         |
+| [Home-Assistant][]          | Y (1)  | Y             | N   | Y     | Y (2)      | N         |
-| [Nextcloud Server][] | Y (1)  | Y             | Y   | Y     | Y (2)      | P (3)     |
+| [Homepage][]                | Y (1)  | Y             | N   | Y     | Y (2)      | N         |
-| [Open WebUI][]       | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Jellyfin][]                | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
-| [Pinchflat][]        | Y      | Y             | Y   | Y (4) | Y (5)      | N         |
+| [Karakeep][]                | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
-| [Vaultwarden][]      | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Nextcloud Server][]        | Y (1)  | Y             | Y   | Y     | Y (2)      | P (3)     |
+| [Open WebUI][]              | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
+| [Pinchflat][]               | Y      | Y             | Y   | Y (4) | Y (5)      | N         |
+| [Simple NixOS Mailserver][] | Y      | Y             | N   | Y     | Y          | N         |
+| [Vaultwarden][]             | Y (1)  | Y             | Y   | Y     | Y (2)      | N         |
 
 Legend: **N**: no but WIP; **P**: partial; **Y**: yes
 
@@ -32,42 +38,80 @@ Legend: **N**: no but WIP; **P**: partial; **Y**: yes
 4. Uses LDAP indirectly through forward auth.
 
 [*Arr]: services-arr.html
+[Firefly-iii]: services-firefly-iii.html
 [Forgejo]: services-forgejo.html
 [Home-Assistant]: services-home-assistant.html
+[Homepage]: services-homepage.html
 [Jellyfin]: services-jellyfin.html
+[Karakeep]: services-karakeep.html
 [Nextcloud Server]: services-nextcloud.html
 [Open WebUI]: services-open-webui.html
 [Pinchflat]: services-pinchflat.html
+[Simple NixOS Mailserver]: services-mailserver.html
 [Vaultwarden]: services-vaultwarden.html
 
-```{=include=} chapters html:into-file=//services-arr.html
+## Dashboard {#services-category-dashboard}
-modules/services/arr/docs/default.md
+
+```{=include=} chapters html:into-file=//services-homepage.html
+modules/services/homepage/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//services-forgejo.html
+## Documents {#services-category-documents}
-modules/services/forgejo/docs/default.md
-```
-
-```{=include=} chapters html:into-file=//services-jellyfin.html
-modules/services/jellyfin/docs/default.md
-```
-
-```{=include=} chapters html:into-file=//services-home-assistant.html
-modules/services/home-assistant/docs/default.md
-```
 
 ```{=include=} chapters html:into-file=//services-nextcloud.html
 modules/services/nextcloud-server/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//services-pinchflat.html
+## Emails {#services-category-emails}
-modules/services/pinchflat/docs/default.md
+
+```{=include=} chapters html:into-file=//services-mailserver.html
+modules/services/mailserver/docs/default.md
+```
+
+## Passwords {#services-category-passwords}
+
+```{=include=} chapters html:into-file=//services-vaultwarden.html
+modules/services/vaultwarden/docs/default.md
+```
+
+## Automation {#services-category-automation}
+
+```{=include=} chapters html:into-file=//services-home-assistant.html
+modules/services/home-assistant/docs/default.md
+```
+
+## AI {#services-category-ai}
+
+```{=include=} chapters html:into-file=//services-karakeep.html
+modules/services/karakeep/docs/default.md
 ```
 
 ```{=include=} chapters html:into-file=//services-open-webui.html
 modules/services/open-webui/docs/default.md
 ```
 
-```{=include=} chapters html:into-file=//services-vaultwarden.html
+## Code {#services-category-code}
-modules/services/vaultwarden/docs/default.md
+
+```{=include=} chapters html:into-file=//services-forgejo.html
+modules/services/forgejo/docs/default.md
+```
+
+## Media {#services-category-media}
+
+```{=include=} chapters html:into-file=//services-arr.html
+modules/services/arr/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//services-jellyfin.html
+modules/services/jellyfin/docs/default.md
+```
+
+```{=include=} chapters html:into-file=//services-pinchflat.html
+modules/services/pinchflat/docs/default.md
+```
+
+## Finance {#services-category-finance}
+
+```{=include=} chapters html:into-file=//services-firefly-iii.html
+modules/services/firefly-iii/docs/default.md
 ```

flake.lock

@@ -20,11 +20,11 @@
     },
     "nix-flake-tests": {
       "locked": {
-        "lastModified": 1677844186,
+        "lastModified": 1775571237,
-        "narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
+        "narHash": "sha256-1f5Uvgcy3gx/eyRp4rqfDRBjcZc9uiHoIlfcFIL7GvI=",
         "owner": "antifuchs",
         "repo": "nix-flake-tests",
-        "rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
+        "rev": "86b789cff8aecbd7c1bed7cd421dd837c3f62201",
         "type": "github"
       },
       "original": {
@@ -35,11 +35,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1758427187,
+        "lastModified": 1781074563,
-        "narHash": "sha256-pHpxZ/IyCwoTQPtFIAG2QaxuSm8jWzrzBGjwQZIttJc=",
+        "narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "554be6495561ff07b6c724047bdd7e0716aa7b46",
+        "rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca",
         "type": "github"
       },
       "original": {

flake.nix

@@ -11,49 +11,89 @@
     };
   };
 
-  outputs = inputs@{ self, nixpkgs, nix-flake-tests, flake-utils, nmdsrc, ... }: flake-utils.lib.eachDefaultSystem (system:
+  outputs =
+    inputs@{
+      self,
+      nixpkgs,
+      nix-flake-tests,
+      flake-utils,
+      nmdsrc,
+      ...
+    }:
     let
-      originPkgs = nixpkgs.legacyPackages.${system};
+      shbPatches =
-      shbPatches = originPkgs.lib.optionals (system == "x86_64-linux") [
+        system:
-        # Get rid of lldap patches when https://github.com/NixOS/nixpkgs/pull/425923 is merged.
+        nixpkgs.legacyPackages.${system}.lib.optionals
-        ./patches/lldap.patch
+          (system == "x86_64-linux" || system == "aarch64-linux")
-        ./patches/use-regex-2025.7.34.patch
+          [
+            # Get rid of lldap patches when https://github.com/NixOS/nixpkgs/pull/425923 is merged.
+            ./patches/lldap.patch
+            ./patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch
 
-        # Leaving commented out as an example.
+            # Leaving commented out as an example.
-        # (originPkgs.fetchpatch {
+            # (originPkgs.fetchpatch {
-        #   url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
+            #   url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
-        #   hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
+            #   hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
-        # })
+            # })
-      ];
+          ];
-      patchNixpkgs = {
-        nixpkgs,
-        patches,
-        system,
-      }: nixpkgs.legacyPackages.${system}.applyPatches {
-        name = "nixpkgs-patched";
-        src = nixpkgs;
-        inherit patches;
-      };
-      patchedNixpkgs = (patchNixpkgs {
-        nixpkgs = inputs.nixpkgs;
-        patches = shbPatches;
-        inherit system;
-      });
-      pkgs = import patchedNixpkgs {
-        inherit system;
-        config.allowUnfree = true;
-      };
 
-      # The contract dummies are used to show options for contracts.
+      patchNixpkgs =
-      contractDummyModules = [
+        {
-        modules/contracts/backup/dummyModule.nix
+          nixpkgs,
-        modules/contracts/ssl/dummyModule.nix
+          patches,
-      ];
+          system,
+        }:
+        nixpkgs.legacyPackages.${system}.applyPatches {
+          name = "nixpkgs-patched";
+          src = nixpkgs;
+          inherit patches;
+        };
+      patchedNixpkgs =
+        system:
+        let
+          patched = patchNixpkgs {
+            nixpkgs = inputs.nixpkgs;
+            patches = shbPatches system;
+            inherit system;
+          };
+        in
+        patched
+        // {
+          nixosSystem = args: import "${patched}/nixos/lib/eval-config.nix" args;
+        };
+      pkgs' =
+        system:
+        import (patchedNixpkgs system) {
+          inherit system;
+          config.allowUnfree = true;
+        };
     in
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = pkgs' system;
+
+        # The contract dummies are used to show options for contracts.
+        contractDummyModules = [
+          modules/contracts/backup/dummyModule.nix
+          modules/contracts/dashboard/dummyModule.nix
+          modules/contracts/databasebackup/dummyModule.nix
+          modules/contracts/datasetbackup/dummyModule.nix
+          modules/contracts/secret/dummyModule.nix
+          modules/contracts/ssl/dummyModule.nix
+        ];
+      in
       {
+        formatter = pkgs.nixfmt-tree;
+
         packages.manualHtml = pkgs.callPackage ./docs {
           inherit nmdsrc;
-          allModules = self.nixosModules.all.imports ++ contractDummyModules;
+          allModules =
+            self.nixosModules.default.imports
+            ++ [
+              self.nixosModules.sops
+            ]
+            ++ contractDummyModules;
           release = builtins.readFile ./VERSION;
 
           substituteVersionIn = [
@@ -62,176 +102,182 @@
           ];
           modules = {
             "blocks/authelia" = ./modules/blocks/authelia.nix;
+            "blocks/borgbackup" = ./modules/blocks/borgbackup.nix;
             "blocks/lldap" = ./modules/blocks/lldap.nix;
-            "blocks/ssl" = {
-              module = ./modules/blocks/ssl.nix;
-              optionRoot = [ "shb" "certs" ];
-            };
             "blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
             "blocks/monitoring" = ./modules/blocks/monitoring.nix;
+            "blocks/nginx" = ./modules/blocks/nginx.nix;
             "blocks/postgresql" = ./modules/blocks/postgresql.nix;
             "blocks/restic" = ./modules/blocks/restic.nix;
+            "blocks/sanoid" = ./modules/blocks/sanoid.nix;
             "blocks/sops" = ./modules/blocks/sops.nix;
+            "blocks/ssl" = {
+              module = ./modules/blocks/ssl.nix;
+              optionRoot = [
+                "shb"
+                "certs"
+              ];
+            };
+            "blocks/zfs" = ./modules/blocks/zfs.nix;
+
             "services/arr" = ./modules/services/arr.nix;
+            "services/firefly-iii" = ./modules/services/firefly-iii.nix;
             "services/forgejo" = [
               ./modules/services/forgejo.nix
               (pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
             ];
             "services/home-assistant" = ./modules/services/home-assistant.nix;
+            "services/homepage" = ./modules/services/homepage.nix;
             "services/jellyfin" = ./modules/services/jellyfin.nix;
+            "services/karakeep" = ./modules/services/karakeep.nix;
+            "services/mailserver" = ./modules/services/mailserver.nix;
             "services/nextcloud-server" = {
               module = ./modules/services/nextcloud-server.nix;
-              optionRoot = [ "shb" "nextcloud" ];
+              optionRoot = [
+                "shb"
+                "nextcloud"
+              ];
             };
             "services/open-webui" = ./modules/services/open-webui.nix;
             "services/pinchflat" = ./modules/services/pinchflat.nix;
             "services/vaultwarden" = ./modules/services/vaultwarden.nix;
+
             "contracts/backup" = {
               module = ./modules/contracts/backup/dummyModule.nix;
-              optionRoot = [ "shb" "contracts" "backup" ];
+              optionRoot = [
+                "shb"
+                "contracts"
+                "backup"
+              ];
+            };
+            "contracts/dashboard" = {
+              module = ./modules/contracts/dashboard/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "dashboard"
+              ];
             };
             "contracts/databasebackup" = {
               module = ./modules/contracts/databasebackup/dummyModule.nix;
-              optionRoot = [ "shb" "contracts" "databasebackup" ];
+              optionRoot = [
+                "shb"
+                "contracts"
+                "databasebackup"
+              ];
+            };
+            "contracts/datasetbackup" = {
+              module = ./modules/contracts/datasetbackup/dummyModule.nix;
+              optionRoot = [
+                "shb"
+                "contracts"
+                "datasetbackup"
+              ];
             };
             "contracts/secret" = {
               module = ./modules/contracts/secret/dummyModule.nix;
-              optionRoot = [ "shb" "contracts" "secret" ];
+              optionRoot = [
+                "shb"
+                "contracts"
+                "secret"
+              ];
             };
             "contracts/ssl" = {
               module = ./modules/contracts/ssl/dummyModule.nix;
-              optionRoot = [ "shb" "contracts" "ssl" ];
+              optionRoot = [
+                "shb"
+                "contracts"
+                "ssl"
+              ];
             };
           };
         };
 
         # Documentation redirect generation tool - scans HTML files for anchor mappings
-        packages.generateRedirects = 
+        packages.generateRedirects =
-        let
-          # Python patch to inject redirect collector
-          pythonPatch = pkgs.writeText "nixos-render-docs-patch.py" ''
-            # Load redirect collector patch
-            try:
-                import sys, os
-                sys.path.insert(0, os.path.dirname(__file__) + '/..')
-                import missing_refs_collector
-            except Exception as e:
-                print(f"Warning: Failed to load redirect collector: {e}", file=sys.stderr)
-          '';
-
-          # Patched nixos-render-docs that collects redirects during HTML generation
-          nixos-render-docs-patched = pkgs.writeShellApplication {
-            name = "nixos-render-docs";
-            runtimeInputs = [ pkgs.nixos-render-docs ];
-            text = ''
-              TEMP_DIR=$(mktemp -d); trap 'rm -rf "$TEMP_DIR"' EXIT
-              
-              cp -r ${pkgs.nixos-render-docs}/${pkgs.python3.sitePackages}/nixos_render_docs "$TEMP_DIR/"
-              chmod -R +w "$TEMP_DIR"
-              cp ${./docs/generate-redirects-nixos-render-docs.py} "$TEMP_DIR/missing_refs_collector.py"
-              echo '{}' > "$TEMP_DIR/empty_redirects.json"
-              cat ${pythonPatch} >> "$TEMP_DIR/nixos_render_docs/__init__.py"
-              
-              ARGS=()
-              while [[ $# -gt 0 ]]; do
-                case $1 in
-                  --redirects) ARGS+=("$1" "$TEMP_DIR/empty_redirects.json"); shift 2 ;;
-                  *) ARGS+=("$1"); shift ;;
-                esac
-              done
-              
-              export PYTHONPATH="$TEMP_DIR:''${PYTHONPATH:-}"
-              nixos-render-docs "''${ARGS[@]}"
-            '';
-          };
-        in
-        (self.packages.${system}.manualHtml.override {
-          nixos-render-docs = nixos-render-docs-patched;
-        }).overrideAttrs (old: {
-          installPhase = ''
-            ${old.installPhase}
-            ln -sf share/doc/selfhostblocks/redirects.json $out/redirects.json
-          '';
-        });
-
-        lib =
-          (pkgs.callPackage ./lib {})
-          // {
-            contracts = pkgs.callPackage ./modules/contracts {};
-            patches = shbPatches;
-            inherit patchNixpkgs patchedNixpkgs;
-          };
-
-        checks =
           let
-            inherit (pkgs.lib) foldl foldlAttrs mergeAttrs optionalAttrs;
+            # Python patch to inject redirect collector
+            pythonPatch = pkgs.writeText "nixos-render-docs-patch.py" ''
+              # Load redirect collector patch
+              try:
+                  import sys, os
+                  sys.path.insert(0, os.path.dirname(__file__) + '/..')
+                  import missing_refs_collector
+              except Exception as e:
+                  print(f"Warning: Failed to load redirect collector: {e}", file=sys.stderr)
+            '';
 
-            importFiles = files:
+            # Patched nixos-render-docs that collects redirects during HTML generation
-              map (m: pkgs.callPackage m {}) files;
+            nixos-render-docs-patched = pkgs.writeShellApplication {
+              name = "nixos-render-docs";
+              runtimeInputs = [ pkgs.nixos-render-docs ];
+              text = ''
+                TEMP_DIR=$(mktemp -d); trap 'rm -rf "$TEMP_DIR"' EXIT
 
-            mergeTests = foldl mergeAttrs {};
+                cp -r ${pkgs.nixos-render-docs}/${pkgs.python3.sitePackages}/nixos_render_docs "$TEMP_DIR/"
+                chmod -R +w "$TEMP_DIR"
+                cp ${./docs/generate-redirects-nixos-render-docs.py} "$TEMP_DIR/missing_refs_collector.py"
+                echo '{}' > "$TEMP_DIR/empty_redirects.json"
+                cat ${pythonPatch} >> "$TEMP_DIR/nixos_render_docs/__init__.py"
 
-            flattenAttrs = root: attrset: foldlAttrs (acc: name: value: acc // {
+                ARGS=()
-              "${root}_${name}" = value;
+                while [[ $# -gt 0 ]]; do
-            }) {} attrset;
+                  case $1 in
+                    --redirects) ARGS+=("$1" "$TEMP_DIR/empty_redirects.json"); shift 2 ;;
+                    *) ARGS+=("$1"); shift ;;
+                  esac
+                done
 
-            vm_test = name: path: flattenAttrs "vm_${name}" (
+                export PYTHONPATH="$TEMP_DIR:''${PYTHONPATH:-}"
-              import path {
+                nixos-render-docs "''${ARGS[@]}"
-                inherit pkgs;
+              '';
-                inherit (pkgs) lib;
-              }
-            );
-
-            shblib = pkgs.callPackage ./lib {};
-          in (optionalAttrs (system == "x86_64-linux") ({
-            modules = shblib.check {
-              inherit pkgs;
-              tests =
-                mergeTests (importFiles [
-                  ./test/modules/davfs.nix
-                  # TODO: Make this not use IFD
-                  ./test/modules/lib.nix
-                ]);
             };
+          in
+          (self.packages.${system}.manualHtml.override {
+            nixos-render-docs = nixos-render-docs-patched;
+          }).overrideAttrs
+            (old: {
+              installPhase = ''
+                ${old.installPhase}
+                ln -sf share/doc/selfhostblocks/redirects.json $out/redirects.json
+              '';
+            });
 
-            # TODO: Make this not use IFD
+        packages.manualHtml-watch = pkgs.writeShellApplication {
-            lib = nix-flake-tests.lib.check {
+          name = "manualHtml-watch";
-              inherit pkgs;
+          runtimeInputs = [
-              tests = pkgs.callPackage ./test/modules/lib.nix {};
+            pkgs.findutils
-            };
+            pkgs.entr
-          }
+          ];
-          // (vm_test "arr" ./test/services/arr.nix)
+          text = ''
-          // (vm_test "audiobookshelf" ./test/services/audiobookshelf.nix)
+            while sleep 1; do
-          // (vm_test "deluge" ./test/services/deluge.nix)
+              find . -name "*.nix" -o -name "*.md" \
-          // (vm_test "forgejo" ./test/services/forgejo.nix)
+                | entr -d sh -c '(nix run --offline .#update-redirects && nix build --offline .#manualHtml)' || :
-          // (vm_test "grocy" ./test/services/grocy.nix)
+            done
-          // (vm_test "hledger" ./test/services/hledger.nix)
+          '';
-          // (vm_test "immich" ./test/services/immich.nix)
+        };
-          // (vm_test "homeassistant" ./test/services/home-assistant.nix)
-          // (vm_test "jellyfin" ./test/services/jellyfin.nix)
-          // (vm_test "monitoring" ./test/services/monitoring.nix)
-          // (vm_test "nextcloud" ./test/services/nextcloud.nix)
-          // (vm_test "open-webui" ./test/services/open-webui.nix)
-          // (vm_test "pinchflat" ./test/services/pinchflat.nix)
-          // (vm_test "vaultwarden" ./test/services/vaultwarden.nix)
 
-          // (vm_test "authelia" ./test/blocks/authelia.nix)
+        packages.update-flake-lock-pr = pkgs.callPackage ./.github/workflows/update-flake-lock-pr.nix { };
-          // (vm_test "lldap" ./test/blocks/lldap.nix)
-          // (vm_test "lib" ./test/blocks/lib.nix)
-          // (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
-          // (vm_test "postgresql" ./test/blocks/postgresql.nix)
-          // (vm_test "restic" ./test/blocks/restic.nix)
-          // (vm_test "ssl" ./test/blocks/ssl.nix)
 
-          // (vm_test "contracts-backup" ./test/contracts/backup.nix)
+        lib = (pkgs.callPackage ./lib { }) // {
-          // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
+          test = pkgs.callPackage ./test/common.nix { };
-          // (vm_test "contracts-secret" ./test/contracts/secret.nix)
+          contracts = pkgs.callPackage ./modules/contracts {
-          ));
+            shb = self.lib.${system};
+          };
+          patches = shbPatches system;
+          inherit patchNixpkgs;
+          patchedNixpkgs = patchedNixpkgs system;
+        };
 
         # To see the traces, run:
         #   nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
-        packages.playwright =
+        packages.playwright = pkgs.callPackage (
-          pkgs.callPackage ({ stdenvNoCC, makeWrapper, playwright }: stdenvNoCC.mkDerivation {
+          {
+            stdenvNoCC,
+            makeWrapper,
+            playwright,
+          }:
+          stdenvNoCC.mkDerivation {
             name = "playwright";
 
             src = playwright;
@@ -246,97 +292,223 @@
                 --set PLAYWRIGHT_BROWSERS_PATH ${pkgs.playwright-driver.browsers} \
                 --set PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS true
             '';
-          }) {};
+          }
+        ) { };
 
         # Run "nix run .#update-redirects" to regenerate docs/redirects.json
         apps.update-redirects = {
           type = "app";
-          program = "${pkgs.writeShellApplication {
+          program = "${
-            name = "update-redirects";
+            pkgs.writeShellApplication {
-            runtimeInputs = [ pkgs.nix pkgs.jq ];
+              name = "update-redirects";
-            text = ''
+              runtimeInputs = [
-              echo "=== SelfHostBlocks Redirects Updater ==="
+                pkgs.nix
-              echo "Generating fresh ./docs/redirects.json..."
+                pkgs.jq
-              
+              ];
-              nix build .#generateRedirects || { echo "Error: Failed to generate redirects" >&2; exit 1; }
+              text = ''
-              [[ -f result/redirects.json ]] || { echo "Error: Generated redirects file not found" >&2; exit 1; }
+                echo "=== SelfHostBlocks Redirects Updater ==="
-              
+                echo "Generating fresh ./docs/redirects.json..."
-              echo "Generated $(jq 'keys | length' result/redirects.json) redirects"
+
-              
+                nix build .#generateRedirects || { echo "Error: Failed to generate redirects" >&2; exit 1; }
-              [[ -f docs/redirects.json ]] && cp docs/redirects.json docs/redirects.json.backup && echo "Created backup"
+                [[ -f result/redirects.json ]] || { echo "Error: Generated redirects file not found" >&2; exit 1; }
-              cp result/redirects.json docs/redirects.json
+
-              echo "  Updated docs/redirects.json"
+                echo "Generated $(jq 'keys | length' result/redirects.json) redirects"
-              echo "To verify: nix build .#manualHtml"
+
-            '';
+                [[ -f docs/redirects.json ]] && cp docs/redirects.json docs/redirects.json.backup && echo "Created backup"
-          }}/bin/update-redirects";
+                cp result/redirects.json docs/redirects.json
+                echo "  Updated docs/redirects.json"
+                echo "To verify: nix build .#manualHtml"
+              '';
+            }
+          }/bin/update-redirects";
         };
       }
-  ) // {
+    )
-    herculesCI.ciSystems = [ "x86_64-linux" ];
+    // flake-utils.lib.eachSystem [ "x86_64-linux" ] (
+      system:
+      let
+        pkgs = pkgs' system;
+      in
+      {
+        checks =
+          let
+            inherit (pkgs.lib)
+              foldl
+              foldlAttrs
+              mergeAttrs
+              ;
 
-    nixosModules.default = self.nixosModules.all;
+            importFiles =
-    nixosModules.all = {
+              files:
-      imports = [
+              map (
-        # blocks
+                m:
-        self.nixosModules.authelia
+                pkgs.callPackage m {
-        self.nixosModules.davfs
+                  shb = self.lib.${system};
-        self.nixosModules.hardcodedsecret
+                }
-        self.nixosModules.lldap
+              ) files;
-        self.nixosModules.mitmdump
-        self.nixosModules.monitoring
-        self.nixosModules.nginx
-        self.nixosModules.postgresql
-        self.nixosModules.restic
-        self.nixosModules.ssl
-        self.nixosModules.sops
-        self.nixosModules.tinyproxy
-        self.nixosModules.vpn
-        self.nixosModules.zfs
 
-        # services
+            mergeTests = foldl mergeAttrs { };
-        self.nixosModules.arr
+
-        self.nixosModules.audiobookshelf
+            flattenAttrs =
-        self.nixosModules.deluge
+              root: attrset:
-        self.nixosModules.forgejo
+              foldlAttrs (
-        self.nixosModules.grocy
+                acc: name: value:
-        self.nixosModules.hledger
+                acc
-        self.nixosModules.immich
+                // {
-        self.nixosModules.home-assistant
+                  "${root}_${name}" = value;
-        self.nixosModules.jellyfin
+                }
-        self.nixosModules.nextcloud-server
+              ) { } attrset;
-        self.nixosModules.open-webui
+
-        self.nixosModules.pinchflat
+            vm_test =
-        self.nixosModules.vaultwarden
+              name: path:
-      ];
+              flattenAttrs "vm_${name}" (
+                removeAttrs
+                  (pkgs.callPackage path {
+                    shb = self.lib.${system};
+                  })
+                  [
+                    "override"
+                    "overrideDerivation"
+                  ]
+              );
+          in
+          (
+            {
+              modules = self.lib.${system}.check {
+                inherit pkgs;
+                tests = mergeTests (importFiles [
+                  ./test/modules/davfs.nix
+                  # TODO: Make this not use IFD
+                  ./test/modules/lib.nix
+                ]);
+              };
+
+              # TODO: Make this not use IFD
+              lib = nix-flake-tests.lib.check {
+                inherit pkgs;
+                tests = pkgs.callPackage ./test/modules/lib.nix {
+                  shb = self.lib.${system};
+                };
+              };
+            }
+            // (vm_test "arr" ./test/services/arr.nix)
+            // (vm_test "audiobookshelf" ./test/services/audiobookshelf.nix)
+            // (vm_test "deluge" ./test/services/deluge.nix)
+            // (vm_test "firefly-iii" ./test/services/firefly-iii.nix)
+            // (vm_test "forgejo" ./test/services/forgejo.nix)
+            // (vm_test "grocy" ./test/services/grocy.nix)
+            // (vm_test "hledger" ./test/services/hledger.nix)
+            // (vm_test "immich" ./test/services/immich.nix)
+            // (vm_test "homeassistant" ./test/services/home-assistant.nix)
+            // (vm_test "homepage" ./test/services/homepage.nix)
+            // (vm_test "jellyfin" ./test/services/jellyfin.nix)
+            // (vm_test "karakeep" ./test/services/karakeep.nix)
+            // (vm_test "mailserver" ./test/services/mailserver.nix)
+            // (vm_test "nextcloud" ./test/services/nextcloud.nix)
+            // (vm_test "open-webui" ./test/services/open-webui.nix)
+            // (vm_test "paperless" ./test/services/paperless.nix)
+            // (vm_test "pinchflat" ./test/services/pinchflat.nix)
+            // (vm_test "vaultwarden" ./test/services/vaultwarden.nix)
+
+            // (vm_test "authelia" ./test/blocks/authelia.nix)
+            // (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
+            // (vm_test "lib" ./test/blocks/lib.nix)
+            // (vm_test "lldap" ./test/blocks/lldap.nix)
+            // (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
+            // (vm_test "monitoring" ./test/blocks/monitoring.nix)
+            // (vm_test "postgresql" ./test/blocks/postgresql.nix)
+            // (vm_test "restic" ./test/blocks/restic.nix)
+            // (vm_test "ssl" ./test/blocks/ssl.nix)
+            // (vm_test "zfs" ./test/blocks/zfs.nix)
+
+            // (vm_test "contracts-backup" ./test/contracts/backup.nix)
+            // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
+            // (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix)
+            // (vm_test "contracts-secret" ./test/contracts/secret.nix)
+          );
+
+      }
+    )
+    // {
+      herculesCI.ciSystems = [ "x86_64-linux" ];
+
+      nixosModules.default = {
+        imports = [
+          # blocks
+          self.nixosModules.authelia
+          self.nixosModules.borgbackup
+          self.nixosModules.davfs
+          self.nixosModules.hardcodedsecret
+          self.nixosModules.lldap
+          self.nixosModules.mitmdump
+          self.nixosModules.monitoring
+          self.nixosModules.nginx
+          self.nixosModules.postgresql
+          self.nixosModules.restic
+          self.nixosModules.sanoid
+          self.nixosModules.ssl
+          self.nixosModules.tinyproxy
+          self.nixosModules.vpn
+          self.nixosModules.zfs
+
+          # services
+          self.nixosModules.arr
+          self.nixosModules.audiobookshelf
+          self.nixosModules.deluge
+          self.nixosModules.firefly-iii
+          self.nixosModules.forgejo
+          self.nixosModules.grocy
+          self.nixosModules.hledger
+          self.nixosModules.immich
+          self.nixosModules.home-assistant
+          self.nixosModules.homepage
+          self.nixosModules.jellyfin
+          self.nixosModules.karakeep
+          self.nixosModules.mailserver
+          self.nixosModules.nextcloud-server
+          self.nixosModules.open-webui
+          self.nixosModules.pinchflat
+          self.nixosModules.paperless
+          self.nixosModules.vaultwarden
+        ];
+      };
+
+      nixosModules.lib = lib/module.nix;
+
+      nixosModules.authelia = modules/blocks/authelia.nix;
+      nixosModules.borgbackup = modules/blocks/borgbackup.nix;
+      nixosModules.davfs = modules/blocks/davfs.nix;
+      nixosModules.hardcodedsecret = modules/blocks/hardcodedsecret.nix;
+      nixosModules.lldap = modules/blocks/lldap.nix;
+      nixosModules.mitmdump = modules/blocks/mitmdump.nix;
+      nixosModules.monitoring = modules/blocks/monitoring.nix;
+      nixosModules.nginx = modules/blocks/nginx.nix;
+      nixosModules.postgresql = modules/blocks/postgresql.nix;
+      nixosModules.restic = modules/blocks/restic.nix;
+      nixosModules.sanoid = modules/blocks/sanoid.nix;
+      nixosModules.sops = modules/blocks/sops.nix;
+      nixosModules.ssl = modules/blocks/ssl.nix;
+      nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
+      nixosModules.vpn = modules/blocks/vpn.nix;
+      nixosModules.zfs = modules/blocks/zfs.nix;
+
+      nixosModules.arr = modules/services/arr.nix;
+      nixosModules.audiobookshelf = modules/services/audiobookshelf.nix;
+      nixosModules.deluge = modules/services/deluge.nix;
+      nixosModules.firefly-iii = modules/services/firefly-iii.nix;
+      nixosModules.forgejo = modules/services/forgejo.nix;
+      nixosModules.grocy = modules/services/grocy.nix;
+      nixosModules.hledger = modules/services/hledger.nix;
+      nixosModules.immich = modules/services/immich.nix;
+      nixosModules.home-assistant = modules/services/home-assistant.nix;
+      nixosModules.homepage = modules/services/homepage.nix;
+      nixosModules.jellyfin = modules/services/jellyfin.nix;
+      nixosModules.karakeep = modules/services/karakeep.nix;
+      nixosModules.mailserver = modules/services/mailserver.nix;
+      nixosModules.nextcloud-server = modules/services/nextcloud-server.nix;
+      nixosModules.open-webui = modules/services/open-webui.nix;
+      nixosModules.paperless = modules/services/paperless.nix;
+      nixosModules.pinchflat = modules/services/pinchflat.nix;
+      nixosModules.vaultwarden = modules/services/vaultwarden.nix;
     };
-
-    nixosModules.authelia = modules/blocks/authelia.nix;
-    nixosModules.davfs = modules/blocks/davfs.nix;
-    nixosModules.hardcodedsecret = modules/blocks/hardcodedsecret.nix;
-    nixosModules.lldap = modules/blocks/lldap.nix;
-    nixosModules.mitmdump = modules/blocks/mitmdump.nix;
-    nixosModules.monitoring = modules/blocks/monitoring.nix;
-    nixosModules.nginx = modules/blocks/nginx.nix;
-    nixosModules.postgresql = modules/blocks/postgresql.nix;
-    nixosModules.restic = modules/blocks/restic.nix;
-    nixosModules.ssl = modules/blocks/ssl.nix;
-    nixosModules.sops = modules/blocks/sops.nix;
-    nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
-    nixosModules.vpn = modules/blocks/vpn.nix;
-    nixosModules.zfs = modules/blocks/zfs.nix;
-
-    nixosModules.arr = modules/services/arr.nix;
-    nixosModules.audiobookshelf = modules/services/audiobookshelf.nix;
-    nixosModules.deluge = modules/services/deluge.nix;
-    nixosModules.forgejo = modules/services/forgejo.nix;
-    nixosModules.grocy = modules/services/grocy.nix;
-    nixosModules.hledger = modules/services/hledger.nix;
-    nixosModules.immich = modules/services/immich.nix;
-    nixosModules.home-assistant = modules/services/home-assistant.nix;
-    nixosModules.jellyfin = modules/services/jellyfin.nix;
-    nixosModules.nextcloud-server = modules/services/nextcloud-server.nix;
-    nixosModules.open-webui = modules/services/open-webui.nix;
-    nixosModules.pinchflat = modules/services/pinchflat.nix;
-    nixosModules.vaultwarden = modules/services/vaultwarden.nix;
-  };
 }

lib/default.nix

@@ -1,300 +1,460 @@
 { pkgs, lib }:
 let
   inherit (builtins) isAttrs hasAttr;
-  inherit (lib) any concatMapStringsSep concatStringsSep;
+  inherit (lib)
-in
+    any
-rec {
+    concatMapStringsSep
-  # Replace secrets in a file.
+    concatStringsSep
-  # - userConfig is an attrset that will produce a config file.
+    escapeShellArg
-  # - resultPath is the location the config file should have on the filesystem.
+    ;
-  # - generator is a function taking two arguments name and value and returning path in the nix
+  shb = rec {
-  #   nix store where the
+    # Replace secrets in a file.
-  replaceSecrets = { userConfig, resultPath, generator, user ? null, permissions ? "u=r,g=r,o=" }:
+    # - userConfig is an attrset that will produce a config file.
-    let
+    # - resultPath is the location the config file should have on the filesystem.
-      configWithTemplates = withReplacements userConfig;
+    # - generator is a function taking two arguments name and value and returning path in the nix
+    #   nix store where the
+    replaceSecrets =
+      {
+        userConfig,
+        resultPath,
+        generator,
+        user ? null,
+        permissions ? "u=r,g=r,o=",
+      }:
+      let
+        configWithTemplates = withReplacements userConfig;
 
-      nonSecretConfigFile = generator "template" configWithTemplates;
+        nonSecretConfigFile = generator "template" configWithTemplates;
 
-      replacements = getReplacements userConfig;
+        replacements = getReplacements userConfig;
-    in
+      in
       replaceSecretsScript {
         file = nonSecretConfigFile;
         inherit resultPath replacements;
         inherit user permissions;
       };
 
-  replaceSecretsFormatAdapter = format: format.generate;
+    replaceSecretsFormatAdapter = format: format.generate;
-  replaceSecretsGeneratorAdapter = generator: name: value: pkgs.writeText "generator " (generator value);
+    replaceSecretsGeneratorAdapter =
-  toEnvVar = replaceSecretsGeneratorAdapter (v: (lib.generators.toINIWithGlobalSection {} { globalSection = v; }));
+      generator: name: value:
+      pkgs.writeText "generator " (generator value);
+    toEnvVar = replaceSecretsGeneratorAdapter (
+      v: (lib.generators.toINIWithGlobalSection { } { globalSection = v; })
+    );
 
-  template = file: newPath: replacements: replaceSecretsScript {
+    template =
-    inherit file replacements;
+      file: newPath: replacements:
-    resultPath = newPath;
+      replaceSecretsScript {
-  };
+        inherit file replacements;
+        resultPath = newPath;
+      };
 
-  genReplacement = secret:
+    genReplacement =
-    let
+      secret:
-      t = { transform ? null, ... }: if isNull transform then x: x else transform;
+      let
-    in
+        t =
+          {
+            transform ? null,
+            ...
+          }:
+          if isNull transform then x: x else transform;
+      in
       lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
 
-  replaceSecretsScript = { file, resultPath, replacements, user ? null, permissions ? "u=r,g=r,o=" }:
+    replaceSecretsScript =
-    let
+      {
-      templatePath = resultPath + ".template";
+        file,
+        resultPath,
+        replacements,
+        user ? null,
+        permissions ? "u=r,g=r,o=",
+      }:
+      let
+        templatePath = resultPath + ".template";
 
-      # We check that the files containing the secrets have the
+        # We check that the files containing the secrets have the
-      # correct permissions for us to read them in this separate
+        # correct permissions for us to read them in this separate
-      # step. Otherwise, the $(cat ...) commands inside the sed
+        # step. Otherwise, the $(cat ...) commands inside the sed
-      # replacements could fail but not fail individually but
+        # replacements could fail but not fail individually but
-      # not fail the whole script.
+        # not fail the whole script.
-      checkPermissions = concatMapStringsSep "\n" (pattern: "cat ${pattern.source} > /dev/null") replacements;
+        checkPermissions = concatMapStringsSep "\n" (
+          pattern: "cat ${pattern.source} > /dev/null"
+        ) replacements;
 
-      sedPatterns = concatMapStringsSep " " (pattern: "-e \"s|${pattern.name}|${pattern.value}|\"") (map genReplacement replacements);
+        generatedReplacements = map genReplacement replacements;
 
-      sedCmd = if replacements == []
+        replaceScript = pkgs.writers.writePython3 "replace-secret" { } ''
-               then "cat"
+          import argparse
-               else "${pkgs.gnused}/bin/sed ${sedPatterns}";
+          import pathlib
-    in
+
+
+          def parse_args() -> argparse.Namespace:
+              parser = argparse.ArgumentParser()
+              parser.add_argument("--file", required=True, type=pathlib.Path)
+              parser.add_argument("--marker", required=True)
+              parser.add_argument("--replacement", required=True)
+              return parser.parse_args()
+
+
+          args = parse_args()
+          content = args.file.read_text()
+          args.file.write_text(content.replace(args.marker, args.replacement))
+        '';
+
+        replaceCommands = concatMapStringsSep "\n" (pattern: ''
+          ${replaceScript} \
+            --file ${escapeShellArg resultPath} \
+            --marker ${escapeShellArg pattern.name} \
+            --replacement "${pattern.value}"
+        '') generatedReplacements;
+
+        replaceCmd =
+          if replacements == [ ] then
+            "cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}"
+          else
+            ''
+              cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}
+              ${replaceCommands}
+            '';
+      in
       ''
-      set -euo pipefail
+        set -euo pipefail
 
-      ${checkPermissions}
+        ${checkPermissions}
 
-      mkdir -p $(dirname ${templatePath})
+        mkdir -p $(dirname ${templatePath})
-      ln -fs ${file} ${templatePath}
+        ln -fs ${file} ${templatePath}
-      rm -f ${resultPath}
+        rm -f ${resultPath}
-      touch ${resultPath}
+        touch ${resultPath}
-      '' + (lib.optionalString (user != null) ''
+      ''
-      chown ${user} ${resultPath}
+      + (lib.optionalString (user != null) ''
-      '') + ''
+        chown ${user} ${resultPath}
-      ${sedCmd} ${templatePath} > ${resultPath}
+      '')
-      chmod ${permissions} ${resultPath}
+      + ''
+        ${replaceCmd}
+        chmod ${permissions} ${resultPath}
       '';
 
-  secretFileType = lib.types.submodule {
+    secretFileType = lib.types.submodule {
-    options = {
+      options = {
-      source = lib.mkOption {
+        source = lib.mkOption {
-        type = lib.types.path;
+          type = lib.types.path;
-        description = "File containing the value.";
+          description = "File containing the value.";
-      };
+        };
 
-      transform = lib.mkOption {
+        transform = lib.mkOption {
-        type = lib.types.raw;
+          type = lib.types.raw;
-        description = "An optional function to transform the secret.";
+          description = "An optional function to transform the secret.";
-        default = null;
+          default = null;
-        example = lib.literalExpression ''
+          example = lib.literalExpression ''
-        v: "prefix-$${v}-suffix"
+            v: "prefix-$${v}-suffix"
-        '';
+          '';
+        };
       };
     };
-  };
 
-  secretName = names:
+    secretName =
-    "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) names)}%";
+      names: "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) names)}%";
 
-  withReplacements = attrs:
+    withReplacements =
-    let
+      attrs:
-      valueOrReplacement = name: value:
+      let
-        if !(builtins.isAttrs value && value ? "source")
+        valueOrReplacement =
-        then value
+          name: value: if !(builtins.isAttrs value && value ? "source") then value else secretName name;
-        else secretName name;
+      in
-    in
+      mapAttrsRecursiveCond (v: !v ? "source") valueOrReplacement attrs;
-      mapAttrsRecursiveCond (v: ! v ? "source") valueOrReplacement attrs;
 
-  getReplacements = attrs:
+    getReplacements =
-    let
+      attrs:
-      addNameField = name: value:
+      let
-        if !(builtins.isAttrs value && value ? "source")
+        addNameField =
-        then value
+          name: value:
-        else value // { name = name; };
+          if !(builtins.isAttrs value && value ? "source") then value else value // { name = name; };
 
-      secretsWithName = mapAttrsRecursiveCond (v: ! v ? "source") addNameField attrs;
+        secretsWithName = mapAttrsRecursiveCond (v: !v ? "source") addNameField attrs;
-    in
+      in
       collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
-      
-  # Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
-  mapAttrsRecursiveCond =
-    # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
-    cond:
-    # A function, given a list of attribute names and a value, returns a new value.
-    f:
-    # Attribute set or list to recursively map over.
-    set:
-    let
-      recurse = path: val:
-        if builtins.isAttrs val && cond val
-        then lib.attrsets.mapAttrs (n: v: recurse (path ++ [n]) v) val
-        else if builtins.isList val && cond val
-        then lib.lists.imap0 (i: v: recurse (path ++ [(builtins.toString i)]) v) val
-        else f path val;
-    in recurse [] set;
 
-  # Like lib.attrsets.collect but also recurses on lists.
+    # Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
-  collect =
+    mapAttrsRecursiveCond =
-  # Given an attribute's value, determine if recursion should stop.
+      # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
-  pred:
+      cond:
-  # The attribute set to recursively collect.
+      # A function, given a list of attribute names and a value, returns a new value.
-  attrs:
+      f:
-    if pred attrs then
+      # Attribute set or list to recursively map over.
-      [ attrs ]
+      set:
-    else if builtins.isAttrs attrs then
+      let
-      lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
+        recurse =
-    else if builtins.isList attrs then
+          path: val:
-      lib.lists.concatMap (collect pred) attrs
+          if builtins.isAttrs val && cond val then
-    else
+            lib.attrsets.mapAttrs (n: v: recurse (path ++ [ n ]) v) val
-      [];
+          else if builtins.isList val && cond val then
+            lib.lists.imap0 (i: v: recurse (path ++ [ (builtins.toString i) ]) v) val
+          else
+            f path val;
+      in
+      recurse [ ] set;
 
-  indent = i: str: lib.concatMapStringsSep "\n" (x: (lib.strings.replicate i " ") + x) (lib.splitString "\n" str);
+    # Like lib.attrsets.collect but also recurses on lists.
+    collect =
+      # Given an attribute's value, determine if recursion should stop.
+      pred:
+      # The attribute set to recursively collect.
+      attrs:
+      if pred attrs then
+        [ attrs ]
+      else if builtins.isAttrs attrs then
+        lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
+      else if builtins.isList attrs then
+        lib.lists.concatMap (collect pred) attrs
+      else
+        [ ];
+
+    indent =
+      i: str:
+      lib.concatMapStringsSep "\n" (x: (lib.strings.replicate i " ") + x) (lib.splitString "\n" str);
+
+    # Generator for XML
+    formatXML =
+      {
+        enclosingRoot ? null,
+      }:
+      {
+        type =
+          with lib.types;
+          let
+            valueType =
+              nullOr (oneOf [
+                bool
+                int
+                float
+                str
+                path
+                (attrsOf valueType)
+                (listOf valueType)
+              ])
+              // {
+                description = "XML value";
+              };
+          in
+          valueType;
+
+        generate =
+          name: value:
+          pkgs.callPackage (
+            { runCommand, python3 }:
+            runCommand "config"
+              {
+                value = builtins.toJSON (if enclosingRoot == null then value else { ${enclosingRoot} = value; });
+                passAsFile = [ "value" ];
+              }
+              (
+                pkgs.writers.writePython3 "dict2xml"
+                  {
+                    libraries = with python3.pkgs; [
+                      python
+                      dict2xml
+                    ];
+                  }
+                  ''
+                    import os
+                    import json
+                    from dict2xml import dict2xml
+
+                    with open(os.environ["valuePath"]) as f:
+                        content = json.loads(f.read())
+                        if content is None:
+                            print("Could not parse env var valuePath as json")
+                            os.exit(2)
+                        with open(os.environ["out"], "w") as out:
+                            out.write(dict2xml(content))
+                  ''
+              )
+          ) { };
 
-  # Generator for XML
-  formatXML = {
-    enclosingRoot ? null
-  }: {
-    type = with lib.types; let
-      valueType = nullOr (oneOf [
-        bool
-        int
-        float
-        str
-        path
-        (attrsOf valueType)
-        (listOf valueType)
-      ]) // {
-        description = "XML value";
       };
-    in valueType;
 
-    generate = name: value: pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
+    parseXML =
-      value = builtins.toJSON (
+      xml:
-        if enclosingRoot == null then
+      let
-          value
+        xmlToJsonFile = pkgs.callPackage (
-        else
+          { runCommand, python3 }:
-          { ${enclosingRoot} = value; });
+          runCommand "config"
-      passAsFile = [ "value" ];
+            {
-    } (pkgs.writers.writePython3 "dict2xml" {
+              inherit xml;
-      libraries = with python3.pkgs; [ python dict2xml ];
+              passAsFile = [ "xml" ];
-    } ''
+            }
-      import os
+            (
-      import json
+              pkgs.writers.writePython3 "xml2json"
-      from dict2xml import dict2xml
+                {
-
+                  libraries = with python3.pkgs; [ python ];
-      with open(os.environ["valuePath"]) as f:
+                }
-          content = json.loads(f.read())
+                ''
-          if content is None:
+                  import os
-              print("Could not parse env var valuePath as json")
+                  import json
-              os.exit(2)
+                  from collections import ChainMap
-          with open(os.environ["out"], "w") as out:
+                  from xml.etree import ElementTree
-              out.write(dict2xml(content))
-    '')) {};
-
-  };
-
-  parseXML = xml:
-    let
-      xmlToJsonFile = pkgs.callPackage ({ runCommand, python3 }: runCommand "config" {
-        inherit xml;
-        passAsFile = [ "xml" ];
-      } (pkgs.writers.writePython3 "xml2json" {
-        libraries = with python3.pkgs; [ python ];
-      } ''
-        import os
-        import json
-        from collections import ChainMap
-        from xml.etree import ElementTree
-        
-
-        def xml_to_dict_recursive(root):
-            all_descendants = list(root)
-            if len(all_descendants) == 0:
-                return {root.tag: root.text}
-            else:
-                merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
-                return {root.tag: dict(merged_dict)}
 
 
-        with open(os.environ["xmlPath"]) as f:
+                  def xml_to_dict_recursive(root):
-            root = ElementTree.XML(f.read())
+                      all_descendants = list(root)
-            xml = xml_to_dict_recursive(root)
+                      if len(all_descendants) == 0:
-            j = json.dumps(xml)
+                          return {root.tag: root.text}
+                      else:
+                          merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
+                          return {root.tag: dict(merged_dict)}
 
-            with open(os.environ["out"], "w") as out:
+
-                out.write(j)
+                  with open(os.environ["xmlPath"]) as f:
-      '')) {};
+                      root = ElementTree.XML(f.read())
-    in
+                      xml = xml_to_dict_recursive(root)
+                      j = json.dumps(xml)
+
+                      with open(os.environ["out"], "w") as out:
+                          out.write(j)
+                ''
+            )
+        ) { };
+      in
       builtins.fromJSON (builtins.readFile xmlToJsonFile);
 
-  renameAttrName = attrset: from: to:
+    renameAttrName =
-    (lib.attrsets.filterAttrs (name: v: name == from) attrset) // {
+      attrset: from: to:
-      ${to} = attrset.${from};
+      (lib.attrsets.filterAttrs (name: v: name == from) attrset)
-    };
+      // {
+        ${to} = attrset.${from};
+      };
 
-  # Taken from https://github.com/antifuchs/nix-flake-tests/blob/main/default.nix
+    # Taken from https://github.com/antifuchs/nix-flake-tests/blob/main/default.nix
-  # with a nicer diff display function.
+    # with a nicer diff display function.
-  check = { pkgs, tests }:
+    check =
-    let
+      { pkgs, tests }:
-      formatValue = val:
+      let
-        if (builtins.isList val || builtins.isAttrs val) then builtins.toJSON val
+        formatValue =
-        else builtins.toString val;
+          val:
+          if (builtins.isList val || builtins.isAttrs val) then
+            builtins.toJSON val
+          else
+            builtins.toString val;
 
-      resultToString = { name, expected, result }:
+        resultToString =
-        builtins.readFile (pkgs.runCommand "nix-flake-tests-error" {
+          {
-          expected = formatValue expected;
+            name,
-          result = formatValue result;
+            expected,
-          passAsFile = [ "expected" "result" ];
+            result,
-        } ''
+          }:
-          echo "${name} failed (- expected, + result)" > $out
+          builtins.readFile (
-          cp ''${expectedPath} ''${expectedPath}.json
+            pkgs.runCommand "nix-flake-tests-error"
-          cp ''${resultPath} ''${resultPath}.json
+              {
-          ${pkgs.deepdiff}/bin/deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
+                expected = formatValue expected;
-        '');
+                result = formatValue result;
+                passAsFile = [
+                  "expected"
+                  "result"
+                ];
 
-      results = pkgs.lib.runTests tests;
+                nativeBuildInputs = [
-    in
+                  (pkgs.python3.withPackages (ps: [ ps.deepdiff ] ++ ps.deepdiff.optional-dependencies.cli))
-    if results != [ ] then
+                ];
-      builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
+              }
-    else
+              ''
-      pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
+                echo "${name} failed (- expected, + result)" > $out
+                cp ''${expectedPath} ''${expectedPath}.json
+                cp ''${resultPath} ''${resultPath}.json
+                deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
+              ''
+          );
 
+        results = pkgs.lib.runTests tests;
+      in
+      if results != [ ] then
+        builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
+      else
+        pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
 
-  genConfigOutOfBandSystemd = { config, configLocation, generator, user ? null, permissions ? "u=r,g=r,o=" }:
+    genConfigOutOfBandSystemd =
-    {
+      {
-      loadCredentials = getLoadCredentials "source" config;
+        config,
-      preStart = lib.mkBefore (replaceSecrets {
+        configLocation,
-        userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
+        generator,
-        resultPath = configLocation;
+        user ? null,
-        inherit generator;
+        permissions ? "u=r,g=r,o=",
-        inherit user permissions;
+      }:
-      });
+      {
-    };
+        loadCredentials = getLoadCredentials "source" config;
+        preStart = lib.mkBefore (replaceSecrets {
+          userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
+          resultPath = configLocation;
+          inherit generator;
+          inherit user permissions;
+        });
+      };
 
-  updateToLoadCredentials = sourceField: rootDir: attrs:
+    updateToLoadCredentials =
-    let
+      sourceField: rootDir: attrs:
-      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+      let
+        hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
 
-      valueOrLoadCredential = path: value:
+        valueOrLoadCredential =
-        if ! (hasPlaceholderField value)
+          path: value:
-        then value
+          if !(hasPlaceholderField value) then
-        else value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
+            value
-    in
+          else
-      mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) valueOrLoadCredential attrs;
+            value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
+      in
+      mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) valueOrLoadCredential attrs;
 
-  getLoadCredentials = sourceField: attrs:
+    getLoadCredentials =
-    let
+      sourceField: attrs:
-      hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
+      let
+        hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
 
-      addPathField = path: value:
+        addPathField =
-        if ! (hasPlaceholderField value)
+          path: value: if !(hasPlaceholderField value) then value else value // { inherit path; };
-        then value
-        else value // { inherit path; };
 
-      secretsWithPath = mapAttrsRecursiveCond (v: ! (hasPlaceholderField v)) addPathField attrs;
+        secretsWithPath = mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) addPathField attrs;
 
-      allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
+        allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
 
-      genLoadCredentials = secret:
+        genLoadCredentials = secret: "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
-        "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
+      in
-    in
       map genLoadCredentials allSecrets;
 
-  anyNotNull = any (x: x != null);
+    anyNotNull = any (x: x != null);
+
+    mkJellyfinPlugin =
+      {
+        pname,
+        version,
+        hash,
+        url,
+      }:
+      pkgs.callPackage (
+        { stdenv, fetchzip }:
+        stdenv.mkDerivation (finalAttrs: {
+          inherit pname version;
+
+          src = fetchzip {
+            inherit url hash;
+            stripRoot = false;
+          };
+
+          dontBuild = true;
+
+          installPhase = ''
+            mkdir $out
+            cp -r . $out
+          '';
+        })
+      ) { };
+
+    update =
+      attr: fn: attrset:
+      attrset // { ${attr} = fn attrset.${attr}; };
+
+  };
+in
+shb
+// {
+  homepage = pkgs.callPackage ./homepage.nix { inherit shb; };
 }

lib/homepage.nix

@@ -0,0 +1,92 @@
+{ lib, shb }:
+let
+  sort =
+    attr: vs:
+    map (v: { ${v.name} = v.${attr}; }) (
+      lib.sortOn (v: v.sortOrder) (lib.mapAttrsToList (n: v: v // { name = n; }) vs)
+    );
+
+  slufigy = builtins.replaceStrings [ "-" ] [ "_" ];
+
+  mkService =
+    groupName: serviceName:
+    {
+      request,
+      ...
+    }:
+    apiKey: settings:
+    lib.recursiveUpdate (
+      {
+        href = request.externalUrl;
+        siteMonitor = if (request.internalUrl == null) then null else request.internalUrl;
+        icon = "sh-${lib.toLower serviceName}";
+      }
+      // lib.optionalAttrs (apiKey != null) {
+        widget = {
+          # Duplicating because widgets call the api key various names
+          # and duplicating is a hacky but easy solution.
+          key = "{{HOMEPAGE_FILE_${slufigy groupName}_${slufigy serviceName}}}";
+          password = "{{HOMEPAGE_FILE_${slufigy groupName}_${slufigy serviceName}}}";
+          type = lib.toLower serviceName;
+          url = if (request.internalUrl != null) then request.internalUrl else request.externalUrl;
+        };
+      }
+    ) settings;
+
+  asServiceGroup =
+    cfg:
+    sort "services" (
+      lib.mapAttrs (
+        groupName: groupCfg:
+        shb.update "services" (
+          services:
+          sort "dashboard" (
+            lib.mapAttrs (
+              serviceName: serviceCfg:
+              shb.update "dashboard" (
+                dashboard:
+                (mkService groupName serviceName) dashboard serviceCfg.apiKey (serviceCfg.settings or { })
+              ) serviceCfg
+            ) services
+          )
+        ) groupCfg
+      ) cfg
+    );
+
+  allKeys =
+    cfg:
+    let
+      flat = lib.flatten (
+        lib.mapAttrsToList (
+          groupName: groupCfg:
+          lib.mapAttrsToList (
+            serviceName: serviceCfg:
+            lib.optionalAttrs (serviceCfg.apiKey != null) {
+              inherit serviceName groupName;
+              inherit (serviceCfg.apiKey.result) path;
+            }
+          ) groupCfg.services
+        ) cfg
+      );
+
+      flatWithApiKey = builtins.filter (v: v != { }) flat;
+    in
+    builtins.listToAttrs (
+      map (
+        {
+          groupName,
+          serviceName,
+          path,
+        }:
+        lib.nameValuePair "${slufigy groupName}_${slufigy serviceName}" path
+      ) flatWithApiKey
+    );
+in
+{
+  inherit
+    allKeys
+    asServiceGroup
+    mkService
+    sort
+    ;
+}

lib/module.nix

@@ -0,0 +1,10 @@
+{ pkgs, lib, ... }:
+let
+  shb = (import ./default.nix { inherit pkgs lib; });
+in
+{
+  _module.args.shb = shb // {
+    test = pkgs.callPackage ../test/common.nix { };
+    contracts = pkgs.callPackage ../modules/contracts { inherit shb; };
+  };
+}

modules/blocks/authelia.nix

@@ -1,12 +1,16 @@
-{ config, options, pkgs, lib, ... }:
+{
+  config,
+  options,
+  pkgs,
+  lib,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.authelia;
   opt = options.shb.authelia;
 
-  contracts = pkgs.callPackage ../contracts {};
-  shblib = pkgs.callPackage ../../lib {};
-
   fqdn = "${cfg.subdomain}.${cfg.domain}";
   fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";
 
@@ -18,6 +22,7 @@ let
 in
 {
   imports = [
+    ../../lib/module.nix
     ./lldap.nix
     ./mitmdump.nix
     ./postgresql.nix
@@ -46,7 +51,7 @@ in
 
     ssl = lib.mkOption {
       description = "Path to SSL files";
-      type = lib.types.nullOr contracts.ssl.certs;
+      type = lib.types.nullOr shb.contracts.ssl.certs;
       default = null;
     };
 
@@ -81,50 +86,50 @@ in
           jwtSecret = lib.mkOption {
             description = "JWT secret.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           ldapAdminPassword = lib.mkOption {
             description = "LDAP admin user password.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           sessionSecret = lib.mkOption {
             description = "Session secret.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           storageEncryptionKey = lib.mkOption {
-            description = "Storage encryption key.";
+            description = "Storage encryption key. Must be >= 20 characters.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
           identityProvidersOIDCHMACSecret = lib.mkOption {
-            description = "Identity provider OIDC HMAC secret.";
+            description = "Identity provider OIDC HMAC secret. Must be >= 40 characters.";
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
@@ -135,10 +140,10 @@ in
               Generate one with `nix run nixpkgs#openssl -- genrsa -out keypair.pem 2048`
             '';
             type = lib.types.submodule {
-              options = contracts.secret.mkRequester {
+              options = shb.contracts.secret.mkRequester {
                 mode = "0400";
                 owner = cfg.autheliaUser;
-                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
+                restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
               };
             };
           };
@@ -149,19 +154,25 @@ in
     extraOidcClaimsPolicies = lib.mkOption {
       description = "Extra OIDC claims policies.";
       type = lib.types.attrsOf lib.types.attrs;
-      default = {};
+      default = { };
     };
 
     extraOidcScopes = lib.mkOption {
       description = "Extra OIDC scopes.";
       type = lib.types.attrsOf lib.types.attrs;
-      default = {};
+      default = { };
+    };
+
+    extraOidcAuthorizationPolicies = lib.mkOption {
+      description = "Extra OIDC authorization policies.";
+      type = lib.types.attrsOf lib.types.attrs;
+      default = { };
     };
 
     extraDefinitions = lib.mkOption {
       description = "Extra definitions.";
       type = lib.types.attrsOf lib.types.attrs;
-      default = {};
+      default = { };
     };
 
     oidcClients = lib.mkOption {
@@ -173,79 +184,92 @@ in
           client_secret.source = pkgs.writeText "dummy.secret" "dummy_client_secret";
           public = false;
           authorization_policy = "one_factor";
-          redirect_uris = [];
+          redirect_uris = [ ];
         }
       ];
-      type = lib.types.listOf (lib.types.submodule {
+      type = lib.types.listOf (
-        freeformType = lib.types.attrsOf lib.types.anything;
+        lib.types.submodule {
+          freeformType = lib.types.attrsOf lib.types.anything;
 
-        options = {
+          options = {
-          client_id = lib.mkOption {
+            client_id = lib.mkOption {
-            type = lib.types.str;
+              type = lib.types.str;
-            description = "Unique identifier of the OIDC client.";
+              description = "Unique identifier of the OIDC client.";
+            };
+
+            client_name = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = "Human readable description of the OIDC client.";
+              default = null;
+            };
+
+            client_secret = lib.mkOption {
+              type = shb.secretFileType;
+              description = ''
+                File containing the shared secret with the OIDC client.
+
+                Generate with:
+
+                ```
+                nix run nixpkgs#authelia -- \
+                    crypto hash generate pbkdf2 \
+                    --variant sha512 \
+                    --random \
+                    --random.length 72 \
+                    --random.charset rfc3986
+                ```
+              '';
+            };
+
+            public = lib.mkOption {
+              type = lib.types.bool;
+              description = "If the OIDC client is public or not.";
+              default = false;
+              apply = v: if v then "true" else "false";
+            };
+
+            authorization_policy = lib.mkOption {
+              type = lib.types.enum (
+                [
+                  "one_factor"
+                  "two_factor"
+                ]
+                ++ lib.attrNames cfg.extraOidcAuthorizationPolicies
+              );
+              description = "Require one factor (password) or two factor (device) authentication.";
+              default = "one_factor";
+            };
+
+            redirect_uris = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = "List of uris that are allowed to be redirected to.";
+            };
+
+            scopes = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = "Scopes to ask for. See https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims";
+              example = [
+                "openid"
+                "profile"
+                "email"
+                "groups"
+              ];
+              default = [ ];
+            };
+
+            claims_policy = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Claim policy.
+
+                Defaults to 'default' to provide a backwards compatible experience.
+                Read [this document](https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims/#restore-functionality-prior-to-claims-parameter) for more information.
+              '';
+              default = "default";
+            };
           };
-
+        }
-          client_name = lib.mkOption {
+      );
-            type = lib.types.nullOr lib.types.str;
-            description = "Human readable description of the OIDC client.";
-            default = null;
-          };
-
-          client_secret = lib.mkOption {
-            type = shblib.secretFileType;
-            description = ''
-            File containing the shared secret with the OIDC client.
-
-            Generate with:
-
-            ```
-            nix run nixpkgs#authelia -- \
-                crypto hash generate pbkdf2 \
-                --variant sha512 \
-                --random \
-                --random.length 72 \
-                --random.charset rfc3986
-            ```
-            '';
-          };
-
-          public = lib.mkOption {
-            type = lib.types.bool;
-            description = "If the OIDC client is public or not.";
-            default = false;
-            apply = v: if v then "true" else "false";
-          };
-
-          authorization_policy = lib.mkOption {
-            type = lib.types.enum [ "one_factor" "two_factor" ];
-            description = "Require one factor (password) or two factor (device) authentication.";
-            default = "one_factor";
-          };
-
-          redirect_uris = lib.mkOption {
-            type = lib.types.listOf lib.types.str;
-            description = "List of uris that are allowed to be redirected to.";
-          };
-
-          scopes = lib.mkOption {
-            type = lib.types.listOf lib.types.str;
-            description = "Scopes to ask for. See https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims";
-            example = [ "openid" "profile" "email" "groups" ];
-            default = [];
-          };
-
-          claims_policy = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
-            description = ''
-              Claim policy.
-
-              Defaults to 'default' to provide a backwards compatible experience.
-              Read [this document](https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims/#restore-functionality-prior-to-claims-parameter) for more information.
-            '';
-            default = "default";
-          };
-        };
-      });
     };
 
     smtp = lib.mkOption {
@@ -258,54 +282,65 @@ in
       default = "/tmp/authelia-notifications";
       type = lib.types.oneOf [
         lib.types.str
-        (lib.types.nullOr (lib.types.submodule {
+        (lib.types.nullOr (
-          options = {
+          lib.types.submodule {
-            from_address = lib.mkOption {
+            options = {
-              type = lib.types.str;
+              from_address = lib.mkOption {
-              description = "SMTP address from which the emails originate.";
+                type = lib.types.str;
-              example = "authelia@mydomain.com";
+                description = "SMTP address from which the emails originate.";
-            };
+                example = "authelia@mydomain.com";
-            from_name = lib.mkOption {
+              };
-              type = lib.types.str;
+              from_name = lib.mkOption {
-              description = "SMTP name from which the emails originate.";
+                type = lib.types.str;
-              default = "Authelia";
+                description = "SMTP name from which the emails originate.";
-            };
+                default = "Authelia";
-            host = lib.mkOption {
+              };
-              type = lib.types.str;
+              scheme = lib.mkOption {
-              description = "SMTP host to send the emails to.";
+                description = "The protocl must be smtp, submission, or submissions. The only difference between these schemes are the default ports and submissions requires a TLS transport per SMTP Ports Security Measures, whereas submission and smtp use a standard TCP transport and typically enforce StartTLS.";
-            };
+                type = lib.types.enum [
-            port = lib.mkOption {
+                  "smtp"
-              type = lib.types.port;
+                  "submission"
-              description = "SMTP port to send the emails to.";
+                  "submissions"
-              default = 25;
+                ];
-            };
+                default = "smtp";
-            username = lib.mkOption {
+              };
-              type = lib.types.str;
+              host = lib.mkOption {
-              description = "Username to connect to the SMTP host.";
+                type = lib.types.str;
-            };
+                description = "SMTP host to send the emails to.";
-            password = lib.mkOption {
+              };
-              description = "File containing the password to connect to the SMTP host.";
+              port = lib.mkOption {
-              type = lib.types.submodule {
+                type = lib.types.port;
-                options = contracts.secret.mkRequester {
+                description = "SMTP port to send the emails to.";
-                  mode = "0400";
+                default = 25;
-                  owner = cfg.autheliaUser;
+              };
-                  restartUnits = [ "authelia-${fqdn}" ];
+              username = lib.mkOption {
+                type = lib.types.str;
+                description = "Username to connect to the SMTP host.";
+              };
+              password = lib.mkOption {
+                description = "File containing the password to connect to the SMTP host.";
+                type = lib.types.submodule {
+                  options = shb.contracts.secret.mkRequester {
+                    mode = "0400";
+                    owner = cfg.autheliaUser;
+                    restartUnits = [ "authelia-${fqdn}.service" ];
+                  };
                 };
               };
             };
-          };
+          }
-        }))
+        ))
       ];
     };
 
     rules = lib.mkOption {
       type = lib.types.listOf lib.types.anything;
       description = "Rule based clients";
-      default = [];
+      default = [ ];
     };
 
     mount = lib.mkOption {
-      type = contracts.mount;
+      type = shb.contracts.mount;
       description = ''
         Mount configuration. This is an output option.
 
@@ -319,12 +354,16 @@ in
         ```
       '';
       readOnly = true;
-      default = { path = "/var/lib/authelia-authelia.${cfg.domain}"; };
+      default = {
-      defaultText = { path = "/var/lib/authelia-authelia.example.com"; };
+        path = "/var/lib/authelia-authelia.${cfg.domain}";
+      };
+      defaultText = {
+        path = "/var/lib/authelia-authelia.example.com";
+      };
     };
 
     mountRedis = lib.mkOption {
-      type = contracts.mount;
+      type = shb.contracts.mount;
       description = ''
         Mount configuration for Redis. This is an output option.
 
@@ -338,7 +377,9 @@ in
         ```
       '';
       readOnly = true;
-      default = { path = "/var/lib/redis-authelia"; };
+      default = {
+        path = "/var/lib/redis-authelia";
+      };
     };
 
     debug = lib.mkOption {
@@ -349,6 +390,20 @@ in
         to see exactly what Authelia receives and sends back.
       '';
     };
+
+    dashboard = lib.mkOption {
+      description = ''
+        Dashboard contract consumer
+      '';
+      default = { };
+      type = lib.types.submodule {
+        options = shb.contracts.dashboard.mkRequester {
+          externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
+          externalUrlText = "https://\${config.shb.authelia.subdomain}.\${config.shb.authelia.domain}";
+          internalUrl = "http://127.0.0.1:${toString listenPort}";
+        };
+      };
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -365,7 +420,7 @@ in
 
     # Overriding the user name so we don't allow any weird characters anywhere. For example, postgres users do not accept the '.'.
     users = {
-      groups.${autheliaCfg.user} = {};
+      groups.${autheliaCfg.user} = { };
       users.${autheliaCfg.user} = {
         isSystemUser = true;
         group = autheliaCfg.user;
@@ -379,18 +434,19 @@ in
       secrets = {
         jwtSecretFile = cfg.secrets.jwtSecret.result.path;
         storageEncryptionKeyFile = cfg.secrets.storageEncryptionKey.result.path;
+        sessionSecretFile = cfg.secrets.sessionSecret.result.path;
+        oidcIssuerPrivateKeyFile = cfg.secrets.identityProvidersOIDCIssuerPrivateKey.result.path;
+        oidcHmacSecretFile = cfg.secrets.identityProvidersOIDCHMACSecret.result.path;
       };
       # See https://www.authelia.com/configuration/methods/secrets/
       environmentVariables = {
         AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD_FILE = toString cfg.secrets.ldapAdminPassword.result.path;
-        AUTHELIA_SESSION_SECRET_FILE = toString cfg.secrets.sessionSecret.result.path;
         # Not needed since we use peer auth.
         # AUTHELIA_STORAGE_POSTGRES_PASSWORD_FILE = "/run/secrets/authelia/postgres_password";
-        AUTHELIA_STORAGE_ENCRYPTION_KEY_FILE = toString cfg.secrets.storageEncryptionKey.result.path;
+        AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE = lib.mkIf (!(builtins.isString cfg.smtp)) (
-        AUTHELIA_IDENTITY_PROVIDERS_OIDC_HMAC_SECRET_FILE = toString cfg.secrets.identityProvidersOIDCHMACSecret.result.path;
+          toString cfg.smtp.password.result.path
-        AUTHELIA_IDENTITY_PROVIDERS_OIDC_ISSUER_PRIVATE_KEY_FILE = toString cfg.secrets.identityProvidersOIDCIssuerPrivateKey.result.path;
+        );
-
+        X_AUTHELIA_CONFIG_FILTERS = "template";
-        AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE = lib.mkIf (!(builtins.isString cfg.smtp)) (toString cfg.smtp.password.result.path);
       };
       settings = {
         server.address = "tcp://127.0.0.1:${toString listenPort}";
@@ -423,10 +479,12 @@ in
         # Inspired from https://www.authelia.com/configuration/session/introduction/ and https://www.authelia.com/configuration/session/redis
         session = {
           name = "authelia_session";
-          cookies = [{
+          cookies = [
-            domain = if isNull cfg.port then cfg.domain else "${cfg.domain}:${toString cfg.port}";
+            {
-            authelia_url = "https://${cfg.subdomain}.${cfg.domain}";
+              domain = if isNull cfg.port then cfg.domain else "${cfg.domain}:${toString cfg.port}";
-          }];
+              authelia_url = "https://${cfg.subdomain}.${cfg.domain}";
+            }
+          ];
           same_site = "lax";
           expiration = "1h";
           inactivity = "5m";
@@ -450,8 +508,7 @@ in
             filename = cfg.smtp;
           };
           smtp = lib.mkIf (!(builtins.isString cfg.smtp)) {
-            host = cfg.smtp.host;
+            address = "${cfg.smtp.scheme}://${cfg.smtp.host}:${toString cfg.smtp.port}";
-            port = cfg.smtp.port;
             username = cfg.smtp.username;
             sender = "${cfg.smtp.from_name} <${cfg.smtp.from_address}>";
             subject = "[Authelia] {title}";
@@ -463,7 +520,11 @@ in
           networks = [
             {
               name = "internal";
-              networks = [ "10.0.0.0/8" "172.16.0.0/12" "192.168.0.0/18" ];
+              networks = [
+                "10.0.0.0/8"
+                "172.16.0.0/12"
+                "192.168.0.0/18"
+              ];
             }
           ];
           rules = [
@@ -474,7 +535,8 @@ in
                 "^/api/.*"
               ];
             }
-          ] ++ cfg.rules;
+          ]
+          ++ cfg.rules;
         };
         telemetry = {
           metrics = {
@@ -484,16 +546,25 @@ in
         };
 
         log.level = if cfg.debug then "debug" else "info";
-      } // {
+      }
+      // {
         identity_providers.oidc = {
           claims_policies = {
             # This default claim should go away at some point.
             # https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims/#restore-functionality-prior-to-claims-parameter
-            default.id_token = [ "email" "preferred_username" "name" "groups" ];
+            default.id_token = [
-          } // cfg.extraOidcClaimsPolicies;
+              "email"
+              "preferred_username"
+              "name"
+              "groups"
+            ];
+          }
+          // cfg.extraOidcClaimsPolicies;
           scopes = cfg.extraOidcScopes;
+          authorization_policies = cfg.extraOidcAuthorizationPolicies;
         };
-      } // lib.optionalAttrs (cfg.extraDefinitions != {}) {
+      }
+      // lib.optionalAttrs (cfg.extraDefinitions != { }) {
         definitions = cfg.extraDefinitions;
       };
 
@@ -502,18 +573,22 @@ in
 
     systemd.services."authelia-${fqdn}".preStart =
       let
-        mkCfg = clients:
+        mkCfg =
-          shblib.replaceSecrets {
+          clients:
+          shb.replaceSecrets {
             userConfig = {
               identity_providers.oidc.clients = clients;
             };
             resultPath = "/var/lib/authelia-${fqdn}/oidc_clients.yaml";
-            generator = shblib.replaceSecretsGeneratorAdapter (lib.generators.toYAML {});
+            generator = shb.replaceSecretsGeneratorAdapter (lib.generators.toYAML { });
           };
       in
-        lib.mkBefore (mkCfg cfg.oidcClients + ''
+      lib.mkBefore (
-        ${pkgs.bash}/bin/bash -c '(while ! ${pkgs.netcat-openbsd}/bin/nc -z -v -w1 ${cfg.ldapHostname} ${toString cfg.ldapPort}; do echo "Waiting for port ${cfg.ldapHostname}:${toString cfg.ldapPort} to open..."; sleep 2; done); sleep 2'
+        mkCfg cfg.oidcClients
-          '');
+        + ''
+          ${pkgs.bash}/bin/bash -c '(while ! ${pkgs.netcat-openbsd}/bin/nc -z -v -w1 ${cfg.ldapHostname} ${toString cfg.ldapPort}; do echo "Waiting for port ${cfg.ldapHostname}:${toString cfg.ldapPort} to open..."; sleep 2; done); sleep 2'
+        ''
+      );
 
     services.nginx.virtualHosts.${fqdn} = {
       forceSSL = !(isNull cfg.ssl);
@@ -548,7 +623,7 @@ in
             error_page 403 = /error/403;
             error_page 404 = /error/404;
         }
-        '';
+      '';
 
       locations."/api/verify".extraConfig = ''
         add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
@@ -561,7 +636,7 @@ in
 
         proxy_set_header Host $http_x_forwarded_host;
         proxy_pass http://127.0.0.1:9091;
-        '';
+      '';
     };
 
     # I would like this to live outside of the Authelia module.
@@ -570,10 +645,12 @@ in
     shb.mitmdump.instances."authelia-${fqdn}" = lib.mkIf cfg.debug {
       listenPort = 9091;
       upstreamPort = 9090;
+      timeout = 30;
       after = [ "authelia-${fqdn}.service" ];
       enabledAddons = [ config.shb.mitmdump.addons.logger ];
       extraArgs = [
-        "--set" "verbose_pattern=/api"
+        "--set"
+        "verbose_pattern=/api"
       ];
     };
 
@@ -594,7 +671,7 @@ in
         job_name = "authelia";
         static_configs = [
           {
-            targets = ["127.0.0.1:9959"];
+            targets = [ "127.0.0.1:9959" ];
             labels = {
               "hostname" = config.networking.hostName;
               "domain" = cfg.domain;
@@ -604,17 +681,20 @@ in
       }
     ];
 
-    systemd.targets."authelia-${fqdn}" = let
+    systemd.targets."authelia-${fqdn}" =
-      services = [
+      let
-        "authelia-${fqdn}.service"
+        services = [
-      ] ++ lib.optionals cfg.debug [
+          "authelia-${fqdn}.service"
-        config.shb.mitmdump.instances."authelia-${fqdn}".serviceName
+        ]
-      ];
+        ++ lib.optionals cfg.debug [
-    in {
+          config.shb.mitmdump.instances."authelia-${fqdn}".serviceName
-      after = services;
+        ];
-      requires = services;
+      in
+      {
+        after = services;
+        requires = services;
 
-      wantedBy = [ "multi-user.target" ];
+        wantedBy = [ "multi-user.target" ];
-    };
+      };
   };
 }

modules/blocks/authelia/docs/default.md

@@ -8,10 +8,17 @@ This block sets up an [Authelia][] service for Single-Sign On integration.
 
 Compared to the upstream nixpkgs module, this module is tightly integrated
 with SHB which allows easy configuration of SSO with [OIDC integration](#blocks-authelia-shb-oidc)
-or with [forward auth integration](#blocks-authelia-shb-forward-auth)
 as well as some extensive [troubleshooting](#blocks-authelia-troubleshooting) features.
 
-## Global Setup {#blocks-authelia-global-setup}
+Note that forward authentication is configured with the [nginx block](blocks-nginx.html#blocks-nginx-usage-shbforwardauth).
+
+## Features {#services-authelia-features}
+
+- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-authelia-usage-applicationdashboard)
+
+## Usage {#services-authelia-usage}
+
+### Initial Configuration {#blocks-authelia-usage-configuration}
 
 Authelia cannot work without SSL and LDAP.
 So setting up the Authelia block requires to setup the [SSL block][] first
@@ -43,31 +50,31 @@ shb.authelia = {
     port = 587;
     username = "postmaster@mg.example.com";
     from_address = "authelia@example.com";
-    password.result = config.shb.sops.secrets."authelia/smtp_password".result;
+    password.result = config.shb.sops.secret."authelia/smtp_password".result;
   };
 
   secrets = {
-    jwtSecret.result = config.shb.sops.secrets."authelia/jwt_secret".result;
+    jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
-    ldapAdminPassword.result = config.shb.sops.secrets."authelia/ldap_admin_password".result;
+    ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
-    sessionSecret.result = config.shb.sops.secrets."authelia/session_secret".result;
+    sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
-    storageEncryptionKey.result = config.shb.sops.secrets."authelia/storage_encryption_key".result;
+    storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
-    identityProvidersOIDCHMACSecret.result = config.shb.sops.secrets."authelia/hmac_secret".result;
+    identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
-    identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secrets."authelia/private_key".result;
+    identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
   };
 };
 
 shb.certs.certs.letsencrypt."example.com".extraDomains = [ "auth.example.com" ];
 
-shb.sops.secrets."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
+shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
-shb.sops.secrets."authelia/ldap_admin_password" = {
+shb.sops.secret."authelia/ldap_admin_password" = {
   request = config.shb.authelia.secrets.ldapAdminPassword.request;
   settings.key = "lldap/user_password";
 };
-shb.sops.secrets."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
+shb.sops.secret."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
-shb.sops.secrets."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
+shb.sops.secret."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
-shb.sops.secrets."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
+shb.sops.secret."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
-shb.sops.secrets."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
+shb.sops.secret."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
-shb.sops.secrets."authelia/smtp_password".request = config.shb.authelia.smtp.password.request;
+shb.sops.secret."authelia/smtp_password".request = config.shb.authelia.smtp.password.request;
 ```
 
 This assumes secrets are setup with SOPS
@@ -79,6 +86,22 @@ Crucially, the `shb.authelia.secrets.ldapAdminPasswordFile` must be the same
 as the `shb.lldap.ldapUserPassword` defined for the [LLDAP block][].
 This is done using Sops' `key` option.
 
+### Application Dashboard {#services-authelia-usage-applicationdashboard}
+
+Integration with the [dashboard contract](contracts-dashboard.html) is provided
+by the [dashboard option](#blocks-authelia-options-shb.authelia.dashboard).
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.Authelia = {
+    sortOrder = 2;
+    dashboard.request = config.shb.authelia.dashboard.request;
+  };
+}
+```
+
 ## SHB OIDC integration {#blocks-authelia-shb-oidc}
 
 For services [provided by SelfHostBlocks][services] that handle [OIDC integration][OIDC],
@@ -94,8 +117,8 @@ shb.<service>.sso = {
   enable = true;
   endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
 
-  secret.result = config.shb.sops.secrets."<service>/sso/secret".result;
+  secret.result = config.shb.sops.secret."<service>/sso/secret".result;
-  secretForAuthelia.result = config.shb.sops.secrets."<service>/sso/secretForAuthelia".result;
+  secretForAuthelia.result = config.shb.sops.secret."<service>/sso/secretForAuthelia".result;
 };
 
 shb.sops.secret."<service>/sso/secret".request = config.shb.<service>.sso.secret.request;
@@ -121,7 +144,7 @@ the necessary configuration is:
 shb.authelia.oidcClients = [
   {
     client_id = "<service>";
-    client_secret.source = shb.sops.secret."<service>/sso/secretForAuthelia".response.path;
+    client_secret.source = config.shb.sops.secret."<service>/sso/secretForAuthelia".response.path;
     scopes = [ "openid" "email" "profile" ];
     redirect_uris = [
       "<provided by service documentation>"
@@ -166,7 +189,7 @@ services.open-webui.environment = {
 shb.authelia.oidcClients = [
   {
     client_id = "open-webui";
-    client_secret.source = shb.sops.secret."open-webui/sso/secretForAuthelia".response.path;
+    client_secret.source = config.shb.sops.secret."open-webui/sso/secretForAuthelia".response.path;
     scopes = [ "openid" "email" "profile" ];
     redirect_uris = [
       "<provided by service documentation>"
@@ -191,40 +214,9 @@ Inspiration can be taken from SelfHostBlocks' source code.
 To access the UI, we will need to create an `open-webui_user` and
 `open-webui_admin` LDAP group and assign our user to it.
 
-## SHB Forward Auth {#blocks-authelia-shb-forward-auth}
-
-For services provided by SelfHostBlocks that do not handle [OIDC integration][OIDC],
-this block can provide [forward authentication][] which still allows the service to be protected by Authelia.
-
-The user could still be required to authenticate to the service itself,
-although some services can automatically users authorized by Authelia.
-
-[forward authentication]: https://doc.traefik.io/traefik/middlewares/http/forwardauth/
-
-Integrating with this block is done with the following code:
-
-```nix
-shb.<services>.authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
-```
-
 ## Forward Auth {#blocks-authelia-forward-auth}
 
-To integrate a service that does not handle OIDC integration
+Forward authentication is provided by the [nginx block](blocks-nginx.html#blocks-nginx-usage-ssl).
-and which is not provided by SelfHostBlocks with this Authelia block,
-the necessary configuration is:
-
-```nix
-shb.nginx.vhosts = [
-  {
-    subdomain = "<service>";
-    domain = "example.com";
-    ssl = config.shb.certs.certs.letsencrypt."example.com";
-    upstream = "http://127.0.0.1:${toString config.services.<service>.port}/";
-  }
-];
-```
-
-This configuration assumes usage of the [SSL block][].
 
 ## Troubleshooting {#blocks-authelia-troubleshooting}
 

modules/blocks/borgbackup.nix

@@ -1,145 +1,241 @@
-{ config, pkgs, lib, utils, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  utils,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.borgbackup;
 
-  instanceOptions = {
+  inherit (lib)
-    enable = lib.mkEnableOption "shb borgbackup";
+    concatStringsSep
+    filterAttrs
+    flatten
+    literalExpression
+    optionals
+    listToAttrs
+    mapAttrsToList
+    mkOption
+    mkMerge
+    ;
+  inherit (lib)
+    mkIf
+    nameValuePair
+    optionalAttrs
+    removePrefix
+    ;
+  inherit (lib.types)
+    attrsOf
+    int
+    oneOf
+    nonEmptyStr
+    nullOr
+    str
+    submodule
+    ;
 
-    keySopsFile = lib.mkOption {
+  commonOptions =
-      description = "Sops file that holds this instance's repository key and passphrase.";
+    {
-      type = lib.types.path;
+      name,
-      example = "secrets/backup.yaml";
+      prefix,
-    };
+      config,
+      ...
+    }:
+    {
+      enable = lib.mkEnableOption ''
+        SelfHostBlocks' BorgBackup block;
 
-    encryptionKeyFile = lib.mkOption {
+        A disabled instance will not backup data anymore
-      description = "Encryption key for the backup.";
+        but still provides the helper tool to restore snapshots
-      type = lib.types.path;
+      '';
-    };
 
-    encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${if isNull instance.secretName then name else instance.secretName}";
+      passphrase = lib.mkOption {
-    borg_keys_directory = "/run/secrets/borgmatic/keys";
+        description = "Encryption key for the backup repository.";
-
+        type = lib.types.submodule {
-    sourceDirectories = lib.mkOption {
+          options = shb.contracts.secret.mkRequester {
-      description = "Source directories.";
+            mode = "0400";
-      type = lib.types.nonEmptyListOf lib.types.str;
+            owner = config.request.user;
-    };
+            ownerText = "[shb.borgbackup.${prefix}.<name>.request.user](#blocks-borgbackup-options-shb.borgbackup.${prefix}._name_.request.user)";
-
+            restartUnits = [ (fullName name config.settings.repository) ];
-    excludePatterns = lib.mkOption {
+            restartUnitsText = "[ [shb.borgbackup.${prefix}.<name>.settings.repository](#blocks-borgbackup-options-shb.borgbackup.${prefix}._name_.settings.repository) ]";
-      description = "Exclude patterns.";
-      type = lib.types.listOf lib.types.str;
-      default = [];
-    };
-
-    secretName = lib.mkOption {
-      description = "Secret name, if null use the name of the backup instance.";
-      type = lib.types.nullOr lib.types.str;
-      default = null;
-    };
-
-    repositories = lib.mkOption {
-      description = "Repositories to back this instance to.";
-      type = lib.types.nonEmptyListOf (lib.types.submodule {
-        options = {
-          path = lib.mkOption {
-            type = lib.types.str;
-            description = "Repository location";
-          };
-
-          timerConfig = lib.mkOption {
-            type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption;
-            default = {
-              OnCalendar = "daily";
-              Persistent = true;
-            };
-            description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
-            example = {
-              OnCalendar = "00:05";
-              RandomizedDelaySec = "5h";
-              Persistent = true;
-            };
-          };
-        };
-      });
-    };
-
-    retention = lib.mkOption {
-      description = "Retention options.";
-      type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
-      default = {
-        keep_within = "1d";
-        keep_hourly = 24;
-        keep_daily = 7;
-        keep_weekly = 4;
-        keep_monthly = 6;
-      };
-    };
-
-    consistency = lib.mkOption {
-      description = "Consistency frequency options.";
-      type = lib.types.attrsOf lib.types.nonEmptyStr;
-      default = {};
-      example = {
-        repository = "2 weeks";
-        archives = "1 month";
-      };
-    };
-
-    hooks = lib.mkOption {
-      description = "Hooks to run before or after the backup.";
-      default = {};
-      type = lib.types.submodule {
-        options = {
-          beforeBackup = lib.mkOption {
-            description = "Hooks to run before backup";
-            type = lib.types.listOf lib.types.str;
-            default = [];
-          };
-
-          afterBackup = lib.mkOption {
-            description = "Hooks to run after backup";
-            type = lib.types.listOf lib.types.str;
-            default = [];
           };
         };
       };
+
+      repository = lib.mkOption {
+        description = "Repository to send the backups to.";
+        type = submodule {
+          options = {
+            path = mkOption {
+              type = str;
+              description = "Repository location";
+            };
+
+            secrets = mkOption {
+              type = attrsOf shb.secretFileType;
+              default = { };
+              description = ''
+                Secrets needed to access the repository where the backups will be stored.
+
+                See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+                and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
+
+              '';
+              example = literalExpression ''
+                {
+                  AWS_ACCESS_KEY_ID.source = <path/to/secret>;
+                  AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
+                }
+              '';
+            };
+
+            timerConfig = mkOption {
+              type = attrsOf utils.systemdUtils.unitOptions.unitOption;
+              default = {
+                OnCalendar = "daily";
+                Persistent = true;
+              };
+              description = "When to run the backup. See {manpage}`systemd.timer(5)` for details.";
+              example = {
+                OnCalendar = "00:05";
+                RandomizedDelaySec = "5h";
+                Persistent = true;
+              };
+            };
+          };
+        };
+      };
+
+      retention = lib.mkOption {
+        description = "Retention options. See {command}`borg help prune` for the available options.";
+        type = attrsOf (oneOf [
+          int
+          nonEmptyStr
+        ]);
+        default = {
+          within = "1d";
+          hourly = 24;
+          daily = 7;
+          weekly = 4;
+          monthly = 6;
+        };
+      };
+
+      consistency = lib.mkOption {
+        description = "Consistency frequency options.";
+        type = lib.types.attrsOf lib.types.nonEmptyStr;
+        default = { };
+        example = {
+          repository = "2 weeks";
+          archives = "1 month";
+        };
+      };
+
+      limitUploadKiBs = mkOption {
+        type = nullOr int;
+        description = "Limit upload bandwidth to the given KiB/s amount.";
+        default = null;
+        example = 8000;
+      };
+
+      stateDir = mkOption {
+        type = nullOr lib.types.str;
+        description = ''
+          Override the directory in which {command}`borg` stores its
+          configuration and cache. By default it uses the user's
+          home directory but is some cases this can cause conflicts.
+        '';
+        default = null;
+      };
     };
 
-    environmentFile = lib.mkOption {
+  repoSlugName = name: builtins.replaceStrings [ "/" ":" ] [ "_" "_" ] (removePrefix "/" name);
-      type = lib.types.bool;
+  fullName = name: repository: "borgbackup-job-${name}_${repoSlugName repository.path}";
-      description = "Add environment file to be read by the systemd service.";
-      default = false;
-      example = true;
-    };
-  };
-
-  repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name);
-
 in
 {
+  imports = [
+    ../../lib/module.nix
+    ../blocks/monitoring.nix
+  ];
+
   options.shb.borgbackup = {
-    user = lib.mkOption {
+    enableDashboard = lib.mkEnableOption "the Backups SHB dashboard" // {
-      description = "Unix user doing the backups.";
+      default = true;
-      type = lib.types.str;
-      default = "backup";
     };
 
-    group = lib.mkOption {
+    instances = mkOption {
-      description = "Unix group doing the backups.";
+      description = "Files to backup following the [backup contract](./shb.contracts-backup.html).";
-      type = lib.types.str;
+      default = { };
-      default = "backup";
+      type = attrsOf (
+        submodule (
+          { name, config, ... }:
+          {
+            options = shb.contracts.backup.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the BorgBackup provider.
+                '';
+
+                type = submodule {
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "instances";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
+            };
+          }
+        )
+      );
     };
 
-    instances = lib.mkOption {
+    databases = mkOption {
-      description = "Each instance is a backup setting";
+      description = "Databases to backup following the [database backup contract](./shb.contracts-databasebackup.html).";
-      default = {};
+      default = { };
-      type = lib.types.attrsOf (lib.types.submodule {
+      type = attrsOf (
-        options = instanceOptions;
+        submodule (
-      });
+          { name, config, ... }:
+          {
+            options = shb.contracts.databasebackup.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the BorgBackup provider.
+                '';
+
+                type = submodule {
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "databases";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
+            };
+          }
+        )
+      );
     };
 
     borgServer = lib.mkOption {
-      description = "Add borgbackup package so external backups can use this server as a remote.";
+      description = "Add borgbackup package to `environment.systemPackages` so external backups can use this server as a remote.";
       default = false;
       example = true;
       type = lib.types.bool;
@@ -148,7 +244,7 @@ in
     # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
     performance = lib.mkOption {
       description = "Reduce performance impact of backup jobs.";
-      default = {};
+      default = { };
       type = lib.types.submodule {
         options = {
           niceness = lib.mkOption {
@@ -157,7 +253,11 @@ in
             default = 15;
           };
           ioSchedulingClass = lib.mkOption {
-            type = lib.types.enum [ "idle" "best-effort" "realtime" ];
+            type = lib.types.enum [
+              "idle"
+              "best-effort"
+              "realtime"
+            ];
             description = "ionice scheduling class, defaults to best-effort IO.";
             default = "best-effort";
           };
@@ -171,125 +271,231 @@ in
     };
   };
 
-  config = lib.mkIf (cfg.instances != {}) (
+  config = lib.mkIf (cfg.instances != { } || cfg.databases != { }) (
     let
-      enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
+      enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances;
-    in lib.mkMerge [
+      enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases;
-      # Secrets configuration
+    in
+    lib.mkMerge [
       {
-        users.users = {
+        environment.systemPackages =
-          ${cfg.user} = {
+          optionals (cfg.borgServer || enabledInstances != { } || enabledDatabases != { })
-            name = cfg.user;
+            [
-            group = cfg.group;
+              pkgs.borgbackup
-            home = "/var/lib/backup";
+            ];
-            createHome = true;
-            isSystemUser = true;
-            extraGroups = [ "keys" ];
-          };
-        };
-        users.groups = {
-          ${cfg.group} = {
-            name = cfg.group;
-          };
-        };
-
-        sops.secrets =
-          let
-            mkSopsSecret = name: instance: (
-              [
-                {
-                  "${instance.backend}/passphrases/${if isNull instance.secretName then name else instance.secretName}" = {
-                    sopsFile = instance.keySopsFile;
-                    mode = "0440";
-                    owner = cfg.user;
-                    group = cfg.group;
-                  };
-                }
-              ] ++ lib.optional ((lib.filter ({path, ...}: lib.strings.hasPrefix "s3" path) instance.repositories) != []) {
-                "${instance.backend}/environmentfiles/${if isNull instance.secretName then name else instance.secretName}" = {
-                  sopsFile = instance.keySopsFile;
-                  mode = "0440";
-                  owner = cfg.user;
-                  group = cfg.group;
-                };
-              } ++ lib.optionals (instance.backend == "borgmatic") (lib.flatten (map ({path, ...}: {
-                "${instance.backend}/keys/${repoSlugName path}" = {
-                  key = "${instance.backend}/keys/${if isNull instance.secretName then name else instance.secretName}";
-                  sopsFile = instance.keySopsFile;
-                  mode = "0440";
-                  owner = cfg.user;
-                  group = cfg.group;
-                };
-              }) instance.repositories))
-            );
-          in
-            lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSopsSecret enabledInstances));
       }
-      # Borgmatic configuration
       {
-        systemd.timers.borgmatic = lib.mkIf (enabledInstances != {}) {
+        services.borgbackup.jobs =
-          timerConfig = {
-            OnCalendar = "hourly";
-          };
-        };
-
-        systemd.services.borgmatic = lib.mkIf (enabledInstances != {}) {
-          serviceConfig = {
-            User = cfg.user;
-            Group = cfg.group;
-            ExecStartPre = [ "" ]; # Do not sleep before starting.
-            ExecStart = [ "" "${pkgs.borgmatic}/bin/borgmatic --verbosity -1 --syslog-verbosity 1" ];
-            # For borgmatic, since we have only one service, we need to merge all environmentFile
-            # from all instances.
-            EnvironmentFile = lib.mapAttrsToList (name: value: value.environmentFile) enabledInstances;
-          };
-        };
-
-        systemd.packages = lib.mkIf (enabledInstances != {}) [ pkgs.borgmatic ];
-        environment.systemPackages = (
-          lib.optionals cfg.borgServer [ pkgs.borgbackup ]
-          ++ lib.optionals (enabledInstances != {}) [ pkgs.borgbackup pkgs.borgmatic ]
-        );
-
-        environment.etc =
           let
-            mkSettings = name: instance: {
+            mkJob = name: instance: {
-              "borgmatic.d/${name}.yaml".text = lib.generators.toYAML {} {
+              "${name}_${repoSlugName instance.settings.repository.path}" = {
-                location =
+                inherit (instance.request) user;
-                  {
-                    source_directories = instance.sourceDirectories;
-                    repositories = map ({path, ...}: path) instance.repositories;
-                  }
-                  // (lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
-                    excludePatterns = instance.excludePatterns;
-                  });
 
-                storage = {
+                repo = instance.settings.repository.path;
-                  encryption_passcommand = "cat ${instance.encryptionKeyFile}";
-                  borg_keys_directory = "/run/secrets/borgmatic/keys";
-                };
 
-                retention = instance.retention;
+                paths = instance.request.sourceDirectories;
-                consistency.checks =
-                  let
-                    mkCheck = name: frequency: {
-                      inherit name frequency;
-                    };
-                  in
-                    lib.attrsets.mapAttrsToList mkCheck instance.consistency;
 
-                # hooks = lib.mkMerge [
+                encryption.mode = "repokey-blake2";
-                #   lib.optionalAttrs (builtins.length instance.hooks.beforeBackup > 0) {
+                # We do not set encryption.passphrase here, we set BORG_PASSPHRASE_FD further down.
-                #     inherit (instance.hooks) beforeBackup;
+                encryption.passCommand = "cat ${instance.settings.passphrase.result.path}";
-                #   }
+
-                #   lib.optionalAttrs (builtins.length instance.hooks.afterBackup > 0) {
+                doInit = true;
-                #     inherit (instance.hooks) afterBackup;
+                failOnWarnings = true;
-                #   }
+                stateDir = instance.settings.stateDir;
-                # ];
+
+                persistentTimer = instance.settings.repository.timerConfig.Persistent or false;
+                startAt = ""; # Some non-empty string value tricks the upstream module in creating the systemd timer.
+
+                prune.keep = instance.settings.retention;
+
+                preHook = concatStringsSep "\n" instance.request.hooks.beforeBackup;
+
+                postHook = concatStringsSep "\n" instance.request.hooks.afterBackup;
+
+                extraArgs = (
+                  optionals (instance.settings.limitUploadKiBs != null) [
+                    "--upload-ratelimit=${toString instance.settings.limitUploadKiBs}"
+                  ]
+                );
+
+                exclude = instance.request.excludePatterns;
               };
             };
           in
-            lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings enabledInstances);
+          mkMerge (mapAttrsToList mkJob enabledInstances);
       }
-    ]);
+      {
+        services.borgbackup.jobs =
+          let
+            mkJob = name: instance: {
+              "${name}_${repoSlugName instance.settings.repository.path}" = {
+                inherit (instance.request) user;
+
+                repo = instance.settings.repository.path;
+
+                dumpCommand = lib.getExe (
+                  pkgs.writeShellApplication {
+                    name = "dump-command";
+                    text = instance.request.backupCmd;
+                  }
+                );
+
+                encryption.mode = "repokey-blake2";
+                # We do not set encryption.passphrase here, we set BORG_PASSPHRASE_FD further down.
+                encryption.passCommand = "cat ${instance.settings.passphrase.result.path}";
+
+                doInit = true;
+                failOnWarnings = true;
+                stateDir = instance.settings.stateDir;
+
+                persistentTimer = instance.settings.repository.timerConfig.Persistent or false;
+                startAt = ""; # Some non-empty list value that tricks upstream in creating the systemd timer.
+
+                prune.keep = instance.settings.retention;
+
+                extraArgs = (
+                  optionals (instance.settings.limitUploadKiBs != null) [
+                    "--upload-ratelimit=${toString instance.settings.limitUploadKiBs}"
+                  ]
+                );
+              };
+            };
+          in
+          mkMerge (mapAttrsToList mkJob enabledDatabases);
+      }
+      {
+        systemd.timers =
+          let
+            mkTimer = name: instance: {
+              ${fullName name instance.settings.repository} = {
+                timerConfig = lib.mkForce instance.settings.repository.timerConfig;
+              };
+            };
+          in
+          mkMerge (mapAttrsToList mkTimer (enabledInstances // enabledDatabases));
+      }
+      {
+        systemd.services =
+          let
+            mkSettings =
+              name: instance:
+              let
+                serviceName = fullName name instance.settings.repository;
+              in
+              {
+                ${serviceName} = mkMerge [
+                  {
+                    serviceConfig = {
+                      # Purposely not a oneshot systemd service otherwise
+                      # the service waits on the completion the backup before deactivating.
+                      # This seems like a nice property at first but there is one annoying
+                      # edge case when deploying. If a backup job somehow is started when
+                      # the deploy happens, the deploy will wait on the service to finish
+                      # before considering the deploy done. And worse, it will consider the
+                      # deploy as failed if the backup fails, which is not what you want.
+                      Nice = lib.mkForce cfg.performance.niceness;
+                      IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass;
+                      IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority;
+                      # BindReadOnlyPaths = instance.sourceDirectories;
+                    };
+                  }
+                  (optionalAttrs (instance.settings.repository.secrets != { }) {
+                    serviceConfig.EnvironmentFile = [
+                      "/run/secrets_borgbackup/${serviceName}"
+                    ];
+                    after = [ "${serviceName}-pre.service" ];
+                    requires = [ "${serviceName}-pre.service" ];
+                  })
+                ];
+
+                "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != { }) (
+                  let
+                    script = shb.genConfigOutOfBandSystemd {
+                      config = instance.settings.repository.secrets;
+                      configLocation = "/run/secrets_borgbackup/${serviceName}";
+                      generator = shb.toEnvVar;
+                      user = instance.request.user;
+                    };
+                  in
+                  {
+                    script = script.preStart;
+                    # Makes the systemd service wait for the backup to be done before changing state to inactive.
+                    serviceConfig.Type = "oneshot";
+                    serviceConfig.LoadCredential = script.loadCredentials;
+                  }
+                );
+              };
+          in
+          mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
+      }
+      {
+        systemd.services =
+          let
+            mkEnv =
+              name: instance:
+              nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
+                enable = true;
+                wantedBy = [ "multi-user.target" ];
+                serviceConfig.Type = "oneshot";
+                script = (
+                  shb.replaceSecrets {
+                    userConfig = instance.settings.repository.secrets // {
+                      BORG_PASSCOMMAND = ''"cat ${instance.settings.passphrase.result.path}"'';
+                      BORG_REPO = instance.settings.repository.path;
+                    };
+                    resultPath = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
+                    generator = shb.toEnvVar;
+                    user = instance.request.user;
+                  }
+                );
+              };
+          in
+          listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases)));
+      }
+      {
+        environment.systemPackages =
+          let
+            mkBorgBackupBinary =
+              name: instance:
+              shb.contracts.backup.mkRestoreScript {
+                name = fullName name instance.settings.repository;
+                user = instance.request.user;
+                sudoPreserveEnvs = [
+                  "BORG_REPO"
+                  "BORG_PASSCOMMAND"
+                ];
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
+                restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
+              };
+          in
+          flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
+      }
+      {
+        environment.systemPackages =
+          let
+            mkBorgBackupBinary =
+              name: instance:
+              shb.contracts.backup.mkRestoreScript {
+                name = fullName name instance.settings.repository;
+                user = instance.request.user;
+                sudoPreserveEnvs = [
+                  "BORG_REPO"
+                  "BORG_PASSCOMMAND"
+                ];
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
+                restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
+              };
+          in
+          flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);
+      }
+
+      (lib.mkIf (cfg.enableDashboard && (cfg.instances != { } || cfg.databases != { })) {
+        shb.monitoring.dashboards = [
+          ./backup/dashboard/Backups.json
+        ];
+      })
+    ]
+  );
 }

modules/blocks/borgbackup/docs/default.md

@@ -0,0 +1,302 @@
+# Borgbackup Block {#blocks-borgbackup}
+
+Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix).
+
+This block sets up a backup job using [BorgBackup][].
+It is heavily based on the nixpkgs BorgBackup module.
+
+[borgbackup]: https://www.borgbackup.org/
+
+## Provider Contracts {#blocks-borgbackup-contract-provider}
+
+This block provides the following contracts:
+
+- [backup contract](contracts-backup.html) under the [`shb.borgbackup.instances`][instances] option.
+  It is tested with [contract tests][backup contract tests].
+- [database backup contract](contracts-databasebackup.html) under the [`shb.borgbackup.databases`][databases] option.
+  It is tested with [contract tests][database backup contract tests].
+
+[instances]: #blocks-borgbackup-options-shb.borgbackup.instances
+[databases]: #blocks-borgbackup-options-shb.borgbackup.databases
+[backup contract tests]: @REPO@/test/contracts/backup.nix
+[database backup contract tests]: @REPO@/test/contracts/databasebackup.nix
+
+As requested by those two contracts, when setting up a backup with BorgBackup,
+a backup Systemd service and a [restore script](#blocks-borgbackup-maintenance) are provided.
+
+## Usage {#blocks-borgbackup-usage}
+
+The following examples assume usage of the [sops block][] to provide secrets
+although any blocks providing the [secrets contract][] works too.
+
+[sops block]: ./blocks-sops.html
+[secrets contract]: ./contracts-secrets.html
+
+### One folder backed up manually {#blocks-borgbackup-usage-provider-manual}
+
+The following snippet shows how to configure
+the backup of 1 folder to 1 repository.
+We assume that the folder `/var/lib/myfolder` of the service `myservice` must be backed up.
+
+```nix
+shb.borgbackup.instances."myservice" = {
+  request = {
+    user = "myservice";
+
+    sourceDirectories = [
+      "/var/lib/myfolder"
+    ];
+  };
+
+  settings = {
+    enable = true;
+
+    passphrase.result = config.shb.sops.secret."passphrase".result;
+
+    repository = {
+      path = "/srv/backups/myservice";
+      timerConfig = {
+        OnCalendar = "00:00:00";
+        RandomizedDelaySec = "3h";
+      };
+    };
+
+    retention = {
+      within = "1d";
+      hourly = 24;
+      daily = 7;
+      weekly = 4;
+      monthly = 6;
+    };
+  };
+};
+
+shb.sops.secret."passphrase".request =
+  config.shb.borgbackup.instances."myservice".settings.passphrase.request;
+```
+
+### One folder backed up with contract {#blocks-borgbackup-usage-provider-contract}
+
+With the same example as before but assuming the `myservice` service
+has a `myservice.backup` option that is a requester for the backup contract,
+the snippet above becomes:
+
+```nix
+shb.borgbackup.instances."myservice" = {
+  request = config.myservice.backup;
+
+  settings = {
+    enable = true;
+
+    passphrase.result = config.shb.sops.secret."passphrase".result;
+
+    repository = {
+      path = "/srv/backups/myservice";
+      timerConfig = {
+        OnCalendar = "00:00:00";
+        RandomizedDelaySec = "3h";
+      };
+    };
+
+    retention = {
+      within = "1d";
+      hourly = 24;
+      daily = 7;
+      weekly = 4;
+      monthly = 6;
+    };
+  };
+};
+
+shb.sops.secret."passphrase".request =
+  config.shb.borgbackup.instances."myservice".settings.passphrase.request;
+```
+
+### One folder backed up to S3 {#blocks-borgbackup-usage-provider-remote}
+
+Here we will only highlight the differences with the previous configuration.
+
+This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/).
+
+```diff
+  shb.test.backup.instances.myservice = {
+
+    repository = {
+-     path = "/srv/pool1/backups/myfolder";
++     path = "s3:s3.us-west-000.backblazeb2.com/backups/myfolder";
+      timerConfig = {
+        OnCalendar = "00:00:00";
+        RandomizedDelaySec = "3h";
+      };
+
++     extraSecrets = {
++       AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
++       AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
++     };
+    };
+  }
+```
+
+### Multiple directories to multiple destinations {#blocks-borgbackup-usage-multiple}
+
+The following snippet shows how to configure backup of any number of folders to 3 repositories,
+each happening at different times to avoid I/O contention.
+
+We will also make sure to be able to re-use as much as the configuration as possible.
+
+A few assumptions:
+- 2 hard drive pools used for backup are mounted respectively on `/srv/pool1` and `/srv/pool2`.
+- You have a backblaze account.
+
+First, let's define a variable to hold all the repositories we want to back up to:
+
+```nix
+repos = [
+  {
+    path = "/srv/pool1/backups";
+    timerConfig = {
+      OnCalendar = "00:00:00";
+      RandomizedDelaySec = "3h";
+    };
+  }
+  {
+    path = "/srv/pool2/backups";
+    timerConfig = {
+      OnCalendar = "08:00:00";
+      RandomizedDelaySec = "3h";
+    };
+  }
+  {
+    path = "s3:s3.us-west-000.backblazeb2.com/backups";
+    timerConfig = {
+      OnCalendar = "16:00:00";
+      RandomizedDelaySec = "3h";
+    };
+  }
+];
+```
+
+Compared to the previous examples, we do not include the name of what we will back up in the
+repository paths.
+
+Now, let's define a function to create a backup configuration. It will take a list of repositories,
+a name identifying the backup and a list of folders to back up.
+
+```nix
+backupcfg = repositories: name: sourceDirectories {
+  enable = true;
+
+  backend = "borgbackup";
+
+  keySopsFile = ../secrets/backup.yaml;
+
+  repositories = builtins.map (r: {
+    path = "${r.path}/${name}";
+    inherit (r) timerConfig;
+  }) repositories;
+
+  inherit sourceDirectories;
+
+  retention = {
+    within = "1d";
+    hourly = 24;
+    daily = 7;
+    weekly = 4;
+    monthly = 6;
+  };
+
+  environmentFile = true;
+};
+```
+
+Now, we can define multiple backup jobs to backup different folders:
+
+```nix
+shb.test.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"];
+shb.test.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"];
+```
+
+The difference between the above snippet and putting all the folders into one configuration (shown
+below) is the former splits the backups into sub-folders on the repositories.
+
+```nix
+shb.test.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
+```
+
+## Monitoring {#blocks-borgbackup-monitoring}
+
+A generic dashboard for all backup solutions is provided.
+See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backup) section in the monitoring chapter.
+
+## Maintenance {#blocks-borgbackup-maintenance}
+
+### Manual Backup {#blocks-borgbackup-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'borgbackup*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start borgbackup<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-borgbackup-maintenance-restore}
+
+One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
+
+The restore script has all the secrets needed to access the repo,
+it will run `sudo` automatically
+and the user running it needs to have correct permissions for privilege escalation
+
+In the [multiple directories example](#blocks-borgbackup-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
+
+```bash
+borgbackup-job-myfolder1_srv_pool1_backups
+borgbackup-job-myfolder1_srv_pool2_backups
+borgbackup-job-myfolder1_s3_s3.us-west-000.backblazeb2.com_backups
+borgbackup-job-myfolder2_srv_pool1_backups
+borgbackup-job-myfolder2_srv_pool2_backups
+borgbackup-job-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups
+```
+
+Discovering those is easy thanks to tab-completion.
+
+One can then restore a backup from a given repository with:
+
+```bash
+borgbackup-job-myfolder1_srv_pool1_backups restore latest
+```
+
+### Troubleshooting {#blocks-borgbackup-maintenance-troubleshooting}
+
+In case something bad happens with a backup, the [official documentation](https://borgbackup.readthedocs.io/en/stable/077_troubleshooting.html) has a lot of tips.
+
+## Tests {#blocks-borgbackup-tests}
+
+Specific integration tests are defined in [`/test/blocks/borgbackup.nix`](@REPO@/test/blocks/borgbackup.nix).
+
+## Options Reference {#blocks-borgbackup-options}
+
+```{=include=} options
+id-prefix: blocks-borgbackup-options-
+list-id: selfhostblocks-block-borgbackup-options
+source: @OPTIONS_JSON@
+```

modules/blocks/davfs.nix

@@ -1,80 +1,104 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, ... }:
 
 let
   cfg = config.shb.davfs;
-
-  shblib = pkgs.callPackage ../../lib {};
 in
 {
   options.shb.davfs = {
     mounts = lib.mkOption {
       description = "List of mounts.";
-      default = [];
+      default = [ ];
-      type = lib.types.listOf (lib.types.submodule {
+      type = lib.types.listOf (
-        options = {
+        lib.types.submodule {
-          remoteUrl = lib.mkOption {
+          options = {
-            type = lib.types.str;
+            remoteUrl = lib.mkOption {
-            description = "Webdav endpoint to connect to.";
+              type = lib.types.str;
-            example = "https://my.domain.com/dav";
+              description = "Webdav endpoint to connect to.";
-          };
+              example = "https://my.domain.com/dav";
+            };
 
-          mountPoint = lib.mkOption {
+            mountPoint = lib.mkOption {
-            type = lib.types.str;
+              type = lib.types.str;
-            description = "Mount point to mount the webdav endpoint on.";
+              description = "Mount point to mount the webdav endpoint on.";
-            example = "/mnt";
+              example = "/mnt";
-          };
+            };
 
-          username = lib.mkOption {
+            username = lib.mkOption {
-            type = lib.types.str;
+              type = lib.types.str;
-            description = "Username to connect to the webdav endpoint.";
+              description = "Username to connect to the webdav endpoint.";
-          };
+            };
 
-          passwordFile = lib.mkOption {
+            passwordFile = lib.mkOption {
-            type = lib.types.str;
+              type = lib.types.str;
-            description = "Password to connect to the webdav endpoint.";
+              description = "Password to connect to the webdav endpoint.";
-          };
+            };
 
-          uid = lib.mkOption {
+            uid = lib.mkOption {
-            type = lib.types.nullOr lib.types.int;
+              type = lib.types.nullOr lib.types.int;
-            description = "User owner of the mount point.";
+              description = "User owner of the mount point.";
-            example = 1000;
+              example = 1000;
-            default = null;
+              default = null;
-          };
+            };
 
-          gid = lib.mkOption {
+            gid = lib.mkOption {
-            type = lib.types.nullOr lib.types.int;
+              type = lib.types.nullOr lib.types.int;
-            description = "Group owner of the mount point.";
+              description = "Group owner of the mount point.";
-            example = 1000;
+              example = 1000;
-            default = null;
+              default = null;
-          };
+            };
 
-          fileMode = lib.mkOption {
+            fileMode = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
+              type = lib.types.nullOr lib.types.str;
-            description = "File creation mode";
+              description = "File creation mode";
-            example = "0664";
+              example = "0664";
-            default = null;
+              default = null;
-          };
+            };
 
-          directoryMode = lib.mkOption {
+            directoryMode = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
+              type = lib.types.nullOr lib.types.str;
-            description = "Directory creation mode";
+              description = "Directory creation mode";
-            example = "2775";
+              example = "2775";
-            default = null;
+              default = null;
-          };
+            };
 
-          automount = lib.mkOption {
+            automount = lib.mkOption {
-            type = lib.types.bool;
+              type = lib.types.bool;
-            description = "Create a systemd automount unit";
+              description = "Create a systemd automount unit";
-            default = true;
+              default = true;
+            };
           };
-        };
+        }
-      });
+      );
     };
   };
 
   config = {
     services.davfs2.enable = builtins.length cfg.mounts > 0;
 
+    systemd.services = lib.optionalAttrs (builtins.length cfg.mounts > 0) {
+      davfs2-password-generation = {
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig.Type = "oneshot";
+        script = ''
+          mkdir -p /etc/davfs2
+          [ -f /etc/davfs2/secrets ] && mv /etc/davfs2/secrets /etc/davfs2/secrets.prev
+          touch /etc/davfs2/secrets
+          chown root: /etc/davfs2
+          chown root: /etc/davfs2/secrets
+          chmod 700 /etc/davfs2
+          chmod 600 /etc/davfs2/secrets
+        ''
+        + (
+          let
+            mkPasswordCmd = cfg': ''
+              printf "%s %s %s\n" "${cfg'.mountPoint}" "${cfg'.username}" "$(cat ${cfg'.passwordFile})" >> /etc/davfs2/secrets
+            '';
+          in
+          lib.concatStringsSep "\n" (map mkPasswordCmd cfg.mounts)
+        );
+      };
+    };
+
     systemd.mounts =
       let
         mkMountCfg = c: {
@@ -82,6 +106,7 @@ in
           description = "Webdav mount point";
           after = [ "network-online.target" ];
           wants = [ "network-online.target" ];
+          wantedBy = [ "multi-user.target" ];
 
           what = c.remoteUrl;
           where = c.mountPoint;
@@ -95,6 +120,6 @@ in
           mountConfig.TimeoutSec = 15;
         };
       in
-        map mkMountCfg cfg.mounts;
+      map mkMountCfg cfg.mounts;
   };
 }

modules/blocks/hardcodedsecret.nix

@@ -1,84 +1,105 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  ...
+}:
 let
   cfg = config.shb.hardcodedsecret;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   inherit (lib) mapAttrs' mkOption nameValuePair;
-  inherit (lib.types) attrsOf nullOr str submodule;
+  inherit (lib.types)
+    attrsOf
+    nullOr
+    str
+    submodule
+    ;
   inherit (pkgs) writeText;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.hardcodedsecret = mkOption {
-    default = {};
+    default = { };
     description = ''
       Hardcoded secrets. These should only be used in tests.
     '';
     example = lib.literalExpression ''
-    {
+      {
-      mySecret = {
+        mySecret = {
-        request = {
+          request = {
-          user = "me";
+            user = "me";
-          mode = "0400";
+            mode = "0400";
-          restartUnits = [ "myservice.service" ];
+            restartUnits = [ "myservice.service" ];
+          };
+          settings.content = "My Secret";
         };
-        settings.content = "My Secret";
+      }
-      };
-    }
     '';
-    type = attrsOf (submodule ({ name, ... }: {
+    type = attrsOf (
-      options = contracts.secret.mkProvider {
+      submodule (
-        settings = mkOption {
+        { name, ... }:
-          description = ''
+        {
-            Settings specific to the hardcoded secret module.
+          options = shb.contracts.secret.mkProvider {
+            settings = mkOption {
+              description = ''
+                Settings specific to the hardcoded secret module.
 
-            Give either `content` or `source`.
+                Give either `content` or `source`.
-          '';
+              '';
 
-          type = submodule {
+              type = submodule {
-            options = {
+                options = {
-              content = mkOption {
+                  content = mkOption {
-                type = nullOr str;
+                    type = nullOr str;
-                description = ''
+                    description = ''
-                  Content of the secret as a string.
+                      Content of the secret as a string.
 
-                  This will be stored in the nix store and should only be used for testing or maybe in dev.
+                      This will be stored in the nix store and should only be used for testing or maybe in dev.
-                '';
+                    '';
-                default = null;
+                    default = null;
-              };
+                  };
 
-              source = mkOption {
+                  source = mkOption {
-                type = nullOr str;
+                    type = nullOr str;
-                description = ''
+                    description = ''
-                  Source of the content of the secret as a path in the nix store.
+                      Source of the content of the secret as a path in the nix store.
-                '';
+                    '';
-                default = null;
+                    default = null;
+                  };
+                };
               };
             };
-          };
-        };
 
-        resultCfg = {
+            resultCfg = {
-          path = "/run/hardcodedsecrets/hardcodedsecret_${name}";
+              path = "/run/hardcodedsecrets/hardcodedsecret_${name}";
-        };
+            };
-      };
+          };
-    }));
+        }
+      )
+    );
   };
 
   config = {
-    system.activationScripts = mapAttrs' (n: cfg':
+    system.activationScripts = mapAttrs' (
+      n: cfg':
       let
-        source = if cfg'.settings.source != null
+        source =
-                 then cfg'.settings.source
+          if cfg'.settings.source != null then
-                 else writeText "hardcodedsecret_${n}_content" cfg'.settings.content;
+            cfg'.settings.source
+          else
+            writeText "hardcodedsecret_${n}_content" cfg'.settings.content;
       in
-        nameValuePair "hardcodedsecret_${n}" ''
+      nameValuePair "hardcodedsecret_${n}" ''
-          mkdir -p "$(dirname "${cfg'.result.path}")"
+        mkdir -p "$(dirname "${cfg'.result.path}")"
-          touch "${cfg'.result.path}"
+        touch "${cfg'.result.path}"
-          chmod ${cfg'.request.mode} "${cfg'.result.path}"
+        chmod ${cfg'.request.mode} "${cfg'.result.path}"
-          chown ${cfg'.request.owner}:${cfg'.request.group} "${cfg'.result.path}"
+        chown ${cfg'.request.owner}:${cfg'.request.group} "${cfg'.result.path}"
-          cp ${source} "${cfg'.result.path}"
+        cp ${source} "${cfg'.result.path}"
-        ''
+      ''
     ) cfg;
   };
 }

modules/blocks/lldap.nix

@@ -1,10 +1,14 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.lldap;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   fqdn = "${cfg.subdomain}.${cfg.domain}";
 
   inherit (lib) mkOption types;
@@ -49,7 +53,7 @@ let
 in
 {
   imports = [
-    ./hardcodedsecret.nix
+    ../../lib/module.nix
     ./mitmdump.nix
 
     (lib.mkRenamedOptionModule [ "shb" "ldap" ] [ "shb" "lldap" ])
@@ -84,7 +88,7 @@ in
 
     ssl = lib.mkOption {
       description = "Path to SSL files";
-      type = lib.types.nullOr contracts.ssl.certs;
+      type = lib.types.nullOr shb.contracts.ssl.certs;
       default = null;
     };
 
@@ -95,9 +99,9 @@ in
     };
 
     ldapUserPassword = lib.mkOption {
-      description = "LDAP admin user secret.";
+      description = "LDAP admin user secret. Must be >= 8 characters.";
       type = lib.types.submodule {
-        options = contracts.secret.mkRequester {
+        options = shb.contracts.secret.mkRequester {
           mode = "0440";
           owner = "lldap";
           group = "lldap";
@@ -109,7 +113,7 @@ in
     jwtSecret = lib.mkOption {
       description = "JWT secret.";
       type = lib.types.submodule {
-        options = contracts.secret.mkRequester {
+        options = shb.contracts.secret.mkRequester {
           mode = "0440";
           owner = "lldap";
           group = "lldap";
@@ -132,7 +136,7 @@ in
     };
 
     mount = lib.mkOption {
-      type = contracts.mount;
+      type = shb.contracts.mount;
       description = ''
         Mount configuration. This is an output option.
 
@@ -146,7 +150,9 @@ in
         ```
       '';
       readOnly = true;
-      default = { path = "/var/lib/lldap"; };
+      default = {
+        path = "/var/lib/lldap";
+      };
     };
 
     backup = lib.mkOption {
@@ -154,7 +160,7 @@ in
         Backup configuration.
       '';
       type = lib.types.submodule {
-        options = contracts.backup.mkRequester {
+        options = shb.contracts.backup.mkRequester {
           # TODO: is there a workaround that avoid needing to use root?
           # root because otherwise we cannot access the private StateDiretory
           user = "root";
@@ -197,7 +203,7 @@ in
               password = mkOption {
                 description = "Password.";
                 type = lib.types.submodule {
-                  options = contracts.secret.mkRequester {
+                  options = shb.contracts.secret.mkRequester {
                     mode = "0440";
                     owner = "lldap";
                     group = "lldap";
@@ -325,10 +331,24 @@ in
     enforceGroups = mkOption {
       description = "Remove groups not set declaratively.";
       type = types.bool;
-      default = true;
+      default = false;
+    };
+
+    dashboard = lib.mkOption {
+      description = ''
+        Dashboard contract consumer
+      '';
+      default = { };
+      type = lib.types.submodule {
+        options = shb.contracts.dashboard.mkRequester {
+          externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
+          externalUrlText = "https://\${config.shb.lldap.subdomain}.\${config.shb.lldap.domain}";
+          internalUrl = "http://127.0.0.1:${toString cfg.webUIListenPort}";
+        };
+      };
     };
   };
-  
+
   config = lib.mkIf cfg.enable {
 
     services.nginx = {
@@ -341,10 +361,16 @@ in
         locations."/" = {
           extraConfig = ''
             proxy_set_header Host $host;
-          '' + (if isNull cfg.restrictAccessIPRange then "" else ''
+          ''
-            allow ${cfg.restrictAccessIPRange};
+          + (
-            deny all;
+            if isNull cfg.restrictAccessIPRange then
-          '');
+              ""
+            else
+              ''
+                allow ${cfg.restrictAccessIPRange};
+                deny all;
+              ''
+          );
           proxyPass = "http://${toString config.services.lldap.settings.http_host}:${toString config.shb.lldap.webUIListenPort}/";
         };
       };
@@ -355,7 +381,7 @@ in
       group = "lldap";
       isSystemUser = true;
     };
-    users.groups.lldap = {};
+    users.groups.lldap = { };
 
     services.lldap = {
       enable = true;
@@ -383,9 +409,48 @@ in
       };
 
       inherit (cfg) ensureGroups ensureUserFields ensureGroupFields;
-      ensureUsers = lib.mapAttrs (n: v: (lib.removeAttrs v [ "password" ]) // {
+      ensureUsers = lib.mapAttrs (
-        "password_file" = toString v.password.result.path;
+        n: v:
-      }) cfg.ensureUsers;
+        (lib.removeAttrs v [ "password" ])
+        // {
+          "password_file" = toString v.password.result.path;
+        }
+      ) cfg.ensureUsers;
+    };
+
+    # Harden lldap following https://github.com/NixOS/nixpkgs/pull/487933
+    systemd.services.lldap.serviceConfig = {
+      RemoveIPC = true;
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      RestrictAddressFamilies = [
+        "AF_UNIX"
+        "AF_INET"
+        "AF_INET6"
+      ];
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+        "~@resources"
+      ];
+      SystemCallArchitectures = "native";
+      CapabilityBoundingSet = "";
+      LockPersonality = true;
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      PrivateDevices = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectSystem = "strict";
+      ProtectProc = "invisible";
+      ProcSubset = "pid";
+      MemoryDenyWriteExecute = true;
     };
 
     shb.mitmdump.instances."lldap-web" = lib.mkIf cfg.debug {
@@ -394,7 +459,8 @@ in
       after = [ "lldap.service" ];
       enabledAddons = [ config.shb.mitmdump.addons.logger ];
       extraArgs = [
-        "--set" "verbose_pattern=/api"
+        "--set"
+        "verbose_pattern=/api"
       ];
     };
   };

modules/blocks/lldap/docs/default.md

@@ -7,7 +7,13 @@ across services.
 
 [LLDAP]: https://github.com/lldap/lldap
 
-## Global Setup {#blocks-lldap-global-setup}
+## Features {#blocks-lldap-features}
+
+- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#blocks-lldap-usage-applicationdashboard)
+
+## Usage {#blocks-lldap-usage}
+
+### Initial Configuration {#blocks-lldap-usage-configuration}
 
 ```nix
 shb.lldap = {
@@ -29,7 +35,7 @@ shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPasswor
 This assumes secrets are setup with SOPS
 as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual.
 
-## SSL {#blocks-lldap-ssl}
+### SSL {#blocks-lldap-usage-ssl}
 
 Using SSL is an important security practice, like always.
 Using the [SSL block][], the configuration to add to the one above is:
@@ -44,7 +50,7 @@ shb.certs.certs.letsencrypt.${domain}.extraDomains = [
 shb.lldap.ssl = config.shb.certs.certs.letsencrypt.${config.shb.lldap.domain};
 ```
 
-## Restrict Access By IP {#blocks-lldap-restrict-access-by-ip}
+### Restrict Access By IP {#blocks-lldap-usage-restrict-access-by-ip}
 
 For added security, you can restrict access to the LLDAP UI
 by adding the following line:
@@ -53,6 +59,22 @@ by adding the following line:
 shb.lldap.restrictAccessIPRange = "192.168.50.0/24";
 ```
 
+### Application Dashboard {#blocks-lldap-usage-applicationdashboard}
+
+Integration with the [dashboard contract](contracts-dashboard.html) is provided
+by the [dashboard option](#blocks-lldap-options-shb.lldap.dashboard).
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.LLDAP = {
+    sortOrder = 2;
+    dashboard.request = config.shb.lldap.dashboard.request;
+  };
+}
+```
+
 ## Manage Groups {#blocks-lldap-manage-groups}
 
 The following snippet will create group named "family" if it does not exist yet.
@@ -61,7 +83,7 @@ Also, all other groups will be deleted and only the "family" group will remain.
 Note that the `lldap_admin`, `lldap_password_manager` and `lldap_strict_readonly` groups, which are internal to LLDAP, will always exist.
 
 If you want existing groups not declared in the `shb.lldap.ensureGroups` to be deleted,
-set [`shb.lldap.enforceGroups`](#blocks-lldap-options-shb.lldap.enforceGroups) to `false`.
+set [`shb.lldap.enforceGroups`](#blocks-lldap-options-shb.lldap.enforceGroups) to `true`.
 
 ```nix
 {
@@ -142,7 +164,7 @@ set [`shb.lldap.enforceUserMemberships`](#blocks-lldap-options-shb.lldap.enforce
   };
 
   shb.sops.secret."dad".request =
-    shb.lldap.ensureUsers.dad.password.request;
+    config.shb.lldap.ensureUsers.dad.password.request;
 }
 ```
 

modules/blocks/mitmdump.nix

@@ -1,143 +1,165 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
-  inherit (lib) mapAttrs' mkOption nameValuePair types;
+  inherit (lib)
-  inherit (types) attrsOf listOf port submodule str;
+    mapAttrs'
+    mkOption
+    nameValuePair
+    types
+    ;
+  inherit (types)
+    attrsOf
+    listOf
+    port
+    submodule
+    str
+    ;
 
   cfg = config.shb.mitmdump;
 
-  mitmdumpScript = pkgs.writers.writePython3Bin "mitmdump"
+  mitmdumpScript =
-    {
+    pkgs.writers.writePython3Bin "mitmdump"
-      libraries = let
+      {
-        p = pkgs.python3Packages;
+        libraries =
-      in [
+          let
-        p.systemd
+            p = pkgs.python3Packages;
-        p.mitmproxy
+          in
-      ];
-      flakeIgnore = [ "E501" ];
-    }
-    ''
-      from systemd.daemon import notify
-      import argparse
-      import logging
-      import os
-      import subprocess
-      import socket
-      import sys
-      import time
-
-
-      logging.basicConfig(level=logging.INFO, format='%(message)s')
-
-
-      def wait_for_port(host, port, timeout=10):
-          deadline = time.time() + timeout
-          while time.time() < deadline:
-              try:
-                  with socket.create_connection((host, port), timeout=0.5):
-                      return True
-              except Exception:
-                  time.sleep(0.1)
-          return False
-
-
-      def flatten(xss):
-          return [x for xs in xss for x in xs]
-
-
-      parser = argparse.ArgumentParser()
-      parser.add_argument("--listen_host", default="127.0.0.1", help="Host mitmdump will listen on")
-      parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
-      parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
-      parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
-      args, rest = parser.parse_known_args()
-
-      MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
-      if MITMDUMP_BIN is None:
-          raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
-
-      logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
-      wait_for_port(args.upstream_host, args.upstream_port, timeout=10)
-      logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
-
-      proc = subprocess.Popen(
           [
-              MITMDUMP_BIN,
+            p.systemd-python
-              "--listen-host", args.listen_host,
+            p.mitmproxy
-              "-p", args.listen_port,
+          ];
-              "--mode", f"reverse:{args.upstream_host}:{args.upstream_port}",
+        flakeIgnore = [ "E501" ];
-          ] + rest,
+      }
-          stdout=sys.stdout,
+      ''
-          stderr=sys.stderr,
+        from systemd.daemon import notify
-      )
+        import argparse
-
+        import logging
-      logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
+        import os
-      if wait_for_port("127.0.0.1", args.listen_port, timeout=10):
+        import subprocess
-          logging.info(f"Mitmdump is started on port {args.listen_port}.")
+        import socket
-          notify("READY=1")
+        import sys
-      else:
+        import time
-          proc.terminate()
-          exit(1)
-
-      proc.wait()
-    '';
-
-  logger = toString (pkgs.writers.writeText "loggerAddon.py"
-    ''
-    import logging
-    from collections.abc import Sequence
-    from mitmproxy import ctx, http
-    import re
 
 
-    logger = logging.getLogger(__name__)
+        logging.basicConfig(level=logging.INFO, format='%(message)s')
 
 
-    class RegexLogger:
+        def wait_for_port(host, port, timeout):
-        def __init__(self):
+            deadline = time.time() + timeout
-            self.verbose_patterns = None
+            while time.time() < deadline:
-
+                try:
-        def load(self, loader):
+                    with socket.create_connection((host, port), timeout=0.5):
-            loader.add_option(
+                        return True
-                name="verbose_pattern",
+                except Exception:
-                typespec=Sequence[str],
+                    time.sleep(0.1)
-                default=[],
+            return False
-                help="Regex patterns for verbose logging",
-            )
-
-        def response(self, flow: http.HTTPFlow):
-            if self.verbose_patterns is None:
-                self.verbose_patterns = [re.compile(p) for p in ctx.options.verbose_pattern]
-
-            matched = any(p.search(flow.request.path) for p in self.verbose_patterns)
-            if matched:
-                logger.info(format_flow(flow))
 
 
-    def format_flow(flow: http.HTTPFlow) -> str:
+        def flatten(xss):
-        return (
+            return [x for xs in xss for x in xs]
-            "\n"
+
-            "RequestHeaders:\n"
+
-            f"    {format_headers(flow.request.headers.items())}\n"
+        parser = argparse.ArgumentParser()
-            f"RequestBody:     {flow.request.get_text()}\n"
+        parser.add_argument("--listen_host", default="127.0.0.1", help="Host mitmdump will listen on")
-            f"Status:          {flow.response.data.status_code}\n"
+        parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
-            "ResponseHeaders:\n"
+        parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
-            f"    {format_headers(flow.response.headers.items())}\n"
+        parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
-            f"ResponseBody:    {flow.response.get_text()}\n"
+        parser.add_argument("--timeout", required=False, type=int, default=10, help="Timeout to wait for port availability")
+        args, rest = parser.parse_known_args()
+
+        MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
+        if MITMDUMP_BIN is None:
+            raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
+
+        logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
+        wait_for_port(args.upstream_host, args.upstream_port, timeout=args.timeout)
+        logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
+
+        proc = subprocess.Popen(
+            [
+                MITMDUMP_BIN,
+                "--listen-host", args.listen_host,
+                "-p", args.listen_port,
+                "--mode", f"reverse:{args.upstream_host}:{args.upstream_port}",
+            ] + rest,
+            stdout=sys.stdout,
+            stderr=sys.stderr,
         )
 
+        logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
+        if wait_for_port("127.0.0.1", args.listen_port, timeout=args.timeout):
+            logging.info(f"Mitmdump is started on port {args.listen_port}.")
+            notify("READY=1")
+        else:
+            print(f"Mitmdump instance did not start before the timeout of {args.timeout} seconds, consider increasing the timeout")
+            proc.terminate()
+            exit(1)
 
-    def format_headers(headers) -> str:
+        proc.wait()
-        return "\n    ".join(k + ": " + v for k, v in headers)
+      '';
+
+  logger = toString (
+    pkgs.writers.writeText "loggerAddon.py" ''
+      import logging
+      from collections.abc import Sequence
+      from mitmproxy import ctx, http
+      import re
 
 
-    addons = [RegexLogger()]
+      logger = logging.getLogger(__name__)
-  '');
+
+
+      class RegexLogger:
+          def __init__(self):
+              self.verbose_patterns = None
+
+          def load(self, loader):
+              loader.add_option(
+                  name="verbose_pattern",
+                  typespec=Sequence[str],
+                  default=[],
+                  help="Regex patterns for verbose logging",
+              )
+
+          def response(self, flow: http.HTTPFlow):
+              if self.verbose_patterns is None:
+                  self.verbose_patterns = [re.compile(p) for p in ctx.options.verbose_pattern]
+
+              matched = any(p.search(flow.request.path) for p in self.verbose_patterns)
+              if matched:
+                  logger.info(format_flow(flow))
+
+
+      def format_flow(flow: http.HTTPFlow) -> str:
+          return (
+              "\n"
+              "RequestHeaders:\n"
+              f"    {format_headers(flow.request.headers.items())}\n"
+              f"RequestBody:     {flow.request.get_text()}\n"
+              f"Status:          {flow.response.data.status_code}\n"
+              "ResponseHeaders:\n"
+              f"    {format_headers(flow.response.headers.items())}\n"
+              f"ResponseBody:    {flow.response.get_text()}\n"
+          )
+
+
+      def format_headers(headers) -> str:
+          return "\n    ".join(k + ": " + v for k, v in headers)
+
+
+      addons = [RegexLogger()]
+    ''
+  );
 in
 {
   options.shb.mitmdump = {
     addons = mkOption {
       type = attrsOf str;
-      default = [];
+      default = [ ];
       description = ''
         Addons available to the be added to the mitmdump instance.
 
@@ -146,120 +168,137 @@ in
     };
 
     instances = mkOption {
-      default = {};
+      default = { };
       description = "Mitmdump instance.";
-      type = attrsOf (submodule ({ name, ... }: {
+      type = attrsOf (
-        options = {
+        submodule (
-          package = lib.mkPackageOption pkgs "mitmproxy" {};
+          { name, ... }:
+          {
+            options = {
+              package = lib.mkPackageOption pkgs "mitmproxy" { };
 
-          serviceName = mkOption {
+              serviceName = mkOption {
-            type = str;
+                type = str;
-            description = ''
+                description = ''
-              Name of the mitmdump system service.
+                  Name of the mitmdump system service.
-            '';
+                '';
-            default = "mitmdump-${name}.service";
+                default = "mitmdump-${name}.service";
-            readOnly = true;
+                readOnly = true;
-          };
+              };
 
-          listenHost = mkOption {
+              listenHost = mkOption {
-            type = str;
+                type = str;
-            default = "127.0.0.1";
+                default = "127.0.0.1";
-            description = ''
+                description = ''
-              Host the mitmdump instance will connect on.
+                  Host the mitmdump instance will connect on.
-            '';
+                '';
-          };
+              };
 
-          listenPort = mkOption {
+              listenPort = mkOption {
-            type = port;
+                type = port;
-            description = ''
+                description = ''
-              Port the mitmdump instance will listen on.
+                  Port the mitmdump instance will listen on.
 
-              The upstream port from the client's perspective.
+                  The upstream port from the client's perspective.
-            '';
+                '';
-          };
+              };
 
-          upstreamHost = mkOption {
+              upstreamHost = mkOption {
-            type = str;
+                type = str;
-            default = "http://127.0.0.1";
+                default = "http://127.0.0.1";
-            description = ''
+                description = ''
-              Host the mitmdump instance will connect to.
+                  Host the mitmdump instance will connect to.
 
-              If only an IP or domain is provided,
+                  If only an IP or domain is provided,
-              mitmdump will default to connect using HTTPS.
+                  mitmdump will default to connect using HTTPS.
-              If this is not wanted, prefix the IP or domain with the 'http://' protocol.
+                  If this is not wanted, prefix the IP or domain with the 'http://' protocol.
-            '';
+                '';
-          };
+              };
 
-          upstreamPort = mkOption {
+              upstreamPort = mkOption {
-            type = port;
+                type = port;
-            description = ''
+                description = ''
-              Port the mitmdump instance will connect to.
+                  Port the mitmdump instance will connect to.
 
-              The port the server is listening on.
+                  The port the server is listening on.
-            '';
+                '';
-          };
+              };
 
-          after = mkOption {
+              timeout = mkOption {
-            type = listOf str;
+                type = types.int;
-            default = [];
+                default = 30;
-            description = ''
+                description = ''
-              Systemd services that must be started before this mitmdump proxy instance.
+                  Time to wait for upstream to start and mitmdump to start.
+                '';
+              };
 
-              You are guaranteed the mitmdump is listening on the `listenPort`
+              after = mkOption {
-              when its systemd service has started.
+                type = listOf str;
-            '';
+                default = [ ];
-          };
+                description = ''
+                  Systemd services that must be started before this mitmdump proxy instance.
 
-          enabledAddons = mkOption {
+                  You are guaranteed the mitmdump is listening on the `listenPort`
-            type = listOf str;
+                  when its systemd service has started.
-            default = [];
+                '';
-            description = ''
+              };
-              Addons to enable on this mitmdump instance.
-            '';
-            example = lib.literalExpression ''[ config.shb.mitmdump.addons.logger ]'';
-          };
 
-          extraArgs = mkOption {
+              enabledAddons = mkOption {
-            type = listOf str;
+                type = listOf str;
-            default = [];
+                default = [ ];
-            description = ''
+                description = ''
-              Extra arguments to pass to the mitmdump instance.
+                  Addons to enable on this mitmdump instance.
+                '';
+                example = lib.literalExpression "[ config.shb.mitmdump.addons.logger ]";
+              };
 
-              See upstream [manual](https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) for all possible options.
+              extraArgs = mkOption {
-            '';
+                type = listOf str;
-            example = lib.literalExpression ''[ "--set" "verbose_pattern=/api" ]'';
+                default = [ ];
-          };
+                description = ''
-        };
+                  Extra arguments to pass to the mitmdump instance.
-      }));
+
+                  See upstream [manual](https://docs.mitmproxy.org/stable/concepts/options/#flow_detail) for all possible options.
+                '';
+                example = lib.literalExpression ''[ "--set" "verbose_pattern=/api" ]'';
+              };
+            };
+          }
+        )
+      );
     };
   };
 
   config = {
-    systemd.services = mapAttrs' (name: cfg': nameValuePair "mitmdump-${name}" {
+    systemd.services = mapAttrs' (
-      environment = {
+      name: cfg':
-        "HOME" = "/var/lib/private/mitmdump-${name}";
+      nameValuePair "mitmdump-${name}" {
-        "MITMDUMP_BIN" = "${cfg'.package}/bin/mitmdump";
+        environment = {
-      };
+          "HOME" = "/var/lib/private/mitmdump-${name}";
-      serviceConfig = {
+          "MITMDUMP_BIN" = "${cfg'.package}/bin/mitmdump";
-        Type = "notify";
+        };
-        Restart = "on-failure";
+        serviceConfig = {
-        StandardOutput = "journal";
+          Type = "notify";
-        StandardError = "journal";
+          Restart = "on-failure";
+          StandardOutput = "journal";
+          StandardError = "journal";
 
-        DynamicUser = true;
+          DynamicUser = true;
-        WorkingDirectory = "/var/lib/mitmdump-${name}";
+          WorkingDirectory = "/var/lib/mitmdump-${name}";
-        StateDirectory = "mitmdump-${name}";
+          StateDirectory = "mitmdump-${name}";
 
-        ExecStart = let
+          ExecStart =
-          addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
+            let
-          extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
+              addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
-        in
+              extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
-          "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}";
+            in
-      };
+            "${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} --timeout ${toString cfg'.timeout} ${addons} ${extraArgs}";
-      requires = cfg'.after;
+        };
-      after = cfg'.after;
+        requires = cfg'.after;
-      wantedBy = [ "multi-user.target" ];
+        after = cfg'.after;
-    }) cfg.instances;
+        wantedBy = [ "multi-user.target" ];
+      }
+    ) cfg.instances;
 
     shb.mitmdump.addons = {
       inherit logger;

modules/blocks/monitoring/dashboards/Health.json

@@ -0,0 +1,872 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "panels": [],
+      "repeat": "hostname",
+      "title": "${hostname}",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "footer": {
+              "reducers": []
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 3,
+      "maxDataPoints": 400,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "node_os_info",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "OS Versions",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {
+            "keepLabels": [
+              "build_id",
+              "domain",
+              "hostname",
+              "id",
+              "instance",
+              "job",
+              "name",
+              "pretty_name",
+              "version",
+              "version_codename",
+              "version_id"
+            ],
+            "mode": "columns"
+          }
+        },
+        {
+          "id": "groupBy",
+          "options": {
+            "fields": {
+              "Time": {
+                "aggregations": [
+                  "firstNotNull"
+                ],
+                "operation": "aggregate"
+              },
+              "pretty_name": {
+                "aggregations": [],
+                "operation": "groupby"
+              }
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "expr": "node_hwmon_temp_celsius{hostname=~\"$hostname\"}",
+          "instant": false,
+          "legendFormat": "{{chip}} - {{sensor}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Temperature",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 0
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "id": 5,
+      "interval": "1m",
+      "maxDataPoints": 200,
+      "options": {
+        "colWidth": 1,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "rowHeight": 0.8,
+        "showValue": "never",
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "none",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "repeat": "hostname",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "node_zfs_zpool_state{hostname=~\"$hostname\", state=\"online\"} > 0",
+          "legendFormat": "{{zpool}} - {{state}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "ZFS Pools",
+      "type": "status-history"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "line+area"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              },
+              {
+                "color": "transparent",
+                "value": 604808
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "expr": "ssl_certificate_expiry_seconds",
+          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Certificate Remaining Validity",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "years"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "editorMode": "builder",
+          "expr": "scrutiny_smart_power_on_hours{hostname=~\"$hostname\"} / (24 * 365)",
+          "legendFormat": "{{device_name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Operating Years",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-YlRd"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 100,
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 21
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "width": 400
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum by(hostname, domain, mountpoint, device) (node_filesystem_free_bytes{hostname=~\"$hostname\"})",
+          "fullMetaSearch": false,
+          "hide": true,
+          "includeNullMetadata": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "builder",
+          "expr": "sum by(hostname, domain, mountpoint, device) (node_filesystem_size_bytes{hostname=~\"$hostname\"})",
+          "hide": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "name": "Expression",
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "(1 - $A / $B) * 100",
+          "refId": "Disk Full",
+          "type": "math"
+        }
+      ],
+      "title": "Filesystem Disk Usage",
+      "transformations": [
+        {
+          "id": "joinByField",
+          "options": {
+            "byField": "Time",
+            "mode": "outer"
+          }
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 21
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "editorMode": "builder",
+          "expr": "scrutiny_smart_power_on_hours{hostname=~\"$hostname\"}",
+          "legendFormat": "{{device_name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Operating Years",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "footer": {
+              "reducers": []
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 29
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "showHeader": true
+      },
+      "pluginVersion": "12.4.0",
+      "targets": [
+        {
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "scrutiny_device_info{hostname=~\"$hostname\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{device_name}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Disk Info",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "domain": true,
+              "hostname": true,
+              "instance": true,
+              "job": true,
+              "wwn": false
+            },
+            "includeByName": {},
+            "indexByName": {},
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 42,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "baryum",
+          "value": "baryum"
+        },
+        "definition": "label_values(up,hostname)",
+        "name": "hostname",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(up,hostname)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "regexApplyTo": "value",
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Node Health",
+  "uid": "edhuvl28vpjwge",
+  "version": 25,
+  "weekStart": ""
+}

modules/blocks/monitoring/dashboards/SSL.json

@@ -1,143 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": 16,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "line+area"
-            }
-          },
-          "mappings": [],
-          "min": 0,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "transparent",
-                "value": 604808
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 12,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true,
-          "sortBy": "Last *",
-          "sortDesc": false
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "11.4.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
-          },
-          "editorMode": "code",
-          "expr": "ssl_certificate_expiry_seconds",
-          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Certificate Remaining Validity",
-      "type": "timeseries"
-    }
-  ],
-  "preload": false,
-  "schemaVersion": 40,
-  "tags": [],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "browser",
-  "title": "SSL Certificates",
-  "uid": "ae818js0bvw8wb",
-  "version": 8,
-  "weekStart": ""
-}

modules/blocks/monitoring/dashboards/Scraping_Jobs.json

@@ -21,6 +21,50 @@
   "id": 5,
   "links": [],
   "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "{unit=~\"prometheus-.*-exporter.service\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Exporter Logs",
+      "type": "logs"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -29,7 +73,8 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "fixedColor": "red",
+            "mode": "shades"
           },
           "custom": {
             "axisBorderShow": false,
@@ -55,6 +100,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -70,7 +116,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -96,11 +142,12 @@
           "showLegend": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
-      "pluginVersion": "11.4.0",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -127,9 +174,10 @@
         "defaults": {
           "color": {
             "fixedColor": "red",
-            "mode": "fixed"
+            "mode": "thresholds"
           },
           "custom": {
+            "axisPlacement": "auto",
             "fillOpacity": 70,
             "hideFrom": {
               "legend": false,
@@ -146,11 +194,11 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 1
               }
             ]
           }
@@ -175,11 +223,12 @@
         "rowHeight": 0.9,
         "showValue": "never",
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
-      "pluginVersion": "11.4.0",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -200,7 +249,7 @@
             "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
           },
           "editorMode": "code",
-          "expr": "label_replace(increase((sum by(dialer_name) (net_conntrack_dialer_conn_failed_total{hostname=~\"$hostname\"}))[15m:1m]), \"config\", \"$1\", \"dialer_name\", \"(.*)\") > 10",
+          "expr": "label_replace(increase((sum by(dialer_name) (net_conntrack_dialer_conn_failed_total{hostname=~\"$hostname\"}))[15m:1m]), \"config\", \"$1\", \"dialer_name\", \"(.*)\")",
           "hide": false,
           "instant": false,
           "legendFormat": "{{dialer_name}}",
@@ -249,7 +298,7 @@
   ],
   "preload": false,
   "refresh": "",
-  "schemaVersion": 40,
+  "schemaVersion": 42,
   "tags": [],
   "templating": {
     "list": [
@@ -281,6 +330,5 @@
   "timezone": "",
   "title": "Scraping Jobs",
   "uid": "debb763d-77aa-47bd-9290-2e02583c8ed2",
-  "version": 15,
+  "version": 24
-  "weekStart": ""
 }

modules/blocks/monitoring/docs/default.md

@@ -8,26 +8,70 @@ This block sets up the monitoring stack for Self Host Blocks. It is composed of:
 - Prometheus as the database for metrics.
 - Loki as the database for logs.
 
-## Configuration {#blocks-monitoring-configuration}
+## Features {#services-monitoring-features}
+
+- Declarative [LDAP](#blocks-monitoring-options-shb.monitoring.ldap) Configuration.
+  - Needed LDAP groups are created automatically.
+- Declarative [SSO](#blocks-monitoring-options-shb.monitoring.sso) Configuration.
+  - When SSO is enabled, login with user and password is disabled.
+  - Registration is enabled through SSO.
+- Access through [subdomain](#blocks-monitoring-options-shb.monitoring.subdomain) using reverse proxy.
+- Access through [HTTPS](#blocks-monitoring-options-shb.monitoring.ssl) using reverse proxy.
+- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#blocks-monitoring-usage-applicationdashboard)
+- Out of the box integration with [Scrutiny](https://github.com/AnalogJ/scrutiny) service for Hard Drives monitoring. [Manual](#blocks-monitoring-usage-scrutiny)
+
+## Usage {#blocks-monitoring-usage}
+
+### Initial Configuration {#blocks-monitoring-usage-configuration}
+
+The following snippet assumes a few blocks have been setup already:
+
+- the [secrets block](usage.html#usage-secrets) with SOPS,
+- the [`shb.ssl` block](blocks-ssl.html#usage),
+- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
+- the [`shb.authelia` block](blocks-authelia.html#blocks-sso-global-setup).
 
 ```nix
-shb.monitoring = {
+{
-  enable = true;
+  shb.monitoring = {
-  subdomain = "grafana";
+    enable = true;
-  inherit domain;
+    subdomain = "grafana";
-  contactPoints = [ "me@example.com" ];
+    inherit domain;
-  adminPassword.result = config.sops.secrets."monitoring/admin_password".result;
+    contactPoints = [ "me@example.com" ];
-  secretKey.result = config.sops.secrets."monitoring/secret_key".result;
+    adminPassword.result = config.shb.sops.secret."monitoring/admin_password".result;
-};
+    secretKey.result = config.shb.sops.secret."monitoring/secret_key".result;
 
-shb.sops.secret."monitoring/admin_password".request = config.shb.monitoring.adminPassword.request;
+    sso = {
-shb.sops.secret."monitoring/secret_key".request = config.shb.monitoring.secretKey.request;
+      enable = true;
+      authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+
+      sharedSecret.result = config.shb.sops.secret."monitoring/oidcSecret".result;
+      sharedSecretForAuthelia.result = config.shb.sops.secret."monitoring/oidcAutheliaSecret".result;
+    };
+  };
+  
+  shb.sops.secret."monitoring/admin_password".request = config.shb.monitoring.adminPassword.request;
+  shb.sops.secret."monitoring/secret_key".request = config.shb.monitoring.secretKey.request;
+  shb.sops.secret."monitoring/oidcSecret".request = config.shb.monitoring.sso.sharedSecret.request;
+  shb.sops.secret."monitoring/oidcAutheliaSecret" = {
+    request = config.shb.monitoring.sso.sharedSecretForAuthelia.request;
+    settings.key = "monitoring/oidcSecret";
+  };
+};
 ```
 
-With that, Grafana, Prometheus, Loki and Promtail are setup! You can access `Grafana` at
+Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.
-`grafana.example.com` with user `admin` and password ``.
 
-I recommend adding a STMP server configuration so you receive alerts by email:
+With that, Grafana, Prometheus, Loki and Promtail are setup! You can access `Grafana` at
+`grafana.example.com` with user `admin` and the password from the sops key `monitoring/admin_password`.
+
+The [user](#blocks-monitoring-options-shb.monitoring.ldap.userGroup)
+and [admin](#blocks-monitoring-options-shb.monitoring.ldap.adminGroup)
+LDAP groups are created automatically.
+
+### SMTP {#blocks-monitoring-usage-smtp}
+
+I recommend adding an SMTP server configuration so you receive alerts by email:
 
 ```nix
 shb.monitoring.smtp = {
@@ -48,6 +92,8 @@ sops.secrets."monitoring/secret_key" = {
 };
 ```
 
+### Log Optimization {#blocks-monitoring-usage-log-optimization}
+
 Since all logs are now stored in Loki, you can probably reduce the systemd journal retention
 time with:
 
@@ -68,6 +114,42 @@ You might for example want to update the metrics retention time with:
 services.prometheus.retentionTime = "60d";
 ```
 
+### Application Dashboard {#blocks-monitoring-usage-applicationdashboard}
+
+Integration with the [dashboard contract](contracts-dashboard.html) is provided
+by the [dashboard option](#blocks-monitoring-options-shb.monitoring.dashboard).
+
+For example using the [Homepage](services-homepage.html) service:
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.Grafana = {
+    sortOrder = 10;
+    dashboard.request = config.shb.monitoring.dashboard.request;
+  };
+}
+```
+
+There is also an integration for the scrutiny service, see next section.
+
+### Scrutiny {#blocks-monitoring-usage-scrutiny}
+
+Integration with the [Scrutiny](https://github.com/AnalogJ/scrutiny) service is enabled by default and setup automatically.
+
+The web interface will be served under the [scrutiny.subdomain](#blocks-monitoring-options-shb.monitoring.scrutiny.subdomain) option.
+If you don't want the web interface, set the option to `null`.
+
+For integration with the [dashboard contract](contracts-dashboard.html):
+
+```nix
+{
+  shb.homepage.servicesGroups.Admin.services.Scrutiny = {
+    sortOrder = 11;
+    dashboard.request = config.shb.monitoring.scrutiny.dashboard.request;
+  };
+}
+```
+
 ## Provisioning {#blocks-monitoring-provisioning}
 
 Self Host Blocks will create automatically the following resources:
@@ -118,6 +200,43 @@ This dashboard is used to monitor a [deluge](./services-deluge.html) instance.
 ![Deluge Dashboard Top Part](./assets/dashboards_Deluge_1.png)
 ![Deluge Dashboard Bottom Part](./assets/dashboards_Deluge_2.png)
 
+## Backups Dashboard and Alert {#blocks-monitoring-backup}
+
+This dashboard shows Restic and BorgBackup backup jobs, or any job with "backup" in the systemd service name.
+
+### Dashboard {#blocks-monitoring-backup-dashboard}
+
+Variables:
+
+- The "Job" variable allows to select one or more backup jobs. "All" is the default.
+- The "mountpoints" variable allows to select only relevant mountpoints for backup. "All" is the default.
+
+The most important graphs are the first three:
+
+- "Backup Jobs in the Past Week": Shows stats on all backup jobs that ran in the past.
+  It is sorted by the "Failed" column in descending order.
+  This way, one can directly see when a job has failures.
+- "Schedule": Shows when a job will run.
+  The unit is "Datetime from Now" meaning it shows when a job ran or will run relative to the current time.
+  An annotation will show up when the "Late Backups" alert fired or resolved.
+- "Backup jobs": Shows when a backup job ran.
+  Normally, jobs running for less than 15 seconds will not show up in the graph.
+  We crafted a query that still shows them but the length is 15 seconds, even if the backup job
+  took less time to run.
+
+![Backups Dashboard Top Part](./assets/dashboards_Backups_1.png)
+![Backups Dashboard Middle Part](./assets/dashboards_Backups_2.png)
+![Backups Dashboard Bottom Part](./assets/dashboards_Backups_3.png)
+
+### Alerts {#blocks-monitoring-backup-alerts}
+
+- The "Late Backups" alert will fire if a backup job did not run at all in the last 24 hours
+  or if all runs were failures in the last 24 hours.
+  It will show up as annotations in the "Schedule" panel of the dashboard.
+
+![Late Backups Alert Firing](./assets/alert_rules_LateBackups_1.png)
+![Backups Alert Showing Up In Dashboard](./assets/dashboards_Backups_alert.png)
+
 ## Requests Error Budget Alert {#blocks-monitoring-budget-alerts}
 
 This alert will fire when the ratio between number of requests getting a 5XX response from a service
@@ -126,6 +245,55 @@ and the total requests to that service exceeds 1%.
 ![Error Dashboard Top Part](./assets/alert_rules_5xx_1.png)
 ![Error Dashboard Bottom Part](./assets/alert_rules_5xx_2.png)
 
+## SSL Certificates Dashboard and Alert {#blocks-monitoring-ssl}
+
+This dashboard shows Let's Encrypt renewal and setup jobs,
+or any job starting with "acme-" in the systemd service name.
+
+### Dashboard {#blocks-monitoring-ssl-dashboard}
+
+Variables:
+
+- The "Job" variable allows to focus on one or more certificate. "All" is the default.
+
+Graphs:
+
+- "Certificate Remaining Validity": Shows in how long will certificates expire.
+  It shows all files under `/var/lib/acme`.
+  An annotation will show up when the "Certificate Did Not Renew" alert fired or resolved.
+- "Schedule": Shows when a job will run.
+  The unit is "Datetime from Now" meaning it shows when a job ran or will run relative to the current time.
+- "Jobs in the Past Week": Shows stats on all renewal jobs that ran in the past.
+  It is sorted by the "Failed" column in descending order.
+  This way, one can directly see when a job has failures.
+  Note, the stats is not accurate because detecting jobs taking taking less than 15 seconds
+  is not supported well.
+- "Job Runs": Shows when a renewal job ran.
+  Normally, jobs running for less than 15 seconds will not show up in the graph.
+  We crafted a query that still shows them but the length is 100 seconds, even if the job
+  took less time to run.
+
+![SSL Dashboard No Filter](./assets/dashboards_SSL_all.png)
+![SSL Dashboard Filter Failing](./assets/dashboards_SSL_fail.png)
+
+### Alerts {#blocks-monitoring-ssl-alerts}
+
+- The "Certificate Did Not Renew" alert will fire if a backup job did not run at all in the last 24 hours
+    or if all runs were failures in the last 24 hours.
+  It will show up as annotations in the "Schedule" panel of the dashboard.
+
+![Late SSL Jobs Alert Firing](./assets/alert_rules_LateSSL_1.png)
+
+## Impermanence {#blocks-monitoring-impermanence}
+
+To save the fluent-bit folder in an impermanence setup, add:
+
+```nix
+{
+  shb.zfs.datasets."safe/monitoring-fluent-bit".path = config.shb.jellyfin.impermanence.fluent-bit;
+}
+```
+
 ## Options Reference {#blocks-monitoring-options}
 
 ```{=include=} options

modules/blocks/monitoring/rules.json

@@ -146,7 +146,7 @@
             "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
           },
           "editorMode": "code",
-          "expr": "ssl_certificate_expiry_seconds",
+          "expr": "min by(subject) (ssl_certificate_expiry_seconds)",
           "interval": "",
           "intervalMs": 15000,
           "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
@@ -254,5 +254,133 @@
       "role": "sysadmin"
     },
     "isPaused": false
+  },
+  {
+    "uid": "df4doj5pomhvkf",
+    "title": "Late Backups",
+    "condition": "C",
+    "data": [
+      {
+        "refId": "A",
+        "relativeTimeRange": {
+          "from": 10800,
+          "to": 0
+        },
+        "datasourceUid": "df80f9f5-97d7-4112-91d8-72f523a02b09",
+        "model": {
+          "adhocFilters": [],
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "(\n  # Timer triggered at least once in the last 24h\n  label_replace((\n    time()\n      -\n    systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"}\n  ) < 24*60*60, \"name\", \"$1.service\", \"name\", \"(.*).timer\")\n  AND on(name)\n  # At least one failure in the last 24h\n  (\n    max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"failed\"}[24h]) > 0\n  )\n  AND on(name)\n  # No successes in the last 24h\n  (\n    max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"inactive\"}[24h]) == 0\n  )\n)",
+          "instant": false,
+          "interval": "",
+          "intervalMs": 15000,
+          "legendFormat": "{{name}}",
+          "maxDataPoints": 43200,
+          "range": true,
+          "refId": "A"
+        }
+      },
+      {
+        "refId": "B",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        },
+        "datasourceUid": "__expr__",
+        "model": {
+          "conditions": [
+            {
+              "evaluator": {
+                "params": [],
+                "type": "gt"
+              },
+              "operator": {
+                "type": "and"
+              },
+              "query": {
+                "params": [
+                  "B"
+                ]
+              },
+              "reducer": {
+                "params": [],
+                "type": "last"
+              },
+              "type": "query"
+            }
+          ],
+          "datasource": {
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "A",
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "reducer": "last",
+          "refId": "B",
+          "type": "reduce"
+        }
+      },
+      {
+        "refId": "C",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        },
+        "datasourceUid": "__expr__",
+        "model": {
+          "conditions": [
+            {
+              "evaluator": {
+                "params": [
+                  0
+                ],
+                "type": "gt"
+              },
+              "operator": {
+                "type": "and"
+              },
+              "query": {
+                "params": [
+                  "C"
+                ]
+              },
+              "reducer": {
+                "params": [],
+                "type": "last"
+              },
+              "type": "query"
+            }
+          ],
+          "datasource": {
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "B",
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "C",
+          "type": "threshold"
+        }
+      }
+    ],
+    "dashboardUid": "f05500d0-15ed-4719-b68d-fb898ca13cc8",
+    "panelId": 15,
+    "noDataState": "OK",
+    "execErrState": "Error",
+    "annotations": {
+      "__dashboardUid__": "f05500d0-15ed-4719-b68d-fb898ca13cc8",
+      "__panelId__": "15",
+      "summary": "A backup did not run in the last 24 hours."
+    },
+    "labels": {
+      "role": "sysadmin"
+    },
+    "isPaused": false
   }
 ]

modules/blocks/nginx.nix

@@ -1,10 +1,13 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  lib,
+  shb,
+  ...
+}:
 
 let
   cfg = config.shb.nginx;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   fqdn = c: "${c.subdomain}.${c.domain}";
 
   vhostConfig = lib.types.submodule {
@@ -23,13 +26,14 @@ let
 
       ssl = lib.mkOption {
         description = "Path to SSL files";
-        type = lib.types.nullOr contracts.ssl.certs;
+        type = lib.types.nullOr shb.contracts.ssl.certs;
         default = null;
       };
 
       upstream = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.nullOr lib.types.str;
         description = "Upstream url to be protected.";
+        default = null;
         example = "http://127.0.0.1:1234";
       };
 
@@ -42,12 +46,43 @@ let
 
       autheliaRules = lib.mkOption {
         type = lib.types.listOf (lib.types.attrsOf lib.types.anything);
-        default = [];
+        default = [ ];
+        description = "Authelia rule configuration";
+        example = lib.literalExpression ''
+          [
+            # Protect /admin endpoint with 2FA
+            # and only allow access to admin users.
+            {
+              domain = "myapp.example.com";
+              policy = "two_factor";
+              subject = [ "group:service_admin" ];
+              resources = [
+                "^/admin"
+              ];
+            }
+            # Leave /api endpoint open - assumes an API key is used to protect it.
+            {
+              domain = "myapp.example.com";
+              policy = "bypass";
+              resources = [
+                "^/api"
+              ];
+            },
+            # Protect rest of app with 1FA
+            # and allow access to normal and admin users.
+            {
+              domain = "myapp.example.com";
+              policy = "one_factor";
+              subject = ["group:service_user"];
+            },
+          ]
+        '';
+      };
+
+      phpForwardAuth = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
         description = "Authelia rule configuration";
-        example = lib.literalExpression ''[{
-        policy = "two_factor";
-        subject = ["group:service_user"];
-        }]'';
       };
 
       extraConfig = lib.mkOption {
@@ -59,6 +94,10 @@ let
   };
 in
 {
+  imports = [
+    ./authelia.nix
+  ];
+
   options.shb.nginx = {
     accessLog = lib.mkOption {
       type = lib.types.bool;
@@ -77,41 +116,44 @@ in
     vhosts = lib.mkOption {
       description = "Endpoints to be protected by authelia.";
       type = lib.types.listOf vhostConfig;
-      default = [];
+      default = [ ];
     };
   };
 
   config = {
-    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
 
     services.nginx.enable = true;
     services.nginx.logError = lib.mkIf cfg.debugLog "stderr warn";
     services.nginx.appendHttpConfig = lib.mkIf cfg.accessLog ''
-        log_format apm
+      log_format apm
-          '{'
+        '{'
-          '"remote_addr":"$remote_addr",'
+        '"remote_addr":"$remote_addr",'
-          '"remote_user":"$remote_user",'
+        '"remote_user":"$remote_user",'
-          '"time_local":"$time_local",'
+        '"time_local":"$time_local",'
-          '"request":"$request",'
+        '"request":"$request",'
-          '"request_length":"$request_length",'
+        '"request_length":"$request_length",'
-          '"server_name":"$server_name",'
+        '"server_name":"$server_name",'
-          '"status":"$status",'
+        '"status":"$status",'
-          '"bytes_sent":"$bytes_sent",'
+        '"bytes_sent":"$bytes_sent",'
-          '"body_bytes_sent":"$body_bytes_sent",'
+        '"body_bytes_sent":"$body_bytes_sent",'
-          '"referrer":"$http_referrer",'
+        '"referrer":"$http_referrer",'
-          '"user_agent":"$http_user_agent",'
+        '"user_agent":"$http_user_agent",'
-          '"gzip_ration":"$gzip_ratio",'
+        '"gzip_ration":"$gzip_ratio",'
-          '"post":"$request_body",'
+        '"post":"$request_body",'
-          '"upstream_addr":"$upstream_addr",'
+        '"upstream_addr":"$upstream_addr",'
-          '"upstream_status":"$upstream_status",'
+        '"upstream_status":"$upstream_status",'
-          '"request_time":"$request_time",'
+        '"request_time":"$request_time",'
-          '"upstream_response_time":"$upstream_response_time",'
+        '"upstream_response_time":"$upstream_response_time",'
-          '"upstream_connect_time":"$upstream_connect_time",'
+        '"upstream_connect_time":"$upstream_connect_time",'
-          '"upstream_header_time":"$upstream_header_time"'
+        '"upstream_header_time":"$upstream_header_time"'
-          '}';
+        '}';
 
-        access_log syslog:server=unix:/dev/log apm;
+      access_log syslog:server=unix:/dev/log apm;
-      '';
+    '';
 
     services.nginx.virtualHosts =
       let
@@ -122,52 +164,58 @@ in
             sslCertificateKey = lib.mkIf (!(isNull c.ssl)) c.ssl.paths.key;
 
             # Taken from https://github.com/authelia/authelia/issues/178
-            locations."/".extraConfig = ''
+            locations."/".extraConfig =
-              add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+              lib.optionalString (c.upstream != null) ''
-              add_header X-Content-Type-Options nosniff;
+                add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
-              add_header X-Frame-Options "SAMEORIGIN";
+                add_header X-Content-Type-Options nosniff;
-              add_header X-XSS-Protection "1; mode=block";
+                add_header X-Frame-Options "SAMEORIGIN";
-              add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
+                add_header X-XSS-Protection "1; mode=block";
-              add_header X-Download-Options noopen;
+                add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
-              add_header X-Permitted-Cross-Domain-Policies none;
+                add_header X-Download-Options noopen;
+                add_header X-Permitted-Cross-Domain-Policies none;
 
-              proxy_set_header Host $host;
+                proxy_set_header Host $host;
-              proxy_set_header X-Real-IP $remote_addr;
+                proxy_set_header X-Real-IP $remote_addr;
-              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-              proxy_set_header X-Forwarded-Proto $scheme;
+                proxy_set_header X-Forwarded-Proto $scheme;
-              proxy_http_version 1.1;
+                proxy_http_version 1.1;
-              proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Upgrade $http_upgrade;
-              proxy_set_header Connection "upgrade";
+                proxy_set_header Connection "upgrade";
-              proxy_cache_bypass $http_upgrade;
+                proxy_cache_bypass $http_upgrade;
 
-              proxy_pass ${c.upstream};
+                proxy_pass ${c.upstream};
-            ''
+              ''
-            + c.extraConfig
+              + c.extraConfig
-            + lib.optionalString (c.authEndpoint != null) ''
+              + lib.optionalString (c.authEndpoint != null) ''
-              auth_request /authelia;
+                auth_request /authelia;
-              auth_request_set $user $upstream_http_remote_user;
+                auth_request_set $user $upstream_http_remote_user;
-              auth_request_set $groups $upstream_http_remote_groups;
+                auth_request_set $groups $upstream_http_remote_groups;
-              proxy_set_header X-Forwarded-User $user;
+                proxy_set_header X-Forwarded-User $user;
-              proxy_set_header X-Forwarded-Groups $groups;
+                proxy_set_header X-Forwarded-Groups $groups;
-              # TODO: Are those needed?
+                # TODO: Are those needed?
-              # auth_request_set $name $upstream_http_remote_name;
+                # auth_request_set $name $upstream_http_remote_name;
-              # auth_request_set $email $upstream_http_remote_email;
+                # auth_request_set $email $upstream_http_remote_email;
-              # proxy_set_header Remote-Name $name;
+                # proxy_set_header Remote-Name $name;
-              # proxy_set_header Remote-Email $email;
+                # proxy_set_header Remote-Email $email;
-              # TODO: Would be nice to have this working, I think.
+                # TODO: Would be nice to have this working, I think.
-              # set $new_cookie $http_cookie;
+                # set $new_cookie $http_cookie;
-              # if ($http_cookie ~ "(.*)(?:^|;)\s*example\.com\.session\.id=[^;]+(.*)") {
+                # if ($http_cookie ~ "(.*)(?:^|;)\s*example\.com\.session\.id=[^;]+(.*)") {
-              #     set $new_cookie $1$2;
+                #     set $new_cookie $1$2;
-              # }
+                # }
-              # proxy_set_header Cookie $new_cookie;
+                # proxy_set_header Cookie $new_cookie;
 
-              auth_request_set $redirect $scheme://$http_host$request_uri;
+                auth_request_set $redirect $scheme://$http_host$request_uri;
-              error_page 401 =302 ${c.authEndpoint}?rd=$redirect;
+                error_page 401 =302 ${c.authEndpoint}?rd=$redirect;
-              error_page 403 = ${c.authEndpoint}/error/403;
+                error_page 403 = ${c.authEndpoint}/error/403;
+              '';
+
+            locations."~ \\.php$".extraConfig = lib.mkIf (c.phpForwardAuth) ''
+              fastcgi_param HTTP_X_FORWARDED_USER   $user;
+              fastcgi_param HTTP_X_FORWARDED_GROUPS $groups;
             '';
 
             # Virtual endpoint created by nginx to forward auth requests.
-            locations."/authelia".extraConfig = lib.mkIf (!(isNull c.authEndpoint)) ''
+            locations."/authelia".extraConfig = lib.mkIf (c.authEndpoint != null) ''
               internal;
               proxy_pass ${c.authEndpoint}/api/verify;
 
@@ -189,13 +237,13 @@ in
           };
         };
       in
-        lib.mkMerge (map vhostCfg cfg.vhosts);
+      lib.mkMerge (map vhostCfg cfg.vhosts);
 
     shb.authelia.rules =
       let
         authConfig = c: map (r: r // { domain = fqdn c; }) c.autheliaRules;
       in
-        lib.flatten (map authConfig cfg.vhosts);
+      lib.flatten (map authConfig cfg.vhosts);
 
     security.acme.defaults.reloadServices = [
       "nginx.service"

modules/blocks/nginx/docs/default.md

@@ -0,0 +1,211 @@
+# Nginx Block {#blocks-nginx}
+
+Defined in [`/modules/blocks/nginx.nix`](@REPO@/modules/blocks/nginx.nix).
+
+This block sets up a [Nginx](https://nginx.org/) instance.
+
+It complements the upstream nixpkgs with some authentication and debugging improvements as shows in the Usage section.
+
+## Usage {#blocks-nginx-usage}
+
+### Access Logging {#blocks-nginx-usage-accesslog}
+
+JSON access logging is enabled with the [`shb.nginx.accessLog`](#blocks-nginx-options-shb.nginx.accessLog) option:
+
+```nix
+{
+  shb.nginx.accessLog = true;
+}
+```
+
+Looking at the systemd logs (`journalctl -fu nginx`) will show for example:
+
+```json
+nginx[969]: server nginx:
+  {
+    "remote_addr":"192.168.1.1",
+    "remote_user":"-",
+    "time_local":"29/Dec/2025:14:22:41 +0000",
+    "request":"POST /api/firstfactor HTTP/2.0",
+    "request_length":"264",
+    "server_name":"auth_example_com",
+    "status":"200",
+    "bytes_sent":"855",
+    "body_bytes_sent":"60",
+    "referrer":"-",
+    "user_agent":"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    "gzip_ration":"-",
+    "post":"{\x22username\x22:\x22charlie\x22,\x22password\x22:\x22CharliePassword\x22,\x22keepMeLoggedIn\x22:false,\x22targetURL\x22:\x22https://f.example.com/\x22,\x22requestMethod\x22:null}",
+    "upstream_addr":"127.0.0.1:9091",
+    "upstream_status":"200",
+    "request_time":"0.873",
+    "upstream_response_time":"0.873",
+    "upstream_connect_time":"0.001",
+    "upstream_header_time":"0.872"
+  }
+```
+
+This _will_ log the body of POST queries so it should only be enabled for debug logging.
+
+### Debug Logging {#blocks-nginx-usage-debuglog}
+
+Debug logging is enabled with the [`shb.nginx.debugLog`](#blocks-nginx-options-shb.nginx.debugLog) option:
+
+```nix
+{
+  shb.nginx.debugLog = true;
+}
+```
+
+If enabled, it sets:
+
+```
+error_log stderr warn;
+```
+
+### Virtual Host Upstream Proxy {#blocks-nginx-usage-upstream}
+
+Easy upstream proxy setup is done with the [`shb.nginx.vhosts.*.upstream`](#blocks-nginx-options-shb.nginx.vhosts._.upstream) option:
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      upstream = "http://127.0.0.1:9090";
+    }
+  ];
+}
+```
+
+This will set also a few headers.
+Some are shown here and others please see in the [nginx](@REPO@/modules/blocks/nginx.nix) module:
+
+- `Host` = `$host`;
+- `X-Real-IP` = `$remote_addr`;
+- `X-Forwarded-For` = `$proxy_add_x_forwarded_for`;
+- `X-Forwarded-Proto` = `$scheme`;
+
+### Virtual Host SSL Generator Contract Integration {#blocks-nginx-usage-ssl}
+
+This module integrates with the [SSL Generator Contract](./contracts-ssl.html)
+to setup HTTPs with the [`shb.nginx.vhosts.*.ssl`](#blocks-nginx-options-shb.nginx.vhosts._.ssl) option:
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      ssl = config.shb.certs.certs.letsencrypt.${domain};;
+    }
+  ];
+
+  shb.certs.certs.letsencrypt.${domain} = {
+    inherit domain;
+  };
+}
+```
+
+### Virtual Host SHB Forward Authentication {#blocks-nginx-usage-shbforwardauth}
+
+For services provided by SelfHostBlocks that do not handle [OIDC integration][OIDC],
+this block can provide [forward authentication][] which still allows the service
+to still be protected by an SSO server.
+
+[OIDC]: blocks-authelia.html#blocks-authelia-shb-oidc
+
+The user could still be required to authenticate to the service itself,
+although some services can automatically users authorized by Authelia.
+
+[forward authentication]: https://doc.traefik.io/traefik/middlewares/http/forwardauth/
+
+Integrating with this block is done with the following code:
+
+```nix
+shb.<services>.authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
+```
+
+### Virtual Host Forward Authentication {#blocks-nginx-usage-forwardauth}
+
+Forward authentication is when Nginx talks with the SSO service directly
+and the user is authenticated before reaching the upstream application.
+
+The SSO service responds with the username, group and more information about the user.
+This is then forwarded to the upstream application by Nginx.
+
+Note that _every_ request is authenticated this way with the SSO server
+so it involves more hops than a direct [OIDC integration](blocks-authelia.html#blocks-authelia-shb-oidc).
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      authEndpoint = "authelia.example.com";
+      autheliaRules = [
+        [
+          # Protect /admin endpoint with 2FA
+          # and only allow access to admin users.
+          {
+            domain = "myapp.example.com";
+            policy = "two_factor";
+            subject = [ "group:service_admin" ];
+            resources = [
+              "^/admin"
+            ];
+          }
+          # Leave /api endpoint open - assumes an API key is used to protect it.
+          {
+            domain = "myapp.example.com";
+            policy = "bypass";
+            resources = [
+              "^/api"
+            ];
+          },
+          # Protect rest of app with 1FA
+          # and allow access to normal and admin users.
+          {
+            domain = "myapp.example.com";
+            policy = "one_factor";
+            subject = ["group:service_user"];
+          },
+        ]
+      ];
+    }
+  ];
+}
+```
+
+If PHP is used with fastCGI,
+extra headers must be added by enabling the [`shb.nginx.vhosts.*.phpForwardAuth`](#blocks-nginx-options-shb.nginx.vhosts._.phpForwardAuth) option.
+
+### Virtual Host Extra Config {#blocks-nginx-usage-extraconfig}
+
+To add extra configuration to a virtual host,
+use the [`shb.nginx.vhosts.*.extraConfig`](#blocks-nginx-options-shb.nginx.vhosts._.extraConfig) option.
+This can be used to add headers, for example:
+
+```nix
+{
+  shb.nginx.vhosts = [
+    {
+      domain = "example.com";
+      subdomain = "mysubdomain";
+      extraConfig = ''
+        add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
+      '';
+    }
+  ];
+}
+```
+
+## Options Reference {#blocks-nginx-options}
+
+```{=include=} options
+id-prefix: blocks-nginx-options-
+list-id: selfhostblocks-block-nginx-options
+source: @OPTIONS_JSON@
+```

modules/blocks/postgresql.nix

@@ -1,9 +1,15 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  ...
+}:
 let
   cfg = config.shb.postgresql;
-  contracts = pkgs.callPackage ../contracts {};
 
-  upgrade-script = old: new:
+  upgrade-script =
+    old: new:
     let
       oldStr = builtins.toString old;
       newStr = builtins.toString new;
@@ -11,37 +17,41 @@ let
       oldPkg = pkgs.${"postgresql_${oldStr}"};
       newPkg = pkgs.${"postgresql_${newStr}"};
     in
-      pkgs.writeScriptBin "upgrade-pg-cluster-${oldStr}-${newStr}" ''
+    pkgs.writeScriptBin "upgrade-pg-cluster-${oldStr}-${newStr}" ''
-        set -eux
+      set -eux
-        # XXX it's perhaps advisable to stop all services that depend on postgresql
+      # XXX it's perhaps advisable to stop all services that depend on postgresql
-        systemctl stop postgresql
+      systemctl stop postgresql
 
-        export NEWDATA="/var/lib/postgresql/${newPkg.psqlSchema}"
+      export NEWDATA="/var/lib/postgresql/${newPkg.psqlSchema}"
-        export NEWBIN="${newPkg}/bin"
+      export NEWBIN="${newPkg}/bin"
 
-        export OLDDATA="/var/lib/postgresql/${oldPkg.psqlSchema}"
+      export OLDDATA="/var/lib/postgresql/${oldPkg.psqlSchema}"
-        export OLDBIN="${oldPkg}/bin"
+      export OLDBIN="${oldPkg}/bin"
 
-        install -d -m 0700 -o postgres -g postgres "$NEWDATA"
+      install -d -m 0700 -o postgres -g postgres "$NEWDATA"
-        cd "$NEWDATA"
+      cd "$NEWDATA"
-        sudo -u postgres $NEWBIN/initdb -D "$NEWDATA"
+      sudo -u postgres $NEWBIN/initdb -D "$NEWDATA"
 
-        sudo -u postgres $NEWBIN/pg_upgrade \
+      sudo -u postgres $NEWBIN/pg_upgrade \
-          --old-datadir "$OLDDATA" --new-datadir "$NEWDATA" \
+        --old-datadir "$OLDDATA" --new-datadir "$NEWDATA" \
-          --old-bindir $OLDBIN --new-bindir $NEWBIN \
+        --old-bindir $OLDBIN --new-bindir $NEWBIN \
-          "$@"
+        "$@"
-      '';
+    '';
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.postgresql = {
     debug = lib.mkOption {
       type = lib.types.bool;
       description = ''
-      Enable debugging options.
+        Enable debugging options.
 
-      Currently enables shared_preload_libraries = "auto_explain, pg_stat_statements"
+        Currently enables shared_preload_libraries = "auto_explain, pg_stat_statements"
 
-      See https://www.postgresql.org/docs/current/pgstatstatements.html'';
+        See https://www.postgresql.org/docs/current/pgstatstatements.html'';
       default = false;
     };
     enableTCPIP = lib.mkOption {
@@ -55,15 +65,15 @@ in
         Backup configuration.
       '';
 
-      default = {};
+      default = { };
       type = lib.types.submodule {
-        options = contracts.databasebackup.mkRequester {
+        options = shb.contracts.databasebackup.mkRequester {
           user = "postgres";
 
           backupName = "postgres.sql";
 
           backupCmd = ''
-            ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
+            ${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
           '';
 
           restoreCmd = ''
@@ -75,27 +85,29 @@ in
 
     ensures = lib.mkOption {
       description = "List of username, database and/or passwords that should be created.";
-      type = lib.types.listOf (lib.types.submodule {
+      type = lib.types.listOf (
-        options = {
+        lib.types.submodule {
-          username = lib.mkOption {
+          options = {
-            type = lib.types.str;
+            username = lib.mkOption {
-            description = "Postgres user name.";
+              type = lib.types.str;
-          };
+              description = "Postgres user name.";
+            };
 
-          database = lib.mkOption {
+            database = lib.mkOption {
-            type = lib.types.str;
+              type = lib.types.str;
-            description = "Postgres database.";
+              description = "Postgres database.";
-          };
+            };
 
-          passwordFile = lib.mkOption {
+            passwordFile = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
+              type = lib.types.nullOr lib.types.str;
-            description = "Optional password file for the postgres user. If not given, only peer auth is accepted for this user, otherwise password auth is allowed.";
+              description = "Optional password file for the postgres user. If not given, only peer auth is accepted for this user, otherwise password auth is allowed.";
-            default = null;
+              default = null;
-            example = "/run/secrets/postgresql/password";
+              example = "/run/secrets/postgresql/password";
+            };
           };
-        };
+        }
-      });
+      );
-      default = [];
+      default = [ ];
     };
   };
 
@@ -123,51 +135,61 @@ in
       dbConfig = ensureCfgs: {
         services.postgresql.enable = lib.mkDefault ((builtins.length ensureCfgs) > 0);
         services.postgresql.ensureDatabases = map ({ database, ... }: database) ensureCfgs;
-        services.postgresql.ensureUsers = map ({ username, database, ... }: {
+        services.postgresql.ensureUsers = map (
-          name = username;
+          { username, database, ... }:
-          ensureDBOwnership = true;
+          {
-          ensureClauses.login = true;
+            name = username;
-        }) ensureCfgs;
+            ensureDBOwnership = true;
+            ensureClauses.login = true;
+          }
+        ) ensureCfgs;
       };
 
       pwdConfig = ensureCfgs: {
-        systemd.services.postgresql-setup.script = lib.mkAfter
+        systemd.services.postgresql-setup.script = lib.mkAfter (
-          (let
+          let
             prefix = ''
-            psql -tA <<'EOF'
+              psql -tA <<'EOF'
-              DO $$
+                DO $$
-              DECLARE password TEXT;
+                DECLARE password TEXT;
-              BEGIN
+                BEGIN
             '';
             suffix = ''
-              END $$;
+                END $$;
-            EOF
+              EOF
-            '';
-            exec = { username, passwordFile, ... }: ''
-            password := trim(both from replace(pg_read_file('${passwordFile}'), E'\n', '''));
-            EXECUTE format('ALTER ROLE ${username} WITH PASSWORD '''%s''';', password);
             '';
+            exec =
+              { username, passwordFile, ... }:
+              ''
+                password := trim(both from replace(pg_read_file('${passwordFile}'), E'\n', '''));
+                EXECUTE format('ALTER ROLE "${username}" WITH PASSWORD '''%s''';', password);
+              '';
             cfgsWithPasswords = builtins.filter (cfg: cfg.passwordFile != null) ensureCfgs;
           in
-            if (builtins.length cfgsWithPasswords) == 0 then "" else
+          if (builtins.length cfgsWithPasswords) == 0 then
-              prefix + (lib.concatStrings (map exec cfgsWithPasswords)) + suffix);
+            ""
+          else
+            prefix + (lib.concatStrings (map exec cfgsWithPasswords)) + suffix
+        );
       };
 
-      debugConfig = enableDebug: lib.mkIf enableDebug {
+      debugConfig =
-        services.postgresql.settings.shared_preload_libraries = "auto_explain, pg_stat_statements";
+        enableDebug:
-      };
+        lib.mkIf enableDebug {
+          services.postgresql.settings.shared_preload_libraries = "auto_explain, pg_stat_statements";
+        };
     in
-      lib.mkMerge ([
+    lib.mkMerge ([
-        commonConfig
+      commonConfig
-        (dbConfig cfg.ensures)
+      (dbConfig cfg.ensures)
-        (pwdConfig cfg.ensures)
+      (pwdConfig cfg.ensures)
-        (lib.mkIf cfg.enableTCPIP tcpConfig)
+      (lib.mkIf cfg.enableTCPIP tcpConfig)
-        (debugConfig cfg.debug)
+      (debugConfig cfg.debug)
-        {
+      {
-          environment.systemPackages = lib.mkIf config.services.postgresql.enable [
+        environment.systemPackages = lib.mkIf config.services.postgresql.enable [
-            (upgrade-script 15 16)
+          (upgrade-script 15 16)
-            (upgrade-script 16 17)
+          (upgrade-script 16 17)
-          ];
+        ];
-        }
+      }
-      ]);
+    ]);
 }

modules/blocks/postgresql/docs/default.md

@@ -6,17 +6,46 @@ This block sets up a [PostgreSQL][] database.
 
 [postgresql]: https://www.postgresql.org/
 
-## Tests {#blocks-postgresql-tests}
+Compared to the upstream nixpkgs module, this module also sets up:
 
-Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
+- Enabling TCP/IP login and also accepting password authentication from localhost with [`shb.postgresql.enableTCPIP`](#blocks-postgresql-options-shb.postgresql.enableTCPIP).
+- Enhance the `ensure*` upstream option by setting up a database's password from a password file with [`shb.postgresql.ensures`](#blocks-postgresql-options-shb.postgresql.ensures).
+- Debug logging with `auto_explain` and `pg_stat_statements` with [`shb.postgresql.debug`](#blocks-postgresql-options-shb.postgresql.debug).
 
-## Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
+## Usage {#blocks-postgresql-usage}
+
+### Ensure User and Database {#blocks-postgresql-ensures}
+
+Ensure a database and user exists:
+
+```nix
+shb.postgresql.ensures = [
+  {
+    username = "firefly-iii";
+    database = "firefly-iii";
+  }
+];
+```
+
+Also set up the database password from a file path:
+
+```nix
+shb.postgresql.ensures = [
+  {
+    username = "firefly-iii";
+    database = "firefly-iii";
+    passwordFile = "/run/secrets/firefly-iii_db_password";
+  }
+];
+```
+
+### Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
 
 This block can be backed up using the [database backup](contracts-databasebackup.html) contract.
 
 Contract integration tests are defined in [`/test/contracts/databasebackup.nix`](@REPO@/test/contracts/databasebackup.nix).
 
-### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
+#### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
 
 ```nix
 {
@@ -30,6 +59,10 @@ Contract integration tests are defined in [`/test/contracts/databasebackup.nix`]
 }
 ```
 
+## Tests {#blocks-postgresql-tests}
+
+Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
+
 ## Options Reference {#blocks-postgresql-options}
 
 ```{=include=} options

modules/blocks/restic.nix

@@ -1,172 +1,234 @@
-{ config, pkgs, lib, utils, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  shb,
+  utils,
+  ...
+}:
 
 let
   cfg = config.shb.restic;
 
-  shblib = pkgs.callPackage ../../lib {};
+  inherit (lib)
-  contracts = pkgs.callPackage ../contracts {};
+    concatStringsSep
+    filterAttrs
+    flatten
+    literalExpression
+    optionals
+    listToAttrs
+    mapAttrsToList
+    mkEnableOption
+    mkOption
+    mkMerge
+    ;
+  inherit (lib)
+    hasPrefix
+    mkIf
+    nameValuePair
+    optionalAttrs
+    removePrefix
+    ;
+  inherit (lib.types)
+    attrsOf
+    enum
+    int
+    ints
+    oneOf
+    nonEmptyStr
+    nullOr
+    str
+    submodule
+    ;
 
-  inherit (lib) concatStringsSep filterAttrs flatten literalExpression optionals listToAttrs mapAttrsToList mkEnableOption mkOption mkMerge;
+  commonOptions =
-  inherit (lib) hasPrefix mkIf nameValuePair optionalAttrs removePrefix;
+    {
-  inherit (lib.types) attrsOf enum int ints oneOf nonEmptyStr nullOr str submodule;
+      name,
+      prefix,
+      config,
+      ...
+    }:
+    {
+      enable = mkEnableOption ''
+        SelfHostBlocks' Restic block
 
-  commonOptions = { name, prefix, config, ... }: {
+        A disabled instance will not backup data anymore
-    enable = mkEnableOption ''
+        but still provides the helper tool to restore snapshots
-      this backup intance.
+      '';
 
-      A disabled instance will not backup data anymore
+      passphrase = lib.mkOption {
-      but still provides the helper tool to restore snapshots
+        description = "Encryption key for the backup repository.";
-    '';
+        type = lib.types.submodule {
-
+          options = shb.contracts.secret.mkRequester {
-    passphrase = lib.mkOption {
+            mode = "0400";
-      description = "Encryption key for the backup repository.";
+            owner = config.request.user;
-      type = lib.types.submodule {
+            ownerText = "[shb.restic.${prefix}.<name>.request.user](#blocks-restic-options-shb.restic.${prefix}._name_.request.user)";
-        options = contracts.secret.mkRequester {
+            restartUnits = [ "${fullName name config.settings.repository}.service" ];
-          mode = "0400";
+            restartUnitsText = "[ [shb.restic.${prefix}.<name>.settings.repository](#blocks-restic-options-shb.restic.${prefix}._name_.settings.repository) ]";
-          owner = config.request.user;
+          };
-          ownerText = "[shb.restic.${prefix}.<name>.request.user](#blocks-restic-options-shb.restic.${prefix}._name_.request.user)";
-          restartUnits = [ (fullName name config.settings.repository) ];
-          restartUnitsText = "[ [shb.restic.${prefix}.<name>.settings.repository](#blocks-restic-options-shb.restic.${prefix}._name_.settings.repository) ]";
         };
       };
-    };
 
-    repository = mkOption {
+      repository = mkOption {
-      description = "Repositories to back this instance to.";
+        description = "Repositories to back this instance to.";
-      type = submodule {
+        type = submodule {
-        options = {
+          options = {
-          path = mkOption {
+            path = mkOption {
-            type = str;
+              type = str;
-            description = "Repository location";
+              description = "Repository location";
-          };
-
-          secrets = mkOption {
-            type = attrsOf shblib.secretFileType;
-            default = {};
-            description = ''
-              Secrets needed to access the repository where the backups will be stored.
-
-              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
-              and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
-
-            '';
-            example = literalExpression ''
-              {
-                AWS_ACCESS_KEY_ID.source = <path/to/secret>;
-                AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
-              }
-            '';
-          };
-
-          timerConfig = mkOption {
-            type = attrsOf utils.systemdUtils.unitOptions.unitOption;
-            default = {
-              OnCalendar = "daily";
-              Persistent = true;
             };
-            description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
+
-            example = {
+            secrets = mkOption {
-              OnCalendar = "00:05";
+              type = attrsOf shb.secretFileType;
-              RandomizedDelaySec = "5h";
+              default = { };
-              Persistent = true;
+              description = ''
+                Secrets needed to access the repository where the backups will be stored.
+
+                See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
+                and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
+
+              '';
+              example = literalExpression ''
+                {
+                  AWS_ACCESS_KEY_ID.source = <path/to/secret>;
+                  AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
+                }
+              '';
+            };
+
+            timerConfig = mkOption {
+              type = attrsOf utils.systemdUtils.unitOptions.unitOption;
+              default = {
+                OnCalendar = "daily";
+                Persistent = true;
+              };
+              description = "When to run the backup. See {manpage}`systemd.timer(5)` for details.";
+              example = {
+                OnCalendar = "00:05";
+                RandomizedDelaySec = "5h";
+                Persistent = true;
+              };
             };
           };
         };
       };
-    };
 
-    retention = mkOption {
+      retention = mkOption {
-      description = "For how long to keep backup files.";
+        description = "For how long to keep backup files.";
-      type = attrsOf (oneOf [ int nonEmptyStr ]);
+        type = attrsOf (oneOf [
-      default = {
+          int
-        keep_within = "1d";
+          nonEmptyStr
-        keep_hourly = 24;
+        ]);
-        keep_daily = 7;
+        default = {
-        keep_weekly = 4;
+          keep_within = "1d";
-        keep_monthly = 6;
+          keep_hourly = 24;
+          keep_daily = 7;
+          keep_weekly = 4;
+          keep_monthly = 6;
+        };
+      };
+
+      limitUploadKiBs = mkOption {
+        type = nullOr int;
+        description = "Limit upload bandwidth to the given KiB/s amount.";
+        default = null;
+        example = 8000;
+      };
+
+      limitDownloadKiBs = mkOption {
+        type = nullOr int;
+        description = "Limit download bandwidth to the given KiB/s amount.";
+        default = null;
+        example = 8000;
       };
     };
 
-    limitUploadKiBs = mkOption {
+  repoSlugName = name: builtins.replaceStrings [ "/" ":" ] [ "_" "_" ] (removePrefix "/" name);
-      type = nullOr int;
-      description = "Limit upload bandwidth to the given KiB/s amount.";
-      default = null;
-      example = 8000;
-    };
-
-    limitDownloadKiBs = mkOption {
-      type = nullOr int;
-      description = "Limit download bandwidth to the given KiB/s amount.";
-      default = null;
-      example = 8000;
-    };
-  };
-
-  repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (removePrefix "/" name);
   fullName = name: repository: "restic-backups-${name}_${repoSlugName repository.path}";
 in
 {
   imports = [
-    ./hardcodedsecret.nix
+    ../../lib/module.nix
+    ../blocks/monitoring.nix
   ];
 
   options.shb.restic = {
+    enableDashboard = lib.mkEnableOption "the Backups SHB dashboard" // {
+      default = true;
+    };
+
     instances = mkOption {
-      description = "Files to backup following the [backup contract](./contracts-backup.html).";
+      description = "Files to backup following the [backup contract](./shb.contracts-backup.html).";
-      default = {};
+      default = { };
-      type = attrsOf (submodule ({ name, config, ... }: {
+      type = attrsOf (
-        options = contracts.backup.mkProvider {
+        submodule (
-          settings = mkOption {
+          { name, config, ... }:
-            description = ''
+          {
-              Settings specific to the Restic provider.
+            options = shb.contracts.backup.mkProvider {
-            '';
+              settings = mkOption {
+                description = ''
+                  Settings specific to the Restic provider.
+                '';
 
-            type = submodule {
+                type = submodule {
-              options = commonOptions { inherit name config; prefix = "instances"; };
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "instances";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
             };
-          };
+          }
-
+        )
-          resultCfg = {
+      );
-            restoreScript = fullName name config.settings.repository;
-            restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
-
-            backupService = "${fullName name config.settings.repository}.service";
-            backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
-          };
-        };
-      }));
     };
 
     databases = mkOption {
-      description = "Databases to backup following the [database backup contract](./contracts-databasebackup.html).";
+      description = "Databases to backup following the [database backup contract](./shb.contracts-databasebackup.html).";
-      default = {};
+      default = { };
-      type = attrsOf (submodule ({ name, config, ... }: {
+      type = attrsOf (
-        options = contracts.databasebackup.mkProvider {
+        submodule (
-          settings = mkOption {
+          { name, config, ... }:
-            description = ''
+          {
-              Settings specific to the Restic provider.
+            options = shb.contracts.databasebackup.mkProvider {
-            '';
+              settings = mkOption {
+                description = ''
+                  Settings specific to the Restic provider.
+                '';
 
-            type = submodule {
+                type = submodule {
-              options = commonOptions { inherit name config; prefix = "databases"; };
+                  options = commonOptions {
+                    inherit name config;
+                    prefix = "databases";
+                  };
+                };
+              };
+
+              resultCfg = {
+                restoreScript = fullName name config.settings.repository;
+                restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
+
+                backupService = "${fullName name config.settings.repository}.service";
+                backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
+              };
             };
-          };
+          }
-
+        )
-          resultCfg = {
+      );
-            restoreScript = fullName name config.settings.repository;
-            restoreScriptText = "${fullName "<name>" { path = "path/to/repository"; }}";
-
-            backupService = "${fullName name config.settings.repository}.service";
-            backupServiceText = "${fullName "<name>" { path = "path/to/repository"; }}.service";
-          };
-        };
-      }));
     };
 
     # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
     performance = mkOption {
       description = "Reduce performance impact of backup jobs.";
-      default = {};
+      default = { };
       type = submodule {
         options = {
           niceness = mkOption {
@@ -175,7 +237,11 @@ in
             default = 15;
           };
           ioSchedulingClass = mkOption {
-            type = enum [ "idle" "best-effort" "realtime" ];
+            type = enum [
+              "idle"
+              "best-effort"
+              "realtime"
+            ];
             description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
             default = "best-effort";
           };
@@ -189,23 +255,28 @@ in
     };
   };
 
-  config = mkIf (cfg.instances != {} || cfg.databases != {}) (
+  config = mkIf (cfg.instances != { } || cfg.databases != { }) (
     let
       enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances;
       enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases;
-    in mkMerge [
+    in
+    mkMerge [
       {
-        environment.systemPackages = optionals (enabledInstances != {} || enabledDatabases != {}) [ pkgs.restic ];
+        environment.systemPackages = optionals (enabledInstances != { } || enabledDatabases != { }) [
+          pkgs.restic
+        ];
       }
       {
         # Create repository if it is a local path.
         systemd.tmpfiles.rules =
           let
-            mkSettings = name: instance: optionals (hasPrefix "/" instance.settings.repository.path) [
+            mkSettings =
-              "d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -"
+              name: instance:
-            ];
+              optionals (hasPrefix "/" instance.settings.repository.path) [
+                "d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -"
+              ];
           in
-            flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
+          flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
       }
       {
         services.restic.backups =
@@ -224,8 +295,8 @@ in
 
                 inherit (instance.settings.repository) timerConfig;
 
-                pruneOpts = mapAttrsToList (name: value:
+                pruneOpts = mapAttrsToList (
-                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+                  name: value: "--${builtins.replaceStrings [ "_" ] [ "-" ] name} ${builtins.toString value}"
                 ) instance.settings.retention;
 
                 backupPrepareCommand = concatStringsSep "\n" instance.request.hooks.beforeBackup;
@@ -239,12 +310,13 @@ in
                   ++ (optionals (instance.settings.limitDownloadKiBs != null) [
                     "--limit-download=${toString instance.settings.limitDownloadKiBs}"
                   ]);
-              } // optionalAttrs (builtins.length instance.request.excludePatterns > 0) {
+              }
+              // optionalAttrs (builtins.length instance.request.excludePatterns > 0) {
                 exclude = instance.request.excludePatterns;
               };
             };
           in
-            mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
+          mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
       }
       {
         services.restic.backups =
@@ -263,8 +335,8 @@ in
 
                 inherit (instance.settings.repository) timerConfig;
 
-                pruneOpts = mapAttrsToList (name: value:
+                pruneOpts = mapAttrsToList (
-                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+                  name: value: "--${builtins.replaceStrings [ "_" ] [ "-" ] name} ${builtins.toString value}"
                 ) instance.settings.retention;
 
                 extraBackupArgs =
@@ -274,120 +346,139 @@ in
                   ++ (optionals (instance.settings.limitDownloadKiBs != null) [
                     "--limit-download=${toString instance.settings.limitDownloadKiBs}"
                   ])
-                  ++ 
+                  ++ (
-                  (let
+                    let
-                    cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd;
+                      cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd;
-                  in
+                    in
                     [
                       "--stdin-filename ${instance.request.backupName} --stdin-from-command -- ${cmd}/bin/dump.sh"
-                    ]);
+                    ]
+                  );
               };
             };
           in
-            mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
+          mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
       }
       {
         systemd.services =
           let
-            mkSettings = name: instance:
+            mkSettings =
+              name: instance:
               let
                 serviceName = fullName name instance.settings.repository;
               in
-                {
+              {
-                  ${serviceName} = mkMerge [
+                ${serviceName} = mkMerge [
-                    {
+                  {
-                      serviceConfig = {
+                    serviceConfig = {
-                        Nice = cfg.performance.niceness;
+                      Nice = cfg.performance.niceness;
-                        IOSchedulingClass = cfg.performance.ioSchedulingClass;
+                      IOSchedulingClass = cfg.performance.ioSchedulingClass;
-                        IOSchedulingPriority = cfg.performance.ioPriority;
+                      IOSchedulingPriority = cfg.performance.ioPriority;
-                        # BindReadOnlyPaths = instance.sourceDirectories;
+                      # BindReadOnlyPaths = instance.sourceDirectories;
-                      };
+                    };
-                    }
+                  }
-                    (optionalAttrs (instance.settings.repository.secrets != {})
+                  (optionalAttrs (instance.settings.repository.secrets != { }) {
-                      {
+                    serviceConfig.EnvironmentFile = [
-                        serviceConfig.EnvironmentFile = [
+                      "/run/secrets_restic/${serviceName}"
-                          "/run/secrets_restic/${serviceName}"
+                    ];
-                        ];
+                    after = [ "${serviceName}-pre.service" ];
-                        after = [ "${serviceName}-pre.service" ];
+                    requires = [ "${serviceName}-pre.service" ];
-                        requires = [ "${serviceName}-pre.service" ];
+                  })
-                      })
+                ];
-                  ];
 
-                  "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != {})
+                "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != { }) (
-                    (let
+                  let
-                      script = shblib.genConfigOutOfBandSystemd {
+                    script = shb.genConfigOutOfBandSystemd {
-                        config = instance.settings.repository.secrets;
+                      config = instance.settings.repository.secrets;
-                        configLocation = "/run/secrets_restic/${serviceName}";
+                      configLocation = "/run/secrets_restic/${serviceName}";
-                        generator = shblib.toEnvVar;
+                      generator = shb.toEnvVar;
-                        user = instance.request.user;
+                      user = instance.request.user;
-                      };
+                    };
-                    in
+                  in
-                      {
+                  {
-                        script = script.preStart;
+                    script = script.preStart;
-                        serviceConfig.Type = "oneshot";
+                    serviceConfig.Type = "oneshot";
-                        serviceConfig.LoadCredential = script.loadCredentials;
+                    serviceConfig.LoadCredential = script.loadCredentials;
-                      });
+                  }
-            };
+                );
+              };
           in
-            mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
+          mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
       }
       {
-        systemd.services = let
+        systemd.services =
-          mkEnv = name: instance:
+          let
-            nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
+            mkEnv =
-              enable = true;
+              name: instance:
-              wantedBy = [ "multi-user.target" ];
+              nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
-              serviceConfig.Type = "oneshot";
+                enable = true;
-              script = (shblib.replaceSecrets {
+                wantedBy = [ "multi-user.target" ];
-                userConfig = instance.settings.repository.secrets // {
+                # Purposely not a oneshot systemd service otherwise
-                  RESTIC_PASSWORD_FILE = toString instance.settings.passphrase.result.path;
+                # the service waits on the completion the backup before deactivating.
-                  RESTIC_REPOSITORY = instance.settings.repository.path;
+                # This seems like a nice property at first but there is one annoying
-                };
+                # edge case when deploying. If a backup job somehow is started when
-                resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
+                # the deploy happens, the deploy will wait on the service to finish
-                generator = shblib.toEnvVar;
+                # before considering the deploy done. And worse, it will consider the
-                user = instance.request.user;
+                # deploy as failed if the backup fails, which is not what you want.
-              });
+                script = (
-            };
+                  shb.replaceSecrets {
-        in
+                    userConfig = instance.settings.repository.secrets // {
+                      RESTIC_PASSWORD_FILE = toString instance.settings.passphrase.result.path;
+                      RESTIC_REPOSITORY = instance.settings.repository.path;
+                    };
+                    resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
+                    generator = shb.toEnvVar;
+                    user = instance.request.user;
+                  }
+                );
+              };
+          in
           listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases)));
       }
       {
-        environment.systemPackages = let
+        environment.systemPackages =
-          mkResticBinary = name: instance:
+          let
-            pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
+            mkResticBinary =
-              set -euo pipefail
+              name: instance:
-
+              shb.contracts.backup.mkRestoreScript {
-              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
+                name = fullName name instance.settings.repository;
-                       | xargs -d '\n')
+                user = instance.request.user;
-
+                sudoPreserveEnvs = [
-              if ! [ "$1" = "restore" ]; then
+                  "RESTIC_REPOSITORY"
-                sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
+                  "RESTIC_PASSWORD_FILE"
-              else
+                ];
-                shift
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic restore $@ --target /"
+                restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
-              fi
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
-              '';
+              };
-        in
+          in
           flatten (mapAttrsToList mkResticBinary cfg.instances);
       }
       {
-        environment.systemPackages = let
+        environment.systemPackages =
-          mkResticBinary = name: instance:
+          let
-            pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
+            mkResticBinary =
-              set -euo pipefail
+              name: instance:
-
+              shb.contracts.backup.mkRestoreScript {
-              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
+                name = fullName name instance.settings.repository;
-                       | xargs -d '\n')
+                user = instance.request.user;
-
+                sudoPreserveEnvs = [
-              if ! [ "$1" = "restore" ]; then
+                  "RESTIC_REPOSITORY"
-                sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
+                  "RESTIC_PASSWORD_FILE"
-              else
+                ];
-                shift
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupName} | ${instance.request.restoreCmd}"
+                restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
-              fi
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
-              '';
+              };
-        in
+          in
           flatten (mapAttrsToList mkResticBinary cfg.databases);
       }
-    ]);
+
+      (lib.mkIf (cfg.enableDashboard && (cfg.instances != { } || cfg.databases != { })) {
+        shb.monitoring.dashboards = [
+          ./backup/dashboard/Backups.json
+        ];
+      })
+    ]
+  );
 }

modules/blocks/restic/docs/default.md

@@ -3,13 +3,10 @@
 Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
 
 This block sets up a backup job using [Restic][].
+It is heavily based on the nixpkgs Restic module.
 
 [restic]: https://restic.net/
 
-## Tests {#blocks-restic-tests}
-
-Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
-
 ## Provider Contracts {#blocks-restic-contract-provider}
 
 This block provides the following contracts:
@@ -25,9 +22,7 @@ This block provides the following contracts:
 [database backup contract tests]: @REPO@/test/contracts/databasebackup.nix
 
 As requested by those two contracts, when setting up a backup with Restic,
-a backup Systemd service and restore script are provided.
+a backup Systemd service and a [restore script](#blocks-restic-maintenance) are provided.
-The restore script has all the secrets needed to access the repo,
-the only requirement to run it is to be able to `sudo` in the expected user.
 
 ## Usage {#blocks-restic-usage}
 
@@ -56,7 +51,7 @@ shb.restic.instances."myservice" = {
   settings = {
     enable = true;
 
-    passphrase.result = shb.sops.secret."passphrase".result;
+    passphrase.result = config.shb.sops.secret."passphrase".result;
 
     repository = {
       path = "/srv/backups/myservice";
@@ -77,7 +72,7 @@ shb.restic.instances."myservice" = {
 };
 
 shb.sops.secret."passphrase".request =
-  shb.restic.instances."myservice".settings.passphrase.request;
+  config.shb.restic.instances."myservice".settings.passphrase.request;
 ```
 
 ### One folder backed up with contract {#blocks-restic-usage-provider-contract}
@@ -88,12 +83,12 @@ the snippet above becomes:
 
 ```nix
 shb.restic.instances."myservice" = {
-  request = config.myservice.backup;
+  request = config.myservice.backup.request;
 
   settings = {
     enable = true;
 
-    passphrase.result = shb.sops.secret."passphrase".result;
+    passphrase.result = config.shb.sops.secret."passphrase".result;
 
     repository = {
       path = "/srv/backups/myservice";
@@ -114,7 +109,7 @@ shb.restic.instances."myservice" = {
 };
 
 shb.sops.secret."passphrase".request =
-  shb.restic.instances."myservice".settings.passphrase.request;
+  config.shb.restic.instances."myservice".settings.passphrase.request;
 ```
 
 ### One folder backed up to S3 {#blocks-restic-usage-provider-remote}
@@ -124,7 +119,7 @@ Here we will only highlight the differences with the previous configuration.
 This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/).
 
 ```diff
-  shb.backup.instances.myservice = {
+  shb.test.backup.instances.myservice = {
 
     repository = {
 -     path = "/srv/pool1/backups/myfolder";
@@ -142,7 +137,7 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
   }
 ```
 
-## Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
+### Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
 
 The following snippet shows how to configure backup of any number of folders to 3 repositories,
 each happening at different times to avoid I/O contention.
@@ -217,28 +212,62 @@ backupcfg = repositories: name: sourceDirectories {
 Now, we can define multiple backup jobs to backup different folders:
 
 ```nix
-shb.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"];
+shb.test.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"];
-shb.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"];
+shb.test.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"];
 ```
 
 The difference between the above snippet and putting all the folders into one configuration (shown
 below) is the former splits the backups into sub-folders on the repositories.
 
 ```nix
-shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
+shb.test.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"];
 ```
 
-## Demo {#blocks-restic-demo}
-
-[WIP]
-
 ## Monitoring {#blocks-restic-monitoring}
 
-[WIP]
+A generic dashboard for all backup solutions is provided.
+See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backup) section in the monitoring chapter.
 
 ## Maintenance {#blocks-restic-maintenance}
 
-One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
+### Manual Backup {#blocks-restic-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'restic*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start restic<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-restic-maintenance-restore}
+
+One command-line helper is provided per backup instance and repository pair which allows to:
+
+- list snapshots: `<script> snapshots`
+- to restore a snapshot: `<script> restore <snapshot>`
+
+The restore script has all the secrets needed to access the repo,
+it will run `sudo` automatically
+and the user running it needs to have correct permissions for privilege escalation.
 
 In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
 
@@ -263,6 +292,10 @@ restic-myfolder1_srv_pool1_backups restore latest
 
 In case something bad happens with a backup, the [official documentation](https://restic.readthedocs.io/en/stable/077_troubleshooting.html) has a lot of tips.
 
+## Tests {#blocks-restic-tests}
+
+Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
+
 ## Options Reference {#blocks-restic-options}
 
 ```{=include=} options

modules/blocks/sanoid.nix

@@ -0,0 +1,164 @@
+{
+  config,
+  lib,
+  pkgs,
+  shb,
+  utils,
+  ...
+}:
+let
+  cfg = config.shb.sanoid;
+
+  restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore";
+  backupScriptBase = "sanoid";
+
+  restoreScript =
+    name:
+    pkgs.writers.writePython3Bin (restoreScriptName name)
+      {
+        flakeIgnore = [ "E501" ];
+      }
+      ''
+        import argparse
+        import subprocess
+        import sys
+
+
+        dataset = "${name}"
+
+
+        class ZFSError(Exception):
+            pass
+
+
+        def run_command(cmd: list[str]) -> str:
+            try:
+                result = subprocess.run(
+                    cmd,
+                    check=True,
+                    text=True,
+                    capture_output=True,
+                )
+                return result.stdout.strip()
+            except subprocess.CalledProcessError as e:
+                raise ZFSError(
+                    f"Command failed: {' '.join(cmd)}\n"
+                    f"Exit code: {e.returncode}\n"
+                    f"stderr: {e.stderr.strip()}"
+                ) from e
+
+
+        def list_snapshots() -> None:
+            """List all ZFS snapshots."""
+            output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset])
+            if not output:
+                return []
+            return output.splitlines()
+
+
+        def restore_snapshot(snapshot: str) -> None:
+            """Rollback to a given snapshot."""
+            if not snapshot:
+                raise ValueError("Snapshot name must not be empty")
+
+            print(f"Rolling back to snapshot: {snapshot}")
+            run_command(["zfs", "rollback", "-r", snapshot])
+            print("Rollback completed successfully.")
+
+
+        def build_parser() -> argparse.ArgumentParser:
+            parser = argparse.ArgumentParser(description=f"Restore script for {dataset}")
+
+            subparsers = parser.add_subparsers(dest="command", required=True)
+
+            subparsers.add_parser(
+                "snapshots",
+                help="List all ZFS snapshots",
+            )
+
+            restore_parser = subparsers.add_parser(
+                "restore",
+                help="Rollback to a specific snapshot",
+            )
+            restore_parser.add_argument(
+                "snapshot",
+                help="Snapshot name (e.g. pool/dataset@snapname)",
+            )
+
+            return parser
+
+
+        def main():
+            parser = build_parser()
+            args = parser.parse_args()
+
+            try:
+                if args.command == "snapshots":
+                    snapshots = list_snapshots()
+                    for s in snapshots:
+                        print(s)
+                elif args.command == "restore":
+                    restore_snapshot(args.snapshot)
+                else:
+                    parser.print_help()
+                    sys.exit(1)
+
+            except ZFSError as e:
+                print(f"ERROR: {e}", file=sys.stderr)
+                sys.exit(1)
+            except Exception as e:
+                print(f"Unexpected error: {e}", file=sys.stderr)
+                sys.exit(1)
+
+
+        if __name__ == "__main__":
+            main()
+      '';
+in
+{
+  imports = [
+    ../../lib/module.nix
+  ];
+
+  options.shb.sanoid.backup = lib.mkOption {
+    description = "Sanoid prodiver for file backup contract";
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, ... }:
+        {
+          options = shb.contracts.backup.mkProvider {
+            resultCfg = {
+              restoreScript = restoreScriptName name;
+              restoreScriptText = restoreScriptName "<name>";
+
+              backupService = "${backupScriptBase}.service";
+              backupServiceText = "${backupScriptBase}.service";
+            };
+
+            settings = lib.mkOption {
+              description = "Options passed to the `services.sanoid.datasets.<name>` option.";
+              default = { };
+              type = lib.types.attrsOf lib.types.anything;
+            };
+          };
+        }
+      )
+    );
+  };
+
+  config = lib.mkIf (cfg.backup != { }) {
+    services.sanoid.enable = true;
+
+    services.sanoid.datasets =
+      let
+        mkDataset = name: cfg': {
+          inherit name;
+          value = cfg'.settings;
+        };
+      in
+      lib.mapAttrs' mkDataset cfg.backup;
+
+    environment.systemPackages = map restoreScript (lib.attrNames cfg.backup);
+  };
+}

modules/blocks/sanoid/docs/default.md

@@ -0,0 +1,97 @@
+# Sanoid Block {#blocks-sanoid}
+
+Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix):
+
+```nix
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.sanoid
+  ];
+}
+```
+
+## Provider Contracts {#blocks-sanoid-contract-provider}
+
+This block provides the following contracts:
+
+- [dataset backup contract](contracts-datasetbackup.html) under the [`shb.sanoid.backup`][backup] option.
+  It is tested with the [generic contract tests][backup contract tests].
+
+[backup]: #blocks-sanoid-options-shb.sanoid.backup
+[backup contract tests]: @REPO@/test/contracts/backup.nix
+
+## Usage {#blocks-sanoid-usage}
+
+Sanoid uses templates to know when snapshots should be kept or pruned.
+
+### Default Template {#blocks-sanoid-usage-default-template}
+
+Backup a dataset using the default Sanoid template:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+  };
+}
+```
+
+This uses the dataset backup contract which is exposed through the ZFS module's [`shb.zfs.pools.<name>.datasets.<name>.datasetBackup`](blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup) option.
+
+### Custom Template {#blocks-sanoid-usage-custom-template}
+
+Create a custom template and use it:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  services.sanoid.templates."yearly" = {
+    hourly = 10;
+    daily = 3;
+    monthly = 3;
+    yearly = 2;
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+    template = "yearly";
+  };
+}
+```
+
+Note we use the upstream `services.sanoid.templates` option to define the templates.
+
+### Without contract {#blocks-sanoid-usage-without-contract}
+
+To backup a dataset that does not provide the dataset backup contract,
+we can just set the request manually:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request.dataset = "root/home";
+  };
+}
+```
+
+Note the attr name under the `shb.sanoid.backup` option does not
+set the dataset name.
+
+## Options Reference {#blocks-sanoid-options}
+
+```{=include=} options
+id-prefix: blocks-sanoid-options-
+list-id: selfhostblocks-blocks-sanoid-options
+source: @OPTIONS_JSON@
+```

modules/blocks/sops.nix

@@ -1,45 +1,59 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  shb,
+  ...
+}:
 let
   inherit (lib) mapAttrs mkOption;
   inherit (lib.types) attrsOf anything submodule;
 
-  contracts = pkgs.callPackage ../contracts {};
-
   cfg = config.shb.sops;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.sops = {
     secret = mkOption {
       description = "Secret following the [secret contract](./contracts-secret.html).";
-      default = {};
+      default = { };
-      type = attrsOf (submodule ({ name, options, ... }: {
+      type = attrsOf (
-        options = contracts.secret.mkProvider {
+        submodule (
-          settings = mkOption {
+          { name, options, ... }:
-            description = ''
+          {
-              Settings specific to the Sops provider.
+            options = shb.contracts.secret.mkProvider {
+              settings = mkOption {
+                description = ''
+                  Settings specific to the Sops provider.
 
-              This is a passthrough option to set [sops-nix options](https://github.com/Mic92/sops-nix/blob/master/modules/sops/default.nix).
+                  This is a passthrough option to set [sops-nix options](https://github.com/Mic92/sops-nix/blob/master/modules/sops/default.nix).
 
-              Note though that the `mode`, `owner`, `group`, and `restartUnits`
+                  Note though that the `mode`, `owner`, `group`, and `restartUnits`
-              are managed by the [shb.sops.secret.<name>.request](#blocks-sops-options-shb.sops.secret._name_.request) option.
+                  are managed by the [shb.sops.secret.<name>.request](#blocks-sops-options-shb.sops.secret._name_.request) option.
-            '';
+                '';
 
-            type = attrsOf anything;
+                type = attrsOf anything;
-            default = {};
+                default = { };
-          };
+              };
 
-          resultCfg = {
+              resultCfg = {
-            path = "/run/secrets/${name}";
+                path = "/run/secrets/${name}";
-            pathText = "/run/secrets/<name>";
+                pathText = "/run/secrets/<name>";
-          };
+              };
-        };
+            };
-      }));
+          }
+        )
+      );
     };
   };
 
   config = {
-    sops.secrets = let
+    sops.secrets =
-      mkSecret = n: secretCfg: secretCfg.request // secretCfg.settings;
+      let
-    in mapAttrs mkSecret cfg.secret;
+        mkSecret = n: secretCfg: secretCfg.request // secretCfg.settings;
+      in
+      mapAttrs mkSecret cfg.secret;
   };
 }

modules/blocks/sops/docs/default.md

@@ -13,7 +13,7 @@ to adapt it to the [secret contract](./contracts-secret.html).
 
 This block provides the following contracts:
 
-- [secret contract][] under the [`shb.sops.secrets`][secret] option.
+- [secret contract][] under the [`shb.sops.secret`][secret] option.
   It is not yet tested with [contract tests][secret contract tests] but it is used extensively on several machines.
 
 [secret]: #blocks-sops-options-shb.sops.secret
@@ -21,7 +21,7 @@ This block provides the following contracts:
 [secret contract tests]: @REPO@/test/contracts/secret.nix
 
 As requested by the contract, when asking for a secret with the `shb.sops` module,
-the path where the secret will be located can be found under the [`shb.sops.secrets.<name>.result`][result] option.
+the path where the secret will be located can be found under the [`shb.sops.secret.<name>.result`][result] option.
 
 [result]: #blocks-sops-options-shb.sops.secret._name_.result
 

modules/blocks/ssl/dashboard/SSL.json

@@ -0,0 +1,727 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 16,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": 0,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "line+area"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              },
+              {
+                "color": "transparent",
+                "value": 604808
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": false
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "expr": "min by(exported_hostname, subject, path) (ssl_certificate_expiry_seconds{subject=~\"CN=$job\"})",
+          "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Certificate Remaining Validity",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "fieldMinMax": false,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "dateTimeFromNow"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true,
+          "width": 300
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "systemd_timer_next_trigger_seconds{name=~\"acme-renew-$job.timer\"} * 1000",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Schedule",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "footer": {
+              "reducers": []
+            },
+            "inspect": false
+          },
+          "decimals": 0,
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "green",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "% failed"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percentunit"
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "color",
+                "value": {
+                  "mode": "continuous-GrYlRd"
+                }
+              },
+              {
+                "id": "max",
+                "value": 1
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "total"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "color-background"
+                }
+              },
+              {
+                "id": "color",
+                "value": {
+                  "mode": "thresholds"
+                }
+              },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "red",
+                      "value": 0
+                    },
+                    {
+                      "color": "transparent",
+                      "value": 1
+                    }
+                  ]
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byType",
+              "options": "string"
+            },
+            "properties": [
+              {
+                "id": "custom.minWidth",
+                "value": 150
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byType",
+              "options": "number"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 100
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "cellHeight": "sm",
+        "enablePagination": true,
+        "frozenColumns": {},
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "increase(systemd_unit_state{name=~\"acme-(order-renew-)?[[job]].service\", state=\"activating\"}[7d])",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "increase(systemd_unit_state{name=~\"acme-(order-renew-)?[[job]].service\", state=\"failed\"}[7d])",
+          "hide": false,
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "B"
+        }
+      ],
+      "title": "Jobs in the Past Week",
+      "transformations": [
+        {
+          "id": "labelsToFields",
+          "options": {
+            "mode": "columns"
+          }
+        },
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "groupingToMatrix",
+          "options": {
+            "columnField": "state",
+            "rowField": "name",
+            "valueField": "Value"
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "total",
+            "binary": {
+              "left": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "activating"
+                }
+              },
+              "operator": "+",
+              "right": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "failed"
+                }
+              }
+            },
+            "mode": "binary",
+            "reduce": {
+              "include": [
+                "activating",
+                "failed"
+              ],
+              "reducer": "sum"
+            }
+          }
+        },
+        {
+          "id": "calculateField",
+          "options": {
+            "alias": "% failed",
+            "binary": {
+              "left": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "failed"
+                }
+              },
+              "operator": "/",
+              "right": {
+                "matcher": {
+                  "id": "byName",
+                  "options": "total"
+                }
+              }
+            },
+            "mode": "binary",
+            "reduce": {
+              "reducer": "sum"
+            }
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "total"
+              }
+            ]
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "failed"
+              }
+            ]
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "includeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "activating": "success",
+              "name\\state": "Job"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "description": "",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 7,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "If the log panel is empty, it may be because the amount of lines is too high. Try filtering a few jobs first.",
+        "mode": "markdown"
+      },
+      "pluginVersion": "12.2.0",
+      "title": "",
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "default": false,
+        "type": "loki",
+        "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 14,
+        "w": 12,
+        "x": 12,
+        "y": 10
+      },
+      "id": 8,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": false,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "cd6cc53e-840c-484d-85f7-96fede324006"
+          },
+          "direction": "backward",
+          "editorMode": "code",
+          "expr": "{unit=~\"acme-(order-renew-)?($job).service\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Logs - $job",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+      },
+      "description": "The job duration is not accurate. Jobs taking less than 15s to run will sometimes appear as taking 100s.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineWidth": 0,
+            "spanNulls": false
+          },
+          "mappings": [
+            {
+              "options": {
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "Running"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "#EAB839",
+                "value": 0
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "never",
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "max(label_replace(systemd_unit_state{name=~\"acme-(order-renew-)?$job.service\", state=\"activating\"}, \"name\", \"$1\", \"name\", \"acme-order-renew-(.*).service\") > 0 or on(name) label_replace(clamp(systemd_timer_last_trigger_seconds{name=~\"acme-renew-$job.timer\"} - (systemd_timer_last_trigger_seconds{name=~\"acme-renew-$job.timer\"} offset 100s) > 0, 0, 1), \"name\", \"$1\", \"name\", \"acme-renew-(.*).timer\")) by (name)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "key": "Q-e1d5c07a-8dcc-4f34-aa5c-cdebcbdda322-0",
+          "legendFormat": "{{name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Job Runs",
+      "type": "state-timeline"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 42,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "definition": "label_values(systemd_unit_state{name=~\"acme-renew-.*.timer\"},name)",
+        "includeAll": true,
+        "label": "Job",
+        "multi": true,
+        "name": "job",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(systemd_unit_state{name=~\"acme-renew-.*.timer\"},name)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "/acme-renew-(?<value>.*).timer/",
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "SSL Certificates",
+  "uid": "ae818js0bvw8wb",
+  "version": 25
+}

modules/blocks/ssl/docs/default.md

@@ -62,6 +62,7 @@ We can ask Let's Encrypt to generate a certificate with:
 shb.certs.certs.letsencrypt."example.com" = {
   domain = "example.com";
   group = "nginx";
+  reloadServices = [ "nginx.service" ];
   dnsProvider = "linode";
   adminEmail = "admin@example.com";
   credentialsFile = /path/to/secret/file;
@@ -79,6 +80,13 @@ The credential file's content would be a key-value pair:
 LINODE_TOKEN=XYZ...
 ```
 
+If you use one subdomain per service,
+asking for certificates for a subdomain is done with:
+
+```nix
+shb.certs.certs.letsencrypt."example.com".extraDomains = [ "nextcloud.${domain}" ];
+```
+
 For other providers, see the [official instruction](https://go-acme.github.io/lego/dns/).
 
 ## Usage {#block-ssl-usage}
@@ -110,11 +118,35 @@ config.shb.certs.systemdService
 See also the [SSL certificate generator usage](contracts-ssl.html#ssl-contract-usage) for a more detailed usage
 example.
 
+## Monitoring {#blocks-ssl-monitoring}
+
+A dashboard for SSL certificates is provided.
+See [SSL Certificates Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-ssl) section in the monitoring chapter.
+
 ## Debug {#block-ssl-debug}
 
 Each CA and Cert is generated by a systemd service whose name can be seen in the `systemdService`
 option. You can then see the latest errors messages using `journalctl`.
 
+### Let's Encrypt debug {#blocks-ssl-debug-lets-encrypt}
+
+Since the SHB SSL block uses the [`security.acme`][] module under the hood,
+knowing how that one works can become required if something goes wrong.
+
+For each domain and subdomain, noted as `fqdn` hereunder,
+the following systemd timers and services are created:
+
+- `acme-renew-${fqdn}.timer` triggers the `acme-order-renew-${fqdn}.service` service every day.
+- `acme-${fqdn}.service` (re)generate the initial self-signed certificate,
+   only if the following job never succeeded at least once yet.
+- `acme-order-renew-${fqdn}.service` asks for a new certificate
+   only if the certificate will expire in the next 30 days.
+   Has logic to only renew if the list of domains has not changed.
+
+Also, a global service named `acme-setup.service` is created
+
+[`security.acme`]: https://nixos.org/manual/nixos/stable/#module-security-acme
+
 ## Tests {#block-ssl-tests}
 
 The self-signed implementation is tested in [`/tests/vm/ssl.nix`](@REPO@/tests/vm/ssl.nix).

modules/blocks/tinyproxy.nix

@@ -1,28 +1,41 @@
 # Inspired from https://github.com/NixOS/nixpkgs/pull/231152 but made it so we can have multiple instances.
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 with lib;
 
 let
   cfg = config.shb.tinyproxy;
 
-  mkValueStringTinyproxy = with lib; v:
+  mkValueStringTinyproxy =
-        if true  ==         v then "yes"
+    with lib;
-        else if false ==    v then "no"
+    v:
-        else generators.mkValueStringDefault {} v;
+    if true == v then
+      "yes"
+    else if false == v then
+      "no"
+    else
+      generators.mkValueStringDefault { } v;
 
-  mkKeyValueTinyproxy = {
+  mkKeyValueTinyproxy =
-    mkValueString ? mkValueStringDefault {}
+    {
-  }: sep: k: v:
+      mkValueString ? mkValueStringDefault { },
-    if null     ==  v then ""
+    }:
-    else "${lib.strings.escape [sep] k}${sep}${mkValueString v}";
+    sep: k: v:
+    if null == v then "" else "${lib.strings.escape [ sep ] k}${sep}${mkValueString v}";
 
-  settingsFormat = (pkgs.formats.keyValue {
+  settingsFormat = (
+    pkgs.formats.keyValue {
       mkKeyValue = mkKeyValueTinyproxy {
         mkValueString = mkValueStringTinyproxy;
       } " ";
-      listsAsDuplicateKeys= true;
+      listsAsDuplicateKeys = true;
-  });
+    }
+  );
 
   configFile = name: cfg: settingsFormat.generate "tinyproxy-${name}.conf" cfg.settings;
 in
@@ -33,115 +46,124 @@ in
         options = {
           enable = mkEnableOption "Tinyproxy daemon";
 
-          package = mkPackageOption pkgs "tinyproxy" {};
+          package = mkPackageOption pkgs "tinyproxy" { };
 
           dynamicBindFile = mkOption {
             description = ''
-            File holding the IP to bind to.
+              File holding the IP to bind to.
             '';
             default = "";
           };
 
           settings = mkOption {
             description = ''
-            Configuration for [tinyproxy](https://tinyproxy.github.io/).
+              Configuration for [tinyproxy](https://tinyproxy.github.io/).
             '';
             default = { };
-            example = literalExpression ''{
+            example = literalExpression ''
-            Port 8888;
+              {
-            Listen 127.0.0.1;
+                          Port 8888;
-            Timeout 600;
+                          Listen 127.0.0.1;
-            Allow 127.0.0.1;
+                          Timeout 600;
-            Anonymous = ['"Host"' '"Authorization"'];
+                          Allow 127.0.0.1;
-            ReversePath = '"/example/" "http://www.example.com/"';
+                          Anonymous = ['"Host"' '"Authorization"'];
-            }'';
+                          ReversePath = '"/example/" "http://www.example.com/"';
-            type = types.submodule ({name, ...}: {
+                          }'';
-              freeformType = settingsFormat.type;
+            type = types.submodule (
-              options = {
+              { name, ... }:
-                Listen = mkOption {
+              {
-                  type = types.str;
+                freeformType = settingsFormat.type;
-                  default = "127.0.0.1";
+                options = {
-                  description = ''
+                  Listen = mkOption {
-                  Specify which address to listen to.
+                    type = types.str;
-                  '';
+                    default = "127.0.0.1";
+                    description = ''
+                      Specify which address to listen to.
+                    '';
+                  };
+                  Port = mkOption {
+                    type = types.int;
+                    default = 8888;
+                    description = ''
+                      Specify which port to listen to.
+                    '';
+                  };
+                  Anonymous = mkOption {
+                    type = types.listOf types.str;
+                    default = [ ];
+                    description = ''
+                      If an `Anonymous` keyword is present, then anonymous proxying is enabled. The
+                      headers listed with `Anonymous` are allowed through, while all others are denied.
+                      If no Anonymous keyword is present, then all headers are allowed through. You must
+                      include quotes around the headers.
+                    '';
+                  };
+                  Filter = mkOption {
+                    type = types.nullOr types.path;
+                    default = null;
+                    description = ''
+                      Tinyproxy supports filtering of web sites based on URLs or domains. This option
+                      specifies the location of the file containing the filter rules, one rule per line.
+                    '';
+                  };
                 };
-                Port = mkOption {
+              }
-                  type = types.int;
+            );
-                  default = 8888;
-                  description = ''
-                  Specify which port to listen to.
-                  '';
-                };
-                Anonymous = mkOption {
-                  type = types.listOf types.str;
-                  default = [];
-                  description = ''
-                  If an `Anonymous` keyword is present, then anonymous proxying is enabled. The
-                  headers listed with `Anonymous` are allowed through, while all others are denied.
-                  If no Anonymous keyword is present, then all headers are allowed through. You must
-                  include quotes around the headers.
-                  '';
-                };
-                Filter = mkOption {
-                  type = types.nullOr types.path;
-                  default = null;
-                  description = ''
-                  Tinyproxy supports filtering of web sites based on URLs or domains. This option
-                  specifies the location of the file containing the filter rules, one rule per line.
-                  '';
-                };
-              };
-            });
           };
         };
       };
     in
-      {
+    {
-        shb.tinyproxy = mkOption {
+      shb.tinyproxy = mkOption {
-          description = "Tinyproxy instances.";
+        description = "Tinyproxy instances.";
-          default = {};
+        default = { };
-          type = types.attrsOf instanceOption;
+        type = types.attrsOf instanceOption;
-        };
       };
+    };
 
   config = {
     systemd.services =
       let
-        instanceConfig = name: c: mkIf c.enable {
+        instanceConfig =
-          "tinyproxy-${name}" = {
+          name: c:
-            description = "TinyProxy daemon - instance ${name}";
+          mkIf c.enable {
-            after = [ "network.target" ];
+            "tinyproxy-${name}" = {
-            wantedBy = [ "multi-user.target" ];
+              description = "TinyProxy daemon - instance ${name}";
-            serviceConfig = {
+              after = [ "network.target" ];
-              User = "tinyproxy";
+              wantedBy = [ "multi-user.target" ];
-              Group = "tinyproxy";
+              serviceConfig = {
-              Type = "simple";
+                User = "tinyproxy";
-              ExecStart = "${getExe c.package} -d -c /etc/tinyproxy/${name}.conf";
+                Group = "tinyproxy";
-              ExecReload = "${pkgs.coreutils}/bin/kill -SIGHUP $MAINPID";
+                Type = "simple";
-              KillSignal = "SIGINT";
+                ExecStart = "${getExe c.package} -d -c /etc/tinyproxy/${name}.conf";
-              TimeoutStopSec = "30s";
+                ExecReload = "${pkgs.coreutils}/bin/kill -SIGHUP $MAINPID";
-              Restart = "on-failure";
+                KillSignal = "SIGINT";
-              RestartSec = "1s";
+                TimeoutStopSec = "30s";
-              RestartSteps = "3";
+                Restart = "on-failure";
-              RestartMaxDelaySec = "10s";
+                RestartSec = "1s";
-              ConfigurationDirectory = "tinyproxy";
+                RestartSteps = "3";
+                RestartMaxDelaySec = "10s";
+                ConfigurationDirectory = "tinyproxy";
+              };
+              preStart = concatStringsSep "\n" (
+                [
+                  "cat ${configFile name c} > /etc/tinyproxy/${name}.conf"
+                ]
+                ++ optionals (c.dynamicBindFile != "") [
+                  "echo -n 'Bind ' >> /etc/tinyproxy/${name}.conf"
+                  "cat ${c.dynamicBindFile} >> /etc/tinyproxy/${name}.conf"
+                ]
+              );
             };
-            preStart = concatStringsSep "\n" ([
-              "cat ${configFile name c} > /etc/tinyproxy/${name}.conf"
-            ] ++ optionals (c.dynamicBindFile != "") [
-              "echo -n 'Bind ' >> /etc/tinyproxy/${name}.conf"
-              "cat ${c.dynamicBindFile} >> /etc/tinyproxy/${name}.conf"
-            ]);
           };
-        };
       in
-        mkMerge (mapAttrsToList instanceConfig cfg);
+      mkMerge (mapAttrsToList instanceConfig cfg);
 
     users.users.tinyproxy = {
       group = "tinyproxy";
       isSystemUser = true;
     };
-    users.groups.tinyproxy = {};
+    users.groups.tinyproxy = { };
   };
 
   meta.maintainers = with maintainers; [ tcheronneau ];

modules/blocks/vpn.nix

@@ -1,4 +1,9 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
 
 let
   cfg = config.shb.vpn;
@@ -6,197 +11,212 @@ let
   quoteEach = lib.concatMapStrings (x: ''"${x}"'');
 
   nordvpnConfig =
-    { name
+    {
-    , dev
+      name,
-    , authFile
+      dev,
-    , remoteServerIP
+      authFile,
-    , dependentServices ? []
+      remoteServerIP,
-    }: ''
+      dependentServices ? [ ],
-    client
+    }:
-    dev ${dev}
+    ''
-    proto tcp
+      client
-    remote ${remoteServerIP} 443
+      dev ${dev}
-    resolv-retry infinite
+      proto tcp
-    remote-random
+      remote ${remoteServerIP} 443
-    nobind
+      resolv-retry infinite
-    tun-mtu 1500
+      remote-random
-    tun-mtu-extra 32
+      nobind
-    mssfix 1450
+      tun-mtu 1500
-    persist-key
+      tun-mtu-extra 32
-    persist-tun
+      mssfix 1450
-    ping 15
+      persist-key
-    ping-restart 0
+      persist-tun
-    ping-timer-rem
+      ping 15
-    reneg-sec 0
+      ping-restart 0
-    comp-lzo no
+      ping-timer-rem
+      reneg-sec 0
+      comp-lzo no
 
-    status /tmp/openvpn/${name}.status
+      status /tmp/openvpn/${name}.status
 
-    remote-cert-tls server
+      remote-cert-tls server
 
-    auth-user-pass ${authFile}
+      auth-user-pass ${authFile}
-    verb 3
+      verb 3
-    pull
+      pull
-    fast-io
+      fast-io
-    cipher AES-256-CBC
+      cipher AES-256-CBC
-    auth SHA512
+      auth SHA512
 
-    script-security 2
+      script-security 2
-    route-noexec
+      route-noexec
-    route-up ${routeUp name dependentServices}/bin/routeUp.sh
+      route-up ${routeUp name dependentServices}/bin/routeUp.sh
-    down ${routeDown name dependentServices}/bin/routeDown.sh
+      down ${routeDown name dependentServices}/bin/routeDown.sh
 
-    <ca>
+      <ca>
-    -----BEGIN CERTIFICATE-----
+      -----BEGIN CERTIFICATE-----
-    MIIFCjCCAvKgAwIBAgIBATANBgkqhkiG9w0BAQ0FADA5MQswCQYDVQQGEwJQQTEQ
+      MIIFCjCCAvKgAwIBAgIBATANBgkqhkiG9w0BAQ0FADA5MQswCQYDVQQGEwJQQTEQ
-    MA4GA1UEChMHTm9yZFZQTjEYMBYGA1UEAxMPTm9yZFZQTiBSb290IENBMB4XDTE2
+      MA4GA1UEChMHTm9yZFZQTjEYMBYGA1UEAxMPTm9yZFZQTiBSb290IENBMB4XDTE2
-    MDEwMTAwMDAwMFoXDTM1MTIzMTIzNTk1OVowOTELMAkGA1UEBhMCUEExEDAOBgNV
+      MDEwMTAwMDAwMFoXDTM1MTIzMTIzNTk1OVowOTELMAkGA1UEBhMCUEExEDAOBgNV
-    BAoTB05vcmRWUE4xGDAWBgNVBAMTD05vcmRWUE4gUm9vdCBDQTCCAiIwDQYJKoZI
+      BAoTB05vcmRWUE4xGDAWBgNVBAMTD05vcmRWUE4gUm9vdCBDQTCCAiIwDQYJKoZI
-    hvcNAQEBBQADggIPADCCAgoCggIBAMkr/BYhyo0F2upsIMXwC6QvkZps3NN2/eQF
+      hvcNAQEBBQADggIPADCCAgoCggIBAMkr/BYhyo0F2upsIMXwC6QvkZps3NN2/eQF
-    kfQIS1gql0aejsKsEnmY0Kaon8uZCTXPsRH1gQNgg5D2gixdd1mJUvV3dE3y9FJr
+      kfQIS1gql0aejsKsEnmY0Kaon8uZCTXPsRH1gQNgg5D2gixdd1mJUvV3dE3y9FJr
-    XMoDkXdCGBodvKJyU6lcfEVF6/UxHcbBguZK9UtRHS9eJYm3rpL/5huQMCppX7kU
+      XMoDkXdCGBodvKJyU6lcfEVF6/UxHcbBguZK9UtRHS9eJYm3rpL/5huQMCppX7kU
-    eQ8dpCwd3iKITqwd1ZudDqsWaU0vqzC2H55IyaZ/5/TnCk31Q1UP6BksbbuRcwOV
+      eQ8dpCwd3iKITqwd1ZudDqsWaU0vqzC2H55IyaZ/5/TnCk31Q1UP6BksbbuRcwOV
-    skEDsm6YoWDnn/IIzGOYnFJRzQH5jTz3j1QBvRIuQuBuvUkfhx1FEwhwZigrcxXu
+      skEDsm6YoWDnn/IIzGOYnFJRzQH5jTz3j1QBvRIuQuBuvUkfhx1FEwhwZigrcxXu
-    MP+QgM54kezgziJUaZcOM2zF3lvrwMvXDMfNeIoJABv9ljw969xQ8czQCU5lMVmA
+      MP+QgM54kezgziJUaZcOM2zF3lvrwMvXDMfNeIoJABv9ljw969xQ8czQCU5lMVmA
-    37ltv5Ec9U5hZuwk/9QO1Z+d/r6Jx0mlurS8gnCAKJgwa3kyZw6e4FZ8mYL4vpRR
+      37ltv5Ec9U5hZuwk/9QO1Z+d/r6Jx0mlurS8gnCAKJgwa3kyZw6e4FZ8mYL4vpRR
-    hPdvRTWCMJkeB4yBHyhxUmTRgJHm6YR3D6hcFAc9cQcTEl/I60tMdz33G6m0O42s
+      hPdvRTWCMJkeB4yBHyhxUmTRgJHm6YR3D6hcFAc9cQcTEl/I60tMdz33G6m0O42s
-    Qt/+AR3YCY/RusWVBJB/qNS94EtNtj8iaebCQW1jHAhvGmFILVR9lzD0EzWKHkvy
+      Qt/+AR3YCY/RusWVBJB/qNS94EtNtj8iaebCQW1jHAhvGmFILVR9lzD0EzWKHkvy
-    WEjmUVRgCDd6Ne3eFRNS73gdv/C3l5boYySeu4exkEYVxVRn8DhCxs0MnkMHWFK6
+      WEjmUVRgCDd6Ne3eFRNS73gdv/C3l5boYySeu4exkEYVxVRn8DhCxs0MnkMHWFK6
-    MyzXCCn+JnWFDYPfDKHvpff/kLDobtPBf+Lbch5wQy9quY27xaj0XwLyjOltpiST
+      MyzXCCn+JnWFDYPfDKHvpff/kLDobtPBf+Lbch5wQy9quY27xaj0XwLyjOltpiST
-    LWae/Q4vAgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqG
+      LWae/Q4vAgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqG
-    SIb3DQEBDQUAA4ICAQC9fUL2sZPxIN2mD32VeNySTgZlCEdVmlq471o/bDMP4B8g
+      SIb3DQEBDQUAA4ICAQC9fUL2sZPxIN2mD32VeNySTgZlCEdVmlq471o/bDMP4B8g
-    nQesFRtXY2ZCjs50Jm73B2LViL9qlREmI6vE5IC8IsRBJSV4ce1WYxyXro5rmVg/
+      nQesFRtXY2ZCjs50Jm73B2LViL9qlREmI6vE5IC8IsRBJSV4ce1WYxyXro5rmVg/
-    k6a10rlsbK/eg//GHoJxDdXDOokLUSnxt7gk3QKpX6eCdh67p0PuWm/7WUJQxH2S
+      k6a10rlsbK/eg//GHoJxDdXDOokLUSnxt7gk3QKpX6eCdh67p0PuWm/7WUJQxH2S
-    DxsT9vB/iZriTIEe/ILoOQF0Aqp7AgNCcLcLAmbxXQkXYCCSB35Vp06u+eTWjG0/
+      DxsT9vB/iZriTIEe/ILoOQF0Aqp7AgNCcLcLAmbxXQkXYCCSB35Vp06u+eTWjG0/
-    pyS5V14stGtw+fA0DJp5ZJV4eqJ5LqxMlYvEZ/qKTEdoCeaXv2QEmN6dVqjDoTAo
+      pyS5V14stGtw+fA0DJp5ZJV4eqJ5LqxMlYvEZ/qKTEdoCeaXv2QEmN6dVqjDoTAo
-    k0t5u4YRXzEVCfXAC3ocplNdtCA72wjFJcSbfif4BSC8bDACTXtnPC7nD0VndZLp
+      k0t5u4YRXzEVCfXAC3ocplNdtCA72wjFJcSbfif4BSC8bDACTXtnPC7nD0VndZLp
-    +RiNLeiENhk0oTC+UVdSc+n2nJOzkCK0vYu0Ads4JGIB7g8IB3z2t9ICmsWrgnhd
+      +RiNLeiENhk0oTC+UVdSc+n2nJOzkCK0vYu0Ads4JGIB7g8IB3z2t9ICmsWrgnhd
-    NdcOe15BincrGA8avQ1cWXsfIKEjbrnEuEk9b5jel6NfHtPKoHc9mDpRdNPISeVa
+      NdcOe15BincrGA8avQ1cWXsfIKEjbrnEuEk9b5jel6NfHtPKoHc9mDpRdNPISeVa
-    wDBM1mJChneHt59Nh8Gah74+TM1jBsw4fhJPvoc7Atcg740JErb904mZfkIEmojC
+      wDBM1mJChneHt59Nh8Gah74+TM1jBsw4fhJPvoc7Atcg740JErb904mZfkIEmojC
-    VPhBHVQ9LHBAdM8qFI2kRK0IynOmAZhexlP/aT/kpEsEPyaZQlnBn3An1CRz8h0S
+      VPhBHVQ9LHBAdM8qFI2kRK0IynOmAZhexlP/aT/kpEsEPyaZQlnBn3An1CRz8h0S
-    PApL8PytggYKeQmRhl499+6jLxcZ2IegLfqq41dzIjwHwTMplg+1pKIOVojpWA==
+      PApL8PytggYKeQmRhl499+6jLxcZ2IegLfqq41dzIjwHwTMplg+1pKIOVojpWA==
-    -----END CERTIFICATE-----
+      -----END CERTIFICATE-----
-    </ca>
+      </ca>
-    key-direction 1
+      key-direction 1
-    <tls-auth>
+      <tls-auth>
-    #
+      #
-    # 2048 bit OpenVPN static key
+      # 2048 bit OpenVPN static key
-    #
+      #
-    -----BEGIN OpenVPN Static key V1-----
+      -----BEGIN OpenVPN Static key V1-----
-    e685bdaf659a25a200e2b9e39e51ff03
+      e685bdaf659a25a200e2b9e39e51ff03
-    0fc72cf1ce07232bd8b2be5e6c670143
+      0fc72cf1ce07232bd8b2be5e6c670143
-    f51e937e670eee09d4f2ea5a6e4e6996
+      f51e937e670eee09d4f2ea5a6e4e6996
-    5db852c275351b86fc4ca892d78ae002
+      5db852c275351b86fc4ca892d78ae002
-    d6f70d029bd79c4d1c26cf14e9588033
+      d6f70d029bd79c4d1c26cf14e9588033
-    cf639f8a74809f29f72b9d58f9b8f5fe
+      cf639f8a74809f29f72b9d58f9b8f5fe
-    fc7938eade40e9fed6cb92184abb2cc1
+      fc7938eade40e9fed6cb92184abb2cc1
-    0eb1a296df243b251df0643d53724cdb
+      0eb1a296df243b251df0643d53724cdb
-    5a92a1d6cb817804c4a9319b57d53be5
+      5a92a1d6cb817804c4a9319b57d53be5
-    80815bcfcb2df55018cc83fc43bc7ff8
+      80815bcfcb2df55018cc83fc43bc7ff8
-    2d51f9b88364776ee9d12fc85cc7ea5b
+      2d51f9b88364776ee9d12fc85cc7ea5b
-    9741c4f598c485316db066d52db4540e
+      9741c4f598c485316db066d52db4540e
-    212e1518a9bd4828219e24b20d88f598
+      212e1518a9bd4828219e24b20d88f598
-    a196c9de96012090e333519ae18d3509
+      a196c9de96012090e333519ae18d3509
-    9427e7b372d348d352dc4c85e18cd4b9
+      9427e7b372d348d352dc4c85e18cd4b9
-    3f8a56ddb2e64eb67adfc9b337157ff4
+      3f8a56ddb2e64eb67adfc9b337157ff4
-    -----END OpenVPN Static key V1-----
+      -----END OpenVPN Static key V1-----
-    </tls-auth>
+      </tls-auth>
     '';
 
-  routeUp = name: dependentServices: pkgs.writeShellApplication {
+  routeUp =
-    name = "routeUp.sh";
+    name: dependentServices:
+    pkgs.writeShellApplication {
+      name = "routeUp.sh";
 
-    runtimeInputs = [ pkgs.iproute2 pkgs.systemd pkgs.nettools ];
+      runtimeInputs = [
+        pkgs.iproute2
+        pkgs.systemd
+        pkgs.nettools
+      ];
 
-    text = ''
+      text = ''
-    echo "Running route-up..."
+        echo "Running route-up..."
 
-    echo "dev=''${dev:?}"
+        echo "dev=''${dev:?}"
-    echo "ifconfig_local=''${ifconfig_local:?}"
+        echo "ifconfig_local=''${ifconfig_local:?}"
-    echo "route_vpn_gateway=''${route_vpn_gateway:?}"
+        echo "route_vpn_gateway=''${route_vpn_gateway:?}"
 
-    set -x
+        set -x
 
-    ip rule
+        ip rule
-    ip rule add from "''${ifconfig_local:?}/32" table ${name}
+        ip rule add from "''${ifconfig_local:?}/32" table ${name}
-    ip rule add to "''${route_vpn_gateway:?}/32" table ${name}
+        ip rule add to "''${route_vpn_gateway:?}/32" table ${name}
-    ip rule
+        ip rule
 
-    ip route list table ${name} || :
+        ip route list table ${name} || :
-    retVal=$?
+        retVal=$?
-    if [ $retVal -eq 2 ]; then
+        if [ $retVal -eq 2 ]; then
-      echo "table is empty"
+          echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
+        elif [ $retVal -ne 0 ]; then
-      exit 1
+          exit 1
-    fi
+        fi
-    ip route add default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name}
+        ip route add default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name}
-    ip route flush cache
+        ip route flush cache
-    ip route list table ${name} || :
+        ip route list table ${name} || :
-    retVal=$?
+        retVal=$?
-    if [ $retVal -eq 2 ]; then
+        if [ $retVal -eq 2 ]; then
-      echo "table is empty"
+          echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
+        elif [ $retVal -ne 0 ]; then
-      exit 1
+          exit 1
-    fi
+        fi
 
-    echo "''${ifconfig_local:?}" > /run/openvpn/${name}/ifconfig_local
+        echo "''${ifconfig_local:?}" > /run/openvpn/${name}/ifconfig_local
 
-    dependencies=(${quoteEach dependentServices})
+        dependencies=(${quoteEach dependentServices})
-    for i in "''${dependencies[@]}"; do
+        for i in "''${dependencies[@]}"; do
-        systemctl restart "$i" || :
+            systemctl restart "$i" || :
-    done
+        done
 
-    echo "Running route-up DONE"
+        echo "Running route-up DONE"
-    '';
+      '';
-  };
+    };
 
-  routeDown = name: dependentServices: pkgs.writeShellApplication {
+  routeDown =
-    name = "routeDown.sh";
+    name: dependentServices:
+    pkgs.writeShellApplication {
+      name = "routeDown.sh";
 
-    runtimeInputs = [ pkgs.iproute2 pkgs.systemd pkgs.nettools pkgs.coreutils ];
+      runtimeInputs = [
+        pkgs.iproute2
+        pkgs.systemd
+        pkgs.nettools
+        pkgs.coreutils
+      ];
 
-    text = ''
+      text = ''
-    echo "Running route-down..."
+        echo "Running route-down..."
 
-    echo "dev=''${dev:?}"
+        echo "dev=''${dev:?}"
-    echo "ifconfig_local=''${ifconfig_local:?}"
+        echo "ifconfig_local=''${ifconfig_local:?}"
-    echo "route_vpn_gateway=''${route_vpn_gateway:?}"
+        echo "route_vpn_gateway=''${route_vpn_gateway:?}"
 
-    set -x
+        set -x
 
-    ip rule
+        ip rule
-    ip rule del from "''${ifconfig_local:?}/32" table ${name}
+        ip rule del from "''${ifconfig_local:?}/32" table ${name}
-    ip rule del to "''${route_vpn_gateway:?}/32" table ${name}
+        ip rule del to "''${route_vpn_gateway:?}/32" table ${name}
-    ip rule
+        ip rule
 
-    # This will probably fail because the dev is already gone.
+        # This will probably fail because the dev is already gone.
-    ip route list table ${name} || :
+        ip route list table ${name} || :
-    retVal=$?
+        retVal=$?
-    if [ $retVal -eq 2 ]; then
+        if [ $retVal -eq 2 ]; then
-      echo "table is empty"
+          echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
+        elif [ $retVal -ne 0 ]; then
-      exit 1
+          exit 1
-    fi
+        fi
-    ip route del default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name} || :
+        ip route del default via "''${route_vpn_gateway:?}" dev "''${dev:?}" table ${name} || :
-    ip route flush cache
+        ip route flush cache
-    ip route list table ${name} || :
+        ip route list table ${name} || :
-    retVal=$?
+        retVal=$?
-    if [ $retVal -eq 2 ]; then
+        if [ $retVal -eq 2 ]; then
-      echo "table is empty"
+          echo "table is empty"
-    elif [ $retVal -ne 0 ]; then
+        elif [ $retVal -ne 0 ]; then
-      exit 1
+          exit 1
-    fi
+        fi
 
-    rm /run/openvpn/${name}/ifconfig_local
+        rm /run/openvpn/${name}/ifconfig_local
 
-    dependencies=(${quoteEach dependentServices})
+        dependencies=(${quoteEach dependentServices})
-    for i in "''${dependencies[@]}"; do
+        for i in "''${dependencies[@]}"; do
-        systemctl stop "$i" || :
+            systemctl stop "$i" || :
-    done
+        done
 
-    echo "Running route-down DONE"
+        echo "Running route-down DONE"
-    '';
+      '';
-  };
+    };
 in
 {
   options =
@@ -205,7 +225,7 @@ in
         options = {
           enable = lib.mkEnableOption "OpenVPN config";
 
-          package = lib.mkPackageOption pkgs "openvpn" {};
+          package = lib.mkPackageOption pkgs "openvpn" { };
 
           provider = lib.mkOption {
             description = "VPN provider, if given uses ready-made configuration.";
@@ -243,68 +263,76 @@ in
         };
       };
     in
-      {
+    {
-        shb.vpn = lib.mkOption {
+      shb.vpn = lib.mkOption {
-          description = "OpenVPN instances.";
+        description = "OpenVPN instances.";
-          default = {};
+        default = { };
-          type = lib.types.attrsOf instanceOption;
+        type = lib.types.attrsOf instanceOption;
-        };
       };
+    };
 
   config = {
     services.openvpn.servers =
       let
-        instanceConfig = name: c: lib.mkIf c.enable {
+        instanceConfig =
-          ${name} = {
+          name: c:
-            autoStart = true;
+          lib.mkIf c.enable {
+            ${name} = {
+              autoStart = true;
 
-            up = "mkdir -p /run/openvpn/${name}";
+              up = "mkdir -p /run/openvpn/${name}";
 
-            config = nordvpnConfig {
+              config = nordvpnConfig {
-              inherit name;
+                inherit name;
-              inherit (c) dev remoteServerIP authFile;
+                inherit (c) dev remoteServerIP authFile;
-              dependentServices = lib.optional (c.proxyPort != null) "tinyproxy-${name}.service";
+                dependentServices = lib.optional (c.proxyPort != null) "tinyproxy-${name}.service";
+              };
             };
           };
-        };
       in
-        lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
+      lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
 
-    systemd.tmpfiles.rules = map (name:
+    systemd.tmpfiles.rules = map (name: "d /tmp/openvpn/${name}.status 0700 root root") (
-      "d /tmp/openvpn/${name}.status 0700 root root"
+      lib.attrNames cfg
-    ) (lib.attrNames cfg);
+    );
 
     networking.iproute2.enable = true;
-    networking.iproute2.rttablesExtraConfig =
+    networking.iproute2.rttablesExtraConfig = lib.concatStringsSep "\n" (
-      lib.concatStringsSep "\n" (lib.mapAttrsToList (name: c: "${toString c.routingNumber} ${name}") cfg);
+      lib.mapAttrsToList (name: c: "${toString c.routingNumber} ${name}") cfg
+    );
 
     shb.tinyproxy =
       let
-        instanceConfig = name: c: lib.mkIf (c.enable && c.proxyPort != null) {
+        instanceConfig =
-          ${name} = {
+          name: c:
-            enable = true;
+          lib.mkIf (c.enable && c.proxyPort != null) {
-            # package = pkgs.tinyproxy.overrideAttrs (old: {
+            ${name} = {
-            #   withDebug = false;
+              enable = true;
-            #   patches = old.patches ++ [
+              # package = pkgs.tinyproxy.overrideAttrs (old: {
-            #     (pkgs.fetchpatch {
+              #   withDebug = false;
-            #       name = "";
+              #   patches = old.patches ++ [
-            #       url = "https://github.com/tinyproxy/tinyproxy/pull/494/commits/2532ba09896352b31f3538d7819daa1fc3f829f1.patch";
+              #     (pkgs.fetchpatch {
-            #       sha256 = "sha256-Q0MkHnttW8tH3+hoCt9ACjHjmmZQgF6pC/menIrU0Co=";
+              #       name = "";
-            #     })
+              #       url = "https://github.com/tinyproxy/tinyproxy/pull/494/commits/2532ba09896352b31f3538d7819daa1fc3f829f1.patch";
-            #   ];
+              #       sha256 = "sha256-Q0MkHnttW8tH3+hoCt9ACjHjmmZQgF6pC/menIrU0Co=";
-            # });
+              #     })
-            dynamicBindFile = "/run/openvpn/${name}/ifconfig_local";
+              #   ];
-            settings = {
+              # });
-              Port = c.proxyPort;
+              dynamicBindFile = "/run/openvpn/${name}/ifconfig_local";
-              Listen = "127.0.0.1";
+              settings = {
-              Syslog = "On";
+                Port = c.proxyPort;
-              LogLevel = "Info";
+                Listen = "127.0.0.1";
-              Allow = [ "127.0.0.1" "::1" ];
+                Syslog = "On";
-              ViaProxyName = ''"tinyproxy"'';
+                LogLevel = "Info";
+                Allow = [
+                  "127.0.0.1"
+                  "::1"
+                ];
+                ViaProxyName = ''"tinyproxy"'';
+              };
             };
           };
-        };
       in
-        lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
+      lib.mkMerge (lib.mapAttrsToList instanceConfig cfg);
   };
 }

modules/blocks/zfs.nix

@@ -1,73 +1,259 @@
-{ config, pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  shb,
+  utils,
+  ...
+}:
 
 let
   cfg = config.shb.zfs;
+
+  datasets =
+    if cfg.snapshotBeforeActivation.datasets != null then
+      cfg.snapshotBeforeActivation.datasets
+    else
+      config.boot.zfs.extraPools;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.zfs = {
-    defaultPoolName = lib.mkOption {
+    pools = lib.mkOption {
-      type = lib.types.nullOr lib.types.str;
+      description = ''
-      default = null;
+        Attrset of ZFS pools under which datasets will be created.
-      description = "ZFS pool name datasets should be created on if no pool name is given in the dataset.";
+
+        The ZFS pools are not managed by this module, they should already exist.
+
+        Each pool named here will be added to the [`boot.zfs.extraPools`](https://search.nixos.org/options?channel=unstable&include_modular_service_options=0&include_nixos_options=1&query=boot.zfs.extrapools&show=option:boot.zfs.extraPools) option.
+      '';
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule {
+          options = {
+            datasets = lib.mkOption {
+              description = ''
+                ZFS Datasets.
+
+                Each entry in the attrset will be created and mounted in the given path.
+                The attrset name is the dataset name.
+
+                This block implements the following contracts:
+                  - mount
+              '';
+              default = { };
+              example = lib.literalExpression ''
+                shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
+              '';
+              type = lib.types.attrsOf (
+                lib.types.submodule (
+                  { config, name, ... }:
+                  {
+                    options = {
+                      enable = lib.mkEnableOption "shb.zfs.datasets";
+
+                      path = lib.mkOption {
+                        type = lib.types.str;
+                        description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
+                      };
+
+                      mode = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix mode to apply to the dataset root folder.";
+                        default = null;
+                        example = "ug=rwx,g+s";
+                      };
+
+                      owner = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix user to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      group = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix group to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      defaultACLs = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = ''
+                          If non null, default ACL to set on the dataset root folder.
+
+                          Executes "setfacl -d -m $acl $path"
+                        '';
+                        default = null;
+                        example = "g:syncthing:rwX";
+                      };
+
+                      after = lib.mkOption {
+                        type = lib.types.listOf lib.types.str;
+                        description = ''
+                          Order creating this dataset after the mentioned ones.
+                          This only works with datasets managed by this module.
+
+                          Use the name of the dataset without the pool name.
+                        '';
+                        default = [ ];
+                        example = lib.literalExpression ''
+                          [
+                            "backup"
+                          ]
+                        '';
+                      };
+
+                      backup = lib.mkOption {
+                        description = ''
+                          Backup contract consumer configuration.
+
+                          This contract will backup the files inside the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.backup.mkRequester {
+                            user = if config.owner == null then "root" else config.owner;
+                            sourceDirectories = [
+                              config.path
+                            ];
+                            sourceDirectoriesText = ''
+                              [
+                                shb.zfs.pools.<name>.datasets.<name>.path
+                              ]
+                            '';
+                          };
+                        };
+                      };
+
+                      datasetbackup = lib.mkOption {
+                        description = ''
+                          ZFS dataset backup contract configuration.
+
+                          This contract will take snaphots of the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.datasetbackup.mkRequester {
+                            dataset = name;
+                          };
+                        };
+                      };
+                    };
+                  }
+                )
+              );
+            };
+          };
+        }
+
+      );
     };
 
-    datasets = lib.mkOption {
+    snapshotBeforeActivation = {
-      description = ''
+      enable = lib.mkOption {
-        ZFS Datasets.
+        type = lib.types.bool;
+        description = "Take a snapshot of all datasets before activation";
+        default = false;
+      };
 
-        Each entry in the attrset will be created and mounted in the given path.
+      template = lib.mkOption {
-        The attrset name is the dataset name.
+        type = lib.types.str;
+        description = "Bash command to generate the snapshot name. $1 is the path to the new generation.";
+        default = ''pre-$(date --utc '+%y%m%dT%H%M%S')-$(basename "$1")'';
+      };
 
-        This block implements the following contracts:
+      datasets = lib.mkOption {
-          - mount
+        type = lib.types.nullOr (lib.types.listOf lib.types.str);
-      '';
+        description = ''
-      default = {};
+          Defines all datasets to take a snapshot of.
-      example = lib.literalExpression ''
-        shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
-      '';
-      type = lib.types.attrsOf (lib.types.submodule {
-        options = {
-          enable = lib.mkEnableOption "shb.zfs.datasets";
 
-          poolName = lib.mkOption {
+          If set to `null`, the default, take the list of datasets from `config.boot.zfs.extraPools`.
-            type = lib.types.nullOr lib.types.str;
-            default = null;
-            description = "ZFS pool name this dataset should be created on. Overrides the defaultPoolName.";
-          };
 
-          path = lib.mkOption {
+          The snapshots are not recursive unless specificed in the `recursive` option.
-            type = lib.types.str;
+        '';
-            description = "Path this dataset should be mounted on.";
+      };
-          };
+
-        };
+      recursive = lib.mkOption {
-      });
+        type = lib.types.bool;
+        description = "If true, take snapshots recurisevly on the given datasets.";
+        default = false;
+      };
     };
   };
 
+  # The implementation is greatly inspired by https://discourse.nixos.org/t/configure-zfs-filesystems-after-install/48633/3
   config = {
-    assertions = [
+    boot.zfs.extraPools = lib.uniqueStrings (builtins.attrNames cfg.pools);
-      {
-        assertion = lib.any (x: x.poolName == null) (lib.mapAttrsToList (n: v: v) cfg.datasets) -> cfg.defaultPoolName != null;
-        message = "Cannot have both datasets.poolName and defaultPoolName set to null";
-      }
-    ];
 
-    system.activationScripts = lib.mapAttrs' (name: cfg':
+    systemd.services =
       let
-        dataset = (if cfg'.poolName != null then cfg'.poolName else cfg.defaultPoolName) + "/" + name;
+        mkPool =
-      in
+          poolName: poolCfg: lib.listToAttrs (lib.mapAttrsToList (mkDataset poolName) poolCfg.datasets);
-      lib.attrsets.nameValuePair "zfsCreate-${name}" {
-        text = ''
-          ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
-            || ${pkgs.zfs}/bin/zfs create \
-               -o mountpoint=none \
-               ${dataset} || :
 
-          [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
+        mkDataset =
-            || ${pkgs.zfs}/bin/zfs set \
+          poolName: name: cfg':
-               mountpoint=${cfg'.path} \
+          let
-               ${dataset}
+            dataset = poolName + "/" + name;
-        '';
+          in
-      }) cfg.datasets;
+          lib.attrsets.nameValuePair "zfs-create-${utils.escapeSystemdPath dataset}" {
+            # oneshot is used to make the systemd service wait on completion of the script.
+            serviceConfig.Type = "oneshot";
+            unitConfig.DefaultDependencies = false;
+            requiredBy = [ "local-fs.target" ];
+            before = [ "local-fs.target" ];
+            after = [
+              "zfs-import-${poolName}.service"
+              "zfs-mount.service"
+            ]
+            ++ map (n: "zfs-create-${poolName}-${n}.service") cfg'.after;
+
+            unitConfig.ConditionPathIsMountPoint = lib.mkIf (cfg'.path != "none") "!${cfg'.path}";
+
+            script = ''
+              ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
+                || ${pkgs.zfs}/bin/zfs create \
+                   -o mountpoint=none \
+                   ${dataset} || :
+
+              [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
+                || ${pkgs.zfs}/bin/zfs set \
+                   mountpoint="${cfg'.path}" \
+                   ${dataset}
+
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
+              chmod "${cfg'.mode}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
+              chown "${cfg'.owner}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
+              chown :"${cfg'.group}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
+              ${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
+            '';
+          };
+        mergeAttrs = lib.foldl lib.mergeAttrs { };
+      in
+      mergeAttrs (lib.mapAttrsToList mkPool cfg.pools);
+
+    system.preSwitchChecks."shb-zfs-snapshot" = lib.mkIf cfg.snapshotBeforeActivation.enable (
+      ''
+        name="${cfg.snapshotBeforeActivation.template}"
+      ''
+      + (
+        let
+          recursiveFlag = lib.optionalString cfg.snapshotBeforeActivation.recursive "-r";
+        in
+        lib.concatMapStringsSep "\n" (ds: ''
+          echo "Taking ZFS snapshot of ${ds}@$name"
+          zfs snapshot ${recursiveFlag} ${ds}@"$name"
+        '') (lib.uniqueStrings datasets)
+      )
+    );
   };
 }

modules/blocks/zfs/docs/default.md

@@ -0,0 +1,138 @@
+# ZFS Block {#blocks-zfs}
+
+Defined in [`/modules/blocks/zfs.nix`](@REPO@/modules/blocks/zfs.nix):
+
+```nix
+{
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.zfs
+  ];
+}
+```
+
+This block creates ZFS datasets, optionally mounts them and sets permissions on the mount point.
+It also enables taking snapshots on activation.
+
+## Features {#blocks-zfs-features}
+
+- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
+- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
+- Backup of the files in the dataset [`shb.zfs.<name>.backup`][backup] through the [backup contract](./contracts-backup.html).
+- Backup of the dataset itself [`shb.zfs.<name>.datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html).
+- Take a snapshot [before activating or deploying][activationsnapshot] allowing a safe rollback.
+
+[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup
+[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup
+[activationsnapshot]: #blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable
+
+## Usage {#blocks-zfs-usage}
+
+Create a dataset at `root/safe/users` mounted on `/var/lib/nixos`:
+
+```nix
+shb.zfs.pools.root.datasets."safe/users".path = "/var/lib/nixos";
+```
+
+Create a dataset at `backup/syncoid` but do not mount it:
+
+```nix
+shb.zfs.pools.backup.datasets."syncoid".path = "none";
+```
+
+Create a dataset at `root/syncthing` and set custom permissions and ACL.
+Permission and ACL are only enforced for the mount point.
+
+```nix
+shb.zfs.pools.root.datasets."syncthing" = {
+  path = "/srv/syncthing";
+
+  mode = "ug=rwx,g+s,o=";
+  owner = "syncthing";
+  group = "syncthing";
+  defaultACLs = "g:syncthing:rwX";
+};
+```
+
+### Backup dataset {#blocks-zfs-usage-backup-dataset}
+
+To backup the dataset directly, use the [dataset backup contract](contracts-datasetbackup.html).
+For example, with the sanoid module as the dataset backup contract provider:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.sanoid.backup."root/home" = {
+    request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
+    template = "yearly";
+  };
+}
+```
+
+See the [Sanoid block](blocks-sanoid.html) for more examples.
+
+### Backup files {#blocks-zfs-usage-backup-files}
+
+To backup the files in the dataset, use the [file backup contract](contracts-backup.html).
+For example, with the restic module as the file backup contract provider:
+
+```nix
+{
+  shb.zfs.pools.root.datasets.home = {
+    path = "/home";
+  };
+
+  shb.restic.instances."myservice" = {
+    request = shb.zfs.pools.root.datasets.home.backup.request;
+  };
+}
+```
+
+See the [Restic block](blocks-restic.html) for more examples.
+
+### Snapshot before activation {#blocks-zfs-usage-snapshot-before-activation}
+
+To take a snapshot of all datasets before activating a new configuration:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation.enable = true;
+}
+```
+
+This does not take a recursive snapshot by default, to do it recursively, do:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation = {
+    enable = true;
+    recursive = true;
+  };
+}
+```
+
+We did not specify datasets manually, which means the datasets list comes from `boot.zfs.extraPools`.
+To specify the datasets manually, you can use:
+
+```nix
+{
+  shb.zfs.snapshotBeforeActivation = {
+    enable = true;
+    datasets = [ "root" "root/var" ];
+  };
+}
+```
+
+## Options Reference {#blocks-zfs-options}
+
+```{=include=} options
+id-prefix: blocks-zfs-options-
+list-id: selfhostblocks-block-zfs-options
+source: @OPTIONS_JSON@
+```

modules/contracts/backup.nix

@@ -1,24 +1,40 @@
-{ pkgs, lib, ... }:
+{
+  lib,
+  shb,
+  pkgs,
+  ...
+}:
 let
-  inherit (lib) concatStringsSep literalMD mkOption optionalAttrs optionalString;
+  inherit (lib)
-  inherit (lib.types) listOf nonEmptyListOf submodule str;
+    concatStringsSep
-
+    literalMD
-  shblib = pkgs.callPackage ../../lib {};
+    mkOption
-  inherit (shblib) anyNotNull;
+    optionalAttrs
+    optionalString
+    ;
+  inherit (lib.types)
+    listOf
+    nonEmptyListOf
+    submodule
+    str
+    ;
+  inherit (shb) anyNotNull;
 in
 {
   mkRequest =
-    { user ? "",
+    {
+      user ? "",
       userText ? null,
       sourceDirectories ? [ "/var/lib/example" ],
       sourceDirectoriesText ? null,
-      excludePatterns ? [],
+      excludePatterns ? [ ],
       excludePatternsText ? null,
-      beforeBackup ? [],
+      beforeBackup ? [ ],
       beforeBackupText ? null,
-      afterBackup ? [],
+      afterBackup ? [ ],
       afterBackupText ? null,
-    }: mkOption {
+    }:
+    mkOption {
       description = ''
         Request part of the backup contract.
 
@@ -33,72 +49,102 @@ in
         };
       };
 
-      defaultText = optionalString (anyNotNull [
+      defaultText =
-        userText
+        optionalString
-        sourceDirectoriesText
+          (anyNotNull [
-        excludePatternsText
+            userText
-        beforeBackupText
+            sourceDirectoriesText
-        afterBackupText
+            excludePatternsText
-      ]) (literalMD ''
+            beforeBackupText
-      {
+            afterBackupText
-        user = ${if userText != null then userText else user};
+          ])
-        sourceDirectories = ${if sourceDirectoriesText != null then sourceDirectoriesText else "[ " + concatStringsSep " " sourceDirectories + " ]"};
+          (literalMD ''
-        excludePatterns = ${if excludePatternsText != null then excludePatternsText else "[ " + concatStringsSep " " excludePatterns + " ]"};
+            {
-        hooks.beforeBackup = ${if beforeBackupText != null then beforeBackupText else "[ " + concatStringsSep " " beforeBackup + " ]"};
+              user = ${if userText != null then userText else user};
-        hooks.afterBackup = ${if afterBackupText != null then afterBackupText else "[ " + concatStringsSep " " afterBackup + " ]"};
+              sourceDirectories = ${
-      };
+                if sourceDirectoriesText != null then
-      '');
+                  sourceDirectoriesText
+                else
+                  "[ " + concatStringsSep " " sourceDirectories + " ]"
+              };
+              excludePatterns = ${
+                if excludePatternsText != null then
+                  excludePatternsText
+                else
+                  "[ " + concatStringsSep " " excludePatterns + " ]"
+              };
+              hooks.beforeBackup = ${
+                if beforeBackupText != null then
+                  beforeBackupText
+                else
+                  "[ " + concatStringsSep " " beforeBackup + " ]"
+              };
+              hooks.afterBackup = ${
+                if afterBackupText != null then afterBackupText else "[ " + concatStringsSep " " afterBackup + " ]"
+              };
+            };
+          '');
 
       type = submodule {
         options = {
-          user = mkOption {
+          user =
-            description = ''
+            mkOption {
-              Unix user doing the backups.
+              description = ''
-            '';
+                Unix user doing the backups.
-            type = str;
+              '';
-            example = "vaultwarden";
+              type = str;
-            default = user;
+              example = "vaultwarden";
-          } // optionalAttrs (userText != null) {
+              default = user;
-            defaultText = literalMD userText;
+            }
-          };
+            // optionalAttrs (userText != null) {
+              defaultText = literalMD userText;
+            };
 
-          sourceDirectories = mkOption {
+          sourceDirectories =
-            description = "Directories to backup.";
+            mkOption {
-            type = nonEmptyListOf str;
+              description = "Directories to backup.";
-            example = "/var/lib/vaultwarden";
+              type = nonEmptyListOf str;
-            default = sourceDirectories;
+              example = "/var/lib/vaultwarden";
-          } // optionalAttrs (sourceDirectoriesText != null) {
+              default = sourceDirectories;
-            defaultText = literalMD sourceDirectoriesText;
+            }
-          };
+            // optionalAttrs (sourceDirectoriesText != null) {
+              defaultText = literalMD sourceDirectoriesText;
+            };
 
-          excludePatterns = mkOption {
+          excludePatterns =
-            description = "File patterns to exclude.";
+            mkOption {
-            type = listOf str;
+              description = "File patterns to exclude.";
-            default = excludePatterns;
+              type = listOf str;
-          } // optionalAttrs (excludePatternsText != null) {
+              default = excludePatterns;
-            defaultText = literalMD excludePatternsText;
+            }
-          };
+            // optionalAttrs (excludePatternsText != null) {
+              defaultText = literalMD excludePatternsText;
+            };
 
           hooks = mkOption {
             description = "Hooks to run around the backup.";
-            default = {};
+            default = { };
             type = submodule {
               options = {
-                beforeBackup = mkOption {
+                beforeBackup =
-                  description = "Hooks to run before backup.";
+                  mkOption {
-                  type = listOf str;
+                    description = "Hooks to run before backup.";
-                  default = beforeBackup;
+                    type = listOf str;
-                } // optionalAttrs (beforeBackupText != null) {
+                    default = beforeBackup;
-                  defaultText = literalMD beforeBackupText;
+                  }
-                };
+                  // optionalAttrs (beforeBackupText != null) {
+                    defaultText = literalMD beforeBackupText;
+                  };
 
-                afterBackup = mkOption {
+                afterBackup =
-                  description = "Hooks to run after backup.";
+                  mkOption {
-                  type = listOf str;
+                    description = "Hooks to run after backup.";
-                  default = afterBackup;
+                    type = listOf str;
-                } // optionalAttrs (afterBackupText != null) {
+                    default = afterBackup;
-                  defaultText = literalMD afterBackupText;
+                  }
-                };
+                  // optionalAttrs (afterBackupText != null) {
+                    defaultText = literalMD afterBackupText;
+                  };
               };
             };
           };
@@ -106,70 +152,133 @@ in
       };
     };
 
-  mkResult = {
+  mkResult =
-    restoreScript ? "restore",
-    restoreScriptText ? null,
-    backupService ? "backup.service",
-    backupServiceText ? null,
-  }: mkOption {
-    description = ''
-      Result part of the backup contract.
-
-      Options set by the provider module that indicates the name of the backup and restor scripts.
-    '';
-    default = {
-      inherit restoreScript backupService;
-    };
-
-    defaultText = optionalString (anyNotNull [
-      restoreScriptText
-      backupServiceText
-    ]) (literalMD ''
     {
-      restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
+      restoreScript ? "restore",
-      backupService = ${if backupServiceText != null then backupServiceText else backupService};
+      restoreScriptText ? null,
-    }
+      backupService ? "backup.service",
-    '');
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the backup contract.
 
-    type = submodule {
+        Options set by the provider module that indicates the name of the backup and restore scripts.
-      options = {
+      '';
-        restoreScript = mkOption {
+      default = {
-          description = ''
+        inherit restoreScript backupService;
-            Name of script that can restore the database.
+      };
-            One can then list snapshots with:
 
-            ```bash
+      defaultText =
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+        optionalString
-            ```
+          (anyNotNull [
+            restoreScriptText
+            backupServiceText
+          ])
+          (literalMD ''
+            {
+              restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
+              backupService = ${if backupServiceText != null then backupServiceText else backupService};
+            }
+          '');
 
-            And restore the database with:
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
 
-            ```bash
+                ```bash
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
-            ```
+                <snapshot 1> <metadata>
-          '';
+                <snapshot 2> <metadata>
-          type = str;
+                ```
-          default = restoreScript;
-        } // optionalAttrs (restoreScriptText != null) {
-          defaultText = literalMD restoreScriptText;
-        };
 
-        backupService = mkOption {
+                And restore the database with:
-          description = ''
-            Name of service backing up the database.
 
-            This script can be ran manually to backup the database:
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
+                ```
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
 
-            ```bash
+          backupService =
-            $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+            mkOption {
-            ```
+              description = ''
-          '';
+                Name of service backing up the database.
-          type = str;
+
-          default = backupService;
+                This script can be ran manually to backup the database:
-        } // optionalAttrs (backupServiceText != null) {
+
-          defaultText = literalMD backupServiceText;
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
         };
       };
     };
-  };
+
+  passthru.mkRestoreScript =
+    {
+      name,
+      user,
+      sudoPreserveEnvs ? [ ],
+      secretsFile,
+      restoreCmd,
+      listCmd,
+    }:
+    pkgs.writeShellApplication {
+      inherit name;
+
+      text = ''
+        usage() {
+          echo "$0 snapshots"
+          echo "$0 restore <snapshot>"
+        }
+
+        sudocmd() {
+          sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
+        }
+
+        loadenv() {
+          set -a
+          # shellcheck disable=SC1090
+          source <(sudocmd cat "${secretsFile}")
+          set +a
+        }
+
+        if [ "$1" = "restore" ]; then
+          shift
+          snapshot="$1"
+
+          loadenv
+
+          echo "Will restore snapshot '$snapshot'"
+
+          sudocmd sh -c "${restoreCmd}"
+        elif [ "$1" = "snapshots" ]; then
+          shift
+
+          loadenv
+
+          sudocmd sh -c "${listCmd}"
+        else
+          usage
+          exit 1
+        fi
+      '';
+    };
+
 }

modules/contracts/backup/docs/default.md

@@ -20,25 +20,22 @@ source: @OPTIONS_JSON@
 ## Usage {#contract-backup-usage}
 
 A service that can be backed up will provide a `backup` option.
-Such a service is a `requester` providing a `request` for a module `provider` of this contract. 
 
-What this option defines is, from the user perspective - that is _you_ - an implementation detail
+Here is an example module defining such a `backup` option,
-but it will at least define what directories to backup,
+which defines what directories to backup (`sourceDirectories`)
-the user to backup with
+and the user to backup with (`user`).
-and possibly hooks to run before or after the backup job runs.
-
-Here is an example module defining such a `backup` option:
 
 ```nix
 {
   options = {
     myservice.backup = mkOption {
-      type = contracts.backup.request;
+      type = lib.types.submodule {
-      default = {
+        options = shb.contracts.backup.mkRequester {
-        user = "myservice";
+          user = "nextcloud";
-        sourceDirectories = [
+          sourceDirectories = [
-          "/var/lib/myservice"
+            "/var/lib/nextcloud"
-        ];
+          ];
+        };
       };
     };
   };
@@ -48,13 +45,13 @@ Here is an example module defining such a `backup` option:
 Now, on the other side we have a service that uses this `backup` option and actually backs up files.
 This service is a `provider` of this contract and will provide a `result` option.
 
-Let's assume such a module is available under the `backupservice` option
+Let's assume such a module is available under the `backupService` option
-and that one can create multiple backup instances under `backupservice.instances`.
+and that one can create multiple backup instances under `backupService.instances`.
 Then, to actually backup the `myservice` service, one would write:
 
 ```nix
-backupservice.instances.myservice = {
+backupService.instances.myservice = {
-  request = myservice.backup;
+  request = myservice.backup.request;
   
   settings = {
     enable = true;
@@ -63,17 +60,17 @@ backupservice.instances.myservice = {
       path = "/srv/backup/myservice";
     };
 
-    # ... Other options specific to backupservice like scheduling.
+    # ... Other options specific to backupService like scheduling.
   };
 };
 ```
 
 It is advised to backup files to different location, to improve redundancy.
-Thanks to using contracts, this can be made easily either with the same `backupservice`:
+Thanks to using contracts, this can be made easily either with the same `backupService`:
 
 ```nix
-backupservice.instances.myservice_2 = {
+backupService.instances.myservice_2 = {
-  request = myservice.backup;
+  request = myservice.backup.request;
   
   settings = {
     enable = true;
@@ -85,12 +82,13 @@ backupservice.instances.myservice_2 = {
 };
 ```
 
-Or with another module `backupservice_2`!
+Or with another module `backupService_2`!
 
 ## Providers of the Backup Contract {#contract-backup-providers}
 
 - [Restic block](blocks-restic.html).
-- [Borgbackup block](blocks-borgbackup.html) [WIP].
+- [Borgbackup block](blocks-borgbackup.html).
+- [ZFS block](blocks-zfs.html).
 
 ## Requester Blocks and Services {#contract-backup-requesters}
 

modules/contracts/backup/dummyModule.nix

@@ -1,11 +1,13 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  contracts = pkgs.callPackage ../. {};
-
   inherit (lib) mkOption;
   inherit (lib.types) submodule;
 in
 {
+  imports = [
+    ../../../lib/module.nix
+  ];
+
   options.shb.contracts.backup = mkOption {
     description = ''
       Contract for backing up files
@@ -23,7 +25,7 @@ in
     '';
 
     type = submodule {
-      options = contracts.backup.contract;
+      options = shb.contracts.backup.contract;
     };
   };
 }

modules/contracts/backup/test.nix

@@ -1,120 +1,179 @@
-{ pkgs, lib, ... }:
+{
+  pkgs,
+  lib,
+  shb,
+}:
 let
-  testLib = pkgs.callPackage ../../../test/common.nix {};
+  inherit (lib)
-
+    concatMapStringsSep
-  inherit (lib) concatMapStringsSep getAttrFromPath mkIf optionalAttrs setAttrByPath;
+    getAttrFromPath
+    mkIf
+    optionalAttrs
+    setAttrByPath
+    ;
 in
-{ name,
+{
+  name,
   providerRoot,
-  modules ? [],
+  modules ? [ ],
   username ? "me",
   sourceDirectories ? [
     "/opt/files/A"
     "/opt/files/B"
   ],
-  settings, # { repository, config } -> attrset
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
-  extraConfig ? null, # { username, config } -> attrset
+  extraConfig ? null, # { filesRoot, username, config } -> attrset
-}: pkgs.testers.runNixOSTest {
+}:
+shb.test.runNixOSTest {
   inherit name;
 
-  nodes.machine = { config, ... }: {
+  nodes.machine =
-    imports = [ testLib.baseImports ] ++ modules;
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
 
-    config = lib.mkMerge [
+      config = lib.mkMerge [
-      (setAttrByPath providerRoot {
+        (setAttrByPath providerRoot {
-        request = {
+          request = {
-          inherit sourceDirectories;
+            inherit sourceDirectories;
-          user = username;
+            user = username;
-        };
+          };
-        settings = settings {
+          settings = settings {
-          inherit config;
+            inherit config;
-          repository = "/opt/repos/${name}";
+            filesRoot = "/opt/files";
-        };
+          };
-      })
+        })
-      (mkIf (username != "root") {
+        (mkIf (username != "root") {
-        users.users.${username} = {
+          users.users.${username} = {
-          isSystemUser = true;
+            isSystemUser = true;
-          extraGroups = [ "sudoers" ];
+            extraGroups = [ "sudoers" ];
-          group = "root";
+            group = "root";
-        };
+          };
-      })
+        })
-      (optionalAttrs (extraConfig != null) (extraConfig { inherit username config; }))
+        (optionalAttrs (extraConfig != null) (extraConfig {
-    ];
+          inherit username config;
-  };
+          filesRoot = "/opt/files";
+        }))
+      ];
+    };
 
   extraPythonPackages = p: [ p.dictdiffer ];
   skipTypeCheck = true;
 
-  testScript = { nodes, ... }: let
+  testScript =
-    provider = (getAttrFromPath providerRoot nodes.machine).result;
+    { nodes, ... }:
-  in ''
+    let
-    from dictdiffer import diff
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      from datetime import datetime, timedelta
+      from dictdiffer import diff
+      import re
 
-    username = "${username}"
+      username = "${username}"
-    sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
+      sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
 
-    def list_files(dir):
+      def list_files(dir):
-        files_and_content = {}
+          files_and_content = {}
 
-        files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
+          files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
 
-        for f in files:
+          for f in files:
-            content = machine.succeed(f"""cat {f}""").strip()
+              content = machine.succeed(f"""cat {f}""").strip()
-            files_and_content[f] = content
+              files_and_content[f] = content
 
-        return files_and_content
+          return files_and_content
 
-    def assert_files(dir, files):
+      def assert_files(dir, files):
-        result = list(diff(list_files(dir), files))
+          result = list(diff(list_files(dir), files))
-        if len(result) > 0:
+          if len(result) > 0:
-            raise Exception("Unexpected files:", result)
+              raise Exception("Unexpected files:", result)
 
-    with subtest("Create initial content"):
+      with subtest("Create initial content"):
-        for path in sourceDirectories:
+          for path in sourceDirectories:
-            machine.succeed(f"""
+              machine.succeed(f"""
-                mkdir -p {path}
+                  mkdir -p {path}
-                echo repo_fileA_1 > {path}/fileA
+                  echo repo_fileA_1 > {path}/fileA
-                echo repo_fileB_1 > {path}/fileB
+                  echo repo_fileB_1 > {path}/fileB
 
-                chown {username}: -R {path}
+                  chown {username}: -R {path}
-                chmod go-rwx -R {path}
+                  chmod go-rwx -R {path}
-            """)
-
-        for path in sourceDirectories:
-            assert_files(path, {
-                f'{path}/fileA': 'repo_fileA_1',
-                f'{path}/fileB': 'repo_fileB_1',
-            })
-
-    with subtest("First backup in repo"):
-        print(machine.succeed("systemctl cat ${provider.backupService}"))
-        machine.succeed("systemctl start ${provider.backupService}")
-
-    with subtest("New content"):
-        for path in sourceDirectories:
-            machine.succeed(f"""
-              echo repo_fileA_2 > {path}/fileA
-              echo repo_fileB_2 > {path}/fileB
               """)
 
-            assert_files(path, {
+          for path in sourceDirectories:
-                f'{path}/fileA': 'repo_fileA_2',
+              assert_files(path, {
-                f'{path}/fileB': 'repo_fileB_2',
+                  f'{path}/fileA': 'repo_fileA_1',
-            })
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
 
-    with subtest("Delete content"):
+      with subtest("Initial snapshot"):
-        for path in sourceDirectories:
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
-            machine.succeed(f"""rm -r {path}/*""")
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
 
-            assert_files(path, {})
+      with subtest("First backup in repo"):
+          print(machine.succeed("systemctl cat ${provider.backupService}"))
+          machine.succeed("systemctl start --wait ${provider.backupService}")
 
-    with subtest("Restore initial content from repo"):
+      with subtest("One snapshot"):
-        machine.succeed("""${provider.restoreScript} restore latest""")
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
 
-        for path in sourceDirectories:
+      # To accomodate for snapshot orchestrators which keep only a given amount
-            assert_files(path, {
+      # of snapshots per unit of time, we set the time to now + 2h.
-                f'{path}/fileA': 'repo_fileA_1',
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
-                f'{path}/fileB': 'repo_fileB_1',
+      machine.succeed(f"timedatectl set-time '{new_date}'")
-            })
+
+      with subtest("New content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                echo repo_fileA_2 > {path}/fileA
+                echo repo_fileB_2 > {path}/fileB
+                """)
+
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("Delete content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""rm -r {path}/*""")
+
+              assert_files(path, {})
+
+      with subtest("Restore second backup"):
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
     '';
 }

modules/contracts/dashboard.nix

@@ -0,0 +1,77 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types)
+    nullOr
+    submodule
+    str
+    ;
+in
+{
+  mkRequest =
+    {
+      serviceName ? "",
+      externalUrl ? "",
+      externalUrlText ? null,
+      internalUrl ? null,
+      internalUrlText ? null,
+      apiKey ? null,
+    }:
+    mkOption {
+      description = ''
+        Request part of the dashboard contract.
+      '';
+      default = { };
+      type = submodule {
+        options = {
+          externalUrl =
+            mkOption {
+              description = ''
+                URL at which the service can be accessed.
+
+                This URL should go through the reverse proxy.
+              '';
+              type = str;
+              default = externalUrl;
+              example = "https://jellyfin.example.com";
+            }
+            // (lib.optionalAttrs (externalUrlText != null) {
+              defaultText = externalUrlText;
+            });
+
+          internalUrl =
+            mkOption {
+              description = ''
+                URL at which the service can be accessed directly.
+
+                This URL should bypass the reverse proxy.
+                It can be used for example to ping the service
+                and making sure it is up and running correctly.
+              '';
+              type = nullOr str;
+              default = internalUrl;
+              example = "http://127.0.0.1:8081";
+            }
+            // (lib.optionalAttrs (internalUrlText != null) {
+              defaultText = internalUrlText;
+            });
+        };
+      };
+    };
+
+  mkResult =
+    {
+    }:
+    mkOption {
+      description = ''
+        Result part of the dashboard contract.
+
+        No option is provided here.
+      '';
+      default = { };
+      type = submodule {
+        options = {
+        };
+      };
+    };
+}

modules/contracts/dashboard/docs/default.md

@@ -0,0 +1,65 @@
+# Dashboard Contract {#contract-dashboard}
+
+This NixOS contract is used for user-facing services
+that want to be displayed on a dashboard.
+
+It is a contract between a service that can be accessed through an URL
+and a service that wants to show a list of those services.
+
+## Providers {#contract-dashboard-providers}
+
+The providers of this contract in SHB are:
+
+<!-- Somehow generate this list -->
+
+- The homepage service under its [shb.homepage.servicesGroups.<name>.services.<name>.dashboard](#services-homepage-options-shb.homepage.servicesGroups._name_.services._name_.dashboard) option.
+
+## Usage {#contracts-dashboard-usage}
+
+A service that can be shown on a dashboard will provide a `dashboard` option.
+
+Here is an example module defining such a requester option for this dashboard contract:
+
+```nix
+{
+  options = {
+    myservice.dashboard = lib.mkOption {
+      description = ''
+        Dashboard contract consumer
+      '';
+      default = { };
+      type = lib.types.submodule {
+        options = shb.contracts.dashboard.mkRequester {
+          externalUrl = "https://${config.myservice.subdomain}.${config.myservice.domain}";
+          internalUrl = "http://127.0.0.1:${config.myservice.port}";
+        };
+      };
+    };
+  };
+};
+```
+
+Then, plug both consumer and provider together in the `config`:
+
+```nix
+{
+  config = {
+    <provider-module> = {
+      dashboard.request = config.myservice.dashboard.request;
+    };
+  };
+}
+```
+
+And that's it for the contract part.
+For more specific details on each provider, go to their respective manual pages.
+
+## Contract Reference {#contract-dashboard-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-dashboard-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```

modules/contracts/dashboard/dummyModule.nix

@@ -0,0 +1,30 @@
+{ lib, shb, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) submodule;
+in
+{
+  imports = [
+    ../../../lib/module.nix
+  ];
+
+  options.shb.contracts.dashboard = mkOption {
+    description = ''
+      Contract for user-facing services that want to 
+      be displayed on a dashboard.
+
+      The requester communicates to the provider
+      how to access the service
+      through the `request` options.
+
+      The provider reads from the `request` options
+      and configures what is necessary on its side
+      to show the service and check its availability.
+      It does not communicate back to the requester.
+    '';
+
+    type = submodule {
+      options = shb.contracts.dashboard.contract;
+    };
+  };
+}

modules/contracts/databasebackup.nix

@@ -1,14 +1,19 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  inherit (lib) mkOption literalExpression literalMD optionalAttrs optionalString;
+  inherit (lib)
+    mkOption
+    literalExpression
+    literalMD
+    optionalAttrs
+    optionalString
+    ;
   inherit (lib.types) submodule str;
-
+  inherit (shb) anyNotNull;
-  shblib = pkgs.callPackage ../../lib {};
-  inherit (shblib) anyNotNull;
 in
 {
   mkRequest =
-    { user ? "root",
+    {
+      user ? "root",
       userText ? null,
       backupName ? "dump",
       backupNameText ? null,
@@ -16,146 +21,171 @@ in
       backupCmdText ? null,
       restoreCmd ? "",
       restoreCmdText ? null,
-    }: mkOption {
+    }:
+    mkOption {
       description = ''
-        Request part of the backup contract.
+        Request part of the database backup contract.
 
         Options set by the requester module
         enforcing how to backup files.
       '';
 
       default = {
-        inherit user backupName backupCmd restoreCmd;
+        inherit
+          user
+          backupName
+          backupCmd
+          restoreCmd
+          ;
       };
 
-      defaultText = optionalString (anyNotNull [
+      defaultText =
-        userText
+        optionalString
-        backupNameText
+          (anyNotNull [
-        backupCmdText
+            userText
-        restoreCmdText
+            backupNameText
-      ]) (literalMD ''
+            backupCmdText
-      {
+            restoreCmdText
-        user = ${if userText != null then userText else user};
+          ])
-        backupName = ${if backupNameText != null then backupNameText else backupName};
+          (literalMD ''
-        backupCmd = ${if backupCmdText != null then backupCmdText else backupCmd};
+            {
-        restoreCmd = ${if restoreCmdText != null then restoreCmdText else restoreCmd};
+              user = ${if userText != null then userText else user};
-      }
+              backupName = ${if backupNameText != null then backupNameText else backupName};
-      '');
+              backupCmd = ${if backupCmdText != null then backupCmdText else backupCmd};
+              restoreCmd = ${if restoreCmdText != null then restoreCmdText else restoreCmd};
+            }
+          '');
 
       type = submodule {
         options = {
-          user = mkOption {
+          user =
-            description = ''
+            mkOption {
-              Unix user doing the backups.
+              description = ''
+                Unix user doing the backups.
 
-              This should be an admin user having access to all databases.
+                This should be an admin user having access to all databases.
-            '';
+              '';
-            type = str;
+              type = str;
-            example = "postgres";
+              example = "postgres";
-            default = user;
+              default = user;
-          } // optionalAttrs (userText != null) {
+            }
-            defaultText = literalMD userText;
+            // optionalAttrs (userText != null) {
-          };
+              defaultText = literalMD userText;
+            };
 
-          backupName = mkOption {
+          backupName =
-            description = "Name of the backup in the repository.";
+            mkOption {
-            type = str;
+              description = "Name of the backup in the repository.";
-            example = "postgresql.sql";
+              type = str;
-            default = backupName;
+              example = "postgresql.sql";
-          } // optionalAttrs (backupNameText != null) {
+              default = backupName;
-            defaultText = literalMD backupNameText;
+            }
-          };
+            // optionalAttrs (backupNameText != null) {
+              defaultText = literalMD backupNameText;
+            };
 
-          backupCmd = mkOption {
+          backupCmd =
-            description = "Command that produces the database dump on stdout.";
+            mkOption {
-            type = str;
+              description = "Command that produces the database dump on stdout.";
-            example = literalExpression ''
+              type = str;
-              ''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable
+              example = literalExpression ''
-            '';
+                ''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable
-            default = backupCmd;
+              '';
-          } // optionalAttrs (backupCmdText != null) {
+              default = backupCmd;
-            defaultText = literalMD backupCmdText;
+            }
-          };
+            // optionalAttrs (backupCmdText != null) {
+              defaultText = literalMD backupCmdText;
+            };
 
-          restoreCmd = mkOption {
+          restoreCmd =
-            description = "Command that reads the database dump on stdin and restores the database.";
+            mkOption {
-            type = str;
+              description = "Command that reads the database dump on stdin and restores the database.";
-            example = literalExpression ''
+              type = str;
-              ''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres
+              example = literalExpression ''
-            '';
+                ''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres
-            default = restoreCmd;
+              '';
-          } // optionalAttrs (restoreCmdText != null) {
+              default = restoreCmd;
-            defaultText = literalMD restoreCmdText;
+            }
-          };
+            // optionalAttrs (restoreCmdText != null) {
+              defaultText = literalMD restoreCmdText;
+            };
         };
       };
     };
 
-
+  mkResult =
-  mkResult = {
-    restoreScript ? "restore",
-    restoreScriptText ? null,
-    backupService ? "backup.service",
-    backupServiceText ? null,
-  }: mkOption {
-    description = ''
-      Result part of the backup contract.
-
-      Options set by the provider module that indicates the name of the backup and restor scripts.
-    '';
-    default = {
-      inherit restoreScript backupService;
-    };
-
-    defaultText = optionalString (anyNotNull [
-      restoreScriptText
-      backupServiceText
-    ]) (literalMD ''
     {
-      restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
+      restoreScript ? "restore",
-      backupService = ${if backupServiceText != null then backupServiceText else backupService};
+      restoreScriptText ? null,
-    }
+      backupService ? "backup.service",
-    '');
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the database backup contract.
 
-    type = submodule {
+        Options set by the provider module that indicates the name of the backup and restore scripts.
-      options = {
+      '';
-        restoreScript = mkOption {
+      default = {
-          description = ''
+        inherit restoreScript backupService;
-            Name of script that can restore the database.
+      };
-            One can then list snapshots with:
 
-            ```bash
+      defaultText =
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+        optionalString
-            ```
+          (anyNotNull [
+            restoreScriptText
+            backupServiceText
+          ])
+          (literalMD ''
+            {
+              restoreScript = ${if restoreScriptText != null then restoreScriptText else restoreScript};
+              backupService = ${if backupServiceText != null then backupServiceText else backupService};
+            }
+          '');
 
-            And restore the database with:
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
 
-            ```bash
+                ```bash
-            $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
-            ```
+                ```
-          '';
-          type = str;
-          default = restoreScript;
-        } // optionalAttrs (restoreScriptText != null) {
-          defaultText = literalMD restoreScriptText;
-        };
 
-        backupService = mkOption {
+                And restore the database with:
-          description = ''
-            Name of service backing up the database.
 
-            This script can be ran manually to backup the database:
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
+                ```
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
 
-            ```bash
+          backupService =
-            $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+            mkOption {
-            ```
+              description = ''
-          '';
+                Name of service backing up the database.
-          type = str;
+
-          default = backupService;
+                This script can be ran manually to backup the database:
-        } // optionalAttrs (backupServiceText != null) {
+
-          defaultText = literalMD backupServiceText;
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
         };
       };
     };
-  };
 }

modules/contracts/databasebackup/docs/default.md

@@ -20,21 +20,21 @@ source: @OPTIONS_JSON@
 ## Usage {#contract-databasebackup-usage}
 
 A database that can be backed up will provide a `databasebackup` option.
-Such a service is a `requester` providing a `request` for a module `provider` of this contract. 
 
 What this option defines is, from the user perspective - that is _you_ - an implementation detail
 but it will at least define how to create a database dump,
 the user to backup with
 and how to restore from a database dump.
 
-Here is an example module defining such a `databasebackup` option:
+Here is an example module defining such a `databasebackup` option,
+which defines the user to backup with (`user`)
+and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
 
 ```nix
 {
   options = {
     myservice.databasebackup = mkOption {
-      type = contracts.databasebackup.request;
+      type = shb.contracts.databasebackup.mkRequester = {
-      default = {
         user = "myservice";
         backupCmd = ''
           ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
@@ -48,16 +48,16 @@ Here is an example module defining such a `databasebackup` option:
 };
 ```
 
-Now, on the other side we have a service that uses this `backup` option and actually backs up files.
+Now, on the other side we have a service that uses this `databasebackup` option and actually backs up files.
 This service is a `provider` of this contract and will provide a `result` option.
 
-Let's assume such a module is available under the `databasebackupservice` option
+Let's assume such a module is available under the `databaseBackupService` option
-and that one can create multiple backup instances under `databasebackupservice.instances`.
+and that one can create multiple backup instances under `databaseBackupService.instances`.
 Then, to actually backup the `myservice` service, one would write:
 
 ```nix
-databasebackupservice.instances.myservice = {
+databaseBackupService.instances.myservice = {
-  request = myservice.databasebackup;
+  request = myservice.databasebackup.request;
   
   settings = {
     enable = true;
@@ -72,11 +72,11 @@ databasebackupservice.instances.myservice = {
 ```
 
 It is advised to backup files to different location, to improve redundancy.
-Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
+Thanks to using contracts, this can be made easily either with the same `databaseBackupService`:
 
 ```nix
-databasebackupservice.instances.myservice_2 = {
+databaseBackupService.instances.myservice_2 = {
-  request = myservice.backup;
+  request = myservice.databasebackup.request;
   
   settings = {
     enable = true;
@@ -88,12 +88,12 @@ databasebackupservice.instances.myservice_2 = {
 };
 ```
 
-Or with another module `databasebackupservice_2`!
+Or with another module `databaseBackupService_2`!
 
 ## Providers of the Database Backup Contract {#contract-databasebackup-providers}
 
 - [Restic block](blocks-restic.html).
-- [Borgbackup block](blocks-borgbackup.html) [WIP].
+- [Borgbackup block](blocks-borgbackup.html).
 
 ## Requester Blocks and Services {#contract-databasebackup-requesters}
 

modules/contracts/databasebackup/dummyModule.nix

@@ -1,11 +1,13 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  contracts = pkgs.callPackage ../. {};
-
   inherit (lib) mkOption;
   inherit (lib.types) submodule;
 in
 {
+  imports = [
+    ../../../lib/module.nix
+  ];
+
   options.shb.contracts.databasebackup = mkOption {
     description = ''
       Contract for database backup between a requester module
@@ -23,7 +25,7 @@ in
     '';
 
     type = submodule {
-      options = contracts.databasebackup.contract;
+      options = shb.contracts.databasebackup.contract;
     };
   };
 }

modules/contracts/databasebackup/test.nix

@@ -1,86 +1,142 @@
-{ pkgs, lib, ... }:
+{
+  pkgs,
+  lib,
+  shb,
+}:
 let
-  testLib = pkgs.callPackage ../../../test/common.nix {};
+  inherit (lib)
-
+    getAttrFromPath
-  inherit (lib) getAttrFromPath mkIf optionalAttrs setAttrByPath;
+    mkIf
+    optionalAttrs
+    setAttrByPath
+    ;
 in
-{ name,
+{
+  name,
   requesterRoot,
   providerRoot,
   extraConfig ? null, # { config, database } -> attrset
-  modules ? [],
+  modules ? [ ],
   database ? "me",
   settings, # { repository, config } -> attrset
-}: pkgs.testers.runNixOSTest {
+}:
+shb.test.runNixOSTest {
   inherit name;
 
-  nodes.machine = { config, ... }: {
+  nodes.machine =
-    imports = [ testLib.baseImports ] ++ modules;
+    { config, ... }:
-    config = lib.mkMerge [
+    {
-      (setAttrByPath providerRoot {
+      imports = [ shb.test.baseImports ] ++ modules;
-        request = (getAttrFromPath requesterRoot config).request;
+      config = lib.mkMerge [
-        settings = settings {
+        (setAttrByPath providerRoot {
-          inherit config;
+          request = (getAttrFromPath requesterRoot config).request;
-          repository = "/opt/repos/database";
+          settings = settings {
-        };
+            inherit config;
-      })
+            repository = "/opt/repos/database";
-      (mkIf (database != "root") {
+          };
-        users.users.${database} = {
+        })
-          isSystemUser = true;
+        (mkIf (database != "root") {
-          extraGroups = [ "sudoers" ];
+          users.users.${database} = {
-          group = "root";
+            isSystemUser = true;
-        };
+            extraGroups = [ "sudoers" ];
-      })
+            group = "root";
-      (optionalAttrs (extraConfig != null) (extraConfig { inherit config database; }))
+          };
-    ];
+        })
-  };
+        (optionalAttrs (extraConfig != null) (extraConfig {
+          inherit config database;
+        }))
+      ];
+    };
 
-  testScript = { nodes, ... }: let
+  testScript =
-    provider = getAttrFromPath providerRoot nodes.machine;
+    { nodes, ... }:
-  in ''
+    let
-    import csv
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      import csv
+      import re
 
-    start_all()
+      start_all()
-    machine.wait_for_unit("postgresql.service")
+      machine.wait_for_unit("postgresql.service")
-    machine.wait_for_open_port(5432)
+      machine.wait_for_unit("postgresql-setup.service")
+      machine.wait_for_open_port(5432)
 
-    def peer_cmd(cmd, db="me"):
+      def peer_cmd(cmd, db="${database}"):
-        return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
+          return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
 
-    def query(query):
+      def query(query):
-        res = machine.succeed(peer_cmd(query))
+          res = machine.succeed(peer_cmd(query))
-        return list(dict(l) for l in csv.DictReader(res.splitlines()))
+          return list(dict(l) for l in csv.DictReader(res.splitlines()))
 
-    def cmp_tables(a, b):
+      def cmp_tables(a, b):
-        for i in range(max(len(a), len(b))):
+          for i in range(max(len(a), len(b))):
-            diff = set(a[i]) ^ set(b[i])
+              diff = set(a[i]) ^ set(b[i])
-            if len(diff) > 0:
+              if len(diff) > 0:
-                raise Exception(i, diff)
+                  raise Exception(i, diff)
 
-    table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+      with subtest("create fixture"):
+          machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
 
-    with subtest("create fixture"):
+          res = query("SELECT * FROM test")
-        machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
+          table = [{'name': 'car', 'count': '1'}]
-        machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
+          cmp_tables(res, table)
 
-        res = query("SELECT * FROM test")
+      with subtest("Initial snapshots"):
-        cmp_tables(res, table)
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 0:
+              raise Exception(f"Unexpected snapshots:\n{out}")
 
-    with subtest("backup"):
+      with subtest("backup"):
-        print(machine.succeed("systemctl cat ${provider.result.backupService}"))
+          machine.succeed("systemctl start --wait ${provider.backupService}")
-        print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
-        machine.succeed("systemctl start ${provider.result.backupService}")
 
-    with subtest("drop database"):
+      with subtest("One snapshot"):
-        machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
-        machine.fail(peer_cmd("SELECT * FROM test"))
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+              raise Exception(f"Unexpected snapshots:\n{out}")
 
-    with subtest("restore"):
+      with subtest("New content"):
-        print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
-        machine.succeed("${provider.result.restoreScript} restore latest ")
 
-    with subtest("check restoration"):
+          res = query("SELECT * FROM test")
-        res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
-        cmp_tables(res, table)
+          cmp_tables(res, table)
-  '';
+
+      with subtest("backup"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("Two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("drop database"):
+          machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
+          machine.fail(peer_cmd("SELECT * FROM test"))
+
+      with subtest("restore second snapshot"):
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+          cmp_tables(res, table)
+
+      with subtest("restore first snapshot"):
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
+          cmp_tables(res, table)
+    '';
 }

modules/contracts/datasetbackup.nix

@@ -0,0 +1,113 @@
+{ lib, ... }:
+let
+  inherit (lib)
+    literalMD
+    mkOption
+    optionalAttrs
+    ;
+  inherit (lib.types)
+    submodule
+    str
+    ;
+in
+{
+  mkRequest =
+    {
+      dataset ? "",
+      datasetText ? null,
+    }:
+    mkOption {
+      description = ''
+        Request part of the backup contract.
+
+        Options set by the requester module
+        enforcing how to backup files.
+      '';
+
+      default = { };
+
+      type = submodule {
+        options = {
+          dataset =
+            mkOption {
+              description = ''
+                Dataset to backup, including the pool name.
+              '';
+              type = str;
+              example = "root/home";
+              default = dataset;
+            }
+            // optionalAttrs (datasetText != null) {
+              defaultText = literalMD datasetText;
+            };
+        };
+      };
+    };
+
+  mkResult =
+    {
+      restoreScript ? "restore",
+      restoreScriptText ? null,
+      backupService ? "backup.service",
+      backupServiceText ? null,
+    }:
+    mkOption {
+      description = ''
+        Result part of the backup contract.
+
+        Options set by the provider module that indicates the name of the backup and restore scripts.
+      '';
+      default = { };
+
+      type = submodule {
+        options = {
+          restoreScript =
+            mkOption {
+              description = ''
+                Name of script that can restore the database.
+                One can then list snapshots with:
+
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                <snapshot 1> <metadata>
+                <snapshot 2> <metadata>
+                ```
+
+                And restore the database with:
+
+                ```bash
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
+                ```
+
+                It is not garanteed to be able to restore back to a snapshot in the future.
+                With the above example, it may not be possible to restore `<snapshot 2>`
+                after having restored `<snapshot 1>`.
+              '';
+              type = str;
+              default = restoreScript;
+            }
+            // optionalAttrs (restoreScriptText != null) {
+              defaultText = literalMD restoreScriptText;
+            };
+
+          backupService =
+            mkOption {
+              description = ''
+                Name of service backing up the database.
+
+                This script can be ran manually to backup the database:
+
+                ```bash
+                $ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
+                ```
+              '';
+              type = str;
+              default = backupService;
+            }
+            // optionalAttrs (backupServiceText != null) {
+              defaultText = literalMD backupServiceText;
+            };
+        };
+      };
+    };
+}

modules/contracts/datasetbackup/docs/default.md

@@ -0,0 +1,85 @@
+# Dataset Backup Contract {#contract-dataset-backup}
+
+This NixOS contract represents a backup job
+that will backup one ZFS dataset.
+
+It is a contract between a service that manages ZFS datasets
+and a service that backs up ZFS datasets.
+
+## Contract Reference {#contract-datatsetbackup-options}
+
+These are all the options that are expected to exist for this contract to be respected.
+
+```{=include=} options
+id-prefix: contracts-datasetbackup-options-
+list-id: selfhostblocks-options
+source: @OPTIONS_JSON@
+```
+
+## Usage {#contract-datasetbackup-usage}
+
+A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option.
+
+Here is an example module defining such a `backup` option,
+which defines what directories to backup (`sourceDirectories`)
+and the user to backup with (`user`).
+
+```nix
+{
+  options = {
+    myservice.datasetbackup = mkOption {
+      type = lib.types.submodule {
+        options = shb.contracts.datasetbackup.mkRequester {
+          dataset = "root/test";
+        };
+      };
+    };
+  };
+};
+```
+
+Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset.
+This service is a `provider` of this contract and will provide a `result` option.
+
+Let's assume such a module is available under the `backupService` option
+and that one can create multiple backup instances under `backupService.instances`.
+Then, to actually backup the `myservice` service, one would write:
+
+```nix
+backupService.instances.myservice = {
+  request = myservice.datasetbackup.request;
+  
+  settings = {
+    enable = true;
+
+    targetDataset = "backup/myservice";
+
+    # ... Other options specific to backupService like scheduling.
+  };
+};
+```
+
+It is advised to backup files to different location, to improve redundancy.
+Thanks to using contracts, this can be made easily either with the same `backupService`:
+
+```nix
+backupService.instances.myservice_2 = {
+  request = myservice.datasetbackup.request;
+  
+  settings = {
+    enable = true;
+  
+    targetDataset = "backup2/myservice";
+  };
+};
+```
+
+Or with another module `backupService_2`!
+
+## Providers of the Backup Contract {#contract-datasetbackup-providers}
+
+- [Sanoid block](blocks-sanoid.html).
+
+## Requester Blocks and Services {#contract-datasetbackup-requesters}
+
+- [ZFS](blocks-zfs.html#blocks-zfs-backup).

modules/contracts/datasetbackup/dummyModule.nix

@@ -0,0 +1,30 @@
+{ lib, shb, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) submodule;
+in
+{
+  imports = [
+    ../../../lib/module.nix
+  ];
+
+  options.shb.contracts.datasetbackup = mkOption {
+    description = ''
+      Contract for backing up ZFS datasets.
+
+      The requester communicates to the provider
+      the dataset to backup
+      through the `request` options.
+
+      The provider reads from the `request` options
+      and backs up the requested dataset.
+      It communicates to the requester what script is used
+      to backup and restore the files
+      through the `result` options.
+    '';
+
+    type = submodule {
+      options = shb.contracts.datasetbackup.contract;
+    };
+  };
+}

modules/contracts/datasetbackup/test.nix

@@ -0,0 +1,164 @@
+{
+  pkgs,
+  lib,
+  shb,
+}:
+let
+  inherit (lib)
+    concatMapStringsSep
+    getAttrFromPath
+    setAttrByPath
+    ;
+in
+{
+  name,
+  providerRoot,
+  modules ? [ ],
+  dataset ? "root/test",
+  sourceDirectories ? [
+    "/opt/files/A"
+    "/opt/files/B"
+  ],
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
+  extraConfig ? null, # { filesRoot, config } -> attrset
+}:
+shb.test.runNixOSTest {
+  inherit name;
+
+  nodes.machine =
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
+
+      config = lib.mkMerge [
+        (setAttrByPath providerRoot {
+          request = {
+            inherit dataset;
+          };
+          settings = settings {
+            inherit config;
+            filesRoot = "/opt/files";
+          };
+        })
+        (lib.optionalAttrs (extraConfig != null) (extraConfig {
+          inherit config;
+          filesRoot = "/opt/files";
+        }))
+      ];
+    };
+
+  extraPythonPackages = p: [ p.dictdiffer ];
+  skipTypeCheck = true;
+
+  testScript =
+    { nodes, ... }:
+    let
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
+    in
+    ''
+      from datetime import datetime, timedelta
+      from dictdiffer import diff
+      import re
+
+      sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
+
+      def list_files(dir):
+          files_and_content = {}
+
+          files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
+
+          for f in files:
+              content = machine.succeed(f"""cat {f}""").strip()
+              files_and_content[f] = content
+
+          return files_and_content
+
+      def assert_files(dir, files):
+          result = list(diff(list_files(dir), files))
+          if len(result) > 0:
+              raise Exception("Unexpected files:", result)
+
+      with subtest("Create initial content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                  mkdir -p {path}
+                  echo repo_fileA_1 > {path}/fileA
+                  echo repo_fileB_1 > {path}/fileB
+              """)
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
+
+      with subtest("Initial snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("First backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      # To accomodate for snapshot orchestrators which keep only a given amount
+      # of snapshots per unit of time, we set the time to now + 2h.
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
+      machine.succeed(f"timedatectl set-time '{new_date}'")
+
+      with subtest("New content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""
+                echo repo_fileA_2 > {path}/fileA
+                echo repo_fileB_2 > {path}/fileB
+                """)
+
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
+      with subtest("Delete content"):
+          for path in sourceDirectories:
+              machine.succeed(f"""rm -r {path}/*""")
+
+              assert_files(path, {})
+
+      with subtest("Restore second backup"):
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_1',
+                  f'{path}/fileB': 'repo_fileB_1',
+              })
+    '';
+}

modules/contracts/default.nix

@@ -1,58 +1,78 @@
-{ pkgs, lib }:
+{
+  pkgs,
+  lib,
+  shb,
+}:
 let
   inherit (lib) mkOption optionalAttrs;
   inherit (lib.types) anything;
 
   mkContractFunctions =
-    { mkRequest,
+    {
+      mkRequest,
       mkResult,
-    }: {
+    }:
+    {
       mkRequester = requestCfg: {
         request = mkRequest requestCfg;
 
-        result = mkResult {};
+        result = mkResult { };
       };
 
       mkProvider =
-        { resultCfg,
+        {
-          settings ? {},
+          resultCfg,
-        }: {
+          settings ? { },
-          request = mkRequest {};
+        }:
+        {
+          request = mkRequest { };
 
           result = mkResult resultCfg;
-        } // optionalAttrs (settings != {}) { inherit settings; };
+        }
+        // optionalAttrs (settings != { }) { inherit settings; };
 
       contract = {
-        request = mkRequest {};
+        request = mkRequest { };
 
-        result = mkResult {};
+        result = mkResult { };
 
         settings = mkOption {
           description = ''
-          Optional attribute set with options specific to the provider.
+            Optional attribute set with options specific to the provider.
           '';
           type = anything;
         };
       };
     };
 
-  importContract = module:
+  importContract =
+    module:
     let
-      importedModule = pkgs.callPackage module {};
+      importedModule = pkgs.callPackage module {
-    in
+        shb = shb // {
-      mkContractFunctions {
+          inherit contracts;
-        inherit (importedModule) mkRequest mkResult;
+        };
       };
-in
+    in
-{
+    (mkContractFunctions {
-  databasebackup = importContract ./databasebackup.nix;
+      inherit (importedModule) mkRequest mkResult;
-  backup = importContract ./backup.nix;
+    })
-  mount = import ./mount.nix { inherit lib; };
+    // (importedModule.passthru or { });
-  secret = importContract ./secret.nix;
+
-  ssl = import ./ssl.nix { inherit lib; };
+  contracts = {
-  test = {
+    databasebackup = importContract ./databasebackup.nix;
-    secret = import ./secret/test.nix { inherit pkgs lib; };
+    datasetbackup = importContract ./datasetbackup.nix;
-    databasebackup = import ./databasebackup/test.nix { inherit pkgs lib; };
+    dashboard = importContract ./dashboard.nix;
-    backup = import ./backup/test.nix { inherit pkgs lib; };
+    backup = importContract ./backup.nix;
+    mount = pkgs.callPackage ./mount.nix { };
+    secret = importContract ./secret.nix;
+    ssl = pkgs.callPackage ./ssl.nix { };
+    test = {
+      secret = pkgs.callPackage ./secret/test.nix { inherit shb; };
+      databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; };
+      datasetbackup = pkgs.callPackage ./datasetbackup/test.nix { inherit shb; };
+      backup = pkgs.callPackage ./backup/test.nix { inherit shb; };
+    };
   };
-}
+in
+contracts

modules/contracts/secret.nix

@@ -1,22 +1,28 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  inherit (lib) concatStringsSep literalMD mkOption optionalAttrs optionalString;
+  inherit (lib)
+    concatStringsSep
+    literalMD
+    mkOption
+    optionalAttrs
+    optionalString
+    ;
   inherit (lib.types) listOf submodule str;
-
+  inherit (shb) anyNotNull;
-  shblib = pkgs.callPackage ../../lib {};
-  inherit (shblib) anyNotNull;
 in
 {
   mkRequest =
-    { mode ? "0400",
+    {
+      mode ? "0400",
       modeText ? null,
       owner ? "root",
       ownerText ? null,
       group ? "root",
       groupText ? null,
-      restartUnits ? [],
+      restartUnits ? [ ],
       restartUnitsText ? null,
-    }: mkOption {
+    }:
+    mkOption {
       description = ''
         Request part of the secret contract.
 
@@ -25,64 +31,87 @@ in
       '';
 
       default = {
-        inherit mode owner group restartUnits;
+        inherit
+          mode
+          owner
+          group
+          restartUnits
+          ;
       };
 
-      defaultText = optionalString (anyNotNull [
+      defaultText =
-        modeText
+        optionalString
-        ownerText
+          (anyNotNull [
-        groupText
+            modeText
-        restartUnitsText
+            ownerText
-      ]) (literalMD ''
+            groupText
-      {
+            restartUnitsText
-        mode = ${if modeText != null then modeText else mode};
+          ])
-        owner = ${if ownerText != null then ownerText else owner};
+          (literalMD ''
-        group = ${if groupText != null then groupText else group};
+            {
-        restartUnits = ${if restartUnitsText != null then restartUnitsText else "[ " + concatStringsSep " " restartUnits + " ]"};
+              mode = ${if modeText != null then modeText else mode};
-      }
+              owner = ${if ownerText != null then ownerText else owner};
-      '');
+              group = ${if groupText != null then groupText else group};
+              restartUnits = ${
+                if restartUnitsText != null then
+                  restartUnitsText
+                else
+                  "[ " + concatStringsSep " " restartUnits + " ]"
+              };
+            }
+          '');
 
       type = submodule {
         options = {
-          mode = mkOption {
+          mode =
-            description = ''
+            mkOption {
-              Mode of the secret file.
+              description = ''
-            '';
+                Mode of the secret file.
-            type = str;
+              '';
-            default = mode;
+              type = str;
-          } // optionalAttrs (modeText != null) {
+              default = mode;
-            defaultText = literalMD modeText;
+            }
-          };
+            // optionalAttrs (modeText != null) {
+              defaultText = literalMD modeText;
+            };
 
-          owner = mkOption ({
+          owner = mkOption (
-            description = ''
+            {
-              Linux user owning the secret file.
+              description = ''
-            '';
+                Linux user owning the secret file.
-            type = str;
+              '';
-            default = owner;
+              type = str;
-          } // optionalAttrs (ownerText != null) {
+              default = owner;
-            defaultText = literalMD ownerText;
+            }
-          });
+            // optionalAttrs (ownerText != null) {
+              defaultText = literalMD ownerText;
+            }
+          );
 
-          group = mkOption {
+          group =
-            description = ''
+            mkOption {
-              Linux group owning the secret file.
+              description = ''
-            '';
+                Linux group owning the secret file.
-            type = str;
+              '';
-            default = group;
+              type = str;
-          } // optionalAttrs (groupText != null) {
+              default = group;
-            defaultText = literalMD groupText;
+            }
-          };
+            // optionalAttrs (groupText != null) {
+              defaultText = literalMD groupText;
+            };
 
-          restartUnits = mkOption ({
+          restartUnits = mkOption (
-            description = ''
+            {
-              Systemd units to restart after the secret is updated.
+              description = ''
-            '';
+                Systemd units to restart after the secret is updated.
-            type = listOf str;
+              '';
-            default = restartUnits;
+              type = listOf str;
-          } // optionalAttrs (restartUnitsText != null) {
+              default = restartUnits;
-            defaultText = literalMD restartUnitsText;
+            }
-          });
+            // optionalAttrs (restartUnitsText != null) {
+              defaultText = literalMD restartUnitsText;
+            }
+          );
         };
       };
     };
@@ -92,34 +121,39 @@ in
       path ? "/run/secrets/secret",
       pathText ? null,
     }:
-    mkOption ({
+    mkOption (
-      description = ''
+      {
-        Result part of the secret contract.
+        description = ''
+          Result part of the secret contract.
 
-        Options set by the provider module that indicates where the secret can be found.
+          Options set by the provider module that indicates where the secret can be found.
-      '';
+        '';
-      default = {
+        default = {
-        inherit path;
+          inherit path;
-      };
+        };
-      type = submodule {
+        type = submodule {
-        options = {
+          options = {
-          path = mkOption {
+            path =
-            type = lib.types.path;
+              mkOption {
-            description = ''
+                type = lib.types.path;
-              Path to the file containing the secret generated out of band.
+                description = ''
+                  Path to the file containing the secret generated out of band.
 
-              This path will exist after deploying to a target host,
+                  This path will exist after deploying to a target host,
-              it is not available through the nix store.
+                  it is not available through the nix store.
-            '';
+                '';
-            default = path;
+                default = path;
-          } // optionalAttrs (pathText != null) {
+              }
-            defaultText = pathText;
+              // optionalAttrs (pathText != null) {
+                defaultText = pathText;
+              };
           };
         };
-      };
+      }
-    } // optionalAttrs (pathText != null) {
+      // optionalAttrs (pathText != null) {
-      defaultText = {
+        defaultText = {
-        path = pathText;
+          path = pathText;
-      };
+        };
-    });
+      }
+    );
 }

modules/contracts/secret/docs/default.md

@@ -42,12 +42,12 @@ Now, with this contract, a layer on top of `sops` is added which is found under
 The configuration then becomes:
 
 ```nix
-shb.sops.secrets."ldap/user_password" = {
+shb.sops.secret."ldap/user_password" = {
   request = config.shb.lldap.userPassword.request;
   settings.sopsFile = ./secrets.yaml;
 };
 
-shb.lldap.userPassword.result = config.shb.sops.secrets."ldap/user_password".result;
+shb.lldap.userPassword.result = config.shb.sops.secret."ldap/user_password".result;
 ```
 
 The issue is now gone as the responsibility falls
@@ -63,9 +63,9 @@ sops.defaultSopsFile = ./secrets.yaml;
 Then the snippet above is even more simplified:
 
 ```nix
-shb.sops.secrets."ldap/user_password".request = config.shb.lldap.userPassword.request;
+shb.sops.secret."ldap/user_password".request = config.shb.lldap.userPassword.request;
 
-shb.lldap.userPassword.result = config.shb.sops.secrets."ldap/user_password".result;
+shb.lldap.userPassword.result = config.shb.sops.secret."ldap/user_password".result;
 ```
 
 ## Contract Reference {#contract-secret-options}

modules/contracts/secret/dummyModule.nix

@@ -1,11 +1,13 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
 let
-  contracts = pkgs.callPackage ../. {};
-
   inherit (lib) mkOption;
   inherit (lib.types) submodule;
 in
 {
+  imports = [
+    ../../../lib/module.nix
+  ];
+
   options.shb.contracts.secret = mkOption {
     description = ''
       Contract for secrets between a requester module
@@ -21,7 +23,7 @@ in
       through the `result.*` options.
     '';
     type = submodule {
-      options = contracts.secret.contract;
+      options = shb.contracts.secret.contract;
     };
   };
 }

modules/contracts/secret/test.nix

@@ -1,28 +1,39 @@
-{ pkgs, lib, ... }:
+{
+  pkgs,
+  lib,
+  shb,
+}:
 let
-  testLib = pkgs.callPackage ../../../test/common.nix {};
-
   inherit (lib) getAttrFromPath setAttrByPath;
   inherit (lib) mkIf;
 in
-  { name,
+{
-    configRoot,
+  name,
-    settingsCfg, # str -> attrset
+  configRoot,
-    modules ? [],
+  settingsCfg, # str -> attrset
-    owner ? "root",
+  modules ? [ ],
-    group ? "root",
+  owner ? "root",
-    mode ? "0400",
+  group ? "root",
-    restartUnits ? [ "myunit.service" ],
+  mode ? "0400",
-  }: pkgs.testers.runNixOSTest {
+  restartUnits ? [ "myunit.service" ],
-    name = "secret_${name}_${owner}_${group}_${mode}";
+}:
+shb.test.runNixOSTest {
+  name = "secret_${name}_${owner}_${group}_${mode}";
 
-    nodes.machine = { config, ... }: {
+  nodes.machine =
-      imports = [ testLib.baseImports ] ++ modules;
+    { config, ... }:
+    {
+      imports = [ shb.test.baseImports ] ++ modules;
       config = lib.mkMerge [
         (setAttrByPath configRoot {
           A = {
             request = {
-              inherit owner group mode restartUnits;
+              inherit
+                owner
+                group
+                mode
+                restartUnits
+                ;
             };
             settings = settingsCfg "secretA";
           };
@@ -31,35 +42,36 @@ in
           users.users.${owner}.isNormalUser = true;
         })
         (mkIf (group != "root") {
-          users.groups.${group} = {};
+          users.groups.${group} = { };
         })
       ];
     };
 
-    testScript = { nodes, ... }:
+  testScript =
-      let
+    { nodes, ... }:
-        result = (getAttrFromPath configRoot nodes.machine)."A".result;
+    let
-      in
+      result = (getAttrFromPath configRoot nodes.machine)."A".result;
-        ''
+    in
-          owner = machine.succeed("stat -c '%U' ${result.path}").strip()
+    ''
-          print(f"Got owner {owner}")
+      owner = machine.succeed("stat -c '%U' ${result.path}").strip()
-          if owner != "${owner}":
+      print(f"Got owner {owner}")
-              raise Exception(f"Owner should be '${owner}' but got '{owner}'")
+      if owner != "${owner}":
+          raise Exception(f"Owner should be '${owner}' but got '{owner}'")
 
-          group = machine.succeed("stat -c '%G' ${result.path}").strip()
+      group = machine.succeed("stat -c '%G' ${result.path}").strip()
-          print(f"Got group {group}")
+      print(f"Got group {group}")
-          if group != "${group}":
+      if group != "${group}":
-              raise Exception(f"Group should be '${group}' but got '{group}'")
+          raise Exception(f"Group should be '${group}' but got '{group}'")
 
-          mode = str(int(machine.succeed("stat -c '%a' ${result.path}").strip()))
+      mode = str(int(machine.succeed("stat -c '%a' ${result.path}").strip()))
-          print(f"Got mode {mode}")
+      print(f"Got mode {mode}")
-          wantedMode = str(int("${mode}"))
+      wantedMode = str(int("${mode}"))
-          if mode != wantedMode:
+      if mode != wantedMode:
-              raise Exception(f"Mode should be '{wantedMode}' but got '{mode}'")
+          raise Exception(f"Mode should be '{wantedMode}' but got '{mode}'")
 
-          content = machine.succeed("cat ${result.path}").strip()
+      content = machine.succeed("cat ${result.path}").strip()
-          print(f"Got content {content}")
+      print(f"Got content {content}")
-          if content != "secretA":
+      if content != "secretA":
-              raise Exception(f"Content should be 'secretA' but got '{content}'")
+          raise Exception(f"Content should be 'secretA' but got '{content}'")
-        '';
+    '';
 }

modules/contracts/ssl/dummyModule.nix

@@ -1,10 +1,11 @@
-{ pkgs, lib, ... }:
+{ lib, shb, ... }:
-let
-  contracts = pkgs.callPackage ../. {};
-in
 {
+  imports = [
+    ../../../lib/module.nix
+  ];
+
   options.shb.contracts.ssl = lib.mkOption {
     description = "Contract for SSL Certificate generator.";
-    type = contracts.ssl.certs;
+    type = shb.contracts.ssl.certs;
   };
 }