danvics/selfhostblocks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Compare commits

base: danvics:main
Branches Tags
danvics:main
danvics:users2
danvics:dependabot/github_actions/actions/checkout-7
danvics:ci/nixpkgs-update-auto
danvics:fix/-add-Jellyfin-OIDC-scopes
danvics:nextcloud_mediadc
danvics:nextcloud_declarative
danvics:v0.9.0
danvics:v0.8.0
danvics:v0.7.3
danvics:v0.7.2
danvics:v0.7.1
danvics:v0.7.0
danvics:v0.6.1
danvics:v0.6.0
danvics:v0.5.1
danvics:v0.5.0
danvics:v0.4.4
danvics:v0.4.3
danvics:v0.4.2
danvics:v0.4.1
danvics:v0.4.0
danvics:v0.3.1
danvics:v0.3.0
danvics:v0.2.10
danvics:v0.2.9
danvics:v0.2.8
danvics:v0.2.7
danvics:v0.2.6
danvics:v0.2.5
danvics:v0.2.4
danvics:v0.2.3
danvics:v0.2.2
danvics:v0.2.1
danvics:v0.2.0
danvics:v0.1.0
..
compare: danvics:v0.8.0
Branches Tags
danvics:main
danvics:users2
danvics:dependabot/github_actions/actions/checkout-7
danvics:ci/nixpkgs-update-auto
danvics:fix/-add-Jellyfin-OIDC-scopes
danvics:nextcloud_mediadc
danvics:nextcloud_declarative
danvics:v0.9.0
danvics:v0.8.0
danvics:v0.7.3
danvics:v0.7.2
danvics:v0.7.1
danvics:v0.7.0
danvics:v0.6.1
danvics:v0.6.0
danvics:v0.5.1
danvics:v0.5.0
danvics:v0.4.4
danvics:v0.4.3
danvics:v0.4.2
danvics:v0.4.1
danvics:v0.4.0
danvics:v0.3.1
danvics:v0.3.0
danvics:v0.2.10
danvics:v0.2.9
danvics:v0.2.8
danvics:v0.2.7
danvics:v0.2.6
danvics:v0.2.5
danvics:v0.2.4
danvics:v0.2.3
danvics:v0.2.2
danvics:v0.2.1
danvics:v0.2.0
danvics:v0.1.0

No commits in common. "main" and "v0.8.0" have entirely different histories.
main ... v0.8.0

63 changed files with 496 additions and 3059 deletions
Show stats Expand all files Collapse all files

15
.github/workflows/auto-merge.yaml vendored
View file

@ -23,7 +23,7 @@ on:
required: false
jobs:
automerge-rebase:
automerge:
runs-on: ubuntu-latest
steps:
- uses: reitermarkus/automerge@v2
@ -35,16 +35,3 @@ jobs:
pull-request: ${{ github.event.inputs.pull-request }}
review: ${{ github.event.inputs.review }}
dry-run: false
automerge-merge:
runs-on: ubuntu-latest
steps:
- uses: reitermarkus/automerge@v2
with:
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
merge-method: merge
do-not-merge-labels: never-merge
required-labels: automerge-merge
pull-request: ${{ github.event.inputs.pull-request }}
review: ${{ github.event.inputs.review }}
dry-run: false

34
.github/workflows/lock-update.yaml vendored
View file

@ -1,16 +1,9 @@
# See shb.skarabox.com/misc.html
name: Update Flake Lock
on:
schedule:
- cron: '0 */3 * * *' # runs every 3 hours at :00
workflow_dispatch:
inputs:
bisect_future:
description: "Bisect in the future instead of the past"
required: false
default: "false"
schedule:
- cron: '0 0 * * *' # runs daily at 00:00
jobs:
lockfile:
@ -22,22 +15,9 @@ jobs:
uses: cachix/install-nix-action@v31
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: Cache nixpkgs mirror
uses: actions/cache@v5
with:
path: .cache/nixpkgs.git
key: nixpkgs-mirror-v1
- name: Update flake.lock
env:
GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
BISECT_FUTURE: ${{ inputs.bisect_future }}
run: |
git config user.email "update-flake-lock-pr@selfhostblocks.com"
git config user.name "Update Flake Lock PR"
if [ "$BISECT_FUTURE" = false ]; then
BISECT_FUTURE=
elif [ -n "$BISECT_FUTURE" ]; then
BISECT_FUTURE=future
fi
nix run .#update-flake-lock-pr $BISECT_FUTURE
uses: DeterminateSystems/update-flake-lock@main
with:
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
pr-labels: |
automerge

2
.github/workflows/pages.yml vendored
View file

@ -62,7 +62,7 @@ jobs:
uses: actions/configure-pages@v6
- name: Upload artifact
uses: actions/upload-pages-artifact@v5
uses: actions/upload-pages-artifact@v4
with:
path: ./public

223
.github/workflows/update-flake-lock-pr.nix vendored
View file

@ -1,223 +0,0 @@
{
gh,
writeShellApplication,
}:
writeShellApplication {
name = "update-flake-lock-pr";
runtimeInputs = [
gh
];
text = ''
branch=ci/nixpkgs-update
if [ "''${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
branch=ci/nixpkgs-update-auto
fi
goto_future=no
if [ -n "''${1:-}" ]; then
goto_future=yes
fi
main () {
PR=$(get_pr)
if [ -z "$PR" ]; then
create_pr
exit 0
fi
case "$(get_pr_check_status "$PR")" in
pending)
echo "Checks are running on PR $PR, nothing to do yet."
exit 0
;;
succeeded)
echo "Checks succeeded, nothing to do."
exit 0
;;
failing)
echo "Checks failed, updating PR to midpoint commit."
local CURRENT_COMMIT
local LAST_WORKING_COMMIT
local FAILING_COMMIT
local NEW_COMMIT
CURRENT_COMMIT="$(get_main_branch_commit)"
if [ "$goto_future" = "no" ]; then
LAST_WORKING_COMMIT="$CURRENT_COMMIT"
FAILING_COMMIT="$(get_pr_commit "$PR")"
else
LAST_WORKING_COMMIT="$(get_pr_commit "$PR")"
FAILING_COMMIT="$(get_latest_nixpkgs_commit)"
fi
NEW_COMMIT="$(midpoint_commit "$LAST_WORKING_COMMIT" "$FAILING_COMMIT")"
if [ "$NEW_COMMIT" = "$LAST_WORKING_COMMIT" ] || [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
NEW_COMMIT="$(get_latest_nixpkgs_commit)"
fi
if [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
echo "Latest nixpkgs commit is still same failing commit, will retry later."
exit 0
fi
update_pr "$PR" "$CURRENT_COMMIT" "$NEW_COMMIT"
exit 0
;;
esac
}
get_main_branch_commit() {
git fetch origin main
git show origin/main:flake.lock \
| jq -r '.nodes.nixpkgs.locked.rev'
}
midpoint_commit() {
local good="$1"
local bad="$2"
mkdir -p .cache
if [ ! -d .cache/nixpkgs.git ]; then
git clone --mirror https://github.com/NixOS/nixpkgs.git .cache/nixpkgs.git
else
git --git-dir=.cache/nixpkgs.git fetch origin
fi
local commits=()
mapfile -t commits < <(
git --git-dir=.cache/nixpkgs.git \
rev-list \
--reverse \
--ancestry-path \
"''${good}..''${bad}"
)
local count="''${#commits[@]}"
if [ "$count" -eq 0 ]; then
echo "$bad"
return
fi
local midpoint_index=$((count / 2))
echo "''${commits[$midpoint_index]}"
}
get_latest_nixpkgs_commit () {
git ls-remote https://github.com/NixOS/nixpkgs nixos-unstable | cut -f1
}
get_pr () {
gh pr list \
--head "$branch" \
--state open \
--json number \
--jq '.[0].number // empty'
}
create_pr() {
local test_commit
test_commit="$(get_latest_nixpkgs_commit)"
echo "Creating PR on nixpkgs' latest commit $test_commit"
git checkout -B "$branch"
nix flake lock \
--override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
git add flake.lock
git commit -m "update nixpkgs to $test_commit"
git push -u origin "$branch" --force
current_commit="$(get_main_branch_commit)"
gh pr create \
--title "update nixpkgs to $test_commit" \
--body "$(printf "%s\n\n%s" "Automated nixpkgs update. Latest tries:" " - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit")" \
--label automerge-merge
}
append_pr_body() {
local pr="$1"
local text="$2"
local current_body
current_body="$(gh pr view "$pr" --json body --jq .body)"
gh pr edit "$pr" \
--body "$(printf '%s%b' "$current_body" "$text")"
}
update_pr() {
local pr="$1"
local current_commit="$2"
local test_commit="$3"
echo "Updating PR to nixpkgs' commit $test_commit"
git checkout "$branch"
nix flake lock \
--override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
git add flake.lock
git commit -m "update nixpkgs to $test_commit"
git push --force-with-lease origin "$branch"
local future_text
if [ "$goto_future" = "yes" ]; then
future_text="bisected in the future"
else
future_text="bisected in the past"
fi
gh pr edit "$pr" \
--title "update nixpkgs to $test_commit"
append_pr_body "$pr" \
" ❌\n - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit ($future_text)"
}
get_pr_check_status() {
local pr="$1"
local status
status="$(gh pr checks "$pr" --json state --jq '.[].state' || true)"
if echo "$status" | grep -qE 'PENDING|QUEUED|IN_PROGRESS'; then
echo "pending"
return
fi
if echo "$status" | grep -qE 'FAILURE|TIMED_OUT|CANCELLED'; then
echo "failing"
return
fi
echo "success"
}
get_pr_commit() {
local pr="$1"
git fetch origin "$branch"
git show "origin/$branch:flake.lock" \
| jq -r '.nodes.nixpkgs.locked.rev'
}
main
'';
}

39
CHANGELOG.md
View file

@ -16,45 +16,6 @@ Template:
# Upcoming Release
# v0.9.0
Commits: https://github.com/ibizaman/selfhostblocks/compare/v0.8.0...v0.9.0
## Breaking Changes
- Updated simple-nixos-mailserver. Update all options following this pattern:
```diff
- mailserver.mailboxes.<name>.specialUse = "value"
+ mailserver.mailboxes.<name>.special_use = "\\value"
```
Note that simple-nixos-mailserver wanted to upgrade the location of the storage path
to include the ldap UID instead of the email but I kept the email address.
This means there is no migration to do.
- Update nixpkgs [from 6201e2 to abd1ea](https://github.com/ibizaman/selfhostblocks/compare/6201e203d09599479a3b3450ed24fa81537ebc4e...abd1ea17fdbedd48cbca770847b39e3a7a09ab5b).
## New Features
- Add `shb.zfs.snapshotBeforeActivation` option to take a ZFS snapshot of given datasets before activation.
See [the manual](https://shb.skarabox.com/blocks-zfs.html) for more details.
- Add [sanoid block](https://shb.skarabox.com/blocks-sanoid.html)
- Add option to take snapshot before activation with ZFS block.
## Fixes
- Fix oidc_login build for Nextcloud
- Fix mailserver build
## Other Changes
- Harden lldap configuration
- Convert ZFS block to system and [add documentation](https://shb.skarabox.com/blocks-zfs.html)
- Switch monitoring log fetching from deprecated promtail to fluent-bit
- Add test to catch drift for Let's Encrypt DNS provider
# [BROKEN] v0.8.0
## Breaking Changes
- Bump of Nextcloud version to 32 and 33 because of nixpkgs bump. All provided apps are verified compatible with Nextcloud 33 thanks to new tests.

2
VERSION
View file

@ -1 +1 @@
0.9.0
0.8.0

5
demo/minimal/flake.nix
View file

@ -24,10 +24,7 @@
# This module makes the assertions happy and the build succeed.
# This is of course wrong and will not work on any real system.
filesystemModule = {
fileSystems."/" = {
device = "/dev/null";
fsType = "none";
};
fileSystems."/".device = "/dev/null";
boot.loader.grub.devices = [ "/dev/null" ];
};
in

10
docs/blocks.md
View file

@ -31,10 +31,6 @@ modules/blocks/borgbackup/docs/default.md
modules/blocks/restic/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-sanoid.html
modules/blocks/sanoid/docs/default.md
```
## Database {#blocks-category-database}
```{=include=} chapters html:into-file=//blocks-postgresql.html
@ -47,12 +43,6 @@ modules/blocks/postgresql/docs/default.md
modules/blocks/sops/docs/default.md
```
## Filesystem {#blocks-category-filesystem}
```{=include=} chapters html:into-file=//blocks-zfs.html
modules/blocks/zfs/docs/default.md
```
## Network {#blocks-category-network}
```{=include=} chapters html:into-file=//blocks-ssl.html

9
docs/contracts.md
View file

@ -39,9 +39,7 @@ Provided contracts are:
- [Backup contract][] to backup directories.
Two providers are implemented: [BorgBackup][] and [Restic][].
- [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
Two providers are implemented: [BorgBackup][] and [Restic][].
- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets.
One provider is implemented: [Sanoid][]
One provider is implemented: [BorgBackup][] and [Restic][].
- [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
One provider is implemented: [SOPS][].
- [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard.
@ -51,7 +49,6 @@ Provided contracts are:
[borgbackup]: blocks-borgbackup.html
[homepage]: services-homepage.html
[restic]: blocks-restic.html
[sanoid]: blocks-sanoid.html
[sops]: blocks-sops.html
```{=include=} chapters html:into-file=//contracts-ssl.html
@ -66,10 +63,6 @@ modules/contracts/backup/docs/default.md
modules/contracts/databasebackup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-datasetbackup.html
modules/contracts/datasetbackup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-secret.html
modules/contracts/secret/docs/default.md
```

19
docs/contributing.md
View file

@ -61,25 +61,6 @@ If the test includes playwright tests, you can see the playwright trace with:
$ nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
```
If the test fails, you won't have the output directory available.
Instead, run the test with `--keep-failed` and at the end you will see a line saying
```
Keeping build directory '/nix/var/nix/builds/nix-31776-4270051001/build'
```
Copy the path and run:
```bash
sudo cp -r /nix/var/nix/builds/nix-31776-4270051001/build/shared-xchg/trace trace && sudo chown -R $USER: trace
```
Now, open that file with playwright:
```bash
nix run .#playwright -- show-trace trace/0.zip
```
### Debug Tests {#contributing-debug-tests}
Run the test in driver interactive mode:

4
docs/manual.md
View file

@ -28,10 +28,6 @@ blocks.md
recipes.md
```
```{=include=} chapters html:into-file=//misc.html
misc.md
```
```{=include=} chapters html:into-file=//demos.html
demos.md
```

39
docs/misc.md
View file

@ -1,39 +0,0 @@
<!-- Read these docs at https://shb.skarabox.com -->
# Miscellaneous {#misc}
Here goes meta-discussions around the repository and other topics not related directly to the usage of SHB.
## Lock file update {#misc-lock-file-update}
*SHB has an unusual strategy to keep up with its flake inputs. This section explains why.*
SHB only depends on `nixpkgs` as far as flake inputs goes.
There are others but they only matter for tasks around taking care of the repository itself.
So only the `nixpkgs` input is important for downstream consumers of SHB.
To keep up to date with nixpkgs unstable,
initially a job was updating the nixpkgs input by simply updating the lock file and creating a PR.
Now, this job failed most of the time because the tip of unstable often breaks modules and packages, resulting in failing SHB tests.
To fix this, we usually needed to choose a commit around 2 weeks prior to the latest unstable commit.
This usually did not lead to successful tests but the reason was then not
because of failing packages but because nixpkgs modules evolved and SHB needs to adapt to that.
The net effect was SHB was lagging too far behind unstable and updating it was becoming annoying.
The new strategy now is as follows. On every job run:
- If there is no PR, update nixpkgs input to latest unstable commit and create a PR which auto-merges when tests are successful.
- If there is a PR:
- If the checks succeeded, do nothing because the PR will auto-merge.
- If the checks are pending, do nothing yet.
- If the checks failed, bisect and update nixpkgs input to between the commit on main and the commit in the PR.
The effect here is as long as the tests are failing, we'll try a commit further in the past from nixpkgs unstable
and closer to the tip of main branch.
This could be made smarter by distinguishing between types of failures.
We will see if this is needed.
One can also run the bisect manually with `nix run .#update-flake-lock-pr`.
This will create a PR on a different branch than the one used for the automated workflow.
It accepts also one argument `future` which instead of trying a commit in between the tip of SHB main and the PR's failing one,
it tries a commit between the PR's failing one and the nixpkgs unstable latest commit.

237
docs/redirects.json
View file

@ -347,12 +347,6 @@
"blocks-borgbackup-maintenance": [
"blocks-borgbackup.html#blocks-borgbackup-maintenance"
],
"blocks-borgbackup-maintenance-manuql": [
"blocks-borgbackup.html#blocks-borgbackup-maintenance-manuql"
],
"blocks-borgbackup-maintenance-restore": [
"blocks-borgbackup.html#blocks-borgbackup-maintenance-restore"
],
"blocks-borgbackup-maintenance-troubleshooting": [
"blocks-borgbackup.html#blocks-borgbackup-maintenance-troubleshooting"
],
@ -587,9 +581,6 @@
"blocks-category-database": [
"blocks.html#blocks-category-database"
],
"blocks-category-filesystem": [
"blocks.html#blocks-category-filesystem"
],
"blocks-category-introspection": [
"blocks.html#blocks-category-introspection"
],
@ -926,9 +917,6 @@
"blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.serviceName"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.timeout": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.timeout"
],
"blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost": [
"blocks-mitmdump.html#blocks-mitmdump-options-shb.mitmdump.instances._name_.upstreamHost"
],
@ -971,9 +959,6 @@
"blocks-monitoring-error-dashboard": [
"blocks-monitoring.html#blocks-monitoring-error-dashboard"
],
"blocks-monitoring-impermanence": [
"blocks-monitoring.html#blocks-monitoring-impermanence"
],
"blocks-monitoring-nextcloud-dashboard": [
"blocks-monitoring.html#blocks-monitoring-nextcloud-dashboard"
],
@ -1037,9 +1022,6 @@
"blocks-monitoring-options-shb.monitoring.grafanaPort": [
"blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.grafanaPort"
],
"blocks-monitoring-options-shb.monitoring.impermanence": [
"blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.impermanence"
],
"blocks-monitoring-options-shb.monitoring.ldap": [
"blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ldap"
],
@ -1391,12 +1373,6 @@
"blocks-restic-maintenance": [
"blocks-restic.html#blocks-restic-maintenance"
],
"blocks-restic-maintenance-manuql": [
"blocks-restic.html#blocks-restic-maintenance-manuql"
],
"blocks-restic-maintenance-restore": [
"blocks-restic.html#blocks-restic-maintenance-restore"
],
"blocks-restic-maintenance-troubleshooting": [
"blocks-restic.html#blocks-restic-maintenance-troubleshooting"
],
@ -1613,63 +1589,6 @@
"blocks-restic-usage-provider-remote": [
"blocks-restic.html#blocks-restic-usage-provider-remote"
],
"blocks-sanoid": [
"blocks-sanoid.html#blocks-sanoid"
],
"blocks-sanoid-contract-provider": [
"blocks-sanoid.html#blocks-sanoid-contract-provider"
],
"blocks-sanoid-options": [
"blocks-sanoid.html#blocks-sanoid-options"
],
"blocks-sanoid-options-shb.sanoid.backup": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request.excludePatterns": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.excludePatterns"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.afterBackup": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.afterBackup"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.beforeBackup": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.hooks.beforeBackup"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request.sourceDirectories": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.sourceDirectories"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.request.user": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.request.user"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.result": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.result"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.result.backupService": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.result.backupService"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.result.restoreScript": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.result.restoreScript"
],
"blocks-sanoid-options-shb.sanoid.backup._name_.settings": [
"blocks-sanoid.html#blocks-sanoid-options-shb.sanoid.backup._name_.settings"
],
"blocks-sanoid-usage": [
"blocks-sanoid.html#blocks-sanoid-usage"
],
"blocks-sanoid-usage-custom-template": [
"blocks-sanoid.html#blocks-sanoid-usage-custom-template"
],
"blocks-sanoid-usage-default-template": [
"blocks-sanoid.html#blocks-sanoid-usage-default-template"
],
"blocks-sanoid-usage-without-contract": [
"blocks-sanoid.html#blocks-sanoid-usage-without-contract"
],
"blocks-sops": [
"blocks-sops.html#blocks-sops"
],
@ -1841,117 +1760,6 @@
"blocks-ssl-options-shb.certs.systemdService": [
"blocks-ssl.html#blocks-ssl-options-shb.certs.systemdService"
],
"blocks-zfs": [
"blocks-zfs.html#blocks-zfs"
],
"blocks-zfs-features": [
"blocks-zfs.html#blocks-zfs-features"
],
"blocks-zfs-options": [
"blocks-zfs.html#blocks-zfs-options"
],
"blocks-zfs-options-shb.zfs.pools": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.after": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.after"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.excludePatterns": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.excludePatterns"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.afterBackup": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.afterBackup"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.beforeBackup": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.beforeBackup"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.sourceDirectories": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.sourceDirectories"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.user": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.user"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.backupService": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.backupService"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.enable": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.enable"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.mode": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.mode"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner"
],
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path"
],
"blocks-zfs-options-shb.zfs.snapshotBeforeActivation.datasets": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.datasets"
],
"blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable"
],
"blocks-zfs-options-shb.zfs.snapshotBeforeActivation.recursive": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.recursive"
],
"blocks-zfs-options-shb.zfs.snapshotBeforeActivation.template": [
"blocks-zfs.html#blocks-zfs-options-shb.zfs.snapshotBeforeActivation.template"
],
"blocks-zfs-usage": [
"blocks-zfs.html#blocks-zfs-usage"
],
"blocks-zfs-usage-backup-dataset": [
"blocks-zfs.html#blocks-zfs-usage-backup-dataset"
],
"blocks-zfs-usage-backup-files": [
"blocks-zfs.html#blocks-zfs-usage-backup-files"
],
"blocks-zfs-usage-snapshot-before-activation": [
"blocks-zfs.html#blocks-zfs-usage-snapshot-before-activation"
],
"build-time-validation": [
"service-implementation-guide.html#build-time-validation"
],
@ -2012,21 +1820,6 @@
"contract-databasebackup-usage": [
"contracts-databasebackup.html#contract-databasebackup-usage"
],
"contract-dataset-backup": [
"contracts-datasetbackup.html#contract-dataset-backup"
],
"contract-datasetbackup-providers": [
"contracts-datasetbackup.html#contract-datasetbackup-providers"
],
"contract-datasetbackup-requesters": [
"contracts-datasetbackup.html#contract-datasetbackup-requesters"
],
"contract-datasetbackup-usage": [
"contracts-datasetbackup.html#contract-datasetbackup-usage"
],
"contract-datatsetbackup-options": [
"contracts-datasetbackup.html#contract-datatsetbackup-options"
],
"contract-secret": [
"contracts-secret.html#contract-secret"
],
@ -2156,27 +1949,6 @@
"contracts-databasebackup-options-shb.contracts.databasebackup.settings": [
"contracts-databasebackup.html#contracts-databasebackup-options-shb.contracts.databasebackup.settings"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup.request": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup.result": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript"
],
"contracts-datasetbackup-options-shb.contracts.datasetbackup.settings": [
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.settings"
],
"contracts-nixpkgs": [
"contracts.html#contracts-nixpkgs"
],
@ -2411,12 +2183,6 @@
"local-testing": [
"service-implementation-guide.html#local-testing"
],
"misc": [
"misc.html#misc"
],
"misc-lock-file-update": [
"misc.html#misc-lock-file-update"
],
"monitoring-failures": [
"service-implementation-guide.html#monitoring-failures"
],
@ -5024,9 +4790,6 @@
"services-mailserver-options-shb.mailserver.ssl.systemdService": [
"services-mailserver.html#services-mailserver-options-shb.mailserver.ssl.systemdService"
],
"services-mailserver-options-shb.mailserver.stateVersion": [
"services-mailserver.html#services-mailserver-options-shb.mailserver.stateVersion"
],
"services-mailserver-options-shb.mailserver.subdomain": [
"services-mailserver.html#services-mailserver-options-shb.mailserver.subdomain"
],

12
flake.lock
View file

@ -20,11 +20,11 @@
},
"nix-flake-tests": {
"locked": {
"lastModified": 1775571237,
"narHash": "sha256-1f5Uvgcy3gx/eyRp4rqfDRBjcZc9uiHoIlfcFIL7GvI=",
"lastModified": 1677844186,
"narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
"owner": "antifuchs",
"repo": "nix-flake-tests",
"rev": "86b789cff8aecbd7c1bed7cd421dd837c3f62201",
"rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
"type": "github"
},
"original": {
@ -35,11 +35,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1781074563,
"narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=",
"lastModified": 1775036866,
"narHash": "sha256-ZojAnPuCdy657PbTq5V0Y+AHKhZAIwSIT2cb8UgAz/U=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca",
"rev": "6201e203d09599479a3b3450ed24fa81537ebc4e",
"type": "github"
},
"original": {

36
flake.nix
View file

@ -78,7 +78,6 @@
modules/contracts/backup/dummyModule.nix
modules/contracts/dashboard/dummyModule.nix
modules/contracts/databasebackup/dummyModule.nix
modules/contracts/datasetbackup/dummyModule.nix
modules/contracts/secret/dummyModule.nix
modules/contracts/ssl/dummyModule.nix
];
@ -104,13 +103,6 @@
"blocks/authelia" = ./modules/blocks/authelia.nix;
"blocks/borgbackup" = ./modules/blocks/borgbackup.nix;
"blocks/lldap" = ./modules/blocks/lldap.nix;
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
"blocks/nginx" = ./modules/blocks/nginx.nix;
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
"blocks/restic" = ./modules/blocks/restic.nix;
"blocks/sanoid" = ./modules/blocks/sanoid.nix;
"blocks/sops" = ./modules/blocks/sops.nix;
"blocks/ssl" = {
module = ./modules/blocks/ssl.nix;
optionRoot = [
@ -118,8 +110,12 @@
"certs"
];
};
"blocks/zfs" = ./modules/blocks/zfs.nix;
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
"blocks/nginx" = ./modules/blocks/nginx.nix;
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
"blocks/restic" = ./modules/blocks/restic.nix;
"blocks/sops" = ./modules/blocks/sops.nix;
"services/arr" = ./modules/services/arr.nix;
"services/firefly-iii" = ./modules/services/firefly-iii.nix;
"services/forgejo" = [
@ -141,7 +137,6 @@
"services/open-webui" = ./modules/services/open-webui.nix;
"services/pinchflat" = ./modules/services/pinchflat.nix;
"services/vaultwarden" = ./modules/services/vaultwarden.nix;
"contracts/backup" = {
module = ./modules/contracts/backup/dummyModule.nix;
optionRoot = [
@ -166,14 +161,6 @@
"databasebackup"
];
};
"contracts/datasetbackup" = {
module = ./modules/contracts/datasetbackup/dummyModule.nix;
optionRoot = [
"shb"
"contracts"
"datasetbackup"
];
};
"contracts/secret" = {
module = ./modules/contracts/secret/dummyModule.nix;
optionRoot = [
@ -257,8 +244,6 @@
'';
};
packages.update-flake-lock-pr = pkgs.callPackage ./.github/workflows/update-flake-lock-pr.nix { };
lib = (pkgs.callPackage ./lib { }) // {
test = pkgs.callPackage ./test/common.nix { };
contracts = pkgs.callPackage ./modules/contracts {
@ -403,7 +388,6 @@
// (vm_test "homepage" ./test/services/homepage.nix)
// (vm_test "jellyfin" ./test/services/jellyfin.nix)
// (vm_test "karakeep" ./test/services/karakeep.nix)
// (vm_test "mailserver" ./test/services/mailserver.nix)
// (vm_test "nextcloud" ./test/services/nextcloud.nix)
// (vm_test "open-webui" ./test/services/open-webui.nix)
// (vm_test "paperless" ./test/services/paperless.nix)
@ -412,18 +396,16 @@
// (vm_test "authelia" ./test/blocks/authelia.nix)
// (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
// (vm_test "lib" ./test/blocks/lib.nix)
// (vm_test "lldap" ./test/blocks/lldap.nix)
// (vm_test "lib" ./test/blocks/lib.nix)
// (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
// (vm_test "monitoring" ./test/blocks/monitoring.nix)
// (vm_test "postgresql" ./test/blocks/postgresql.nix)
// (vm_test "restic" ./test/blocks/restic.nix)
// (vm_test "ssl" ./test/blocks/ssl.nix)
// (vm_test "zfs" ./test/blocks/zfs.nix)
// (vm_test "contracts-backup" ./test/contracts/backup.nix)
// (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
// (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix)
// (vm_test "contracts-secret" ./test/contracts/secret.nix)
);
@ -445,7 +427,6 @@
self.nixosModules.nginx
self.nixosModules.postgresql
self.nixosModules.restic
self.nixosModules.sanoid
self.nixosModules.ssl
self.nixosModules.tinyproxy
self.nixosModules.vpn
@ -485,9 +466,8 @@
nixosModules.nginx = modules/blocks/nginx.nix;
nixosModules.postgresql = modules/blocks/postgresql.nix;
nixosModules.restic = modules/blocks/restic.nix;
nixosModules.sanoid = modules/blocks/sanoid.nix;
nixosModules.sops = modules/blocks/sops.nix;
nixosModules.ssl = modules/blocks/ssl.nix;
nixosModules.sops = modules/blocks/sops.nix;
nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
nixosModules.vpn = modules/blocks/vpn.nix;
nixosModules.zfs = modules/blocks/zfs.nix;

53
lib/default.nix
View file

@ -1,12 +1,7 @@
{ pkgs, lib }:
let
inherit (builtins) isAttrs hasAttr;
inherit (lib)
any
concatMapStringsSep
concatStringsSep
escapeShellArg
;
inherit (lib) any concatMapStringsSep concatStringsSep;
shb = rec {
# Replace secrets in a file.
# - userConfig is an attrset that will produce a config file.
@ -81,41 +76,11 @@ let
pattern: "cat ${pattern.source} > /dev/null"
) replacements;
generatedReplacements = map genReplacement replacements;
sedPatterns = concatMapStringsSep " " (pattern: "-e \"s|${pattern.name}|${pattern.value}|\"") (
map genReplacement replacements
);
replaceScript = pkgs.writers.writePython3 "replace-secret" { } ''
import argparse
import pathlib
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--file", required=True, type=pathlib.Path)
parser.add_argument("--marker", required=True)
parser.add_argument("--replacement", required=True)
return parser.parse_args()
args = parse_args()
content = args.file.read_text()
args.file.write_text(content.replace(args.marker, args.replacement))
'';
replaceCommands = concatMapStringsSep "\n" (pattern: ''
${replaceScript} \
--file ${escapeShellArg resultPath} \
--marker ${escapeShellArg pattern.name} \
--replacement "${pattern.value}"
'') generatedReplacements;
replaceCmd =
if replacements == [ ] then
"cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}"
else
''
cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}
${replaceCommands}
'';
sedCmd = if replacements == [ ] then "cat" else "${pkgs.gnused}/bin/sed ${sedPatterns}";
in
''
set -euo pipefail
@ -131,7 +96,7 @@ let
chown ${user} ${resultPath}
'')
+ ''
${replaceCmd}
${sedCmd} ${templatePath} > ${resultPath}
chmod ${permissions} ${resultPath}
'';
@ -352,16 +317,12 @@ let
"expected"
"result"
];
nativeBuildInputs = [
(pkgs.python3.withPackages (ps: [ ps.deepdiff ] ++ ps.deepdiff.optional-dependencies.cli))
];
}
''
echo "${name} failed (- expected, + result)" > $out
cp ''${expectedPath} ''${expectedPath}.json
cp ''${resultPath} ''${resultPath}.json
deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
${pkgs.deepdiff}/bin/deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
''
);

1
modules/blocks/authelia.nix
View file

@ -645,7 +645,6 @@ in
shb.mitmdump.instances."authelia-${fqdn}" = lib.mkIf cfg.debug {
listenPort = 9091;
upstreamPort = 9090;
timeout = 30;
after = [ "authelia-${fqdn}.service" ];
enabledAddons = [ config.shb.mitmdump.addons.logger ];
extraArgs = [

92
modules/blocks/borgbackup.nix
View file

@ -386,13 +386,8 @@ in
${serviceName} = mkMerge [
{
serviceConfig = {
# Purposely not a oneshot systemd service otherwise
# the service waits on the completion the backup before deactivating.
# This seems like a nice property at first but there is one annoying
# edge case when deploying. If a backup job somehow is started when
# the deploy happens, the deploy will wait on the service to finish
# before considering the deploy done. And worse, it will consider the
# deploy as failed if the backup fails, which is not what you want.
# Makes the systemd service wait for the backup to be done before changing state to inactive.
Type = "oneshot";
Nice = lib.mkForce cfg.performance.niceness;
IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass;
IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority;
@ -419,7 +414,6 @@ in
in
{
script = script.preStart;
# Makes the systemd service wait for the backup to be done before changing state to inactive.
serviceConfig.Type = "oneshot";
serviceConfig.LoadCredential = script.loadCredentials;
}
@ -457,16 +451,39 @@ in
let
mkBorgBackupBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"BORG_REPO"
"BORG_PASSCOMMAND"
];
secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
set +a
archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
echo "Will restore archive $archive"
(cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive")
'';
};
in
flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
@ -476,16 +493,39 @@ in
let
mkBorgBackupBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"BORG_REPO"
"BORG_PASSCOMMAND"
];
secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
set +a
archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
echo "Will restore archive $archive"
sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}"
'';
};
in
flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);

31
modules/blocks/borgbackup/docs/default.md
View file

@ -3,7 +3,6 @@
Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix).
This block sets up a backup job using [BorgBackup][].
It is heavily based on the nixpkgs BorgBackup module.
[borgbackup]: https://www.borgbackup.org/
@ -230,36 +229,6 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
## Maintenance {#blocks-borgbackup-maintenance}
### Manual Backup {#blocks-borgbackup-maintenance-manuql}
To launch a backup manually, just run:
```bash
systemctl start <systemd-service-name>
```
You can easily discover the systemd service name you need by either listing the units:
```bash
systemctl list-units 'borgbackup*'
```
Or by autocompleting the unit name with `<TAB>`:
```bash
systemctl start borgbackup<TAB><TAB>
```
Note that the systemd services are of `Type=simple` which means the systemd service
will not wait for the backup completion to terminate.
If you want instead to wait for the backup to complete, use the `--wait` flag:
```bash
systemctl start --wait <systemd-service-name>
```
### Restore {#blocks-borgbackup-maintenance-restore}
One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
The restore script has all the secrets needed to access the repo,

25
modules/blocks/davfs.nix
View file

@ -75,30 +75,6 @@ in
config = {
services.davfs2.enable = builtins.length cfg.mounts > 0;
systemd.services = lib.optionalAttrs (builtins.length cfg.mounts > 0) {
davfs2-password-generation = {
wantedBy = [ "multi-user.target" ];
serviceConfig.Type = "oneshot";
script = ''
mkdir -p /etc/davfs2
[ -f /etc/davfs2/secrets ] && mv /etc/davfs2/secrets /etc/davfs2/secrets.prev
touch /etc/davfs2/secrets
chown root: /etc/davfs2
chown root: /etc/davfs2/secrets
chmod 700 /etc/davfs2
chmod 600 /etc/davfs2/secrets
''
+ (
let
mkPasswordCmd = cfg': ''
printf "%s %s %s\n" "${cfg'.mountPoint}" "${cfg'.username}" "$(cat ${cfg'.passwordFile})" >> /etc/davfs2/secrets
'';
in
lib.concatStringsSep "\n" (map mkPasswordCmd cfg.mounts)
);
};
};
systemd.mounts =
let
mkMountCfg = c: {
@ -106,7 +82,6 @@ in
description = "Webdav mount point";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
what = c.remoteUrl;
where = c.mountPoint;

35
modules/blocks/lldap.nix
View file

@ -418,41 +418,6 @@ in
) cfg.ensureUsers;
};
# Harden lldap following https://github.com/NixOS/nixpkgs/pull/487933
systemd.services.lldap.serviceConfig = {
RemoveIPC = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [
"AF_UNIX"
"AF_INET"
"AF_INET6"
];
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
SystemCallArchitectures = "native";
CapabilityBoundingSet = "";
LockPersonality = true;
NoNewPrivileges = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
ProtectProc = "invisible";
ProcSubset = "pid";
MemoryDenyWriteExecute = true;
};
shb.mitmdump.instances."lldap-web" = lib.mkIf cfg.debug {
listenPort = config.shb.lldap.webUIListenPort;
upstreamPort = config.shb.lldap.webUIListenPort + 1;

18
modules/blocks/mitmdump.nix
View file

@ -48,7 +48,7 @@ let
logging.basicConfig(level=logging.INFO, format='%(message)s')
def wait_for_port(host, port, timeout):
def wait_for_port(host, port, timeout=10):
deadline = time.time() + timeout
while time.time() < deadline:
try:
@ -68,7 +68,6 @@ let
parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
parser.add_argument("--timeout", required=False, type=int, default=10, help="Timeout to wait for port availability")
args, rest = parser.parse_known_args()
MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
@ -76,7 +75,7 @@ let
raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
wait_for_port(args.upstream_host, args.upstream_port, timeout=args.timeout)
wait_for_port(args.upstream_host, args.upstream_port, timeout=10)
logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
proc = subprocess.Popen(
@ -91,11 +90,10 @@ let
)
logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
if wait_for_port("127.0.0.1", args.listen_port, timeout=args.timeout):
if wait_for_port("127.0.0.1", args.listen_port, timeout=10):
logging.info(f"Mitmdump is started on port {args.listen_port}.")
notify("READY=1")
else:
print(f"Mitmdump instance did not start before the timeout of {args.timeout} seconds, consider increasing the timeout")
proc.terminate()
exit(1)
@ -224,14 +222,6 @@ in
'';
};
timeout = mkOption {
type = types.int;
default = 30;
description = ''
Time to wait for upstream to start and mitmdump to start.
'';
};
after = mkOption {
type = listOf str;
default = [ ];
@ -292,7 +282,7 @@ in
addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
in
"${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} --timeout ${toString cfg'.timeout} ${addons} ${extraArgs}";
"${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}";
};
requires = cfg'.after;
after = cfg'.after;

87
modules/blocks/monitoring.nix
View file

@ -293,16 +293,6 @@ in
};
};
};
impermanence = lib.mkOption {
description = ''
Paths to save when using impermanence setup.
'';
type = lib.types.attrsOf lib.types.str;
default = {
fluent-bit = "/var/fluent-bit";
};
};
};
config = lib.mkMerge [
@ -585,62 +575,41 @@ in
};
};
# I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
services.fluent-bit = {
services.promtail = {
enable = true;
settings = {
service = {
flush = 1;
log_level = "info";
http_server = "true";
http_listen = "127.0.0.1";
http_port = 9080;
grace = 30;
configuration = {
server = {
http_listen_port = 9080;
grpc_listen_port = 0;
};
pipeline = {
inputs = [
{
name = "systemd";
positions.filename = "/tmp/positions.yaml";
# The asterisk appends the _SYSTEMD_UNIT to the prefix.
tag = "systemd.*";
client.url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/api/prom/push";
# Read logs from this systemd journal directory.
scrape_configs = [
{
job_name = "systemd";
journal = {
json = false;
max_age = "12h";
path = "/var/log/journal";
# Database file to keep track of the journald cursor.
db = "/var/fluent-bit/systemd.db";
# Start reading new entries. Skip entries already stored in journald.
read_from_tail = true;
# Max entries to lookback on start.
max_entries = 10000;
}
];
outputs = [
{
name = "loki";
match = "systemd.*";
host = "localhost";
port = config.services.loki.configuration.server.http_listen_port;
labels = lib.concatMapAttrsStringSep ", " (n: v: "${n}=${v}") {
job = "systemd-journal";
# matches = "_TRANSPORT=kernel";
labels = {
domain = cfg.domain;
hostname = config.networking.hostName;
job = "systemd-journal";
};
label_keys = "$unit";
}
];
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
graceLimit = "1m";
};
services.nginx = {
@ -926,11 +895,11 @@ in
src = pkgs.fetchFromGitHub {
owner = "ibizaman";
repo = "scrutiny";
rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
rev = "7ff9a0530d3e54dd1323c2de34f32be330bfb48c";
hash = "sha256-dE4HuZzaGZKBEkzXwBLQL3h+D55tJMm/EOTpr3wqGAI=";
};
vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
vendorHash = "sha256-j3aGTeHNTr/FoVfFLwASkS96Ks0B/Ka9hPuLAKGZECs=";
});
settings = {

10
modules/blocks/monitoring/docs/default.md
View file

@ -284,16 +284,6 @@ Graphs:
![Late SSL Jobs Alert Firing](./assets/alert_rules_LateSSL_1.png)
## Impermanence {#blocks-monitoring-impermanence}
To save the fluent-bit folder in an impermanence setup, add:
```nix
{
shb.zfs.datasets."safe/monitoring-fluent-bit".path = config.shb.jellyfin.impermanence.fluent-bit;
}
```
## Options Reference {#blocks-monitoring-options}
```{=include=} options

2
modules/blocks/postgresql.nix
View file

@ -73,7 +73,7 @@ in
backupName = "postgres.sql";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
'';
restoreCmd = ''

88
modules/blocks/restic.nix
View file

@ -413,13 +413,7 @@ in
nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
enable = true;
wantedBy = [ "multi-user.target" ];
# Purposely not a oneshot systemd service otherwise
# the service waits on the completion the backup before deactivating.
# This seems like a nice property at first but there is one annoying
# edge case when deploying. If a backup job somehow is started when
# the deploy happens, the deploy will wait on the service to finish
# before considering the deploy done. And worse, it will consider the
# deploy as failed if the backup fails, which is not what you want.
serviceConfig.Type = "oneshot";
script = (
shb.replaceSecrets {
userConfig = instance.settings.repository.secrets // {
@ -440,16 +434,38 @@ in
let
mkResticBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"RESTIC_REPOSITORY"
"RESTIC_PASSWORD_FILE"
];
secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
set +a
echo "Will restore archive 'latest'"
sudocmd ${pkgs.restic}/bin/restic restore latest --target /
'';
};
in
flatten (mapAttrsToList mkResticBinary cfg.instances);
@ -459,16 +475,38 @@ in
let
mkResticBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"RESTIC_REPOSITORY"
"RESTIC_PASSWORD_FILE"
];
secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
set +a
echo "Will restore archive 'latest'"
sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}"
'';
};
in
flatten (mapAttrsToList mkResticBinary cfg.databases);

41
modules/blocks/restic/docs/default.md
View file

@ -3,12 +3,12 @@
Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
This block sets up a backup job using [Restic][].
It is heavily based on the nixpkgs Restic module.
[restic]: https://restic.net/
## Provider Contracts {#blocks-restic-contract-provider}
This block provides the following contracts:
- [backup contract](contracts-backup.html) under the [`shb.restic.instances`][instances] option.
@ -26,6 +26,7 @@ a backup Systemd service and a [restore script](#blocks-restic-maintenance) are
## Usage {#blocks-restic-usage}
The following examples assume usage of the [sops block][] to provide secrets
although any blocks providing the [secrets contract][] works too.
@ -34,6 +35,7 @@ although any blocks providing the [secrets contract][] works too.
### One folder backed up manually {#blocks-restic-usage-provider-manual}
The following snippet shows how to configure
the backup of 1 folder to 1 repository.
We assume that the folder `/var/lib/myfolder` of the service `myservice` must be backed up.
@ -230,44 +232,11 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
## Maintenance {#blocks-restic-maintenance}
### Manual Backup {#blocks-restic-maintenance-manuql}
To launch a backup manually, just run:
```bash
systemctl start <systemd-service-name>
```
You can easily discover the systemd service name you need by either listing the units:
```bash
systemctl list-units 'restic*'
```
Or by autocompleting the unit name with `<TAB>`:
```bash
systemctl start restic<TAB><TAB>
```
Note that the systemd services are of `Type=simple` which means the systemd service
will not wait for the backup completion to terminate.
If you want instead to wait for the backup to complete, use the `--wait` flag:
```bash
systemctl start --wait <systemd-service-name>
```
### Restore {#blocks-restic-maintenance-restore}
One command-line helper is provided per backup instance and repository pair which allows to:
- list snapshots: `<script> snapshots`
- to restore a snapshot: `<script> restore <snapshot>`
One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
The restore script has all the secrets needed to access the repo,
it will run `sudo` automatically
and the user running it needs to have correct permissions for privilege escalation.
and the user running it needs to have correct permissions for privilege escalation
In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:

164
modules/blocks/sanoid.nix
View file

@ -1,164 +0,0 @@
{
config,
lib,
pkgs,
shb,
utils,
...
}:
let
cfg = config.shb.sanoid;
restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore";
backupScriptBase = "sanoid";
restoreScript =
name:
pkgs.writers.writePython3Bin (restoreScriptName name)
{
flakeIgnore = [ "E501" ];
}
''
import argparse
import subprocess
import sys
dataset = "${name}"
class ZFSError(Exception):
pass
def run_command(cmd: list[str]) -> str:
try:
result = subprocess.run(
cmd,
check=True,
text=True,
capture_output=True,
)
return result.stdout.strip()
except subprocess.CalledProcessError as e:
raise ZFSError(
f"Command failed: {' '.join(cmd)}\n"
f"Exit code: {e.returncode}\n"
f"stderr: {e.stderr.strip()}"
) from e
def list_snapshots() -> None:
"""List all ZFS snapshots."""
output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset])
if not output:
return []
return output.splitlines()
def restore_snapshot(snapshot: str) -> None:
"""Rollback to a given snapshot."""
if not snapshot:
raise ValueError("Snapshot name must not be empty")
print(f"Rolling back to snapshot: {snapshot}")
run_command(["zfs", "rollback", "-r", snapshot])
print("Rollback completed successfully.")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=f"Restore script for {dataset}")
subparsers = parser.add_subparsers(dest="command", required=True)
subparsers.add_parser(
"snapshots",
help="List all ZFS snapshots",
)
restore_parser = subparsers.add_parser(
"restore",
help="Rollback to a specific snapshot",
)
restore_parser.add_argument(
"snapshot",
help="Snapshot name (e.g. pool/dataset@snapname)",
)
return parser
def main():
parser = build_parser()
args = parser.parse_args()
try:
if args.command == "snapshots":
snapshots = list_snapshots()
for s in snapshots:
print(s)
elif args.command == "restore":
restore_snapshot(args.snapshot)
else:
parser.print_help()
sys.exit(1)
except ZFSError as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"Unexpected error: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()
'';
in
{
imports = [
../../lib/module.nix
];
options.shb.sanoid.backup = lib.mkOption {
description = "Sanoid prodiver for file backup contract";
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = shb.contracts.backup.mkProvider {
resultCfg = {
restoreScript = restoreScriptName name;
restoreScriptText = restoreScriptName "<name>";
backupService = "${backupScriptBase}.service";
backupServiceText = "${backupScriptBase}.service";
};
settings = lib.mkOption {
description = "Options passed to the `services.sanoid.datasets.<name>` option.";
default = { };
type = lib.types.attrsOf lib.types.anything;
};
};
}
)
);
};
config = lib.mkIf (cfg.backup != { }) {
services.sanoid.enable = true;
services.sanoid.datasets =
let
mkDataset = name: cfg': {
inherit name;
value = cfg'.settings;
};
in
lib.mapAttrs' mkDataset cfg.backup;
environment.systemPackages = map restoreScript (lib.attrNames cfg.backup);
};
}

97
modules/blocks/sanoid/docs/default.md
View file

@ -1,97 +0,0 @@
# Sanoid Block {#blocks-sanoid}
Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix):
```nix
{
imports = [
inputs.selfhostblocks.nixosModules.sanoid
];
}
```
## Provider Contracts {#blocks-sanoid-contract-provider}
This block provides the following contracts:
- [dataset backup contract](contracts-datasetbackup.html) under the [`shb.sanoid.backup`][backup] option.
It is tested with the [generic contract tests][backup contract tests].
[backup]: #blocks-sanoid-options-shb.sanoid.backup
[backup contract tests]: @REPO@/test/contracts/backup.nix
## Usage {#blocks-sanoid-usage}
Sanoid uses templates to know when snapshots should be kept or pruned.
### Default Template {#blocks-sanoid-usage-default-template}
Backup a dataset using the default Sanoid template:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.sanoid.backup."root/home" = {
request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
};
}
```
This uses the dataset backup contract which is exposed through the ZFS module's [`shb.zfs.pools.<name>.datasets.<name>.datasetBackup`](blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup) option.
### Custom Template {#blocks-sanoid-usage-custom-template}
Create a custom template and use it:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
services.sanoid.templates."yearly" = {
hourly = 10;
daily = 3;
monthly = 3;
yearly = 2;
};
shb.sanoid.backup."root/home" = {
request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
template = "yearly";
};
}
```
Note we use the upstream `services.sanoid.templates` option to define the templates.
### Without contract {#blocks-sanoid-usage-without-contract}
To backup a dataset that does not provide the dataset backup contract,
we can just set the request manually:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.sanoid.backup."root/home" = {
request.dataset = "root/home";
};
}
```
Note the attr name under the `shb.sanoid.backup` option does not
set the dataset name.
## Options Reference {#blocks-sanoid-options}
```{=include=} options
id-prefix: blocks-sanoid-options-
list-id: selfhostblocks-blocks-sanoid-options
source: @OPTIONS_JSON@
```

2
modules/blocks/ssl.nix
View file

@ -551,7 +551,7 @@ in
// lib.optionalAttrs (certCfg.dnsProvider != null) {
inherit (certCfg) dnsProvider dnsResolver;
inherit (certCfg) group reloadServices;
environmentFile = certCfg.credentialsFile;
credentialsFile = certCfg.credentialsFile;
};
}
]

314
modules/blocks/zfs.nix
View file

@ -2,258 +2,128 @@
config,
pkgs,
lib,
shb,
utils,
...
}:
let
cfg = config.shb.zfs;
datasets =
if cfg.snapshotBeforeActivation.datasets != null then
cfg.snapshotBeforeActivation.datasets
else
config.boot.zfs.extraPools;
in
{
imports = [
../../lib/module.nix
];
options.shb.zfs = {
pools = lib.mkOption {
defaultPoolName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "ZFS pool name datasets should be created on if no pool name is given in the dataset.";
};
datasets = lib.mkOption {
description = ''
Attrset of ZFS pools under which datasets will be created.
ZFS Datasets.
The ZFS pools are not managed by this module, they should already exist.
Each entry in the attrset will be created and mounted in the given path.
The attrset name is the dataset name.
Each pool named here will be added to the [`boot.zfs.extraPools`](https://search.nixos.org/options?channel=unstable&include_modular_service_options=0&include_nixos_options=1&query=boot.zfs.extrapools&show=option:boot.zfs.extraPools) option.
This block implements the following contracts:
- mount
'';
default = { };
example = lib.literalExpression ''
shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
'';
type = lib.types.attrsOf (
lib.types.submodule {
options = {
datasets = lib.mkOption {
enable = lib.mkEnableOption "shb.zfs.datasets";
poolName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "ZFS pool name this dataset should be created on. Overrides the defaultPoolName.";
};
path = lib.mkOption {
type = lib.types.str;
description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
};
mode = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix mode to apply to the dataset root folder.";
default = null;
example = "ug=rwx,g+s";
};
owner = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix user to apply to the dataset root folder.";
default = null;
example = "syncthing";
};
group = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix group to apply to the dataset root folder.";
default = null;
example = "syncthing";
};
defaultACLs = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = ''
ZFS Datasets.
If non null, default ACL to set on the dataset root folder.
Each entry in the attrset will be created and mounted in the given path.
The attrset name is the dataset name.
This block implements the following contracts:
- mount
Executes "setfacl -d -m $acl $path"
'';
default = { };
example = lib.literalExpression ''
shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
'';
type = lib.types.attrsOf (
lib.types.submodule (
{ config, name, ... }:
{
options = {
enable = lib.mkEnableOption "shb.zfs.datasets";
path = lib.mkOption {
type = lib.types.str;
description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
};
mode = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix mode to apply to the dataset root folder.";
default = null;
example = "ug=rwx,g+s";
};
owner = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix user to apply to the dataset root folder.";
default = null;
example = "syncthing";
};
group = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix group to apply to the dataset root folder.";
default = null;
example = "syncthing";
};
defaultACLs = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = ''
If non null, default ACL to set on the dataset root folder.
Executes "setfacl -d -m $acl $path"
'';
default = null;
example = "g:syncthing:rwX";
};
after = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
Order creating this dataset after the mentioned ones.
This only works with datasets managed by this module.
Use the name of the dataset without the pool name.
'';
default = [ ];
example = lib.literalExpression ''
[
"backup"
]
'';
};
backup = lib.mkOption {
description = ''
Backup contract consumer configuration.
This contract will backup the files inside the dataset.
'';
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = if config.owner == null then "root" else config.owner;
sourceDirectories = [
config.path
];
sourceDirectoriesText = ''
[
shb.zfs.pools.<name>.datasets.<name>.path
]
'';
};
};
};
datasetbackup = lib.mkOption {
description = ''
ZFS dataset backup contract configuration.
This contract will take snaphots of the dataset.
'';
type = lib.types.submodule {
options = shb.contracts.datasetbackup.mkRequester {
dataset = name;
};
};
};
};
}
)
);
default = null;
example = "g:syncthing:rwX";
};
};
}
);
};
snapshotBeforeActivation = {
enable = lib.mkOption {
type = lib.types.bool;
description = "Take a snapshot of all datasets before activation";
default = false;
};
template = lib.mkOption {
type = lib.types.str;
description = "Bash command to generate the snapshot name. $1 is the path to the new generation.";
default = ''pre-$(date --utc '+%y%m%dT%H%M%S')-$(basename "$1")'';
};
datasets = lib.mkOption {
type = lib.types.nullOr (lib.types.listOf lib.types.str);
description = ''
Defines all datasets to take a snapshot of.
If set to `null`, the default, take the list of datasets from `config.boot.zfs.extraPools`.
The snapshots are not recursive unless specificed in the `recursive` option.
'';
};
recursive = lib.mkOption {
type = lib.types.bool;
description = "If true, take snapshots recurisevly on the given datasets.";
default = false;
};
};
};
# The implementation is greatly inspired by https://discourse.nixos.org/t/configure-zfs-filesystems-after-install/48633/3
config = {
boot.zfs.extraPools = lib.uniqueStrings (builtins.attrNames cfg.pools);
assertions = [
{
assertion =
lib.any (x: x.poolName == null) (lib.mapAttrsToList (n: v: v) cfg.datasets)
-> cfg.defaultPoolName != null;
message = "Cannot have both datasets.poolName and defaultPoolName set to null";
}
];
systemd.services =
system.activationScripts = lib.mapAttrs' (
name: cfg':
let
mkPool =
poolName: poolCfg: lib.listToAttrs (lib.mapAttrsToList (mkDataset poolName) poolCfg.datasets);
mkDataset =
poolName: name: cfg':
let
dataset = poolName + "/" + name;
in
lib.attrsets.nameValuePair "zfs-create-${utils.escapeSystemdPath dataset}" {
# oneshot is used to make the systemd service wait on completion of the script.
serviceConfig.Type = "oneshot";
unitConfig.DefaultDependencies = false;
requiredBy = [ "local-fs.target" ];
before = [ "local-fs.target" ];
after = [
"zfs-import-${poolName}.service"
"zfs-mount.service"
]
++ map (n: "zfs-create-${poolName}-${n}.service") cfg'.after;
unitConfig.ConditionPathIsMountPoint = lib.mkIf (cfg'.path != "none") "!${cfg'.path}";
script = ''
${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
|| ${pkgs.zfs}/bin/zfs create \
-o mountpoint=none \
${dataset} || :
[ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
|| ${pkgs.zfs}/bin/zfs set \
mountpoint="${cfg'.path}" \
${dataset}
''
+ lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
chmod "${cfg'.mode}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
chown "${cfg'.owner}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
chown :"${cfg'.group}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
'';
};
mergeAttrs = lib.foldl lib.mergeAttrs { };
dataset = (if cfg'.poolName != null then cfg'.poolName else cfg.defaultPoolName) + "/" + name;
in
mergeAttrs (lib.mapAttrsToList mkPool cfg.pools);
lib.attrsets.nameValuePair "zfsCreate-${name}" {
text = ''
${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
|| ${pkgs.zfs}/bin/zfs create \
-o mountpoint=none \
${dataset} || :
system.preSwitchChecks."shb-zfs-snapshot" = lib.mkIf cfg.snapshotBeforeActivation.enable (
''
name="${cfg.snapshotBeforeActivation.template}"
''
+ (
let
recursiveFlag = lib.optionalString cfg.snapshotBeforeActivation.recursive "-r";
in
lib.concatMapStringsSep "\n" (ds: ''
echo "Taking ZFS snapshot of ${ds}@$name"
zfs snapshot ${recursiveFlag} ${ds}@"$name"
'') (lib.uniqueStrings datasets)
)
);
[ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
|| ${pkgs.zfs}/bin/zfs set \
mountpoint="${cfg'.path}" \
${dataset}
''
+ lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
chmod "${cfg'.mode}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
chown "${cfg'.owner}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
chown :"${cfg'.group}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
'';
}
) cfg.datasets;
};
}

138
modules/blocks/zfs/docs/default.md
View file

@ -1,138 +0,0 @@
# ZFS Block {#blocks-zfs}
Defined in [`/modules/blocks/zfs.nix`](@REPO@/modules/blocks/zfs.nix):
```nix
{
inputs,
...
}:
{
imports = [
inputs.selfhostblocks.nixosModules.zfs
];
}
```
This block creates ZFS datasets, optionally mounts them and sets permissions on the mount point.
It also enables taking snapshots on activation.
## Features {#blocks-zfs-features}
- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
- Backup of the files in the dataset [`shb.zfs.<name>.backup`][backup] through the [backup contract](./contracts-backup.html).
- Backup of the dataset itself [`shb.zfs.<name>.datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html).
- Take a snapshot [before activating or deploying][activationsnapshot] allowing a safe rollback.
[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup
[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup
[activationsnapshot]: #blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable
## Usage {#blocks-zfs-usage}
Create a dataset at `root/safe/users` mounted on `/var/lib/nixos`:
```nix
shb.zfs.pools.root.datasets."safe/users".path = "/var/lib/nixos";
```
Create a dataset at `backup/syncoid` but do not mount it:
```nix
shb.zfs.pools.backup.datasets."syncoid".path = "none";
```
Create a dataset at `root/syncthing` and set custom permissions and ACL.
Permission and ACL are only enforced for the mount point.
```nix
shb.zfs.pools.root.datasets."syncthing" = {
path = "/srv/syncthing";
mode = "ug=rwx,g+s,o=";
owner = "syncthing";
group = "syncthing";
defaultACLs = "g:syncthing:rwX";
};
```
### Backup dataset {#blocks-zfs-usage-backup-dataset}
To backup the dataset directly, use the [dataset backup contract](contracts-datasetbackup.html).
For example, with the sanoid module as the dataset backup contract provider:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.sanoid.backup."root/home" = {
request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
template = "yearly";
};
}
```
See the [Sanoid block](blocks-sanoid.html) for more examples.
### Backup files {#blocks-zfs-usage-backup-files}
To backup the files in the dataset, use the [file backup contract](contracts-backup.html).
For example, with the restic module as the file backup contract provider:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.restic.instances."myservice" = {
request = shb.zfs.pools.root.datasets.home.backup.request;
};
}
```
See the [Restic block](blocks-restic.html) for more examples.
### Snapshot before activation {#blocks-zfs-usage-snapshot-before-activation}
To take a snapshot of all datasets before activating a new configuration:
```nix
{
shb.zfs.snapshotBeforeActivation.enable = true;
}
```
This does not take a recursive snapshot by default, to do it recursively, do:
```nix
{
shb.zfs.snapshotBeforeActivation = {
enable = true;
recursive = true;
};
}
```
We did not specify datasets manually, which means the datasets list comes from `boot.zfs.extraPools`.
To specify the datasets manually, you can use:
```nix
{
shb.zfs.snapshotBeforeActivation = {
enable = true;
datasets = [ "root" "root/var" ];
};
}
```
## Options Reference {#blocks-zfs-options}
```{=include=} options
id-prefix: blocks-zfs-options-
list-id: selfhostblocks-block-zfs-options
source: @OPTIONS_JSON@
```

63
modules/contracts/backup.nix
View file

@ -1,9 +1,4 @@
{
lib,
shb,
pkgs,
...
}:
{ lib, shb, ... }:
let
inherit (lib)
concatStringsSep
@ -192,14 +187,12 @@ in
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
<snapshot 1> <metadata>
<snapshot 2> <metadata>
```
And restore the database with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
```
'';
type = str;
@ -229,56 +222,4 @@ in
};
};
};
passthru.mkRestoreScript =
{
name,
user,
sudoPreserveEnvs ? [ ],
secretsFile,
restoreCmd,
listCmd,
}:
pkgs.writeShellApplication {
inherit name;
text = ''
usage() {
echo "$0 snapshots"
echo "$0 restore <snapshot>"
}
sudocmd() {
sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
}
loadenv() {
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "${secretsFile}")
set +a
}
if [ "$1" = "restore" ]; then
shift
snapshot="$1"
loadenv
echo "Will restore snapshot '$snapshot'"
sudocmd sh -c "${restoreCmd}"
elif [ "$1" = "snapshots" ]; then
shift
loadenv
sudocmd sh -c "${listCmd}"
else
usage
exit 1
fi
'';
};
}

44
modules/contracts/backup/docs/default.md
View file

@ -20,22 +20,27 @@ source: @OPTIONS_JSON@
## Usage {#contract-backup-usage}
A service that can be backed up will provide a `backup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
Here is an example module defining such a `backup` option,
which defines what directories to backup (`sourceDirectories`)
and the user to backup with (`user`).
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define what directories to backup,
the user to backup with
and possibly hooks to run before or after the backup job runs.
Here is an example module defining such a `backup` option:
```nix
{
options = {
myservice.backup = mkOption {
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = "nextcloud";
sourceDirectories = [
"/var/lib/nextcloud"
];
};
options = contracts.backup.request;
};
default = {
user = "myservice";
sourceDirectories = [
"/var/lib/myservice"
];
};
};
};
@ -45,13 +50,13 @@ and the user to backup with (`user`).
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `backupService` option
and that one can create multiple backup instances under `backupService.instances`.
Let's assume such a module is available under the `backupservice` option
and that one can create multiple backup instances under `backupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
backupService.instances.myservice = {
request = myservice.backup.request;
backupservice.instances.myservice = {
request = myservice.backup;
settings = {
enable = true;
@ -60,17 +65,17 @@ backupService.instances.myservice = {
path = "/srv/backup/myservice";
};
# ... Other options specific to backupService like scheduling.
# ... Other options specific to backupservice like scheduling.
};
};
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `backupService`:
Thanks to using contracts, this can be made easily either with the same `backupservice`:
```nix
backupService.instances.myservice_2 = {
request = myservice.backup.request;
backupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
@ -82,13 +87,12 @@ backupService.instances.myservice_2 = {
};
```
Or with another module `backupService_2`!
Or with another module `backupservice_2`!
## Providers of the Backup Contract {#contract-backup-providers}
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html).
- [ZFS block](blocks-zfs.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
## Requester Blocks and Services {#contract-backup-requesters}

54
modules/contracts/backup/test.nix
View file

@ -21,8 +21,8 @@ in
"/opt/files/A"
"/opt/files/B"
],
settings ? { ... }: { }, # { filesRoot, config } -> attrset
extraConfig ? null, # { filesRoot, username, config } -> attrset
settings, # { repository, config } -> attrset
extraConfig ? null, # { username, config } -> attrset
}:
shb.test.runNixOSTest {
inherit name;
@ -40,7 +40,7 @@ shb.test.runNixOSTest {
};
settings = settings {
inherit config;
filesRoot = "/opt/files";
repository = "/opt/repos/${name}";
};
})
(mkIf (username != "root") {
@ -52,7 +52,6 @@ shb.test.runNixOSTest {
})
(optionalAttrs (extraConfig != null) (extraConfig {
inherit username config;
filesRoot = "/opt/files";
}))
];
};
@ -66,9 +65,7 @@ shb.test.runNixOSTest {
provider = (getAttrFromPath providerRoot nodes.machine).result;
in
''
from datetime import datetime, timedelta
from dictdiffer import diff
import re
username = "${username}"
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
@ -106,25 +103,9 @@ shb.test.runNixOSTest {
f'{path}/fileB': 'repo_fileB_1',
})
with subtest("Initial snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("First backup in repo"):
print(machine.succeed("systemctl cat ${provider.backupService}"))
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
# To accomodate for snapshot orchestrators which keep only a given amount
# of snapshots per unit of time, we set the time to now + 2h.
new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
machine.succeed(f"timedatectl set-time '{new_date}'")
machine.succeed("systemctl start ${provider.backupService}")
with subtest("New content"):
for path in sourceDirectories:
@ -138,37 +119,14 @@ shb.test.runNixOSTest {
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Second backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
with subtest("Delete content"):
for path in sourceDirectories:
machine.succeed(f"""rm -r {path}/*""")
assert_files(path, {})
with subtest("Restore second backup"):
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Restore first backup"):
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
with subtest("Restore initial content from repo"):
machine.succeed("""${provider.restoreScript} restore latest""")
for path in sourceDirectories:
assert_files(path, {

28
modules/contracts/databasebackup/docs/default.md
View file

@ -20,21 +20,21 @@ source: @OPTIONS_JSON@
## Usage {#contract-databasebackup-usage}
A database that can be backed up will provide a `databasebackup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define how to create a database dump,
the user to backup with
and how to restore from a database dump.
Here is an example module defining such a `databasebackup` option,
which defines the user to backup with (`user`)
and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
Here is an example module defining such a `databasebackup` option:
```nix
{
options = {
myservice.databasebackup = mkOption {
type = shb.contracts.databasebackup.mkRequester = {
type = contracts.databasebackup.request;
default = {
user = "myservice";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
@ -48,16 +48,16 @@ and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
};
```
Now, on the other side we have a service that uses this `databasebackup` option and actually backs up files.
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `databaseBackupService` option
and that one can create multiple backup instances under `databaseBackupService.instances`.
Let's assume such a module is available under the `databasebackupservice` option
and that one can create multiple backup instances under `databasebackupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
databaseBackupService.instances.myservice = {
request = myservice.databasebackup.request;
databasebackupservice.instances.myservice = {
request = myservice.databasebackup;
settings = {
enable = true;
@ -72,11 +72,11 @@ databaseBackupService.instances.myservice = {
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `databaseBackupService`:
Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
```nix
databaseBackupService.instances.myservice_2 = {
request = myservice.databasebackup.request;
databasebackupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
@ -88,12 +88,12 @@ databaseBackupService.instances.myservice_2 = {
};
```
Or with another module `databaseBackupService_2`!
Or with another module `databasebackupservice_2`!
## Providers of the Database Backup Contract {#contract-databasebackup-providers}
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
## Requester Blocks and Services {#contract-databasebackup-requesters}

64
modules/contracts/databasebackup/test.nix
View file

@ -51,18 +51,16 @@ shb.test.runNixOSTest {
testScript =
{ nodes, ... }:
let
provider = (getAttrFromPath providerRoot nodes.machine).result;
provider = getAttrFromPath providerRoot nodes.machine;
in
''
import csv
import re
start_all()
machine.wait_for_unit("postgresql.service")
machine.wait_for_unit("postgresql-setup.service")
machine.wait_for_open_port(5432)
def peer_cmd(cmd, db="${database}"):
def peer_cmd(cmd, db="me"):
return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
def query(query):
@ -75,68 +73,30 @@ shb.test.runNixOSTest {
if len(diff) > 0:
raise Exception(i, diff)
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
with subtest("create fixture"):
machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}]
cmp_tables(res, table)
with subtest("Initial snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("backup"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("New content"):
machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
cmp_tables(res, table)
with subtest("backup"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("Two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
print(machine.succeed("systemctl cat ${provider.result.backupService}"))
print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
machine.succeed("systemctl start ${provider.result.backupService}")
with subtest("drop database"):
machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
machine.fail(peer_cmd("SELECT * FROM test"))
with subtest("restore second snapshot"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
with subtest("restore"):
print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
machine.succeed("${provider.result.restoreScript} restore latest ")
with subtest("check restoration"):
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
cmp_tables(res, table)
with subtest("restore first snapshot"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}]
cmp_tables(res, table)
'';
}

113
modules/contracts/datasetbackup.nix
View file

@ -1,113 +0,0 @@
{ lib, ... }:
let
inherit (lib)
literalMD
mkOption
optionalAttrs
;
inherit (lib.types)
submodule
str
;
in
{
mkRequest =
{
dataset ? "",
datasetText ? null,
}:
mkOption {
description = ''
Request part of the backup contract.
Options set by the requester module
enforcing how to backup files.
'';
default = { };
type = submodule {
options = {
dataset =
mkOption {
description = ''
Dataset to backup, including the pool name.
'';
type = str;
example = "root/home";
default = dataset;
}
// optionalAttrs (datasetText != null) {
defaultText = literalMD datasetText;
};
};
};
};
mkResult =
{
restoreScript ? "restore",
restoreScriptText ? null,
backupService ? "backup.service",
backupServiceText ? null,
}:
mkOption {
description = ''
Result part of the backup contract.
Options set by the provider module that indicates the name of the backup and restore scripts.
'';
default = { };
type = submodule {
options = {
restoreScript =
mkOption {
description = ''
Name of script that can restore the database.
One can then list snapshots with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
<snapshot 1> <metadata>
<snapshot 2> <metadata>
```
And restore the database with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
```
It is not garanteed to be able to restore back to a snapshot in the future.
With the above example, it may not be possible to restore `<snapshot 2>`
after having restored `<snapshot 1>`.
'';
type = str;
default = restoreScript;
}
// optionalAttrs (restoreScriptText != null) {
defaultText = literalMD restoreScriptText;
};
backupService =
mkOption {
description = ''
Name of service backing up the database.
This script can be ran manually to backup the database:
```bash
$ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
```
'';
type = str;
default = backupService;
}
// optionalAttrs (backupServiceText != null) {
defaultText = literalMD backupServiceText;
};
};
};
};
}

85
modules/contracts/datasetbackup/docs/default.md
View file

@ -1,85 +0,0 @@
# Dataset Backup Contract {#contract-dataset-backup}
This NixOS contract represents a backup job
that will backup one ZFS dataset.
It is a contract between a service that manages ZFS datasets
and a service that backs up ZFS datasets.
## Contract Reference {#contract-datatsetbackup-options}
These are all the options that are expected to exist for this contract to be respected.
```{=include=} options
id-prefix: contracts-datasetbackup-options-
list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```
## Usage {#contract-datasetbackup-usage}
A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option.
Here is an example module defining such a `backup` option,
which defines what directories to backup (`sourceDirectories`)
and the user to backup with (`user`).
```nix
{
options = {
myservice.datasetbackup = mkOption {
type = lib.types.submodule {
options = shb.contracts.datasetbackup.mkRequester {
dataset = "root/test";
};
};
};
};
};
```
Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `backupService` option
and that one can create multiple backup instances under `backupService.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
backupService.instances.myservice = {
request = myservice.datasetbackup.request;
settings = {
enable = true;
targetDataset = "backup/myservice";
# ... Other options specific to backupService like scheduling.
};
};
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `backupService`:
```nix
backupService.instances.myservice_2 = {
request = myservice.datasetbackup.request;
settings = {
enable = true;
targetDataset = "backup2/myservice";
};
};
```
Or with another module `backupService_2`!
## Providers of the Backup Contract {#contract-datasetbackup-providers}
- [Sanoid block](blocks-sanoid.html).
## Requester Blocks and Services {#contract-datasetbackup-requesters}
- [ZFS](blocks-zfs.html#blocks-zfs-backup).

30
modules/contracts/datasetbackup/dummyModule.nix
View file

@ -1,30 +0,0 @@
{ lib, shb, ... }:
let
inherit (lib) mkOption;
inherit (lib.types) submodule;
in
{
imports = [
../../../lib/module.nix
];
options.shb.contracts.datasetbackup = mkOption {
description = ''
Contract for backing up ZFS datasets.
The requester communicates to the provider
the dataset to backup
through the `request` options.
The provider reads from the `request` options
and backs up the requested dataset.
It communicates to the requester what script is used
to backup and restore the files
through the `result` options.
'';
type = submodule {
options = shb.contracts.datasetbackup.contract;
};
};
}

164
modules/contracts/datasetbackup/test.nix
View file

@ -1,164 +0,0 @@
{
pkgs,
lib,
shb,
}:
let
inherit (lib)
concatMapStringsSep
getAttrFromPath
setAttrByPath
;
in
{
name,
providerRoot,
modules ? [ ],
dataset ? "root/test",
sourceDirectories ? [
"/opt/files/A"
"/opt/files/B"
],
settings ? { ... }: { }, # { filesRoot, config } -> attrset
extraConfig ? null, # { filesRoot, config } -> attrset
}:
shb.test.runNixOSTest {
inherit name;
nodes.machine =
{ config, ... }:
{
imports = [ shb.test.baseImports ] ++ modules;
config = lib.mkMerge [
(setAttrByPath providerRoot {
request = {
inherit dataset;
};
settings = settings {
inherit config;
filesRoot = "/opt/files";
};
})
(lib.optionalAttrs (extraConfig != null) (extraConfig {
inherit config;
filesRoot = "/opt/files";
}))
];
};
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
testScript =
{ nodes, ... }:
let
provider = (getAttrFromPath providerRoot nodes.machine).result;
in
''
from datetime import datetime, timedelta
from dictdiffer import diff
import re
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
def list_files(dir):
files_and_content = {}
files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
for f in files:
content = machine.succeed(f"""cat {f}""").strip()
files_and_content[f] = content
return files_and_content
def assert_files(dir, files):
result = list(diff(list_files(dir), files))
if len(result) > 0:
raise Exception("Unexpected files:", result)
with subtest("Create initial content"):
for path in sourceDirectories:
machine.succeed(f"""
mkdir -p {path}
echo repo_fileA_1 > {path}/fileA
echo repo_fileB_1 > {path}/fileB
""")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_1',
f'{path}/fileB': 'repo_fileB_1',
})
with subtest("Initial snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("First backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
# To accomodate for snapshot orchestrators which keep only a given amount
# of snapshots per unit of time, we set the time to now + 2h.
new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
machine.succeed(f"timedatectl set-time '{new_date}'")
with subtest("New content"):
for path in sourceDirectories:
machine.succeed(f"""
echo repo_fileA_2 > {path}/fileA
echo repo_fileB_2 > {path}/fileB
""")
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Second backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
with subtest("Delete content"):
for path in sourceDirectories:
machine.succeed(f"""rm -r {path}/*""")
assert_files(path, {})
with subtest("Restore second backup"):
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Restore first backup"):
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_1',
f'{path}/fileB': 'repo_fileB_1',
})
'';
}

7
modules/contracts/default.nix
View file

@ -54,14 +54,12 @@ let
};
};
in
(mkContractFunctions {
mkContractFunctions {
inherit (importedModule) mkRequest mkResult;
})
// (importedModule.passthru or { });
};
contracts = {
databasebackup = importContract ./databasebackup.nix;
datasetbackup = importContract ./datasetbackup.nix;
dashboard = importContract ./dashboard.nix;
backup = importContract ./backup.nix;
mount = pkgs.callPackage ./mount.nix { };
@ -70,7 +68,6 @@ let
test = {
secret = pkgs.callPackage ./secret/test.nix { inherit shb; };
databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; };
datasetbackup = pkgs.callPackage ./datasetbackup/test.nix { inherit shb; };
backup = pkgs.callPackage ./backup/test.nix { inherit shb; };
};
};

18
modules/services/home-assistant.nix
View file

@ -11,13 +11,23 @@ let
fqdn = "${cfg.subdomain}.${cfg.domain}";
ldap_auth_script_repo = pkgs.fetchFromGitHub {
owner = "lldap";
repo = "lldap";
rev = "7d1f5abc137821c500de99c94f7579761fc949d8";
sha256 = "sha256-8D+7ww70Ja6Qwdfa+7MpjAAHewtCWNf/tuTAExoUrg0=";
};
ldap_auth_script = pkgs.writeShellScriptBin "ldap_auth.sh" ''
export PATH=${pkgs.gnused}/bin:${pkgs.curl}/bin:${pkgs.jq}/bin
exec ${pkgs.bash}/bin/bash ${ldap_auth_script_repo}/example_configs/lldap-ha-auth.sh $@
'';
# Filter secrets from config. Secrets are those of the form { source = <path>; }
secrets = lib.attrsets.filterAttrs (k: v: builtins.isAttrs v) cfg.config;
nonSecrets = (lib.attrsets.filterAttrs (k: v: !(builtins.isAttrs v)) cfg.config);
lldap_ha_auth = pkgs.callPackage ./home-assistant/lldap_ha_auth.nix { };
configWithSecretsIncludes = nonSecrets // (lib.attrsets.mapAttrs (k: v: "!secret ${k}") secrets);
in
{
@ -253,6 +263,7 @@ in
config = {
# Includes dependencies for a basic setup
# https://www.home-assistant.io/integrations/default_config/
default_config = { };
http = {
use_x_forwarded_for = true;
server_host = "127.0.0.1";
@ -270,10 +281,9 @@ in
}
])
++ (lib.optionals cfg.ldap.enable [
# https://www.home-assistant.io/docs/authentication/providers/#command-line
{
type = "command_line";
command = lldap_ha_auth + "/bin/lldap-ha-auth";
command = ldap_auth_script + "/bin/ldap_auth.sh";
args = [
"http://${cfg.ldap.host}:${toString cfg.ldap.port}"
cfg.ldap.userGroup

37
modules/services/home-assistant/lldap_ha_auth.nix
View file

@ -1,37 +0,0 @@
{
lib,
pkgs,
stdenvNoCC,
}:
stdenvNoCC.mkDerivation {
name = "lldap-ha-auth";
src = pkgs.fetchFromGitHub {
owner = "lldap";
repo = "lldap";
rev = "7d1f5abc137821c500de99c94f7579761fc949d8";
sha256 = "sha256-8D+7ww70Ja6Qwdfa+7MpjAAHewtCWNf/tuTAExoUrg0=";
};
nativeBuildInputs = [
pkgs.makeWrapper
];
buildPhase = ''
mkdir -p $out/bin
cp example_configs/lldap-ha-auth.sh $out/bin/lldap-ha-auth
chmod a+x $out/bin/lldap-ha-auth
'';
installPhase = ''
wrapProgram $out/bin/lldap-ha-auth \
--prefix PATH : ${
lib.makeBinPath [
pkgs.gnused
pkgs.curl
pkgs.jq
]
}
'';
}

107
modules/services/mailserver.nix
View file

@ -14,7 +14,7 @@ in
builtins.fetchGit {
url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver.git";
ref = "master";
rev = "e33fbde199eaad513ef5d0746db19d5878150232";
rev = "7d433bf89882f61621f95082e90a4ab91eb0bdd3";
}
+ "/default.nix"
)
@ -105,7 +105,7 @@ in
description = ''
Accounts to sync emails from using IMAP.
Emails will be stored under `''${config.mailserver.storage.path}/''${name}/''${username}`
Emails will be stored under `''${config.mailserver.mailDirectory}/''${name}/''${username}`
'';
type = lib.types.attrsOf (
lib.types.submodule {
@ -137,7 +137,7 @@ in
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.mailserver.storage.owner;
owner = config.mailserver.vmailUserName;
restartUnits = [ "mbsync.service" ];
};
};
@ -267,7 +267,7 @@ in
Enabling this app will create a new LDAP configuration or update one that exists with
the given host.
'';
default = null;
default = { };
type = lib.types.nullOr (
lib.types.submodule {
options = {
@ -333,26 +333,6 @@ in
);
};
stateVersion = lib.mkOption {
type = lib.types.ints.positive;
# SHB started at stateVersion 3.
default = 4;
description = ''
Tracking stateful version changes as an incrementing number.
When a new release comes out we may require manual migration steps to
be completed, before the new version can be put into production.
If your `stateVersion` is too low one or multiple assertions may
trigger to give you instructions on what migrations steps are required
to continue. Increase the `stateVersion` as instructed by the assertion
message.
See https://nixos-mailserver.readthedocs.io/en/latest/release-notes.html
and https://nixos-mailserver.readthedocs.io/en/latest/migrations.html
'';
};
backup = lib.mkOption {
description = ''
Backup emails, index and sieve.
@ -360,16 +340,16 @@ in
default = { };
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = config.mailserver.storage.owner;
user = config.mailserver.vmailUserName;
sourceDirectories = builtins.filter (x: x != null) [
config.mailserver.indexDir
config.mailserver.storage.path
config.mailserver.mailDirectory
config.mailserver.sieveDirectory
];
sourceDirectoriesText = ''
[
config.mailserver.indexDir
config.mailserver.storage.path
config.mailserver.mailDirectory
config.mailserver.sieveDirectory
]
'';
@ -387,7 +367,7 @@ in
user = config.services.rspamd.user;
userText = "services.rspamd.user";
sourceDirectories = builtins.filter (x: x != null) [
config.mailserver.dkim.keyDirectory
config.mailserver.dkimKeyDirectory
];
sourceDirectoriesText = ''
[
@ -405,14 +385,14 @@ in
type = lib.types.attrsOf lib.types.str;
default = {
index = config.mailserver.indexDir;
mail = config.mailserver.storage.path;
mail = config.mailserver.mailDirectory;
sieve = config.mailserver.sieveDirectory;
dkim = config.mailserver.dkimKeyDirectory;
};
defaultText = lib.literalExpression ''
{
index = config.mailserver.indexDir;
mail = config.mailserver.storage.path;
mail = config.mailserver.mailDirectory;
sieve = config.mailserver.sieveDirectory;
dkim = config.mailserver.dkimKeyDirectory;
}
@ -437,8 +417,8 @@ in
(lib.mkIf cfg.enable {
mailserver = {
enable = true;
stateVersion = 3;
fqdn = "${cfg.subdomain}.${cfg.domain}";
inherit (cfg) stateVersion;
domains = [ cfg.domain ];
localDnsResolver = false;
@ -453,22 +433,18 @@ in
# Using / is needed for iOS mail.
# Both following options are used to organize subfolders in subdirectories.
hierarchySeparator = "/";
storage = {
directoryLayout = "fs";
};
useFsLayout = true;
};
services.postfix.settings.main = {
services.postfix.config = {
smtpd_tls_security_level = lib.mkForce "encrypt";
};
# Is probably needed for iOS mail.
services.dovecot2.settings = {
ssl_min_protocol = "TLSv1.2";
# This conflicts with the default setting which seems much better than this one.
# So I'm leaving the default settings and we'll see if something breaks.
# ssl_cipher_list = "HIGH:!aNULL:!MD5";
};
services.dovecot2.extraConfig = ''
ssl_min_protocol = TLSv1.2
ssl_cipher_list = HIGH:!aNULL:!MD5
'';
services.nginx = {
enable = true;
@ -506,8 +482,6 @@ in
in
{
forceSSL = true; # Redirect HTTP → HTTPS
sslCertificate = cfg.ssl.paths.cert;
sslCertificateKey = cfg.ssl.paths.key;
root = "/var/www"; # Dummy root
locations."/.well-known/autoconfig/mail/" = {
alias = "${announce}/";
@ -527,8 +501,6 @@ in
in
{
forceSSL = true; # Redirect HTTP → HTTPS
sslCertificate = cfg.ssl.paths.cert;
sslCertificateKey = cfg.ssl.paths.key;
root = "/var/www"; # Dummy root
locations."/" = {
alias = "${landingPage}/";
@ -576,44 +548,37 @@ in
uris = [
"ldap://${cfg.ldap.host}:${toString cfg.ldap.port}"
];
base = "ou=people,${cfg.ldap.dcdomain}";
scope = "sub";
searchBase = "ou=people,${cfg.ldap.dcdomain}";
searchScope = "sub";
bind = {
dn = "uid=${cfg.ldap.adminName},ou=people,${cfg.ldap.dcdomain}";
passwordFile = cfg.ldap.adminPassword.result.path;
};
# Note that nixos simple mailserver sets auth_bind=yes
# which means authentication binds are used.
# https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_bind/#authentication-ldap-bind
dovecot =
let
filter = "(&(objectClass=inetOrgPerson)(mail=%{user})(memberOf=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain}))";
in
{
passAttrs = "user=user";
passFilter = filter;
userAttrs = lib.concatStringsSep "," [
"=home=${config.mailserver.mailDirectory}/${cfg.ldap.account}/%u"
# "mail=maildir:${config.mailserver.mailDirectory}/${cfg.ldap.account}/%u/mail"
"uid=${config.mailserver.vmailUserName}"
"gid=${config.mailserver.vmailGroupName}"
];
userFilter = filter;
};
# username needs to be set to mail so postfix maps correctly the mail address.
# Otherwise we get error messages when sending
# Sender address rejected: not owned by user
attributes = {
username = "mail";
};
postfix = {
filter = "(&(objectClass=inetOrgPerson)(mail=%s)(memberOf=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain}))";
mailAttribute = "mail";
uidAttribute = "mail";
};
};
};
services.dovecot2.settings = {
"userdb ldap" = {
fields.home = lib.mkForce "${config.mailserver.storage.path}/${cfg.ldap.account}/%{user}";
fields.mail_index_path = lib.mkForce "${config.mailserver.storage.path}/${cfg.ldap.account}/%{user}";
};
"passdb ldap" = {
# LLDAP does not understand ldap user password setup.
fields.password = lib.mkForce null;
# We use bind DN auth.
# https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_bind/#authentication-ldap-bind
bind = true;
};
};
})
(lib.mkIf (cfg.enable && cfg.imapSync != null) {
systemd.services.mbsync =
@ -636,11 +601,11 @@ in
Account ${name}
MaildirStore ${name}-local
INBOX ${config.mailserver.storage.path}/${name}/${acct.username}/mail/
INBOX ${config.mailserver.mailDirectory}/${name}/${acct.username}/mail/
# Maps subfolders on far side to actual subfolders on disk.
# The other option is Maildir++ but then the mailserver.hierarchySeparator must be set to a dot '.'
SubFolders Verbatim
Path ${config.mailserver.storage.path}/${name}/${acct.username}/mail/
Path ${config.mailserver.mailDirectory}/${name}/${acct.username}/mail/
Channel ${name}-main
Far :${name}-remote:
@ -707,7 +672,7 @@ in
description = "Sync mailbox";
serviceConfig = {
Type = "oneshot";
User = config.mailserver.storage.owner;
User = config.mailserver.vmailUserName;
};
script =
let
@ -724,8 +689,8 @@ in
name: acct:
# The equal sign makes sure parent directories have the corret user and group too.
[
"d '${config.mailserver.storage.path}/${name}' 0750 ${config.mailserver.storage.owner} ${config.mailserver.storage.group} - -"
"d '${config.mailserver.storage.path}/${name}/${acct.username}' 0750 ${config.mailserver.storage.owner} ${config.mailserver.storage.group} - -"
"d '${config.mailserver.mailDirectory}/${name}' 0750 ${config.mailserver.vmailUserName} ${config.mailserver.vmailGroupName} - -"
"d '${config.mailserver.mailDirectory}/${name}/${acct.username}' 0750 ${config.mailserver.vmailUserName} ${config.mailserver.vmailGroupName} - -"
];
in
lib.flatten (lib.mapAttrsToList mkAccount cfg.imapSync.accounts);

2
modules/services/mailserver/docs/default.md
View file

@ -128,8 +128,6 @@ in
}
```
With the example above, the mails are stored under `/var/vmail/fastmail/<email address>`.
### Secrets {#services-mailserver-usage-secrets}
Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.

2
modules/services/open-webui.nix
View file

@ -59,12 +59,12 @@ in
WEBUI_NAME = "SelfHostBlocks";
OLLAMA_BASE_URL = "http://127.0.0.1:''${toString config.services.ollama.port}";
RAG_EMBEDDING_ENGINE = "ollama";
RAG_EMBEDDING_MODEL = "nomic-embed-text:v1.5";
ENABLE_OPENAI_API = "True";
OPENAI_API_BASE_URL = "http://127.0.0.1:''${toString config.services.llama-cpp.port}";
ENABLE_WEB_SEARCH = "True";
RAG_EMBEDDING_ENGINE = "openai";
}
'';
};

31
patches/0001-selfhostblocks-never-onboard.patch
View file

@ -1,24 +1,29 @@
From 8d38721322bc47ce089f8d246513be610e22c187 Mon Sep 17 00:00:00 2001
From 6897dd86a41b336c7c03a466990f7e981c5c649c Mon Sep 17 00:00:00 2001
From: ibizaman <ibizaman@tiserbox.com>
Date: Sun, 10 May 2026 08:58:10 +0200
Subject: [PATCH] never onboard
Date: Tue, 23 Sep 2025 11:36:24 +0200
Subject: [PATCH] selfhostblocks: never onboard
---
backend/open_webui/main.py | 2 --
1 file changed, 2 deletions(-)
backend/open_webui/main.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
index e05497c..826af7b 100644
index 5630a5883..5c7c88a64 100644
--- a/backend/open_webui/main.py
+++ b/backend/open_webui/main.py
@@ -2391,8 +2391,6 @@ async def get_app_config(request: Request):
user = await Users.get_user_by_id(data['id'])
@@ -1654,11 +1654,9 @@ async def get_app_config(request: Request):
user = Users.get_user_by_id(data["id"])
user_count = Users.get_num_users()
+ # Never onboard
onboarding = False
- if user is None:
- onboarding = not await Users.has_users()
user_count = await Users.get_num_users() if app.state.LICENSE_METADATA else None
- onboarding = user_count == 0
-
return {
**({"onboarding": True} if onboarding else {}),
"status": True,
--
2.53.0
2.50.1

25
test/blocks/authelia.nix
View file

@ -16,7 +16,6 @@ in
(pkgs'.path + "/nixos/modules/profiles/qemu-guest.nix")
../../modules/blocks/authelia.nix
../../modules/blocks/hardcodedsecret.nix
../../modules/blocks/ssl.nix
];
networking.hosts = {
@ -29,29 +28,11 @@ in
];
};
shb.certs.cas.selfsigned.myca = {
name = "My CA";
};
shb.certs.certs.selfsigned = {
"machine.com" = {
ca = config.shb.certs.cas.selfsigned.myca;
domain = "*.machine.com";
group = "nginx";
};
};
systemd.services.nginx.after = [ config.shb.certs.certs.selfsigned."machine.com".systemdService ];
systemd.services.nginx.requires = [
config.shb.certs.certs.selfsigned."machine.com".systemdService
];
shb.lldap = {
enable = true;
dcdomain = "dc=example,dc=com";
subdomain = "ldap";
domain = "machine.com";
ssl = config.shb.certs.certs.selfsigned."machine.com";
ldapUserPassword.result = config.shb.hardcodedsecret.ldapUserPassword.result;
jwtSecret.result = config.shb.hardcodedsecret.jwtSecret.result;
};
@ -69,8 +50,6 @@ in
enable = true;
subdomain = "authelia";
domain = "machine.com";
ssl = config.shb.certs.certs.selfsigned."machine.com";
ldapHostname = "${config.shb.lldap.subdomain}.${config.shb.lldap.domain}";
ldapPort = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
@ -157,10 +136,10 @@ in
machine.wait_for_unit("authelia-authelia.machine.com.target")
machine.wait_for_open_port(9091)
endpoints = json.loads(machine.succeed("curl -s https://authelia.machine.com/.well-known/openid-configuration"))
endpoints = json.loads(machine.succeed("curl -s http://machine.com/.well-known/openid-configuration"))
auth_endpoint = endpoints['authorization_endpoint']
print(f"auth_endpoint: {auth_endpoint}")
if auth_endpoint != "https://authelia.machine.com/api/oidc/authorization":
if auth_endpoint != "http://machine.com/api/oidc/authorization":
raise Exception("Unexpected auth_endpoint")
resp = machine.succeed(

10
test/blocks/borgbackup.nix
View file

@ -143,7 +143,7 @@ let
})
with subtest("First backup in repo A"):
machine.succeed("systemctl start --wait ${backupService}")
machine.succeed("systemctl start ${backupService}")
with subtest("New content"):
machine.succeed("""
@ -168,12 +168,8 @@ let
assert_files("/opt/files", {})
with subtest("Restore initial content from repo A"):
snapshot = machine.succeed("""
${restoreScript} snapshots
""")
print(snapshot)
machine.succeed(f"""
${restoreScript} restore {snapshot}
machine.succeed("""
${restoreScript} restore latest
""")
assert_files("/opt/files", {

54
test/blocks/monitoring.nix
View file

@ -9,9 +9,6 @@ let
{ ... }:
[
"grafana.service"
"fluent-bit.service"
"loki.service"
"netdata.service"
];
waitForPorts =
{ node, ... }:
@ -104,19 +101,7 @@ let
"page.get_by_role('button', name=re.compile('Accept')).click()"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
"expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
''
assert page.evaluate("""
async () => {
const r = await fetch('/api/user');
if (!r.ok) return false;
const u = await r.json();
return u.login === 'alice'
&& u.email === 'alice@example.com'
&& u.orgId === 1;
}
""")
''
"expect(page.get_by_text('Welcome to Grafana')).to_be_visible()"
];
}
{
@ -133,19 +118,7 @@ let
"page.get_by_role('button', name=re.compile('Accept')).click()"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
"expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
''
assert page.evaluate("""
async () => {
const r = await fetch('/api/user');
if (!r.ok) return false;
const u = await r.json();
return u.login === 'bob'
&& u.email === 'bob@example.com'
&& u.orgId === 1;
}
""")
''
"expect(page.get_by_text('Welcome to Grafana')).to_be_visible()"
];
}
{
@ -159,19 +132,8 @@ let
username = "charlie";
password = "CharliePassword";
nextPageExpect = [
"page.get_by_role('button', name=re.compile('Accept')).click()"
"expect(page).to_have_url('https://${config.test.fqdn}/login', timeout=10000)"
''
assert page.evaluate("""
async () => {
const r = await fetch('/api/user');
if (r.status !== 401) return false;
const u = await r.json();
return u.message === 'Unauthorized';
}
""")
''
"page.get_by_role('button', name=re.compile('Accept')).click()" # I don't understand why this is not needed. Maybe it keeps somewhere the previous token?
"expect(page.get_by_text(re.compile('[Ll]ogin failed'))).to_be_visible(timeout=10000)"
];
}
];
@ -220,7 +182,7 @@ let
''
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
"expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
"expect(page.get_by_text('Temperature history for each device')).to_be_visible(timeout=20000)"
"expect(page.get_by_text('Temperature history for each device')).to_be_visible()"
];
}
{
@ -240,7 +202,7 @@ let
''
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible(timeout=10000)"
"expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
"expect(page.get_by_text('Temperature history for each device')).to_be_visible(timeout=20000)"
"expect(page.get_by_text('Temperature history for each device')).to_be_visible()"
];
}
{
@ -254,8 +216,8 @@ let
username = "charlie";
password = "CharliePassword";
nextPageExpect = [
"expect(page.get_by_text(re.compile('Hi'))).to_be_visible(timeout=10000)" # This Hi is the Authelia profile page.
# "expect(page.get_by_text(re.compile('[Ll]ogin failed'))).to_be_visible(timeout=10000)"
"page.get_by_role('button', name=re.compile('Accept')).click()" # I don't understand why this is not needed. Maybe it keeps somewhere the previous token?
"expect(page.get_by_text(re.compile('[Ll]ogin failed'))).to_be_visible(timeout=10000)"
];
}
];

8
test/blocks/restic.nix
View file

@ -168,12 +168,8 @@ let
assert_files("/opt/files", {})
with subtest("Restore initial content from repo A"):
snapshot = machine.succeed("""
${restoreScript} snapshots
""")
print(snapshot)
machine.succeed(f"""
${restoreScript} restore {snapshot}
machine.succeed("""
${restoreScript} restore latest
""")
assert_files("/opt/files", {

22
test/blocks/ssl.nix
View file

@ -3,28 +3,6 @@ let
pkgs' = pkgs;
in
{
letsencryptDnsProvider = shb.test.runNixOSTest {
name = "ssl-letsencrypt-dns-provider";
nodes.server =
{ config, ... }:
{
imports = [
shb.test.baseImports
../../modules/blocks/ssl.nix
];
shb.certs.certs.letsencrypt.dns = {
domain = "dns.example.com";
dnsProvider = "namecheap";
credentialsFile = "/run/this-file-does-not-exist-acme-env";
adminEmail = "admin@example.com";
};
};
testScript = "";
};
test = shb.test.runNixOSTest {
name = "ssl-test";

158
test/blocks/zfs.nix
View file

@ -1,158 +0,0 @@
{
shb,
...
}:
{
default = shb.test.runNixOSTest {
name = "zfs-default";
nodes.machine =
{ config, pkgs, ... }:
{
imports = [
../../modules/blocks/zfs.nix
];
# Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix
networking.hostId = "deadbeef";
boot.supportedFilesystems = [ "zfs" ];
users.users.syncthing = {
isSystemUser = true;
group = "syncthing";
};
users.groups.syncthing = { };
virtualisation = {
emptyDiskImages = [
512
512
];
};
systemd.services."zfs-zpool-create" = {
unitConfig.DefaultDependencies = false;
after = [ "systemd-modules-load.service" ];
requiredBy = [
"zfs-import-root.service"
"zfs-import-data.service"
"zfs-mount.service"
];
before = [
"zfs-import-root.service"
"zfs-import-data.service"
"zfs-mount.service"
];
script = ''
if [ ! -f /var/done ]; then
${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl root /dev/vdb
${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl data /dev/vdc
sync
fi
touch /var/done
'';
};
shb.zfs.pools.root.datasets.one.path = "/var/root/one";
shb.zfs.pools.root.datasets.two.path = "/var/root/two";
shb.zfs.pools.root.datasets.none.path = "none";
shb.zfs.pools.data.datasets.two = {
path = "/var/data/two";
mode = "ug=rwx,g+s,o=";
owner = "syncthing";
group = "syncthing";
defaultACLs = "g:syncthing:rwX";
};
shb.zfs.snapshotBeforeActivation = {
enable = true;
recursive = true;
};
specialisation = {
test.configuration = { };
test2.configuration = {
# A plus sign is not allowed as the snapshot name so it will make it fail.
shb.zfs.snapshotBeforeActivation.template = "+";
};
};
};
testScript =
{ nodes, ... }:
let
specialisations = "${nodes.machine.system.build.toplevel}/specialisation";
switch = name: "${specialisations}/${name}/bin/switch-to-configuration test";
in
''
import difflib
def assert_facl():
out = machine.succeed("getfacl /var/data/two").splitlines()
expect = """\
# file: var/data/two
# owner: syncthing
# group: syncthing
# flags: -s-
user::rwx
group::rwx
other::---
default:user::rwx
default:group::rwx
default:group:syncthing:rwx
default:mask::rwx
default:other::---
""".splitlines()
if out != expect:
diff = difflib.context_diff(expect, out)
raise Exception(f"Unexpected getfacl:\n{"\n".join(diff)}")
def assert_mounts():
out = sorted([l.split()[0] for l in machine.succeed("mount | grep /var/").splitlines()])
expect = ["data/two", "root/one", "root/two"]
if out != expect:
diff = difflib.context_diff(expect, out)
raise Exception(f"Unexpected mounts:\n{"\n".join(diff)}")
def assert_count_snapshots(name, count):
out = machine.succeed("zfs list -Ht snapshot root/one")
print(out)
if count != len(out.splitlines()):
raise Exception(f"Expected {count} dataset")
machine.start(allow_reboot=True)
machine.wait_for_unit("multi-user.target")
assert_facl()
assert_mounts()
with subtest("no snapshot yet"):
print(machine.succeed("zpool list"))
print(machine.succeed("zfs list"))
assert_count_snapshots("root/one", 0)
assert_count_snapshots("root/two", 0)
assert_count_snapshots("root/none", 0)
assert_count_snapshots("data/two", 0)
machine.succeed("${switch "test"}")
with subtest("snapshot created"):
print(machine.succeed("zpool list"))
print(machine.succeed("zfs list"))
assert_count_snapshots("root/one", 1)
assert_count_snapshots("root/two", 1)
assert_count_snapshots("root/none", 1)
assert_count_snapshots("data/two", 1)
machine.fail("${switch "test2"}")
# TODO: make this work. zpool import does not work after reboot.
# machine.crash()
# machine.wait_for_unit("multi-user.target")
# assert_facl()
# assert_mounts()
'';
};
}

39
test/common.nix
View file

@ -152,30 +152,22 @@ let
code, logs = server.execute("login_playwright")
print(logs)
try:
server.succeed("""
mkdir -p /tmp/shared/
cp -r trace /tmp/shared/
""")
server.copy_from_machine("trace")
server.copy_from_vm("trace")
except:
print("No trace found on server")
if code != 0:
raise Exception("login_playwright did not succeed")
# if code != 0:
# raise Exception("login_playwright did not succeed")
'')
+ (optionalString (hasAttr "test" nodes.client && hasAttr "login" nodes.client.test) ''
with subtest("Login from client"):
code, logs = client.execute("login_playwright")
print(logs)
try:
client.succeed("""
mkdir -p /tmp/shared/
cp -r trace /tmp/shared/
""")
client.copy_from_machine("trace")
client.copy_from_vm("trace")
except:
print("No trace found on client")
if code != 0:
raise Exception("login_playwright did not succeed")
# if code != 0:
# raise Exception("login_playwright did not succeed")
'')
);
@ -502,19 +494,6 @@ in
settings.content = "jwtSecrets";
};
shb.hardcodedsecret.alice = {
request = config.shb.lldap.ensureUsers.alice.password.request;
settings.content = "AlicePassword";
};
shb.hardcodedsecret.bob = {
request = config.shb.lldap.ensureUsers.bob.password.request;
settings.content = "BobPassword";
};
shb.hardcodedsecret.charlie = {
request = config.shb.lldap.ensureUsers.charlie.password.request;
settings.content = "CharliePassword";
};
shb.lldap = {
enable = true;
inherit (config.test) domain;
@ -530,7 +509,7 @@ in
alice = {
email = "alice@example.com";
groups = [ "user_group" ];
password.result = config.shb.hardcodedsecret.alice.result;
password.result.path = pkgs.writeText "alicePassword" "AlicePassword";
};
bob = {
email = "bob@example.com";
@ -538,12 +517,12 @@ in
# so we can make sure users only part admins
# can also login normally.
groups = [ "admin_group" ];
password.result = config.shb.hardcodedsecret.bob.result;
password.result.path = pkgs.writeText "bobPassword" "BobPassword";
};
charlie = {
email = "charlie@example.com";
groups = [ "other_group" ];
password.result = config.shb.hardcodedsecret.charlie.result;
password.result.path = pkgs.writeText "charliePassword" "CharliePassword";
};
};

16
test/contracts/backup.nix
View file

@ -14,12 +14,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ config, ... }:
{ repository, config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = "/opt/repos/mytest";
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};
@ -49,12 +49,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ config, ... }:
{ repository, config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = "/opt/repos/mytest";
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};
@ -84,12 +84,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ config, ... }:
{ repository, config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = "/opt/repos/mytest";
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};
@ -119,12 +119,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ config, ... }:
{ repository, config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = "/opt/repos/mytest";
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};

64
test/contracts/datasetbackup.nix
View file

@ -1,64 +0,0 @@
{ shb, pkgs, ... }:
{
sanoid = shb.contracts.test.datasetbackup {
name = "sanoid";
providerRoot = [
"shb"
"sanoid"
"backup"
# This is the name of the dataset
"root/mytest"
];
modules = [
../../modules/blocks/sanoid.nix
# We use the zfs module to test the sanoid one
../../modules/blocks/zfs.nix
];
settings =
{ ... }:
{
useTemplate = [ "test" ];
};
extraConfig =
{ filesRoot, ... }:
{
# Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix
networking.hostId = "deadbeef";
boot.supportedFilesystems = [ "zfs" ];
virtualisation = {
emptyDiskImages = [
512
];
};
# The test expects to keep one snapshot per hour.
services.sanoid.templates."test" = {
hourly = 1;
daily = 0;
monthly = 0;
yearly = 0;
};
systemd.services."zfs-zpool-create" = {
unitConfig.DefaultDependencies = false;
after = [ "systemd-modules-load.service" ];
requiredBy = [
"zfs-import-root.service"
"zfs-mount.service"
];
before = [
"zfs-import-root.service"
"zfs-mount.service"
];
script = ''
${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl -O mountpoint=none root /dev/vdb
'';
};
shb.zfs.pools.root.datasets."mytest" = {
path = filesRoot;
};
};
};
}

1
test/modules/davfs.nix
View file

@ -34,7 +34,6 @@ in
expected = {
services.davfs2.enable = false;
systemd.mounts = [ ];
systemd.services = { };
};
expr = testConfig { };
};

20
test/services/deluge.nix
View file

@ -25,14 +25,6 @@ let
''
print(${node.name}.succeed('journalctl -n100 -u deluged'))
print(${node.name}.succeed('systemctl status deluged'))
${node.name}.succeed("""
restarts="$(systemctl show deluged.service -P NRestarts)"
if [ "$restarts" -gt 1 ]; then
systemctl status deluged.service --no-pager
journalctl -u deluged.service -b --no-pager
exit 1
fi
""")
print(${node.name}.succeed('systemctl status delugeweb'))
with subtest("web connect"):
@ -116,18 +108,6 @@ let
request = config.shb.deluge.localclientPassword.request;
settings.content = "localpw";
};
systemd.services.deluged = {
unitConfig = {
StartLimitIntervalSec = "30s";
StartLimitBurst = 2;
};
serviceConfig = {
Restart = lib.mkForce "on-failure";
RestartSec = "1s";
};
};
};
clientLogin =

38
test/services/home-assistant.nix
View file

@ -40,36 +40,6 @@ let
unit_system = "metric";
};
};
services.home-assistant.extraComponents = [
# this is effecitvely default_config (2026.5.0), but with components
# skipped that would cause ERRORs in the sandbox
"bluetooth"
"cloud"
"conversation"
"dhcp"
"energy"
"file"
# Requires go2rtc service
# "go2rtc"
"history"
# Requires DNS and HTTP queries
# "homeassistant_alerts"
"logbook"
"media_source"
"mobile_app"
"my"
"ssdp"
"stream"
"sun"
"usage_prediction"
"usb"
"webhook"
"zeroconf"
# include some popular integrations, that absolutely shouldn't break
"knx"
"zha"
];
};
clientLogin =
@ -93,10 +63,10 @@ let
"page.get_by_role('button', name=re.compile('Create my smart home')).click()"
"expect(page.get_by_text('Create user')).to_be_visible()"
''page.get_by_role("textbox", name="Name*", exact=True).fill('Admin')''
''page.get_by_role("textbox", name="Username*").fill('admin')''
''page.get_by_role("textbox", name="Password*", exact=True).fill('adminpassword')''
''page.get_by_role("textbox", name="Confirm password*").fill('adminpassword')''
"page.get_by_label(re.compile('Name')).fill('Admin')"
"page.get_by_label(re.compile('Username')).fill('admin')"
"page.get_by_label(re.compile('Password')).fill('adminpassword')"
"page.get_by_label(re.compile('Confirm password')).fill('adminpassword')"
"page.get_by_role('button', name=re.compile('Create account')).click()"
"expect(page.get_by_text('All set!')).to_be_visible()"

65
test/services/jellyfin.nix
View file

@ -5,36 +5,6 @@ let
adminUser = "jellyfin2";
adminPassword = "admin";
commonExtraScript =
{ node, ... }:
''
server.wait_until_succeeds("journalctl --since -1m --unit jellyfin --grep 'Startup complete'")
headers = unline_with(" ", """
-H 'Content-Type: application/json'
-H 'Authorization: MediaBrowser Client="Android TV", Device="Nvidia Shield", DeviceId="ZQ9YQHHrUzk24vV", Version="0.15.3"'
""")
import time
with subtest("api login success"):
ok = False
for i in range(1, 5):
response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
data="""{"Username": "${adminUser}", "Pw": "${adminPassword}"}""",
extra=headers)
if response['code'] == 200:
ok = True
break
time.sleep(5)
if not ok:
raise Exception(f"Expected success, got: {response['code']}")
with subtest("api login failure"):
response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
data="""{"Username": "${adminUser}", "Pw": "badpassword"}""",
extra=headers)
if response['code'] != 401:
raise Exception(f"Expected failure, got: {response['code']}")
'';
commonTestScript = shb.test.mkScripts {
hasSSL = { node, ... }: !(isNull node.config.shb.jellyfin.ssl);
waitForServices =
@ -57,7 +27,35 @@ let
status = 401;
}
];
extraScript = commonExtraScript;
extraScript =
{ node, ... }:
''
server.wait_until_succeeds("journalctl --since -1m --unit jellyfin --grep 'Startup complete'")
headers = unline_with(" ", """
-H 'Content-Type: application/json'
-H 'Authorization: MediaBrowser Client="Android TV", Device="Nvidia Shield", DeviceId="ZQ9YQHHrUzk24vV", Version="0.15.3"'
""")
import time
with subtest("api login success"):
ok = False
for i in range(1, 5):
response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
data="""{"Username": "${adminUser}", "Pw": "${adminPassword}"}""",
extra=headers)
if response['code'] == 200:
ok = True
break
time.sleep(5)
if not ok:
raise Exception(f"Expected success, got: {response['code']}")
with subtest("api login failure"):
response = curl(client, """{"code":%{response_code}}""", "${node.config.test.proto_fqdn}/Users/AuthenticateByName",
data="""{"Username": "${adminUser}", "Pw": "badpassword"}""",
extra=headers)
if response['code'] != 401:
raise Exception(f"Expected failure, got: {response['code']}")
'';
};
basic =
@ -454,13 +452,12 @@ in
testScript = commonTestScript.access.override {
extraScript =
args@{
{
node,
...
}:
(commonExtraScript args)
# I have no idea why the LDAP Authentication_19.0.0.0 plugin disappears.
+ ''
''
r = server.execute('cat "${node.config.services.jellyfin.dataDir}/plugins/LDAP Authentication_19.0.0.0/meta.json"')
if r[0] != 0:
print("meta.json for plugin LDAP Authentication_19.0.0.0 not found")

278
test/services/mailserver.nix
View file

@ -1,278 +0,0 @@
{
lib,
pkgs,
shb,
...
}:
let
domain = "example.com";
subdomain = "imap";
fqdn = "${subdomain}.${domain}";
trySendMail = pkgs.writeShellApplication {
name = "trySendMail";
runtimeInputs = [
pkgs.swaks
];
text = ''
USER="''${1:-}"
PASSWORD="''${2:-}"
if [ -z "$USER" ]; then
echo "No user given"
exit 1
fi
if [ -z "$PASSWORD" ]; then
swaks \
--server ${fqdn} \
--port 465 \
--tls-on-connect \
--from "$USER" \
--to "$USER"
else
swaks \
--server ${fqdn} \
--port 465 \
--tls-on-connect \
--auth LOGIN \
--auth-user "$USER" \
--auth-password "$PASSWORD" \
--from "$USER" \
--to "$USER"
fi
'';
};
tryRcvMail = pkgs.writeShellApplication {
name = "tryRcvMail";
runtimeInputs = [
pkgs.curl
];
text = ''
USER="''${1:-}"
PASSWORD="''${2:-}"
if [ -z "$USER" ]; then
echo "No user given"
exit 1
fi
if [ -z "$PASSWORD" ]; then
echo "No password given"
exit 1
fi
curl -s --url "imaps://imap.example.com/INBOX;UID=1" \
--user "$USER:$PASSWORD"
'';
};
in
{
basic = shb.test.runNixOSTest {
name = "mailserver_basic";
# Connect to the running VM started with driverInteractive:
# ssh-keygen -R 'vsock-mux//run/user/1000/tmpcf3hs4yp/server_host.socket'; ssh -o User=root vsock-mux//run/user/1000/tmpcf3hs4yp/server_host.socket
interactive.sshBackdoor.enable = true;
nodes.server =
{ config, ... }:
{
imports = [
../../modules/blocks/ssl.nix
../../modules/services/mailserver.nix
];
networking.hosts = {
"127.0.0.1" = [ fqdn ];
};
shb.certs.cas.selfsigned.myca = {
name = "My CA";
};
shb.certs.certs.selfsigned = {
${domain} = {
ca = config.shb.certs.cas.selfsigned.myca;
domain = "*.${domain}";
group = "nginx";
};
};
shb.mailserver = {
enable = true;
inherit subdomain domain;
stateVersion = 4;
ssl = config.shb.certs.certs.selfsigned.${domain};
# imapSync = {
# # syncTimer = "10s";
# # debug = false;
# # accounts.fastmail = {
# # host = "imap.fastmail.com";
# # port = 993;
# # username = email;
# # password.result = config.shb.sops.secret."mailserver/imap/fastmail/password".result;
# # mapSpecialJunk = "Spam";
# # };
# };
# smtpRelay = {
# host = "smtp.fastmail.com";
# port = 587;
# username = email;
# password.result = config.shb.sops.secret."mailserver/smtp/fastmail/password".result;
# };
};
mailserver.mailboxes = {
Junk = {
auto = "subscribe";
special_use = "\\Junk";
};
};
environment.systemPackages = [
pkgs.openssl
pkgs.swaks
trySendMail
tryRcvMail
];
specialisation = {
ldap.configuration =
{ config, ... }:
{
imports = [
../../modules/blocks/hardcodedsecret.nix
../../modules/blocks/lldap.nix
];
networking.hosts = {
"127.0.0.1" = [ "ldap.${domain}" ];
};
environment.systemPackages = [
pkgs.openldap
];
shb.hardcodedsecret.ldapUserPassword = {
request = config.shb.lldap.ldapUserPassword.request;
settings.content = "ldapUserPassword";
};
shb.hardcodedsecret.jwtSecret = {
request = config.shb.lldap.jwtSecret.request;
settings.content = "jwtSecrets";
};
shb.hardcodedsecret.alice = {
request = config.shb.lldap.ensureUsers.alice.password.request;
settings.content = "AlicePassword";
};
shb.hardcodedsecret.charlie = {
request = config.shb.lldap.ensureUsers.charlie.password.request;
settings.content = "CharliePassword";
};
shb.lldap = {
enable = true;
inherit domain;
subdomain = "ldap";
ldapPort = 3890;
webUIListenPort = 17170;
dcdomain = "dc=example,dc=com";
ldapUserPassword.result = config.shb.hardcodedsecret.ldapUserPassword.result;
jwtSecret.result = config.shb.hardcodedsecret.jwtSecret.result;
debug = true;
ensureUsers = {
alice = {
email = "alice@example.com";
groups = [ "user_group" ];
password.result = config.shb.hardcodedsecret.alice.result;
};
charlie = {
email = "charlie@example.com";
groups = [ "other_group" ];
password.result = config.shb.hardcodedsecret.charlie.result;
};
};
ensureGroups = {
user_group = { };
admin_group = { };
other_group = { };
};
};
shb.mailserver = {
ldap = {
enable = true;
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminName = "admin";
adminPassword.result = config.shb.hardcodedsecret.ldapUserPassword.result;
account = "fastmail";
userGroup = "user_group";
};
};
};
};
};
testScript =
{ nodes, ... }:
let
specialisations = "${nodes.server.system.build.toplevel}/specialisation";
switch = name: "${specialisations}/${name}/bin/switch-to-configuration test";
ldapSearch =
let
config = nodes.server.specialisation.ldap.configuration;
in
lib.concatStringsSep " " [
"ldapsearch"
"-H ldap://ldap.example.com:${toString config.shb.lldap.ldapPort}"
];
ldapSearchAdmin =
let
config = nodes.server.specialisation.ldap.configuration;
in
lib.concatStringsSep " " [
"ldapsearch"
"-H ldap://ldap.example.com:${toString config.shb.lldap.ldapPort}"
"-D uid=admin,ou=people,${config.shb.lldap.dcdomain}"
"-w ${config.shb.hardcodedsecret.ldapUserPassword.settings.content}"
];
in
''
server.wait_for_unit("multi-user.target")
server.wait_for_unit("dovecot")
with subtest("ldap"):
server.succeed("${switch "ldap"}")
server.wait_for_unit("lldap")
print(server.succeed("${ldapSearch} -LLL -D uid=alice,ou=people,dc=example,dc=com -w AlicePassword -b cn=user_group,ou=groups,dc=example,dc=com | grep uniquemember | grep alice"))
print(server.succeed("${ldapSearchAdmin} -LLL -b cn=user_group,ou=groups,dc=example,dc=com | grep uniquemember | grep alice"))
server.wait_for_unit("dovecot")
print(server.fail("doveadm auth test auser@example.com apassword"))
print(server.fail("doveadm auth test charlie@example.com CharliePassword"))
print(server.succeed("doveadm auth test alice@example.com AlicePassword"))
server.wait_for_unit("postfix")
print(server.fail("trySendMail alice@example.com"))
print(server.fail("trySendMail auser@example.com apassword"))
print(server.fail("trySendMail charlie@example.com CharliePassword"))
print(server.succeed("trySendMail alice@example.com AlicePassword"))
print(server.fail("tryRcvMail charlie@example.com CharliePassword"))
print(server.succeed("tryRcvMail alice@example.com AlicePassword"))
print(server.succeed("find /var/vmail"))
print(server.succeed("find /var/vmail/fastmail/alice@example.com"))
'';
};
}

24
test/services/open-webui.nix
View file

@ -35,13 +35,7 @@ let
inherit (config.test) subdomain domain;
};
# Speeds up tests because models can't be downloaded anyway and that leads to retries.
services.open-webui.environment = {
OFFLINE_MODE = "true";
BYPASS_EMBEDDING_AND_RETRIEVAL = "true";
RAG_EMBEDDING_ENGINE = "ollama";
RAG_EMBEDDING_MODEL = "dummy";
RAG_OLLAMA_BASE_URL = "http://127.0.0.1:9";
};
services.open-webui.environment.OFFLINE_MODE = "true";
networking.hosts = {
"127.0.0.1" = [ "${config.test.subdomain}.${config.test.domain}" ];
@ -97,7 +91,7 @@ let
username = "alice";
password = "NotAlicePassword";
nextPageExpect = [
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible(timeout=20000)"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible()"
];
}
{
@ -107,15 +101,14 @@ let
"page.get_by_role('button', name=re.compile('Accept')).click()"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible()"
"expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
"expect(page).to_have_url(re.compile('https://o.example.com/$'), timeout=20000)"
"assert page.evaluate(\"async () => { const r = await fetch('/api/v1/auths/'); if (!r.ok) return false; const u = await r.json(); return u.email === 'alice@example.com'; }\")"
"expect(page.get_by_text('logged in')).to_be_visible()"
];
}
{
username = "bob";
password = "NotBobPassword";
nextPageExpect = [
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible(timeout=20000)"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible()"
];
}
{
@ -125,15 +118,14 @@ let
"page.get_by_role('button', name=re.compile('Accept')).click()"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).not_to_be_visible()"
"expect(page.get_by_role('button', name=re.compile('Sign In'))).not_to_be_visible()"
"expect(page).to_have_url(re.compile('https://o.example.com/$'), timeout=20000)"
"assert page.evaluate(\"async () => { const r = await fetch('/api/v1/auths/'); if (!r.ok) return false; const u = await r.json(); return u.email === 'bob@example.com'; }\")"
"expect(page.get_by_text('logged in')).to_be_visible()"
];
}
{
username = "charlie";
password = "NotCharliePassword";
nextPageExpect = [
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible(timeout=20000)"
"expect(page.get_by_text(re.compile('[Ii]ncorrect'))).to_be_visible()"
];
}
{
@ -141,7 +133,7 @@ let
password = "CharliePassword";
nextPageExpect = [
"page.get_by_role('button', name=re.compile('Accept')).click()"
"expect(page.get_by_text('do not have permission')).to_be_visible(timeout=20000)"
"expect(page.get_by_text('unauthorized')).to_be_visible()"
];
}
];
@ -151,8 +143,6 @@ let
sso =
{ config, ... }:
{
virtualisation.memorySize = 4096;
shb.open-webui = {
sso = {
enable = true;