Compare commits

..

1 commit

Author SHA1 Message Date
Cody Wright
15613b2eaf fix: add OIDC scopes to Jellyfin service
Also adds <DisabledPushedAuthorization>true</DisablePushedAuthorization>, as
otherwise I get "Error preparing login: Unauthorized - Failed to push
authorization parameters"
2025-12-26 22:45:56 +01:00
136 changed files with 1792 additions and 12180 deletions

View file

@ -23,7 +23,7 @@ on:
required: false
jobs:
automerge-rebase:
automerge:
runs-on: ubuntu-latest
steps:
- uses: reitermarkus/automerge@v2
@ -35,16 +35,3 @@ jobs:
pull-request: ${{ github.event.inputs.pull-request }}
review: ${{ github.event.inputs.review }}
dry-run: false
automerge-merge:
runs-on: ubuntu-latest
steps:
- uses: reitermarkus/automerge@v2
with:
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
merge-method: merge
do-not-merge-labels: never-merge
required-labels: automerge-merge
pull-request: ${{ github.event.inputs.pull-request }}
review: ${{ github.event.inputs.review }}
dry-run: false

View file

@ -61,7 +61,7 @@ jobs:
keep-outputs = true
keep-failed = true
- name: Setup Caching
uses: cachix/cachix-action@v17
uses: cachix/cachix-action@v16
with:
name: selfhostblocks
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -93,7 +93,7 @@ jobs:
keep-outputs = true
keep-failed = true
- name: Setup Caching
uses: cachix/cachix-action@v17
uses: cachix/cachix-action@v16
with:
name: selfhostblocks
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -124,7 +124,7 @@ jobs:
keep-outputs = true
keep-failed = true
- name: Setup Caching
uses: cachix/cachix-action@v17
uses: cachix/cachix-action@v16
with:
name: selfhostblocks
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -133,7 +133,7 @@ jobs:
echo "resultPath=$(nix eval .#checks.x86_64-linux.${{ matrix.check }} --raw)" >> $GITHUB_ENV
nix build --print-build-logs --show-trace .#checks.x86_64-linux.${{ matrix.check }}
- name: Upload Build Result
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
if: always() && startsWith(matrix.check, 'vm_')
with:
name: ${{ matrix.check }}

View file

@ -77,7 +77,7 @@ jobs:
keep-outputs = true
keep-failed = true
- uses: cachix/cachix-action@v17
- uses: cachix/cachix-action@v16
with:
name: selfhostblocks
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'

View file

@ -15,7 +15,7 @@ jobs:
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Caching
uses: cachix/cachix-action@v17
uses: cachix/cachix-action@v16
with:
name: selfhostblocks
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'

View file

@ -1,16 +1,9 @@
# See shb.skarabox.com/misc.html
name: Update Flake Lock
on:
schedule:
- cron: '0 */3 * * *' # runs every 3 hours at :00
workflow_dispatch:
inputs:
bisect_future:
description: "Bisect in the future instead of the past"
required: false
default: "false"
schedule:
- cron: '0 0 * * *' # runs daily at 00:00
jobs:
lockfile:
@ -22,22 +15,9 @@ jobs:
uses: cachix/install-nix-action@v31
with:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: Cache nixpkgs mirror
uses: actions/cache@v5
with:
path: .cache/nixpkgs.git
key: nixpkgs-mirror-v1
- name: Update flake.lock
env:
GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
BISECT_FUTURE: ${{ inputs.bisect_future }}
run: |
git config user.email "update-flake-lock-pr@selfhostblocks.com"
git config user.name "Update Flake Lock PR"
if [ "$BISECT_FUTURE" = false ]; then
BISECT_FUTURE=
elif [ -n "$BISECT_FUTURE" ]; then
BISECT_FUTURE=future
fi
nix run .#update-flake-lock-pr $BISECT_FUTURE
uses: DeterminateSystems/update-flake-lock@main
with:
token: ${{ secrets.GH_TOKEN_FOR_UPDATES }}
pr-labels: |
automerge

View file

@ -37,7 +37,7 @@ jobs:
github_access_token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Caching
uses: cachix/cachix-action@v17
uses: cachix/cachix-action@v16
with:
name: selfhostblocks
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -59,13 +59,13 @@ jobs:
public
- name: Setup Pages
uses: actions/configure-pages@v6
uses: actions/configure-pages@v5
- name: Upload artifact
uses: actions/upload-pages-artifact@v5
uses: actions/upload-pages-artifact@v4
with:
path: ./public
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v5
uses: actions/deploy-pages@v4

View file

@ -1,223 +0,0 @@
{
gh,
writeShellApplication,
}:
writeShellApplication {
name = "update-flake-lock-pr";
runtimeInputs = [
gh
];
text = ''
branch=ci/nixpkgs-update
if [ "''${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
branch=ci/nixpkgs-update-auto
fi
goto_future=no
if [ -n "''${1:-}" ]; then
goto_future=yes
fi
main () {
PR=$(get_pr)
if [ -z "$PR" ]; then
create_pr
exit 0
fi
case "$(get_pr_check_status "$PR")" in
pending)
echo "Checks are running on PR $PR, nothing to do yet."
exit 0
;;
succeeded)
echo "Checks succeeded, nothing to do."
exit 0
;;
failing)
echo "Checks failed, updating PR to midpoint commit."
local CURRENT_COMMIT
local LAST_WORKING_COMMIT
local FAILING_COMMIT
local NEW_COMMIT
CURRENT_COMMIT="$(get_main_branch_commit)"
if [ "$goto_future" = "no" ]; then
LAST_WORKING_COMMIT="$CURRENT_COMMIT"
FAILING_COMMIT="$(get_pr_commit "$PR")"
else
LAST_WORKING_COMMIT="$(get_pr_commit "$PR")"
FAILING_COMMIT="$(get_latest_nixpkgs_commit)"
fi
NEW_COMMIT="$(midpoint_commit "$LAST_WORKING_COMMIT" "$FAILING_COMMIT")"
if [ "$NEW_COMMIT" = "$LAST_WORKING_COMMIT" ] || [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
NEW_COMMIT="$(get_latest_nixpkgs_commit)"
fi
if [ "$NEW_COMMIT" = "$FAILING_COMMIT" ]; then
echo "Latest nixpkgs commit is still same failing commit, will retry later."
exit 0
fi
update_pr "$PR" "$CURRENT_COMMIT" "$NEW_COMMIT"
exit 0
;;
esac
}
get_main_branch_commit() {
git fetch origin main
git show origin/main:flake.lock \
| jq -r '.nodes.nixpkgs.locked.rev'
}
midpoint_commit() {
local good="$1"
local bad="$2"
mkdir -p .cache
if [ ! -d .cache/nixpkgs.git ]; then
git clone --mirror https://github.com/NixOS/nixpkgs.git .cache/nixpkgs.git
else
git --git-dir=.cache/nixpkgs.git fetch origin
fi
local commits=()
mapfile -t commits < <(
git --git-dir=.cache/nixpkgs.git \
rev-list \
--reverse \
--ancestry-path \
"''${good}..''${bad}"
)
local count="''${#commits[@]}"
if [ "$count" -eq 0 ]; then
echo "$bad"
return
fi
local midpoint_index=$((count / 2))
echo "''${commits[$midpoint_index]}"
}
get_latest_nixpkgs_commit () {
git ls-remote https://github.com/NixOS/nixpkgs nixos-unstable | cut -f1
}
get_pr () {
gh pr list \
--head "$branch" \
--state open \
--json number \
--jq '.[0].number // empty'
}
create_pr() {
local test_commit
test_commit="$(get_latest_nixpkgs_commit)"
echo "Creating PR on nixpkgs' latest commit $test_commit"
git checkout -B "$branch"
nix flake lock \
--override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
git add flake.lock
git commit -m "update nixpkgs to $test_commit"
git push -u origin "$branch" --force
current_commit="$(get_main_branch_commit)"
gh pr create \
--title "update nixpkgs to $test_commit" \
--body "$(printf "%s\n\n%s" "Automated nixpkgs update. Latest tries:" " - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit")" \
--label automerge-merge
}
append_pr_body() {
local pr="$1"
local text="$2"
local current_body
current_body="$(gh pr view "$pr" --json body --jq .body)"
gh pr edit "$pr" \
--body "$(printf '%s%b' "$current_body" "$text")"
}
update_pr() {
local pr="$1"
local current_commit="$2"
local test_commit="$3"
echo "Updating PR to nixpkgs' commit $test_commit"
git checkout "$branch"
nix flake lock \
--override-input nixpkgs "github:NixOS/nixpkgs/$test_commit"
git add flake.lock
git commit -m "update nixpkgs to $test_commit"
git push --force-with-lease origin "$branch"
local future_text
if [ "$goto_future" = "yes" ]; then
future_text="bisected in the future"
else
future_text="bisected in the past"
fi
gh pr edit "$pr" \
--title "update nixpkgs to $test_commit"
append_pr_body "$pr" \
" \n - https://github.com/NixOS/nixpkgs/compare/$current_commit...$test_commit ($future_text)"
}
get_pr_check_status() {
local pr="$1"
local status
status="$(gh pr checks "$pr" --json state --jq '.[].state' || true)"
if echo "$status" | grep -qE 'PENDING|QUEUED|IN_PROGRESS'; then
echo "pending"
return
fi
if echo "$status" | grep -qE 'FAILURE|TIMED_OUT|CANCELLED'; then
echo "failing"
return
fi
echo "success"
}
get_pr_commit() {
local pr="$1"
git fetch origin "$branch"
git show "origin/$branch:flake.lock" \
| jq -r '.nodes.nixpkgs.locked.rev'
}
main
'';
}

View file

@ -16,107 +16,6 @@ Template:
# Upcoming Release
# v0.9.0
Commits: https://github.com/ibizaman/selfhostblocks/compare/v0.8.0...v0.9.0
## Breaking Changes
- Updated simple-nixos-mailserver. Update all options following this pattern:
```diff
- mailserver.mailboxes.<name>.specialUse = "value"
+ mailserver.mailboxes.<name>.special_use = "\\value"
```
Note that simple-nixos-mailserver wanted to upgrade the location of the storage path
to include the ldap UID instead of the email but I kept the email address.
This means there is no migration to do.
- Update nixpkgs [from 6201e2 to abd1ea](https://github.com/ibizaman/selfhostblocks/compare/6201e203d09599479a3b3450ed24fa81537ebc4e...abd1ea17fdbedd48cbca770847b39e3a7a09ab5b).
## New Features
- Add `shb.zfs.snapshotBeforeActivation` option to take a ZFS snapshot of given datasets before activation.
See [the manual](https://shb.skarabox.com/blocks-zfs.html) for more details.
- Add [sanoid block](https://shb.skarabox.com/blocks-sanoid.html)
- Add option to take snapshot before activation with ZFS block.
## Fixes
- Fix oidc_login build for Nextcloud
- Fix mailserver build
## Other Changes
- Harden lldap configuration
- Convert ZFS block to system and [add documentation](https://shb.skarabox.com/blocks-zfs.html)
- Switch monitoring log fetching from deprecated promtail to fluent-bit
- Add test to catch drift for Let's Encrypt DNS provider
# [BROKEN] v0.8.0
## Breaking Changes
- Bump of Nextcloud version to 32 and 33 because of nixpkgs bump. All provided apps are verified compatible with Nextcloud 33 thanks to new tests.
## New Features
- Added Immich Public Proxy service
- Add homepage service with dashboard contract implemented by all services
- Add scrutiny service.
- ZFS module now supports setting permissions
- Add landing page for mailserver and dashboard contract integration
## Bug Fixes
- Use configurable dataDir in arr stack
- Forgejo ensures ldap is setup when sso is configured
- Add nixpkgs patches on aarch64-linux too
- Self-signed certs are now idempotent
- Prometheus scrapes metrics at 15s interval instead of 1m
## Other Changes
- Arr stack declares ldap groups, declare ApiKeys and bypasses auth for readarr when sso is enabled
- Forgejo declares ldap group
# v0.7.3
## New Features
- Add [mailserver module](https://shb.skarabox.com/services-mailserver.html) integrating with [Simple NixOS Mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver) and allowing full backup of an email provider.
- Bump nixpkgs from https://github.com/NixOS/nixpkgs/commit/5e2a59a5b1a82f89f2c7e598302a9cacebb72a67 to https://github.com/NixOS/nixpkgs/commit/bfc1b8a4574108ceef22f02bafcf6611380c100d. [Full diff](https://github.com/nixos/nixpkgs/compare/5e2a59a5b1a82f89f2c7e598302a9cacebb72a67...bfc1b8a4574108ceef22f02bafcf6611380c100d).
On top of minor changes, the most notable one was:
- Updated Jellyfin LDAP and SSO plugins and configuration. @Codys-Wright
## Bug Fixes
- Fix Restic and Authelia modules referencing systemd services without the `.service` suffix and leading to
# v0.7.2
## New Features
- Forgejo uses secrets contract for smtp password.
- Add [Firefly-iii](https://shb.skarabox.com/services-firefly-iii.html) service.
- Jellyfin can [install plugins declaratively](https://shb.skarabox.com/services-jellyfin.html#services-jellyfin-options-shb.jellyfin.plugins).
(Support is quite crude and WIP).
- Jellyfin configures LDAP and SSO fully declaratively, including installing necessary plugins.
- Nextcloud 32 is fully supported thanks to tests for version 31 and 32.
## Fixes
- Revert Authelia to continue using dots in systemd service names.
This caused issue with nginx name resolution.
## Other Changes
- Authelia uses non deprecated `smtp.address` option.
- Add documentation for Nginx block
- Now a user which is only member of the admin LDAP group of a service can login.
Before, some services required a user to be member of both the user and admin LDAP group.
This is ensured by regression tests going forward.
# v0.7.1
## New Features

View file

@ -153,7 +153,7 @@ SelfHostBlocks provides building blocks that take care of common self-hosting ne
- Backup for all services.
- Automatic creation of ZFS datasets per service.
- LDAP and SSO integration for most services.
- Monitoring with Grafana and Prometheus stack with provided dashboards and integration with Scrutiny.
- Monitoring with Grafana and Prometheus stack with provided dashboards.
- Automatic reverse proxy and certificate management for HTTPS.
- VPN and proxy tunneling services.
@ -174,8 +174,6 @@ Also, the stack fits together nicely thanks to [contracts](#contracts).
- Nextcloud
- Audiobookshelf
- Deluge + *arr stack
- Simple NixOS Mailserver
- Firefly-iii
- Forgejo
- Grocy
- Hledger
@ -208,7 +206,7 @@ which altogether provides a solid foundation for self-hosting services:
- BorgBackup
- Davfs
- LDAP
- Monitoring (Grafana - Prometheus - Loki stack + Scrutiny)
- Monitoring (Grafana - Prometheus - Loki stack)
- Nginx
- PostgreSQL
- Restic
@ -250,14 +248,14 @@ shb.nextcloud = {
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
};
apps.sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
};
};
```
@ -275,15 +273,15 @@ shb.forgejo = {
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
};
sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secret.result = config.shb.sops.secret."forgejo/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secret."forgejo/sso/secretForAuthelia".result;
secret.result = config.shb.sops.secrets."forgejo/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secrets."forgejo/sso/secretForAuthelia".result;
};
};
```

View file

@ -1 +1 @@
0.9.0
0.7.1

View file

@ -24,10 +24,7 @@
# This module makes the assertions happy and the build succeed.
# This is of course wrong and will not work on any real system.
filesystemModule = {
fileSystems."/" = {
device = "/dev/null";
fsType = "none";
};
fileSystems."/".device = "/dev/null";
boot.loader.grub.devices = [ "/dev/null" ];
};
in
@ -184,40 +181,6 @@
)
];
};
# Test with:
# nix build .#nixosConfigurations.contractsDirect.config.system.build.toplevel
contractsDirect =
let
nixosSystem' = import "${selfhostblocks.inputs.nixpkgs}/nixos/lib/eval-config.nix";
in
nixosSystem' {
inherit system;
modules = [
filesystemModule
(import "${selfhostblocks}/lib/module.nix")
(
{
config,
lib,
shb,
...
}:
{
options.myOption = lib.mkOption {
# Using provided nixosSystem directly.
# SHB's lib is available under `shb` thanks to the overlay.
type = shb.secretFileType;
};
config = {
myOption.source = "/a/path";
# Use the option.
environment.etc.myOption.text = config.myOption.source;
};
}
)
];
};
};
};
}

View file

@ -187,9 +187,8 @@ This section corresponds to the `sso` section of the [Nextcloud
manual](services-nextcloud.html#services-nextcloudserver-usage-oidc).
::::
At this point, it is assumed you already deployed the `sso` demo. This time, we cannot simply edit local
`/etc/hosts`, because Nextcloud SSO addon must be able to connect to Authelia by domain name
(`auth.example.com`). Instead, there is a `dnsmasq` server running in the VM and you must create a
At this point, it is assumed you already deployed the `sso` demo. There is no host to add to
`/etc/hosts` here. Instead, there is a `dnsmasq` server running in the VM and you must create a
SOCKS proxy to connect to it like so:
```bash

View file

@ -1,5 +1,5 @@
{
description = "Nextcloud example for Self Host Blocks";
description = "Home Assistant example for Self Host Blocks";
inputs = {
selfhostblocks.url = "github:ibizaman/selfhostblocks";

View file

@ -5,11 +5,8 @@ Blocks help you self-host apps or services. They implement a specific function l
access through a subdomain. Each block is designed to be usable on its own and to fit nicely with
others.
All blocks are implemented under the blocks folder [in the repository](@REPO@/modules/blocks).
All services in SHB document how to setup the various blocks provided here.
For custom services or those not provided by SHB,
the [Expose a service Recipe](recipes-exposeService.html) explains how to use the blocks here.
Not all blocks are documented yet.
You can find all available blocks [in the repository](@REPO@/modules/blocks).
## Authentication {#blocks-category-authentication}
@ -31,10 +28,6 @@ modules/blocks/borgbackup/docs/default.md
modules/blocks/restic/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-sanoid.html
modules/blocks/sanoid/docs/default.md
```
## Database {#blocks-category-database}
```{=include=} chapters html:into-file=//blocks-postgresql.html
@ -47,22 +40,12 @@ modules/blocks/postgresql/docs/default.md
modules/blocks/sops/docs/default.md
```
## Filesystem {#blocks-category-filesystem}
```{=include=} chapters html:into-file=//blocks-zfs.html
modules/blocks/zfs/docs/default.md
```
## Network {#blocks-category-network}
```{=include=} chapters html:into-file=//blocks-ssl.html
modules/blocks/ssl/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-nginx.html
modules/blocks/nginx/docs/default.md
```
## Introspection {#blocks-category-introspection}
```{=include=} chapters html:into-file=//blocks-monitoring.html

View file

@ -37,21 +37,14 @@ Provided contracts are:
- [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
Two providers are implemented: self-signed and Let's Encrypt.
- [Backup contract][] to backup directories.
Two providers are implemented: [BorgBackup][] and [Restic][].
One provider is implemented: [Restic][].
- [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
Two providers are implemented: [BorgBackup][] and [Restic][].
- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets.
One provider is implemented: [Sanoid][]
- [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
One provider is implemented: [Restic][].
- [Secret contract](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
One provider is implemented: [SOPS][].
- [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard.
One provider is implemented: [Homepage][].
[backup contract]: contracts-backup.html
[borgbackup]: blocks-borgbackup.html
[homepage]: services-homepage.html
[restic]: blocks-restic.html
[sanoid]: blocks-sanoid.html
[sops]: blocks-sops.html
```{=include=} chapters html:into-file=//contracts-ssl.html
@ -66,18 +59,10 @@ modules/contracts/backup/docs/default.md
modules/contracts/databasebackup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-datasetbackup.html
modules/contracts/datasetbackup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-secret.html
modules/contracts/secret/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-dashboard.html
modules/contracts/dashboard/docs/default.md
```
## Problem Statement {#contracts-why}
Currently in nixpkgs, every module accessing a shared resource

View file

@ -53,66 +53,20 @@ $ nix build .#checks.${system}.modules
$ nix build .#checks.${system}.vm_postgresql_peerAuth
```
### Playwright Tests {#contributing-playwright-tests}
If the test includes playwright tests, you can see the playwright trace with:
```bash
$ nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
```
If the test fails, you won't have the output directory available.
Instead, run the test with `--keep-failed` and at the end you will see a line saying
```
Keeping build directory '/nix/var/nix/builds/nix-31776-4270051001/build'
```
Copy the path and run:
```bash
sudo cp -r /nix/var/nix/builds/nix-31776-4270051001/build/shared-xchg/trace trace && sudo chown -R $USER: trace
```
Now, open that file with playwright:
```bash
nix run .#playwright -- show-trace trace/0.zip
```
### Debug Tests {#contributing-debug-tests}
Run the test in driver interactive mode:
Run one VM test interactively:
```bash
$ nix run .#checks.${system}.vm_postgresql_peerAuth.driverInteractive
```
When you get to the shell, start the server and/or client with one of the following commands:
When you get to the shell, run either `start_all()` or `test_script()`. The former just starts all
the VMs and service, then you can introspect. The latter also starts the VMs if they are not yet and
then will run the test script.
If the test includes playwright tests, you can see the playwright trace with:
```bash
server.start()
client.start()
start_all()
```
To run the test from the shell, use `test_script()`.
Note that if the test script ends in error,
the shell will exit and you will need to restart the VMs.
After the shell started, you will see lines like so:
```
SSH backdoor enabled, the machines can be accessed like this:
Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer).
client: ssh -o User=root vsock/3
server: ssh -o User=root vsock/4
```
With the following command, you can directly access the server's nginx instance with your browser at `http://localhost:8000`:
```bash
ssh-keygen -R vsock/4; ssh -o User=root -L 8000:localhost:80 vsock/4
$ nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
```
## Upload test results to CI {#contributing-upload}
@ -126,18 +80,6 @@ After running the `nix-fast-build` command from the previous section, run:
$ find . -type l -name "result-vm_*" | xargs readlink | nix run nixpkgs#cachix -- push selfhostblocks
```
## Upload package to CI {#contributing-upload-package}
In the rare case where a package must be built but cannot in CI,
for example because of not enough memory,
you can push the package directly to the cache with:
```bash
nix build .#checks.x86_64-linux.vm_karakeep_backup.nodes.server.services.karakeep.package
readlink result | nix run nixpkgs#cachix -- push selfhostblocks
```
## Deploy using colmena {#contributing-deploy-colmena}
```bash

View file

@ -124,7 +124,7 @@ let
outputPath = "share/doc/selfhostblocks";
manpage-urls = pkgs.writeText "manpage-urls.json" "{}";
manpage-urls = pkgs.writeText "manpage-urls.json" ''{}'';
in
stdenv.mkDerivation {
name = "self-host-blocks-manual";

View file

@ -28,10 +28,6 @@ blocks.md
recipes.md
```
```{=include=} chapters html:into-file=//misc.html
misc.md
```
```{=include=} chapters html:into-file=//demos.html
demos.md
```

View file

@ -1,39 +0,0 @@
<!-- Read these docs at https://shb.skarabox.com -->
# Miscellaneous {#misc}
Here goes meta-discussions around the repository and other topics not related directly to the usage of SHB.
## Lock file update {#misc-lock-file-update}
*SHB has an unusual strategy to keep up with its flake inputs. This section explains why.*
SHB only depends on `nixpkgs` as far as flake inputs goes.
There are others but they only matter for tasks around taking care of the repository itself.
So only the `nixpkgs` input is important for downstream consumers of SHB.
To keep up to date with nixpkgs unstable,
initially a job was updating the nixpkgs input by simply updating the lock file and creating a PR.
Now, this job failed most of the time because the tip of unstable often breaks modules and packages, resulting in failing SHB tests.
To fix this, we usually needed to choose a commit around 2 weeks prior to the latest unstable commit.
This usually did not lead to successful tests but the reason was then not
because of failing packages but because nixpkgs modules evolved and SHB needs to adapt to that.
The net effect was SHB was lagging too far behind unstable and updating it was becoming annoying.
The new strategy now is as follows. On every job run:
- If there is no PR, update nixpkgs input to latest unstable commit and create a PR which auto-merges when tests are successful.
- If there is a PR:
- If the checks succeeded, do nothing because the PR will auto-merge.
- If the checks are pending, do nothing yet.
- If the checks failed, bisect and update nixpkgs input to between the commit on main and the commit in the PR.
The effect here is as long as the tests are failing, we'll try a commit further in the past from nixpkgs unstable
and closer to the tip of main branch.
This could be made smarter by distinguishing between types of failures.
We will see if this is needed.
One can also run the bisect manually with `nix run .#update-flake-lock-pr`.
This will create a PR on a different branch than the one used for the automated workflow.
It accepts also one argument `future` which instead of trying a commit in between the tip of SHB main and the PR's failing one,
it tries a commit between the PR's failing one and the nixpkgs unstable latest commit.

View file

@ -221,14 +221,14 @@ shb.nextcloud = {
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
};
apps.sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
};
};
```
@ -246,15 +246,15 @@ shb.forgejo = {
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secret."nextcloud/ldap/admin_password".result;
adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/admin_password".result;
};
sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secret.result = config.shb.sops.secret."forgejo/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secret."forgejo/sso/secretForAuthelia".result;
secret.result = config.shb.sops.secrets."forgejo/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secrets."forgejo/sso/secretForAuthelia".result;
};
};
```

View file

@ -1,5 +1,5 @@
<!-- Read these docs at https://shb.skarabox.com -->
# Self-Host a DNS server {#recipes-dnsServer}
# Self-Host a DNS server making {#recipes-dnsServer}
This recipe will show how to setup [dnsmasq][] as a local DNS server
that forwards all queries to your own domain `example.com` to a local IP - your server running SelfHostBlocks for example.

View file

@ -20,7 +20,6 @@ let
fqdn = "${subdomain}.${domain}";
listenPort = 9000;
dataDir = "/var/lib/awesome";
ldapGroup = "awesome_user";
in
```
@ -76,11 +75,11 @@ shb.nginx.vhosts = [
{
inherit subdomain domain;
ssl = config.shb.certs.certs.letsencrypt.${domain};
upstream = "http://127.0.0.1:${toString listenPort}";
upstream = "http://127.0.0.1:${toString config.services.calibre-web.listen.port}";
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
autheliaRules = [{
policy = "one_factor";
subject = [ "group:${ldapGroup}" ];
subject = [ "group:${config.shb.lldap.ensureGroups.calibre_user.name}" ];
}];
}
];
@ -121,36 +120,10 @@ shb.restic.instances.awesome = {
request.sourceDirectories = [ dataDir ];
settings.enable = true;
settings.passphrase.result = config.shb.sops.secret.awesome.result;
settings.repository.path = config.services.awesome.dataDir;
settings.repository.path = "/srv/backup/awesome";
};
shb.sops.secret."awesome" = {
request = config.shb.restic.instances.awesome.settings.passphrase.request;
};
```
## Impermanence {#recipes-exposeService-impermanence}
To save the data folder in an impermanence setup, add:
```nix
{
shb.zfs.datasets."safe/awesome".path = config.services.awesome.dataDir;
}
```
## Application Dashboard {#recipes-exposeService-applicationdashboard}
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.MyServices.services.Awesome = {
sortOrder = 1;
dashboard.request = {
externalUrl = "https://${fqdn}";
internalUrl = "http://127.0.0.1:${toString listenPort}";
};
};
}
```

File diff suppressed because it is too large Load diff

View file

@ -461,13 +461,6 @@ nix flake check
nix build .#manualHtml
```
To continuously rebuild the documentation of file change, run the following command.
To exit, you'll need to do Ctrl-C twice in a row.
```bash
nix run .#manualHtml-watch
```
### Iterative Development Approach {#iterative-development-approach}
1. **Start with basic functionality** - get core service working

View file

@ -1,9 +1,9 @@
<!-- Read these docs at https://shb.skarabox.com -->
# Services {#services}
Services are usually web applications that SHB help you self-host some of your data.
Services are usually web applications that SHB help you self-host.
Configuration of those is purposely made more opinionated than the upstream nixpkgs modules
in exchange for an uniformized configuration experience.
in exchange for requiring less options to define.
That is possible thanks to the extensive use of blocks provided by SHB.
::: {.note}
@ -13,20 +13,17 @@ Not all services are yet documented. You can find all available services [in the
The following table summarizes for each documented service what features it provides. More
information is provided in the respective manual sections.
| Service | Backup | Reverse Proxy | SSO | LDAP | Monitoring | Profiling |
|-----------------------------|--------|---------------|-----|-------|------------|-----------|
| [*Arr][] | Y (1) | Y | Y | Y (4) | Y (2) | N |
| [Firefly-iii][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Forgejo][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Home-Assistant][] | Y (1) | Y | N | Y | Y (2) | N |
| [Homepage][] | Y (1) | Y | N | Y | Y (2) | N |
| [Jellyfin][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Karakeep][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Nextcloud Server][] | Y (1) | Y | Y | Y | Y (2) | P (3) |
| [Open WebUI][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Pinchflat][] | Y | Y | Y | Y (4) | Y (5) | N |
| [Simple NixOS Mailserver][] | Y | Y | N | Y | Y | N |
| [Vaultwarden][] | Y (1) | Y | Y | Y | Y (2) | N |
| Service | Backup | Reverse Proxy | SSO | LDAP | Monitoring | Profiling |
|----------------------|--------|---------------|-----|-------|------------|-----------|
| [*Arr][] | Y (1) | Y | Y | Y (4) | Y (2) | N |
| [Forgejo][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Home-Assistant][] | Y (1) | Y | N | Y | Y (2) | N |
| [Jellyfin][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Karakeep][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Nextcloud Server][] | Y (1) | Y | Y | Y | Y (2) | P (3) |
| [Open WebUI][] | Y (1) | Y | Y | Y | Y (2) | N |
| [Pinchflat][] | Y | Y | Y | Y (4) | Y (5) | N |
| [Vaultwarden][] | Y (1) | Y | Y | Y | Y (2) | N |
Legend: **N**: no but WIP; **P**: partial; **Y**: yes
@ -38,36 +35,21 @@ Legend: **N**: no but WIP; **P**: partial; **Y**: yes
4. Uses LDAP indirectly through forward auth.
[*Arr]: services-arr.html
[Firefly-iii]: services-firefly-iii.html
[Forgejo]: services-forgejo.html
[Home-Assistant]: services-home-assistant.html
[Homepage]: services-homepage.html
[Jellyfin]: services-jellyfin.html
[Karakeep]: services-karakeep.html
[Nextcloud Server]: services-nextcloud.html
[Open WebUI]: services-open-webui.html
[Pinchflat]: services-pinchflat.html
[Simple NixOS Mailserver]: services-mailserver.html
[Vaultwarden]: services-vaultwarden.html
## Dashboard {#services-category-dashboard}
```{=include=} chapters html:into-file=//services-homepage.html
modules/services/homepage/docs/default.md
```
## Documents {#services-category-documents}
```{=include=} chapters html:into-file=//services-nextcloud.html
modules/services/nextcloud-server/docs/default.md
```
## Emails {#services-category-emails}
```{=include=} chapters html:into-file=//services-mailserver.html
modules/services/mailserver/docs/default.md
```
## Passwords {#services-category-passwords}
```{=include=} chapters html:into-file=//services-vaultwarden.html
@ -109,9 +91,3 @@ modules/services/jellyfin/docs/default.md
```{=include=} chapters html:into-file=//services-pinchflat.html
modules/services/pinchflat/docs/default.md
```
## Finance {#services-category-finance}
```{=include=} chapters html:into-file=//services-firefly-iii.html
modules/services/firefly-iii/docs/default.md
```

View file

@ -634,8 +634,8 @@ One way to setup secrets management using `sops-nix`:
Setting the default this way makes all sops instances use that same file.
7. Reference the secrets in nix:
```nix
shb.sops.secret."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
shb.nextcloud.adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result;
shb.sops.secrets."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
shb.nextcloud.adminPass.result = config.shb.sops.secrets."nextcloud/adminpass".result;
```
The above snippet uses the [secrets contract](./contracts-secret.html) and
[sops block](./blocks-sops.html) to ease the configuration.

View file

@ -20,11 +20,11 @@
},
"nix-flake-tests": {
"locked": {
"lastModified": 1775571237,
"narHash": "sha256-1f5Uvgcy3gx/eyRp4rqfDRBjcZc9uiHoIlfcFIL7GvI=",
"lastModified": 1677844186,
"narHash": "sha256-ErJZ/Gs1rxh561CJeWP5bohA2IcTq1rDneu1WT6CVII=",
"owner": "antifuchs",
"repo": "nix-flake-tests",
"rev": "86b789cff8aecbd7c1bed7cd421dd837c3f62201",
"rev": "bbd9216bd0f6495bb961a8eb8392b7ef55c67afb",
"type": "github"
},
"original": {
@ -35,11 +35,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1781074563,
"narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=",
"lastModified": 1760878510,
"narHash": "sha256-K5Osef2qexezUfs0alLvZ7nQFTGS9DL2oTVsIXsqLgs=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca",
"rev": "5e2a59a5b1a82f89f2c7e598302a9cacebb72a67",
"type": "github"
},
"original": {

355
flake.nix
View file

@ -20,66 +20,52 @@
nmdsrc,
...
}:
let
shbPatches =
system:
nixpkgs.legacyPackages.${system}.lib.optionals
(system == "x86_64-linux" || system == "aarch64-linux")
[
# Get rid of lldap patches when https://github.com/NixOS/nixpkgs/pull/425923 is merged.
./patches/lldap.patch
./patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch
# Leaving commented out as an example.
# (originPkgs.fetchpatch {
# url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
# hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
# })
];
patchNixpkgs =
{
nixpkgs,
patches,
system,
}:
nixpkgs.legacyPackages.${system}.applyPatches {
name = "nixpkgs-patched";
src = nixpkgs;
inherit patches;
};
patchedNixpkgs =
system:
let
patched = patchNixpkgs {
nixpkgs = inputs.nixpkgs;
patches = shbPatches system;
inherit system;
};
in
patched
// {
nixosSystem = args: import "${patched}/nixos/lib/eval-config.nix" args;
};
pkgs' =
system:
import (patchedNixpkgs system) {
inherit system;
config.allowUnfree = true;
};
in
flake-utils.lib.eachDefaultSystem (
system:
let
pkgs = pkgs' system;
originPkgs = nixpkgs.legacyPackages.${system};
shbPatches = originPkgs.lib.optionals (system == "x86_64-linux") [
# Get rid of lldap patches when https://github.com/NixOS/nixpkgs/pull/425923 is merged.
./patches/lldap.patch
./patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch
# Leaving commented out as an example.
# (originPkgs.fetchpatch {
# url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
# hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
# })
];
patchNixpkgs =
{
nixpkgs,
patches,
system,
}:
nixpkgs.legacyPackages.${system}.applyPatches {
name = "nixpkgs-patched";
src = nixpkgs;
inherit patches;
};
patchedNixpkgs =
let
patched = patchNixpkgs {
nixpkgs = inputs.nixpkgs;
patches = shbPatches;
inherit system;
};
in
patched
// {
nixosSystem = args: import "${patched}/nixos/lib/eval-config.nix" args;
};
pkgs = import patchedNixpkgs {
inherit system;
config.allowUnfree = true;
};
# The contract dummies are used to show options for contracts.
contractDummyModules = [
modules/contracts/backup/dummyModule.nix
modules/contracts/dashboard/dummyModule.nix
modules/contracts/databasebackup/dummyModule.nix
modules/contracts/datasetbackup/dummyModule.nix
modules/contracts/secret/dummyModule.nix
modules/contracts/ssl/dummyModule.nix
];
in
@ -104,13 +90,6 @@
"blocks/authelia" = ./modules/blocks/authelia.nix;
"blocks/borgbackup" = ./modules/blocks/borgbackup.nix;
"blocks/lldap" = ./modules/blocks/lldap.nix;
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
"blocks/nginx" = ./modules/blocks/nginx.nix;
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
"blocks/restic" = ./modules/blocks/restic.nix;
"blocks/sanoid" = ./modules/blocks/sanoid.nix;
"blocks/sops" = ./modules/blocks/sops.nix;
"blocks/ssl" = {
module = ./modules/blocks/ssl.nix;
optionRoot = [
@ -118,19 +97,19 @@
"certs"
];
};
"blocks/zfs" = ./modules/blocks/zfs.nix;
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
"blocks/restic" = ./modules/blocks/restic.nix;
"blocks/sops" = ./modules/blocks/sops.nix;
"services/arr" = ./modules/services/arr.nix;
"services/firefly-iii" = ./modules/services/firefly-iii.nix;
"services/forgejo" = [
./modules/services/forgejo.nix
(pkgs.path + "/nixos/modules/services/misc/forgejo.nix")
];
"services/home-assistant" = ./modules/services/home-assistant.nix;
"services/homepage" = ./modules/services/homepage.nix;
"services/jellyfin" = ./modules/services/jellyfin.nix;
"services/karakeep" = ./modules/services/karakeep.nix;
"services/mailserver" = ./modules/services/mailserver.nix;
"services/nextcloud-server" = {
module = ./modules/services/nextcloud-server.nix;
optionRoot = [
@ -141,7 +120,6 @@
"services/open-webui" = ./modules/services/open-webui.nix;
"services/pinchflat" = ./modules/services/pinchflat.nix;
"services/vaultwarden" = ./modules/services/vaultwarden.nix;
"contracts/backup" = {
module = ./modules/contracts/backup/dummyModule.nix;
optionRoot = [
@ -150,14 +128,6 @@
"backup"
];
};
"contracts/dashboard" = {
module = ./modules/contracts/dashboard/dummyModule.nix;
optionRoot = [
"shb"
"contracts"
"dashboard"
];
};
"contracts/databasebackup" = {
module = ./modules/contracts/databasebackup/dummyModule.nix;
optionRoot = [
@ -166,14 +136,6 @@
"databasebackup"
];
};
"contracts/datasetbackup" = {
module = ./modules/contracts/datasetbackup/dummyModule.nix;
optionRoot = [
"shb"
"contracts"
"datasetbackup"
];
};
"contracts/secret" = {
module = ./modules/contracts/secret/dummyModule.nix;
optionRoot = [
@ -243,32 +205,108 @@
'';
});
packages.manualHtml-watch = pkgs.writeShellApplication {
name = "manualHtml-watch";
runtimeInputs = [
pkgs.findutils
pkgs.entr
];
text = ''
while sleep 1; do
find . -name "*.nix" -o -name "*.md" \
| entr -d sh -c '(nix run --offline .#update-redirects && nix build --offline .#manualHtml)' || :
done
'';
};
packages.update-flake-lock-pr = pkgs.callPackage ./.github/workflows/update-flake-lock-pr.nix { };
lib = (pkgs.callPackage ./lib { }) // {
test = pkgs.callPackage ./test/common.nix { };
contracts = pkgs.callPackage ./modules/contracts {
shb = self.lib.${system};
};
patches = shbPatches system;
inherit patchNixpkgs;
patchedNixpkgs = patchedNixpkgs system;
patches = shbPatches;
inherit patchNixpkgs patchedNixpkgs;
};
checks =
let
inherit (pkgs.lib)
foldl
foldlAttrs
mergeAttrs
optionalAttrs
;
importFiles =
files:
map (
m:
pkgs.callPackage m {
shb = self.lib.${system};
}
) files;
mergeTests = foldl mergeAttrs { };
flattenAttrs =
root: attrset:
foldlAttrs (
acc: name: value:
acc
// {
"${root}_${name}" = value;
}
) { } attrset;
vm_test =
name: path:
flattenAttrs "vm_${name}" (
removeAttrs
(pkgs.callPackage path {
shb = self.lib.${system};
})
[
"override"
"overrideDerivation"
]
);
in
(optionalAttrs (system == "x86_64-linux") (
{
modules = self.lib.${system}.check {
inherit pkgs;
tests = mergeTests (importFiles [
./test/modules/davfs.nix
# TODO: Make this not use IFD
./test/modules/lib.nix
]);
};
# TODO: Make this not use IFD
lib = nix-flake-tests.lib.check {
inherit pkgs;
tests = pkgs.callPackage ./test/modules/lib.nix {
shb = self.lib.${system};
};
};
}
// (vm_test "arr" ./test/services/arr.nix)
// (vm_test "audiobookshelf" ./test/services/audiobookshelf.nix)
// (vm_test "deluge" ./test/services/deluge.nix)
// (vm_test "forgejo" ./test/services/forgejo.nix)
// (vm_test "grocy" ./test/services/grocy.nix)
// (vm_test "hledger" ./test/services/hledger.nix)
// (vm_test "immich" ./test/services/immich.nix)
// (vm_test "homeassistant" ./test/services/home-assistant.nix)
// (vm_test "jellyfin" ./test/services/jellyfin.nix)
// (vm_test "karakeep" ./test/services/karakeep.nix)
// (vm_test "nextcloud" ./test/services/nextcloud.nix)
// (vm_test "open-webui" ./test/services/open-webui.nix)
// (vm_test "paperless" ./test/services/paperless.nix)
// (vm_test "pinchflat" ./test/services/pinchflat.nix)
// (vm_test "vaultwarden" ./test/services/vaultwarden.nix)
// (vm_test "authelia" ./test/blocks/authelia.nix)
// (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
// (vm_test "lldap" ./test/blocks/lldap.nix)
// (vm_test "lib" ./test/blocks/lib.nix)
// (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
// (vm_test "monitoring" ./test/blocks/monitoring.nix)
// (vm_test "postgresql" ./test/blocks/postgresql.nix)
// (vm_test "restic" ./test/blocks/restic.nix)
// (vm_test "ssl" ./test/blocks/ssl.nix)
// (vm_test "contracts-backup" ./test/contracts/backup.nix)
// (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
// (vm_test "contracts-secret" ./test/contracts/secret.nix)
));
# To see the traces, run:
# nix run .#playwright -- show-trace $(nix eval .#checks.x86_64-linux.vm_grocy_basic --raw)/trace/0.zip
packages.playwright = pkgs.callPackage (
@ -324,111 +362,6 @@
};
}
)
// flake-utils.lib.eachSystem [ "x86_64-linux" ] (
system:
let
pkgs = pkgs' system;
in
{
checks =
let
inherit (pkgs.lib)
foldl
foldlAttrs
mergeAttrs
;
importFiles =
files:
map (
m:
pkgs.callPackage m {
shb = self.lib.${system};
}
) files;
mergeTests = foldl mergeAttrs { };
flattenAttrs =
root: attrset:
foldlAttrs (
acc: name: value:
acc
// {
"${root}_${name}" = value;
}
) { } attrset;
vm_test =
name: path:
flattenAttrs "vm_${name}" (
removeAttrs
(pkgs.callPackage path {
shb = self.lib.${system};
})
[
"override"
"overrideDerivation"
]
);
in
(
{
modules = self.lib.${system}.check {
inherit pkgs;
tests = mergeTests (importFiles [
./test/modules/davfs.nix
# TODO: Make this not use IFD
./test/modules/lib.nix
]);
};
# TODO: Make this not use IFD
lib = nix-flake-tests.lib.check {
inherit pkgs;
tests = pkgs.callPackage ./test/modules/lib.nix {
shb = self.lib.${system};
};
};
}
// (vm_test "arr" ./test/services/arr.nix)
// (vm_test "audiobookshelf" ./test/services/audiobookshelf.nix)
// (vm_test "deluge" ./test/services/deluge.nix)
// (vm_test "firefly-iii" ./test/services/firefly-iii.nix)
// (vm_test "forgejo" ./test/services/forgejo.nix)
// (vm_test "grocy" ./test/services/grocy.nix)
// (vm_test "hledger" ./test/services/hledger.nix)
// (vm_test "immich" ./test/services/immich.nix)
// (vm_test "homeassistant" ./test/services/home-assistant.nix)
// (vm_test "homepage" ./test/services/homepage.nix)
// (vm_test "jellyfin" ./test/services/jellyfin.nix)
// (vm_test "karakeep" ./test/services/karakeep.nix)
// (vm_test "mailserver" ./test/services/mailserver.nix)
// (vm_test "nextcloud" ./test/services/nextcloud.nix)
// (vm_test "open-webui" ./test/services/open-webui.nix)
// (vm_test "paperless" ./test/services/paperless.nix)
// (vm_test "pinchflat" ./test/services/pinchflat.nix)
// (vm_test "vaultwarden" ./test/services/vaultwarden.nix)
// (vm_test "authelia" ./test/blocks/authelia.nix)
// (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
// (vm_test "lib" ./test/blocks/lib.nix)
// (vm_test "lldap" ./test/blocks/lldap.nix)
// (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
// (vm_test "monitoring" ./test/blocks/monitoring.nix)
// (vm_test "postgresql" ./test/blocks/postgresql.nix)
// (vm_test "restic" ./test/blocks/restic.nix)
// (vm_test "ssl" ./test/blocks/ssl.nix)
// (vm_test "zfs" ./test/blocks/zfs.nix)
// (vm_test "contracts-backup" ./test/contracts/backup.nix)
// (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
// (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix)
// (vm_test "contracts-secret" ./test/contracts/secret.nix)
);
}
)
// {
herculesCI.ciSystems = [ "x86_64-linux" ];
@ -445,7 +378,6 @@
self.nixosModules.nginx
self.nixosModules.postgresql
self.nixosModules.restic
self.nixosModules.sanoid
self.nixosModules.ssl
self.nixosModules.tinyproxy
self.nixosModules.vpn
@ -455,16 +387,13 @@
self.nixosModules.arr
self.nixosModules.audiobookshelf
self.nixosModules.deluge
self.nixosModules.firefly-iii
self.nixosModules.forgejo
self.nixosModules.grocy
self.nixosModules.hledger
self.nixosModules.immich
self.nixosModules.home-assistant
self.nixosModules.homepage
self.nixosModules.jellyfin
self.nixosModules.karakeep
self.nixosModules.mailserver
self.nixosModules.nextcloud-server
self.nixosModules.open-webui
self.nixosModules.pinchflat
@ -485,9 +414,8 @@
nixosModules.nginx = modules/blocks/nginx.nix;
nixosModules.postgresql = modules/blocks/postgresql.nix;
nixosModules.restic = modules/blocks/restic.nix;
nixosModules.sanoid = modules/blocks/sanoid.nix;
nixosModules.sops = modules/blocks/sops.nix;
nixosModules.ssl = modules/blocks/ssl.nix;
nixosModules.sops = modules/blocks/sops.nix;
nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
nixosModules.vpn = modules/blocks/vpn.nix;
nixosModules.zfs = modules/blocks/zfs.nix;
@ -495,16 +423,13 @@
nixosModules.arr = modules/services/arr.nix;
nixosModules.audiobookshelf = modules/services/audiobookshelf.nix;
nixosModules.deluge = modules/services/deluge.nix;
nixosModules.firefly-iii = modules/services/firefly-iii.nix;
nixosModules.forgejo = modules/services/forgejo.nix;
nixosModules.grocy = modules/services/grocy.nix;
nixosModules.hledger = modules/services/hledger.nix;
nixosModules.immich = modules/services/immich.nix;
nixosModules.home-assistant = modules/services/home-assistant.nix;
nixosModules.homepage = modules/services/homepage.nix;
nixosModules.jellyfin = modules/services/jellyfin.nix;
nixosModules.karakeep = modules/services/karakeep.nix;
nixosModules.mailserver = modules/services/mailserver.nix;
nixosModules.nextcloud-server = modules/services/nextcloud-server.nix;
nixosModules.open-webui = modules/services/open-webui.nix;
nixosModules.paperless = modules/services/paperless.nix;

View file

@ -1,460 +1,386 @@
{ pkgs, lib }:
let
inherit (builtins) isAttrs hasAttr;
inherit (lib)
any
concatMapStringsSep
concatStringsSep
escapeShellArg
;
shb = rec {
# Replace secrets in a file.
# - userConfig is an attrset that will produce a config file.
# - resultPath is the location the config file should have on the filesystem.
# - generator is a function taking two arguments name and value and returning path in the nix
# nix store where the
replaceSecrets =
{
userConfig,
resultPath,
generator,
user ? null,
permissions ? "u=r,g=r,o=",
}:
let
configWithTemplates = withReplacements userConfig;
inherit (lib) any concatMapStringsSep concatStringsSep;
in
rec {
# Replace secrets in a file.
# - userConfig is an attrset that will produce a config file.
# - resultPath is the location the config file should have on the filesystem.
# - generator is a function taking two arguments name and value and returning path in the nix
# nix store where the
replaceSecrets =
{
userConfig,
resultPath,
generator,
user ? null,
permissions ? "u=r,g=r,o=",
}:
let
configWithTemplates = withReplacements userConfig;
nonSecretConfigFile = generator "template" configWithTemplates;
nonSecretConfigFile = generator "template" configWithTemplates;
replacements = getReplacements userConfig;
in
replaceSecretsScript {
file = nonSecretConfigFile;
inherit resultPath replacements;
inherit user permissions;
};
replaceSecretsFormatAdapter = format: format.generate;
replaceSecretsGeneratorAdapter =
generator: name: value:
pkgs.writeText "generator " (generator value);
toEnvVar = replaceSecretsGeneratorAdapter (
v: (lib.generators.toINIWithGlobalSection { } { globalSection = v; })
);
template =
file: newPath: replacements:
replaceSecretsScript {
inherit file replacements;
resultPath = newPath;
};
genReplacement =
secret:
let
t =
{
transform ? null,
...
}:
if isNull transform then x: x else transform;
in
lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
replaceSecretsScript =
{
file,
resultPath,
replacements,
user ? null,
permissions ? "u=r,g=r,o=",
}:
let
templatePath = resultPath + ".template";
# We check that the files containing the secrets have the
# correct permissions for us to read them in this separate
# step. Otherwise, the $(cat ...) commands inside the sed
# replacements could fail but not fail individually but
# not fail the whole script.
checkPermissions = concatMapStringsSep "\n" (
pattern: "cat ${pattern.source} > /dev/null"
) replacements;
generatedReplacements = map genReplacement replacements;
replaceScript = pkgs.writers.writePython3 "replace-secret" { } ''
import argparse
import pathlib
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--file", required=True, type=pathlib.Path)
parser.add_argument("--marker", required=True)
parser.add_argument("--replacement", required=True)
return parser.parse_args()
args = parse_args()
content = args.file.read_text()
args.file.write_text(content.replace(args.marker, args.replacement))
'';
replaceCommands = concatMapStringsSep "\n" (pattern: ''
${replaceScript} \
--file ${escapeShellArg resultPath} \
--marker ${escapeShellArg pattern.name} \
--replacement "${pattern.value}"
'') generatedReplacements;
replaceCmd =
if replacements == [ ] then
"cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}"
else
''
cat ${escapeShellArg templatePath} > ${escapeShellArg resultPath}
${replaceCommands}
'';
in
''
set -euo pipefail
${checkPermissions}
mkdir -p $(dirname ${templatePath})
ln -fs ${file} ${templatePath}
rm -f ${resultPath}
touch ${resultPath}
''
+ (lib.optionalString (user != null) ''
chown ${user} ${resultPath}
'')
+ ''
${replaceCmd}
chmod ${permissions} ${resultPath}
'';
secretFileType = lib.types.submodule {
options = {
source = lib.mkOption {
type = lib.types.path;
description = "File containing the value.";
};
transform = lib.mkOption {
type = lib.types.raw;
description = "An optional function to transform the secret.";
default = null;
example = lib.literalExpression ''
v: "prefix-$${v}-suffix"
'';
};
};
replacements = getReplacements userConfig;
in
replaceSecretsScript {
file = nonSecretConfigFile;
inherit resultPath replacements;
inherit user permissions;
};
secretName =
names: "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) names)}%";
replaceSecretsFormatAdapter = format: format.generate;
replaceSecretsGeneratorAdapter =
generator: name: value:
pkgs.writeText "generator " (generator value);
toEnvVar = replaceSecretsGeneratorAdapter (
v: (lib.generators.toINIWithGlobalSection { } { globalSection = v; })
);
withReplacements =
attrs:
let
valueOrReplacement =
name: value: if !(builtins.isAttrs value && value ? "source") then value else secretName name;
in
mapAttrsRecursiveCond (v: !v ? "source") valueOrReplacement attrs;
template =
file: newPath: replacements:
replaceSecretsScript {
inherit file replacements;
resultPath = newPath;
};
getReplacements =
attrs:
let
addNameField =
name: value:
if !(builtins.isAttrs value && value ? "source") then value else value // { name = name; };
genReplacement =
secret:
let
t =
{
transform ? null,
...
}:
if isNull transform then x: x else transform;
in
lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
secretsWithName = mapAttrsRecursiveCond (v: !v ? "source") addNameField attrs;
in
collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
replaceSecretsScript =
{
file,
resultPath,
replacements,
user ? null,
permissions ? "u=r,g=r,o=",
}:
let
templatePath = resultPath + ".template";
# Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
mapAttrsRecursiveCond =
# A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
cond:
# A function, given a list of attribute names and a value, returns a new value.
f:
# Attribute set or list to recursively map over.
set:
let
recurse =
path: val:
if builtins.isAttrs val && cond val then
lib.attrsets.mapAttrs (n: v: recurse (path ++ [ n ]) v) val
else if builtins.isList val && cond val then
lib.lists.imap0 (i: v: recurse (path ++ [ (builtins.toString i) ]) v) val
else
f path val;
in
recurse [ ] set;
# We check that the files containing the secrets have the
# correct permissions for us to read them in this separate
# step. Otherwise, the $(cat ...) commands inside the sed
# replacements could fail but not fail individually but
# not fail the whole script.
checkPermissions = concatMapStringsSep "\n" (
pattern: "cat ${pattern.source} > /dev/null"
) replacements;
# Like lib.attrsets.collect but also recurses on lists.
collect =
# Given an attribute's value, determine if recursion should stop.
pred:
# The attribute set to recursively collect.
attrs:
if pred attrs then
[ attrs ]
else if builtins.isAttrs attrs then
lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
else if builtins.isList attrs then
lib.lists.concatMap (collect pred) attrs
else
[ ];
sedPatterns = concatMapStringsSep " " (pattern: "-e \"s|${pattern.name}|${pattern.value}|\"") (
map genReplacement replacements
);
indent =
i: str:
lib.concatMapStringsSep "\n" (x: (lib.strings.replicate i " ") + x) (lib.splitString "\n" str);
sedCmd = if replacements == [ ] then "cat" else "${pkgs.gnused}/bin/sed ${sedPatterns}";
in
''
set -euo pipefail
# Generator for XML
formatXML =
{
enclosingRoot ? null,
}:
{
type =
with lib.types;
let
valueType =
nullOr (oneOf [
bool
int
float
str
path
(attrsOf valueType)
(listOf valueType)
])
// {
description = "XML value";
};
in
valueType;
${checkPermissions}
generate =
name: value:
pkgs.callPackage (
{ runCommand, python3 }:
runCommand "config"
{
value = builtins.toJSON (if enclosingRoot == null then value else { ${enclosingRoot} = value; });
passAsFile = [ "value" ];
}
(
pkgs.writers.writePython3 "dict2xml"
{
libraries = with python3.pkgs; [
python
dict2xml
];
}
''
import os
import json
from dict2xml import dict2xml
with open(os.environ["valuePath"]) as f:
content = json.loads(f.read())
if content is None:
print("Could not parse env var valuePath as json")
os.exit(2)
with open(os.environ["out"], "w") as out:
out.write(dict2xml(content))
''
)
) { };
mkdir -p $(dirname ${templatePath})
ln -fs ${file} ${templatePath}
rm -f ${resultPath}
touch ${resultPath}
''
+ (lib.optionalString (user != null) ''
chown ${user} ${resultPath}
'')
+ ''
${sedCmd} ${templatePath} > ${resultPath}
chmod ${permissions} ${resultPath}
'';
secretFileType = lib.types.submodule {
options = {
source = lib.mkOption {
type = lib.types.path;
description = "File containing the value.";
};
parseXML =
xml:
let
xmlToJsonFile = pkgs.callPackage (
transform = lib.mkOption {
type = lib.types.raw;
description = "An optional function to transform the secret.";
default = null;
example = lib.literalExpression ''
v: "prefix-$${v}-suffix"
'';
};
};
};
secretName =
names: "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) names)}%";
withReplacements =
attrs:
let
valueOrReplacement =
name: value: if !(builtins.isAttrs value && value ? "source") then value else secretName name;
in
mapAttrsRecursiveCond (v: !v ? "source") valueOrReplacement attrs;
getReplacements =
attrs:
let
addNameField =
name: value:
if !(builtins.isAttrs value && value ? "source") then value else value // { name = name; };
secretsWithName = mapAttrsRecursiveCond (v: !v ? "source") addNameField attrs;
in
collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
# Inspired lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
mapAttrsRecursiveCond =
# A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
cond:
# A function, given a list of attribute names and a value, returns a new value.
f:
# Attribute set or list to recursively map over.
set:
let
recurse =
path: val:
if builtins.isAttrs val && cond val then
lib.attrsets.mapAttrs (n: v: recurse (path ++ [ n ]) v) val
else if builtins.isList val && cond val then
lib.lists.imap0 (i: v: recurse (path ++ [ (builtins.toString i) ]) v) val
else
f path val;
in
recurse [ ] set;
# Like lib.attrsets.collect but also recurses on lists.
collect =
# Given an attribute's value, determine if recursion should stop.
pred:
# The attribute set to recursively collect.
attrs:
if pred attrs then
[ attrs ]
else if builtins.isAttrs attrs then
lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
else if builtins.isList attrs then
lib.lists.concatMap (collect pred) attrs
else
[ ];
indent =
i: str:
lib.concatMapStringsSep "\n" (x: (lib.strings.replicate i " ") + x) (lib.splitString "\n" str);
# Generator for XML
formatXML =
{
enclosingRoot ? null,
}:
{
type =
with lib.types;
let
valueType =
nullOr (oneOf [
bool
int
float
str
path
(attrsOf valueType)
(listOf valueType)
])
// {
description = "XML value";
};
in
valueType;
generate =
name: value:
pkgs.callPackage (
{ runCommand, python3 }:
runCommand "config"
{
inherit xml;
passAsFile = [ "xml" ];
value = builtins.toJSON (if enclosingRoot == null then value else { ${enclosingRoot} = value; });
passAsFile = [ "value" ];
}
(
pkgs.writers.writePython3 "xml2json"
pkgs.writers.writePython3 "dict2xml"
{
libraries = with python3.pkgs; [ python ];
libraries = with python3.pkgs; [
python
dict2xml
];
}
''
import os
import json
from collections import ChainMap
from xml.etree import ElementTree
def xml_to_dict_recursive(root):
all_descendants = list(root)
if len(all_descendants) == 0:
return {root.tag: root.text}
else:
merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
return {root.tag: dict(merged_dict)}
with open(os.environ["xmlPath"]) as f:
root = ElementTree.XML(f.read())
xml = xml_to_dict_recursive(root)
j = json.dumps(xml)
from dict2xml import dict2xml
with open(os.environ["valuePath"]) as f:
content = json.loads(f.read())
if content is None:
print("Could not parse env var valuePath as json")
os.exit(2)
with open(os.environ["out"], "w") as out:
out.write(j)
out.write(dict2xml(content))
''
)
) { };
in
builtins.fromJSON (builtins.readFile xmlToJsonFile);
renameAttrName =
attrset: from: to:
(lib.attrsets.filterAttrs (name: v: name == from) attrset)
// {
${to} = attrset.${from};
};
};
# Taken from https://github.com/antifuchs/nix-flake-tests/blob/main/default.nix
# with a nicer diff display function.
check =
{ pkgs, tests }:
let
formatValue =
val:
if (builtins.isList val || builtins.isAttrs val) then
builtins.toJSON val
else
builtins.toString val;
resultToString =
parseXML =
xml:
let
xmlToJsonFile = pkgs.callPackage (
{ runCommand, python3 }:
runCommand "config"
{
name,
expected,
result,
}:
builtins.readFile (
pkgs.runCommand "nix-flake-tests-error"
inherit xml;
passAsFile = [ "xml" ];
}
(
pkgs.writers.writePython3 "xml2json"
{
expected = formatValue expected;
result = formatValue result;
passAsFile = [
"expected"
"result"
];
nativeBuildInputs = [
(pkgs.python3.withPackages (ps: [ ps.deepdiff ] ++ ps.deepdiff.optional-dependencies.cli))
];
libraries = with python3.pkgs; [ python ];
}
''
echo "${name} failed (- expected, + result)" > $out
cp ''${expectedPath} ''${expectedPath}.json
cp ''${resultPath} ''${resultPath}.json
deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
import os
import json
from collections import ChainMap
from xml.etree import ElementTree
def xml_to_dict_recursive(root):
all_descendants = list(root)
if len(all_descendants) == 0:
return {root.tag: root.text}
else:
merged_dict = ChainMap(*map(xml_to_dict_recursive, all_descendants))
return {root.tag: dict(merged_dict)}
with open(os.environ["xmlPath"]) as f:
root = ElementTree.XML(f.read())
xml = xml_to_dict_recursive(root)
j = json.dumps(xml)
with open(os.environ["out"], "w") as out:
out.write(j)
''
);
results = pkgs.lib.runTests tests;
in
if results != [ ] then
builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
else
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
genConfigOutOfBandSystemd =
{
config,
configLocation,
generator,
user ? null,
permissions ? "u=r,g=r,o=",
}:
{
loadCredentials = getLoadCredentials "source" config;
preStart = lib.mkBefore (replaceSecrets {
userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
resultPath = configLocation;
inherit generator;
inherit user permissions;
});
};
updateToLoadCredentials =
sourceField: rootDir: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
valueOrLoadCredential =
path: value:
if !(hasPlaceholderField value) then
value
else
value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
in
mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) valueOrLoadCredential attrs;
getLoadCredentials =
sourceField: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
addPathField =
path: value: if !(hasPlaceholderField value) then value else value // { inherit path; };
secretsWithPath = mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) addPathField attrs;
allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
genLoadCredentials = secret: "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
in
map genLoadCredentials allSecrets;
anyNotNull = any (x: x != null);
mkJellyfinPlugin =
{
pname,
version,
hash,
url,
}:
pkgs.callPackage (
{ stdenv, fetchzip }:
stdenv.mkDerivation (finalAttrs: {
inherit pname version;
src = fetchzip {
inherit url hash;
stripRoot = false;
};
dontBuild = true;
installPhase = ''
mkdir $out
cp -r . $out
'';
})
)
) { };
in
builtins.fromJSON (builtins.readFile xmlToJsonFile);
update =
attr: fn: attrset:
attrset // { ${attr} = fn attrset.${attr}; };
renameAttrName =
attrset: from: to:
(lib.attrsets.filterAttrs (name: v: name == from) attrset)
// {
${to} = attrset.${from};
};
};
in
shb
// {
homepage = pkgs.callPackage ./homepage.nix { inherit shb; };
# Taken from https://github.com/antifuchs/nix-flake-tests/blob/main/default.nix
# with a nicer diff display function.
check =
{ pkgs, tests }:
let
formatValue =
val:
if (builtins.isList val || builtins.isAttrs val) then
builtins.toJSON val
else
builtins.toString val;
resultToString =
{
name,
expected,
result,
}:
builtins.readFile (
pkgs.runCommand "nix-flake-tests-error"
{
expected = formatValue expected;
result = formatValue result;
passAsFile = [
"expected"
"result"
];
}
''
echo "${name} failed (- expected, + result)" > $out
cp ''${expectedPath} ''${expectedPath}.json
cp ''${resultPath} ''${resultPath}.json
${pkgs.deepdiff}/bin/deep diff ''${expectedPath}.json ''${resultPath}.json >> $out
''
);
results = pkgs.lib.runTests tests;
in
if results != [ ] then
builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
else
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";
genConfigOutOfBandSystemd =
{
config,
configLocation,
generator,
user ? null,
permissions ? "u=r,g=r,o=",
}:
{
loadCredentials = getLoadCredentials "source" config;
preStart = lib.mkBefore (replaceSecrets {
userConfig = updateToLoadCredentials "source" "$CREDENTIALS_DIRECTORY" config;
resultPath = configLocation;
inherit generator;
inherit user permissions;
});
};
updateToLoadCredentials =
sourceField: rootDir: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
valueOrLoadCredential =
path: value:
if !(hasPlaceholderField value) then
value
else
value // { ${sourceField} = rootDir + "/" + concatStringsSep "_" path; };
in
mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) valueOrLoadCredential attrs;
getLoadCredentials =
sourceField: attrs:
let
hasPlaceholderField = v: isAttrs v && hasAttr sourceField v;
addPathField =
path: value: if !(hasPlaceholderField value) then value else value // { inherit path; };
secretsWithPath = mapAttrsRecursiveCond (v: !(hasPlaceholderField v)) addPathField attrs;
allSecrets = collect (v: hasPlaceholderField v) secretsWithPath;
genLoadCredentials = secret: "${concatStringsSep "_" secret.path}:${secret.${sourceField}}";
in
map genLoadCredentials allSecrets;
anyNotNull = any (x: x != null);
}

View file

@ -1,92 +0,0 @@
{ lib, shb }:
let
sort =
attr: vs:
map (v: { ${v.name} = v.${attr}; }) (
lib.sortOn (v: v.sortOrder) (lib.mapAttrsToList (n: v: v // { name = n; }) vs)
);
slufigy = builtins.replaceStrings [ "-" ] [ "_" ];
mkService =
groupName: serviceName:
{
request,
...
}:
apiKey: settings:
lib.recursiveUpdate (
{
href = request.externalUrl;
siteMonitor = if (request.internalUrl == null) then null else request.internalUrl;
icon = "sh-${lib.toLower serviceName}";
}
// lib.optionalAttrs (apiKey != null) {
widget = {
# Duplicating because widgets call the api key various names
# and duplicating is a hacky but easy solution.
key = "{{HOMEPAGE_FILE_${slufigy groupName}_${slufigy serviceName}}}";
password = "{{HOMEPAGE_FILE_${slufigy groupName}_${slufigy serviceName}}}";
type = lib.toLower serviceName;
url = if (request.internalUrl != null) then request.internalUrl else request.externalUrl;
};
}
) settings;
asServiceGroup =
cfg:
sort "services" (
lib.mapAttrs (
groupName: groupCfg:
shb.update "services" (
services:
sort "dashboard" (
lib.mapAttrs (
serviceName: serviceCfg:
shb.update "dashboard" (
dashboard:
(mkService groupName serviceName) dashboard serviceCfg.apiKey (serviceCfg.settings or { })
) serviceCfg
) services
)
) groupCfg
) cfg
);
allKeys =
cfg:
let
flat = lib.flatten (
lib.mapAttrsToList (
groupName: groupCfg:
lib.mapAttrsToList (
serviceName: serviceCfg:
lib.optionalAttrs (serviceCfg.apiKey != null) {
inherit serviceName groupName;
inherit (serviceCfg.apiKey.result) path;
}
) groupCfg.services
) cfg
);
flatWithApiKey = builtins.filter (v: v != { }) flat;
in
builtins.listToAttrs (
map (
{
groupName,
serviceName,
path,
}:
lib.nameValuePair "${slufigy groupName}_${slufigy serviceName}" path
) flatWithApiKey
);
in
{
inherit
allKeys
asServiceGroup
mkService
sort
;
}

View file

@ -11,7 +11,7 @@ let
cfg = config.shb.authelia;
opt = options.shb.authelia;
fqdn = "${cfg.subdomain}.${cfg.domain}";
fqdn = builtins.replaceStrings [ "." ] [ "_" ] "${cfg.subdomain}.${cfg.domain}";
fqdnWithPort = if isNull cfg.port then fqdn else "${fqdn}:${toString cfg.port}";
autheliaCfg = config.services.authelia.instances.${fqdn};
@ -89,7 +89,7 @@ in
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
};
};
};
@ -99,7 +99,7 @@ in
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
};
};
};
@ -109,27 +109,27 @@ in
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
};
};
};
storageEncryptionKey = lib.mkOption {
description = "Storage encryption key. Must be >= 20 characters.";
description = "Storage encryption key.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
};
};
};
identityProvidersOIDCHMACSecret = lib.mkOption {
description = "Identity provider OIDC HMAC secret. Must be >= 40 characters.";
description = "Identity provider OIDC HMAC secret.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
};
};
};
@ -143,7 +143,7 @@ in
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}.service" ];
restartUnits = [ "authelia-${opt.subdomain}.${opt.domain}" ];
};
};
};
@ -323,7 +323,7 @@ in
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = cfg.autheliaUser;
restartUnits = [ "authelia-${fqdn}.service" ];
restartUnits = [ "authelia-${fqdn}" ];
};
};
};
@ -390,20 +390,6 @@ in
to see exactly what Authelia receives and sends back.
'';
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.authelia.subdomain}.\${config.shb.authelia.domain}";
internalUrl = "http://127.0.0.1:${toString listenPort}";
};
};
};
};
config = lib.mkIf cfg.enable {
@ -645,7 +631,6 @@ in
shb.mitmdump.instances."authelia-${fqdn}" = lib.mkIf cfg.debug {
listenPort = 9091;
upstreamPort = 9090;
timeout = 30;
after = [ "authelia-${fqdn}.service" ];
enabledAddons = [ config.shb.mitmdump.addons.logger ];
extraArgs = [

View file

@ -8,17 +8,10 @@ This block sets up an [Authelia][] service for Single-Sign On integration.
Compared to the upstream nixpkgs module, this module is tightly integrated
with SHB which allows easy configuration of SSO with [OIDC integration](#blocks-authelia-shb-oidc)
or with [forward auth integration](#blocks-authelia-shb-forward-auth)
as well as some extensive [troubleshooting](#blocks-authelia-troubleshooting) features.
Note that forward authentication is configured with the [nginx block](blocks-nginx.html#blocks-nginx-usage-shbforwardauth).
## Features {#services-authelia-features}
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-authelia-usage-applicationdashboard)
## Usage {#services-authelia-usage}
### Initial Configuration {#blocks-authelia-usage-configuration}
## Global Setup {#blocks-authelia-global-setup}
Authelia cannot work without SSL and LDAP.
So setting up the Authelia block requires to setup the [SSL block][] first
@ -50,31 +43,31 @@ shb.authelia = {
port = 587;
username = "postmaster@mg.example.com";
from_address = "authelia@example.com";
password.result = config.shb.sops.secret."authelia/smtp_password".result;
password.result = config.shb.sops.secrets."authelia/smtp_password".result;
};
secrets = {
jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result;
ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result;
sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result;
storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result;
identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result;
identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result;
jwtSecret.result = config.shb.sops.secrets."authelia/jwt_secret".result;
ldapAdminPassword.result = config.shb.sops.secrets."authelia/ldap_admin_password".result;
sessionSecret.result = config.shb.sops.secrets."authelia/session_secret".result;
storageEncryptionKey.result = config.shb.sops.secrets."authelia/storage_encryption_key".result;
identityProvidersOIDCHMACSecret.result = config.shb.sops.secrets."authelia/hmac_secret".result;
identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secrets."authelia/private_key".result;
};
};
shb.certs.certs.letsencrypt."example.com".extraDomains = [ "auth.example.com" ];
shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
shb.sops.secret."authelia/ldap_admin_password" = {
shb.sops.secrets."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request;
shb.sops.secrets."authelia/ldap_admin_password" = {
request = config.shb.authelia.secrets.ldapAdminPassword.request;
settings.key = "lldap/user_password";
};
shb.sops.secret."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
shb.sops.secret."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
shb.sops.secret."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
shb.sops.secret."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
shb.sops.secret."authelia/smtp_password".request = config.shb.authelia.smtp.password.request;
shb.sops.secrets."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request;
shb.sops.secrets."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request;
shb.sops.secrets."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request;
shb.sops.secrets."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request;
shb.sops.secrets."authelia/smtp_password".request = config.shb.authelia.smtp.password.request;
```
This assumes secrets are setup with SOPS
@ -86,22 +79,6 @@ Crucially, the `shb.authelia.secrets.ldapAdminPasswordFile` must be the same
as the `shb.lldap.ldapUserPassword` defined for the [LLDAP block][].
This is done using Sops' `key` option.
### Application Dashboard {#services-authelia-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#blocks-authelia-options-shb.authelia.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Admin.services.Authelia = {
sortOrder = 2;
dashboard.request = config.shb.authelia.dashboard.request;
};
}
```
## SHB OIDC integration {#blocks-authelia-shb-oidc}
For services [provided by SelfHostBlocks][services] that handle [OIDC integration][OIDC],
@ -117,8 +94,8 @@ shb.<service>.sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secret.result = config.shb.sops.secret."<service>/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secret."<service>/sso/secretForAuthelia".result;
secret.result = config.shb.sops.secrets."<service>/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secrets."<service>/sso/secretForAuthelia".result;
};
shb.sops.secret."<service>/sso/secret".request = config.shb.<service>.sso.secret.request;
@ -144,7 +121,7 @@ the necessary configuration is:
shb.authelia.oidcClients = [
{
client_id = "<service>";
client_secret.source = config.shb.sops.secret."<service>/sso/secretForAuthelia".response.path;
client_secret.source = shb.sops.secret."<service>/sso/secretForAuthelia".response.path;
scopes = [ "openid" "email" "profile" ];
redirect_uris = [
"<provided by service documentation>"
@ -189,7 +166,7 @@ services.open-webui.environment = {
shb.authelia.oidcClients = [
{
client_id = "open-webui";
client_secret.source = config.shb.sops.secret."open-webui/sso/secretForAuthelia".response.path;
client_secret.source = shb.sops.secret."open-webui/sso/secretForAuthelia".response.path;
scopes = [ "openid" "email" "profile" ];
redirect_uris = [
"<provided by service documentation>"
@ -214,9 +191,40 @@ Inspiration can be taken from SelfHostBlocks' source code.
To access the UI, we will need to create an `open-webui_user` and
`open-webui_admin` LDAP group and assign our user to it.
## SHB Forward Auth {#blocks-authelia-shb-forward-auth}
For services provided by SelfHostBlocks that do not handle [OIDC integration][OIDC],
this block can provide [forward authentication][] which still allows the service to be protected by Authelia.
The user could still be required to authenticate to the service itself,
although some services can automatically users authorized by Authelia.
[forward authentication]: https://doc.traefik.io/traefik/middlewares/http/forwardauth/
Integrating with this block is done with the following code:
```nix
shb.<services>.authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
```
## Forward Auth {#blocks-authelia-forward-auth}
Forward authentication is provided by the [nginx block](blocks-nginx.html#blocks-nginx-usage-ssl).
To integrate a service that does not handle OIDC integration
and which is not provided by SelfHostBlocks with this Authelia block,
the necessary configuration is:
```nix
shb.nginx.vhosts = [
{
subdomain = "<service>";
domain = "example.com";
ssl = config.shb.certs.certs.letsencrypt."example.com";
upstream = "http://127.0.0.1:${toString config.services.<service>.port}/";
}
];
```
This configuration assumes usage of the [SSL block][].
## Troubleshooting {#blocks-authelia-troubleshooting}

View file

@ -98,7 +98,7 @@ let
OnCalendar = "daily";
Persistent = true;
};
description = "When to run the backup. See {manpage}`systemd.timer(5)` for details.";
description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
example = {
OnCalendar = "00:05";
RandomizedDelaySec = "5h";
@ -158,14 +158,9 @@ in
{
imports = [
../../lib/module.nix
../blocks/monitoring.nix
];
options.shb.borgbackup = {
enableDashboard = lib.mkEnableOption "the Backups SHB dashboard" // {
default = true;
};
instances = mkOption {
description = "Files to backup following the [backup contract](./shb.contracts-backup.html).";
default = { };
@ -386,13 +381,8 @@ in
${serviceName} = mkMerge [
{
serviceConfig = {
# Purposely not a oneshot systemd service otherwise
# the service waits on the completion the backup before deactivating.
# This seems like a nice property at first but there is one annoying
# edge case when deploying. If a backup job somehow is started when
# the deploy happens, the deploy will wait on the service to finish
# before considering the deploy done. And worse, it will consider the
# deploy as failed if the backup fails, which is not what you want.
# Makes the systemd service wait for the backup to be done before changing state to inactive.
Type = "oneshot";
Nice = lib.mkForce cfg.performance.niceness;
IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass;
IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority;
@ -419,7 +409,6 @@ in
in
{
script = script.preStart;
# Makes the systemd service wait for the backup to be done before changing state to inactive.
serviceConfig.Type = "oneshot";
serviceConfig.LoadCredential = script.loadCredentials;
}
@ -457,16 +446,39 @@ in
let
mkBorgBackupBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"BORG_REPO"
"BORG_PASSCOMMAND"
];
secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
set +a
archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
echo "Will restore archive $archive"
(cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive")
'';
};
in
flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
@ -476,26 +488,43 @@ in
let
mkBorgBackupBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"BORG_REPO"
"BORG_PASSCOMMAND"
];
secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
set +a
archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
echo "Will restore archive $archive"
sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}"
'';
};
in
flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);
}
(lib.mkIf (cfg.enableDashboard && (cfg.instances != { } || cfg.databases != { })) {
shb.monitoring.dashboards = [
./backup/dashboard/Backups.json
];
})
]
);
}

View file

@ -3,7 +3,6 @@
Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix).
This block sets up a backup job using [BorgBackup][].
It is heavily based on the nixpkgs BorgBackup module.
[borgbackup]: https://www.borgbackup.org/
@ -51,7 +50,7 @@ shb.borgbackup.instances."myservice" = {
settings = {
enable = true;
passphrase.result = config.shb.sops.secret."passphrase".result;
passphrase.result = shb.sops.secret."passphrase".result;
repository = {
path = "/srv/backups/myservice";
@ -72,7 +71,7 @@ shb.borgbackup.instances."myservice" = {
};
shb.sops.secret."passphrase".request =
config.shb.borgbackup.instances."myservice".settings.passphrase.request;
shb.borgbackup.instances."myservice".settings.passphrase.request;
```
### One folder backed up with contract {#blocks-borgbackup-usage-provider-contract}
@ -88,7 +87,7 @@ shb.borgbackup.instances."myservice" = {
settings = {
enable = true;
passphrase.result = config.shb.sops.secret."passphrase".result;
passphrase.result = shb.sops.secret."passphrase".result;
repository = {
path = "/srv/backups/myservice";
@ -109,7 +108,7 @@ shb.borgbackup.instances."myservice" = {
};
shb.sops.secret."passphrase".request =
config.shb.borgbackup.instances."myservice".settings.passphrase.request;
shb.borgbackup.instances."myservice".settings.passphrase.request;
```
### One folder backed up to S3 {#blocks-borgbackup-usage-provider-remote}
@ -230,36 +229,6 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
## Maintenance {#blocks-borgbackup-maintenance}
### Manual Backup {#blocks-borgbackup-maintenance-manuql}
To launch a backup manually, just run:
```bash
systemctl start <systemd-service-name>
```
You can easily discover the systemd service name you need by either listing the units:
```bash
systemctl list-units 'borgbackup*'
```
Or by autocompleting the unit name with `<TAB>`:
```bash
systemctl start borgbackup<TAB><TAB>
```
Note that the systemd services are of `Type=simple` which means the systemd service
will not wait for the backup completion to terminate.
If you want instead to wait for the backup to complete, use the `--wait` flag:
```bash
systemctl start --wait <systemd-service-name>
```
### Restore {#blocks-borgbackup-maintenance-restore}
One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
The restore script has all the secrets needed to access the repo,

View file

@ -75,30 +75,6 @@ in
config = {
services.davfs2.enable = builtins.length cfg.mounts > 0;
systemd.services = lib.optionalAttrs (builtins.length cfg.mounts > 0) {
davfs2-password-generation = {
wantedBy = [ "multi-user.target" ];
serviceConfig.Type = "oneshot";
script = ''
mkdir -p /etc/davfs2
[ -f /etc/davfs2/secrets ] && mv /etc/davfs2/secrets /etc/davfs2/secrets.prev
touch /etc/davfs2/secrets
chown root: /etc/davfs2
chown root: /etc/davfs2/secrets
chmod 700 /etc/davfs2
chmod 600 /etc/davfs2/secrets
''
+ (
let
mkPasswordCmd = cfg': ''
printf "%s %s %s\n" "${cfg'.mountPoint}" "${cfg'.username}" "$(cat ${cfg'.passwordFile})" >> /etc/davfs2/secrets
'';
in
lib.concatStringsSep "\n" (map mkPasswordCmd cfg.mounts)
);
};
};
systemd.mounts =
let
mkMountCfg = c: {
@ -106,7 +82,6 @@ in
description = "Webdav mount point";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
what = c.remoteUrl;
where = c.mountPoint;

View file

@ -99,7 +99,7 @@ in
};
ldapUserPassword = lib.mkOption {
description = "LDAP admin user secret. Must be >= 8 characters.";
description = "LDAP admin user secret.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0440";
@ -331,21 +331,7 @@ in
enforceGroups = mkOption {
description = "Remove groups not set declaratively.";
type = types.bool;
default = false;
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.lldap.subdomain}.\${config.shb.lldap.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.webUIListenPort}";
};
};
default = true;
};
};
@ -418,41 +404,6 @@ in
) cfg.ensureUsers;
};
# Harden lldap following https://github.com/NixOS/nixpkgs/pull/487933
systemd.services.lldap.serviceConfig = {
RemoveIPC = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [
"AF_UNIX"
"AF_INET"
"AF_INET6"
];
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
SystemCallArchitectures = "native";
CapabilityBoundingSet = "";
LockPersonality = true;
NoNewPrivileges = true;
PrivateTmp = true;
PrivateDevices = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
ProtectProc = "invisible";
ProcSubset = "pid";
MemoryDenyWriteExecute = true;
};
shb.mitmdump.instances."lldap-web" = lib.mkIf cfg.debug {
listenPort = config.shb.lldap.webUIListenPort;
upstreamPort = config.shb.lldap.webUIListenPort + 1;

View file

@ -7,13 +7,7 @@ across services.
[LLDAP]: https://github.com/lldap/lldap
## Features {#blocks-lldap-features}
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#blocks-lldap-usage-applicationdashboard)
## Usage {#blocks-lldap-usage}
### Initial Configuration {#blocks-lldap-usage-configuration}
## Global Setup {#blocks-lldap-global-setup}
```nix
shb.lldap = {
@ -35,7 +29,7 @@ shb.sops.secret."lldap/user_password".request = config.shb.lldap.ldapUserPasswor
This assumes secrets are setup with SOPS
as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual.
### SSL {#blocks-lldap-usage-ssl}
## SSL {#blocks-lldap-ssl}
Using SSL is an important security practice, like always.
Using the [SSL block][], the configuration to add to the one above is:
@ -50,7 +44,7 @@ shb.certs.certs.letsencrypt.${domain}.extraDomains = [
shb.lldap.ssl = config.shb.certs.certs.letsencrypt.${config.shb.lldap.domain};
```
### Restrict Access By IP {#blocks-lldap-usage-restrict-access-by-ip}
## Restrict Access By IP {#blocks-lldap-restrict-access-by-ip}
For added security, you can restrict access to the LLDAP UI
by adding the following line:
@ -59,22 +53,6 @@ by adding the following line:
shb.lldap.restrictAccessIPRange = "192.168.50.0/24";
```
### Application Dashboard {#blocks-lldap-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#blocks-lldap-options-shb.lldap.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Admin.services.LLDAP = {
sortOrder = 2;
dashboard.request = config.shb.lldap.dashboard.request;
};
}
```
## Manage Groups {#blocks-lldap-manage-groups}
The following snippet will create group named "family" if it does not exist yet.
@ -83,7 +61,7 @@ Also, all other groups will be deleted and only the "family" group will remain.
Note that the `lldap_admin`, `lldap_password_manager` and `lldap_strict_readonly` groups, which are internal to LLDAP, will always exist.
If you want existing groups not declared in the `shb.lldap.ensureGroups` to be deleted,
set [`shb.lldap.enforceGroups`](#blocks-lldap-options-shb.lldap.enforceGroups) to `true`.
set [`shb.lldap.enforceGroups`](#blocks-lldap-options-shb.lldap.enforceGroups) to `false`.
```nix
{
@ -164,7 +142,7 @@ set [`shb.lldap.enforceUserMemberships`](#blocks-lldap-options-shb.lldap.enforce
};
shb.sops.secret."dad".request =
config.shb.lldap.ensureUsers.dad.password.request;
shb.lldap.ensureUsers.dad.password.request;
}
```

View file

@ -29,7 +29,7 @@ let
p = pkgs.python3Packages;
in
[
p.systemd-python
p.systemd
p.mitmproxy
];
flakeIgnore = [ "E501" ];
@ -48,7 +48,7 @@ let
logging.basicConfig(level=logging.INFO, format='%(message)s')
def wait_for_port(host, port, timeout):
def wait_for_port(host, port, timeout=10):
deadline = time.time() + timeout
while time.time() < deadline:
try:
@ -68,7 +68,6 @@ let
parser.add_argument("--listen_port", required=True, help="Port mitmdump will listen on")
parser.add_argument("--upstream_host", default="http://127.0.0.1", help="Host mitmdump will connect to for upstream. Example: http://127.0.0.1 or https://otherhost")
parser.add_argument("--upstream_port", required=True, help="Port mitmdump will connect to for upstream")
parser.add_argument("--timeout", required=False, type=int, default=10, help="Timeout to wait for port availability")
args, rest = parser.parse_known_args()
MITMDUMP_BIN = os.environ.get("MITMDUMP_BIN")
@ -76,7 +75,7 @@ let
raise Exception("MITMDUMP_BIN env var must be set to the path of the mitmdump binary")
logging.info(f"Waiting for upstream address '{args.upstream_host}:{args.upstream_port}' to be up.")
wait_for_port(args.upstream_host, args.upstream_port, timeout=args.timeout)
wait_for_port(args.upstream_host, args.upstream_port, timeout=10)
logging.info(f"Upstream address '{args.upstream_host}:{args.upstream_port}' is up.")
proc = subprocess.Popen(
@ -91,11 +90,10 @@ let
)
logging.info(f"Waiting for mitmdump instance to start on port {args.listen_port}.")
if wait_for_port("127.0.0.1", args.listen_port, timeout=args.timeout):
if wait_for_port("127.0.0.1", args.listen_port, timeout=10):
logging.info(f"Mitmdump is started on port {args.listen_port}.")
notify("READY=1")
else:
print(f"Mitmdump instance did not start before the timeout of {args.timeout} seconds, consider increasing the timeout")
proc.terminate()
exit(1)
@ -224,14 +222,6 @@ in
'';
};
timeout = mkOption {
type = types.int;
default = 30;
description = ''
Time to wait for upstream to start and mitmdump to start.
'';
};
after = mkOption {
type = listOf str;
default = [ ];
@ -249,7 +239,7 @@ in
description = ''
Addons to enable on this mitmdump instance.
'';
example = lib.literalExpression "[ config.shb.mitmdump.addons.logger ]";
example = lib.literalExpression ''[ config.shb.mitmdump.addons.logger ]'';
};
extraArgs = mkOption {
@ -292,7 +282,7 @@ in
addons = lib.concatMapStringsSep " " (addon: "-s ${addon}") cfg'.enabledAddons;
extraArgs = lib.concatStringsSep " " cfg'.extraArgs;
in
"${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} --timeout ${toString cfg'.timeout} ${addons} ${extraArgs}";
"${lib.getExe mitmdumpScript} --listen_host ${cfg'.listenHost} --listen_port ${toString cfg'.listenPort} --upstream_host ${cfg'.upstreamHost} --upstream_port ${toString cfg'.upstreamPort} ${addons} ${extraArgs}";
};
requires = cfg'.after;
after = cfg'.after;

View file

@ -27,10 +27,8 @@ let
in
{
imports = [
../../lib/module.nix
../blocks/authelia.nix
../blocks/lldap.nix
../blocks/nginx.nix
];
options.shb.monitoring = {
@ -98,10 +96,10 @@ in
default = 1;
};
dashboards = lib.mkOption {
type = lib.types.listOf lib.types.path;
description = "Dashboards to provision under 'Self Host Blocks' folder.";
default = [ ];
provisionDashboards = lib.mkOption {
type = lib.types.bool;
description = "Provision Self Host Blocks dashboards under 'Self Host Blocks' folder.";
default = true;
};
contactPoints = lib.mkOption {
@ -110,35 +108,6 @@ in
default = [ ];
};
scrutiny = {
enable = lib.mkEnableOption "scrutiny service" // {
default = true;
};
subdomain = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = ''
If a string, this will be the subdomain under which the scrutiny web interface will be servced.
If null, the web interface will not be served and only the prometheus metrics will be accessible.
'';
default = "scrutiny";
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.scrutiny.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.monitoring.scrutiny.subdomain}.\${config.shb.monitoring.domain}";
internalUrl = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
internalUrlText = "https://127.0.0.1.\${config.services.scrutiny.settings.web.listen.port}";
};
};
};
};
adminPassword = lib.mkOption {
description = "Initial admin password.";
type = lib.types.submodule {
@ -278,38 +247,13 @@ in
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.monitoring.subdomain}.\${config.shb.monitoring.domain}";
internalUrl = "https://${cfg.subdomain}.${cfg.domain}";
internalUrlText = "https://\${config.shb.monitoring.subdomain}.\${config.shb.monitoring.domain}";
};
};
};
impermanence = lib.mkOption {
description = ''
Paths to save when using impermanence setup.
'';
type = lib.types.attrsOf lib.types.str;
default = {
fluent-bit = "/var/fluent-bit";
};
};
};
config = lib.mkMerge [
(lib.mkIf cfg.enable {
assertions = [
{
assertion = builtins.length cfg.contactPoints > 0;
assertion = (!(isNull cfg.smtp)) -> builtins.length cfg.contactPoints > 0;
message = "Must have at least one contact point for alerting";
}
];
@ -358,25 +302,14 @@ in
};
};
};
})
(lib.mkIf cfg.enable {
shb.monitoring.dashboards = [
./monitoring/dashboards/Errors.json
./monitoring/dashboards/Performance.json
./monitoring/dashboards/Scraping_Jobs.json
];
services.grafana.provision = {
dashboards.settings = lib.mkIf (cfg.dashboards != [ ]) {
dashboards.settings = lib.mkIf cfg.provisionDashboards {
apiVersion = 1;
providers = [
{
folder = "Self Host Blocks";
options.path = pkgs.symlinkJoin {
name = "dashboards";
paths = map (p: pkgs.runCommand "dashboard" { } "mkdir $out; cp ${p} $out") cfg.dashboards;
};
options.path = ./monitoring/dashboards;
allowUiUpdates = true;
disableDeletion = true;
}
@ -467,15 +400,10 @@ in
# any updates.
};
};
})
(lib.mkIf cfg.enable {
services.prometheus = {
enable = true;
port = cfg.prometheusPort;
globalConfig = {
scrape_interval = "15s";
};
};
services.loki = {
@ -585,62 +513,41 @@ in
};
};
# I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
services.fluent-bit = {
services.promtail = {
enable = true;
settings = {
service = {
flush = 1;
log_level = "info";
http_server = "true";
http_listen = "127.0.0.1";
http_port = 9080;
grace = 30;
configuration = {
server = {
http_listen_port = 9080;
grpc_listen_port = 0;
};
pipeline = {
inputs = [
{
name = "systemd";
positions.filename = "/tmp/positions.yaml";
# The asterisk appends the _SYSTEMD_UNIT to the prefix.
tag = "systemd.*";
client.url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/api/prom/push";
# Read logs from this systemd journal directory.
scrape_configs = [
{
job_name = "systemd";
journal = {
json = false;
max_age = "12h";
path = "/var/log/journal";
# Database file to keep track of the journald cursor.
db = "/var/fluent-bit/systemd.db";
# Start reading new entries. Skip entries already stored in journald.
read_from_tail = true;
# Max entries to lookback on start.
max_entries = 10000;
}
];
outputs = [
{
name = "loki";
match = "systemd.*";
host = "localhost";
port = config.services.loki.configuration.server.http_listen_port;
labels = lib.concatMapAttrsStringSep ", " (n: v: "${n}=${v}") {
job = "systemd-journal";
# matches = "_TRANSPORT=kernel";
labels = {
domain = cfg.domain;
hostname = config.networking.hostName;
job = "systemd-journal";
};
label_keys = "$unit";
}
];
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
graceLimit = "1m";
};
services.nginx = {
@ -660,9 +567,7 @@ in
};
};
};
})
(lib.mkIf cfg.enable {
services.prometheus.scrapeConfigs = [
{
job_name = "node";
@ -914,77 +819,5 @@ in
}
];
})
(lib.mkIf (cfg.enable && cfg.scrutiny.enable) {
services.scrutiny = {
enable = true;
openFirewall = false;
# This src includes Prometheus metrics exporter.
package = pkgs.scrutiny.overrideAttrs ({
src = pkgs.fetchFromGitHub {
owner = "ibizaman";
repo = "scrutiny";
rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
};
vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
});
settings = {
web = {
metrics.enabled = true; # Enables Prometheus exporter
listenHost = "127.0.0.1";
};
};
collector = {
enable = true;
};
};
services.prometheus.scrapeConfigs = [
{
job_name = "scrutiny";
metrics_path = "/api/metrics";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}" ];
labels = commonLabels;
}
];
}
];
shb.monitoring.dashboards = [
./monitoring/dashboards/Health.json
];
shb.nginx.vhosts = lib.mkIf (cfg.scrutiny.subdomain != null) [
(
{
inherit (cfg) domain ssl;
subdomain = cfg.scrutiny.subdomain;
upstream = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
autheliaRules = lib.optionals (cfg.sso.enable) [
{
domain = "${cfg.subdomain}.${cfg.domain}";
policy = cfg.sso.authorization_policy;
subject = [
"group:${cfg.ldap.userGroup}"
"group:${cfg.ldap.adminGroup}"
];
}
];
}
// lib.optionalAttrs cfg.sso.enable {
inherit (cfg.sso) authEndpoint;
}
)
];
})
];
}

View file

@ -1,872 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"panels": [],
"repeat": "hostname",
"title": "${hostname}",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"footer": {
"reducers": []
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 1
},
"id": 3,
"maxDataPoints": 400,
"options": {
"cellHeight": "sm",
"showHeader": true
},
"pluginVersion": "12.4.0",
"targets": [
{
"disableTextWrap": false,
"editorMode": "builder",
"expr": "node_os_info",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "OS Versions",
"transformations": [
{
"id": "labelsToFields",
"options": {
"keepLabels": [
"build_id",
"domain",
"hostname",
"id",
"instance",
"job",
"name",
"pretty_name",
"version",
"version_codename",
"version_id"
],
"mode": "columns"
}
},
{
"id": "groupBy",
"options": {
"fields": {
"Time": {
"aggregations": [
"firstNotNull"
],
"operation": "aggregate"
},
"pretty_name": {
"aggregations": [],
"operation": "groupby"
}
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 1
},
"id": 1,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"editorMode": "code",
"expr": "node_hwmon_temp_celsius{hostname=~\"$hostname\"}",
"instant": false,
"legendFormat": "{{chip}} - {{sensor}}",
"range": true,
"refId": "A"
}
],
"title": "Temperature",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisPlacement": "auto",
"fillOpacity": 70,
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 0
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 24,
"x": 0,
"y": 9
},
"id": 5,
"interval": "1m",
"maxDataPoints": 200,
"options": {
"colWidth": 1,
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"rowHeight": 0.8,
"showValue": "never",
"tooltip": {
"hideZeros": false,
"mode": "none",
"sort": "none"
}
},
"pluginVersion": "12.4.0",
"repeat": "hostname",
"repeatDirection": "h",
"targets": [
{
"editorMode": "code",
"expr": "node_zfs_zpool_state{hostname=~\"$hostname\", state=\"online\"} > 0",
"legendFormat": "{{zpool}} - {{state}}",
"range": true,
"refId": "A"
}
],
"title": "ZFS Pools",
"type": "status-history"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": 0
},
{
"color": "transparent",
"value": 604808
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 13
},
"id": 2,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"editorMode": "code",
"expr": "ssl_certificate_expiry_seconds",
"legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
"range": true,
"refId": "A"
}
],
"title": "Certificate Remaining Validity",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "years"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 13
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.4.0",
"targets": [
{
"editorMode": "builder",
"expr": "scrutiny_smart_power_on_hours{hostname=~\"$hostname\"} / (24 * 365)",
"legendFormat": "{{device_name}}",
"range": true,
"refId": "A"
}
],
"title": "Operating Years",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-YlRd"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 100,
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 21
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"width": 400
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.4.0",
"targets": [
{
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(hostname, domain, mountpoint, device) (node_filesystem_free_bytes{hostname=~\"$hostname\"})",
"fullMetaSearch": false,
"hide": true,
"includeNullMetadata": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"editorMode": "builder",
"expr": "sum by(hostname, domain, mountpoint, device) (node_filesystem_size_bytes{hostname=~\"$hostname\"})",
"hide": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "B"
},
{
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "(1 - $A / $B) * 100",
"refId": "Disk Full",
"type": "math"
}
],
"title": "Filesystem Disk Usage",
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "Time",
"mode": "outer"
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 21
},
"id": 9,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.4.0",
"targets": [
{
"editorMode": "builder",
"expr": "scrutiny_smart_power_on_hours{hostname=~\"$hostname\"}",
"legendFormat": "{{device_name}}",
"range": true,
"refId": "A"
}
],
"title": "Operating Years",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"footer": {
"reducers": []
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 29
},
"id": 8,
"options": {
"cellHeight": "sm",
"showHeader": true
},
"pluginVersion": "12.4.0",
"targets": [
{
"editorMode": "builder",
"exemplar": false,
"expr": "scrutiny_device_info{hostname=~\"$hostname\"}",
"format": "table",
"instant": true,
"legendFormat": "{{device_name}}",
"range": false,
"refId": "A"
}
],
"title": "Disk Info",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": true,
"__name__": true,
"domain": true,
"hostname": true,
"instance": true,
"job": true,
"wwn": false
},
"includeByName": {},
"indexByName": {},
"renameByName": {}
}
}
],
"type": "table"
}
],
"preload": false,
"schemaVersion": 42,
"tags": [],
"templating": {
"list": [
{
"current": {
"text": "baryum",
"value": "baryum"
},
"definition": "label_values(up,hostname)",
"name": "hostname",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(up,hostname)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"regexApplyTo": "value",
"type": "query"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Node Health",
"uid": "edhuvl28vpjwge",
"version": 25,
"weekStart": ""
}

View file

@ -17,13 +17,9 @@ This block sets up the monitoring stack for Self Host Blocks. It is composed of:
- Registration is enabled through SSO.
- Access through [subdomain](#blocks-monitoring-options-shb.monitoring.subdomain) using reverse proxy.
- Access through [HTTPS](#blocks-monitoring-options-shb.monitoring.ssl) using reverse proxy.
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#blocks-monitoring-usage-applicationdashboard)
- Out of the box integration with [Scrutiny](https://github.com/AnalogJ/scrutiny) service for Hard Drives monitoring. [Manual](#blocks-monitoring-usage-scrutiny)
## Usage {#blocks-monitoring-usage}
### Initial Configuration {#blocks-monitoring-usage-configuration}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
@ -38,16 +34,16 @@ The following snippet assumes a few blocks have been setup already:
subdomain = "grafana";
inherit domain;
contactPoints = [ "me@example.com" ];
adminPassword.result = config.shb.sops.secret."monitoring/admin_password".result;
secretKey.result = config.shb.sops.secret."monitoring/secret_key".result;
sso = {
enable = true;
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
sharedSecret.result = config.shb.sops.secret."monitoring/oidcSecret".result;
sharedSecretForAuthelia.result = config.shb.sops.secret."monitoring/oidcAutheliaSecret".result;
};
adminPassword.result = config.sops.secrets."monitoring/admin_password".result;
secretKey.result = config.sops.secrets."monitoring/secret_key".result;
sso = {
enable = true;
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
sharedSecret.result = config.shb.sops.secret.oidcSecret.result;
sharedSecretForAuthelia.result = config.shb.sops.secret.oidcAutheliaSecret.result;
};
};
shb.sops.secret."monitoring/admin_password".request = config.shb.monitoring.adminPassword.request;
@ -71,7 +67,7 @@ LDAP groups are created automatically.
### SMTP {#blocks-monitoring-usage-smtp}
I recommend adding an SMTP server configuration so you receive alerts by email:
I recommend adding a STMP server configuration so you receive alerts by email:
```nix
shb.monitoring.smtp = {
@ -114,42 +110,6 @@ You might for example want to update the metrics retention time with:
services.prometheus.retentionTime = "60d";
```
### Application Dashboard {#blocks-monitoring-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#blocks-monitoring-options-shb.monitoring.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Admin.services.Grafana = {
sortOrder = 10;
dashboard.request = config.shb.monitoring.dashboard.request;
};
}
```
There is also an integration for the scrutiny service, see next section.
### Scrutiny {#blocks-monitoring-usage-scrutiny}
Integration with the [Scrutiny](https://github.com/AnalogJ/scrutiny) service is enabled by default and setup automatically.
The web interface will be served under the [scrutiny.subdomain](#blocks-monitoring-options-shb.monitoring.scrutiny.subdomain) option.
If you don't want the web interface, set the option to `null`.
For integration with the [dashboard contract](contracts-dashboard.html):
```nix
{
shb.homepage.servicesGroups.Admin.services.Scrutiny = {
sortOrder = 11;
dashboard.request = config.shb.monitoring.scrutiny.dashboard.request;
};
}
```
## Provisioning {#blocks-monitoring-provisioning}
Self Host Blocks will create automatically the following resources:
@ -284,16 +244,6 @@ Graphs:
![Late SSL Jobs Alert Firing](./assets/alert_rules_LateSSL_1.png)
## Impermanence {#blocks-monitoring-impermanence}
To save the fluent-bit folder in an impermanence setup, add:
```nix
{
shb.zfs.datasets."safe/monitoring-fluent-bit".path = config.shb.jellyfin.impermanence.fluent-bit;
}
```
## Options Reference {#blocks-monitoring-options}
```{=include=} options

View file

@ -275,7 +275,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "(\n # Timer triggered at least once in the last 24h\n label_replace((\n time()\n -\n systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"}\n ) < 24*60*60, \"name\", \"$1.service\", \"name\", \"(.*).timer\")\n AND on(name)\n # At least one failure in the last 24h\n (\n max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"failed\"}[24h]) > 0\n )\n AND on(name)\n # No successes in the last 24h\n (\n max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"inactive\"}[24h]) == 0\n )\n)",
"expr": "(\n # No run in the last 24 hours\n systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"}\n ==\n systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"} offset 24h\n)\nOR\n(\n # All runs in last 24 hours failed\n label_replace((\n time()\n -\n systemd_timer_last_trigger_seconds{name=~\".*backup.*.timer\"}\n ) < 24*60*60, \"name\", \"$1.service\", \"name\", \"(.*).timer\")\n AND on(name)\n (\n max_over_time(systemd_unit_state{name=~\".*backup.*.service\", state=\"failed\"}[24h]) > 0\n )\n)",
"instant": false,
"interval": "",
"intervalMs": 15000,

View file

@ -31,9 +31,8 @@ let
};
upstream = lib.mkOption {
type = lib.types.nullOr lib.types.str;
type = lib.types.str;
description = "Upstream url to be protected.";
default = null;
example = "http://127.0.0.1:1234";
};
@ -49,40 +48,10 @@ let
default = [ ];
description = "Authelia rule configuration";
example = lib.literalExpression ''
[
# Protect /admin endpoint with 2FA
# and only allow access to admin users.
{
domain = "myapp.example.com";
policy = "two_factor";
subject = [ "group:service_admin" ];
resources = [
"^/admin"
];
}
# Leave /api endpoint open - assumes an API key is used to protect it.
{
domain = "myapp.example.com";
policy = "bypass";
resources = [
"^/api"
];
},
# Protect rest of app with 1FA
# and allow access to normal and admin users.
{
domain = "myapp.example.com";
policy = "one_factor";
subject = ["group:service_user"];
},
]
'';
};
phpForwardAuth = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Authelia rule configuration";
[{
policy = "two_factor";
subject = ["group:service_user"];
}]'';
};
extraConfig = lib.mkOption {
@ -164,58 +133,52 @@ in
sslCertificateKey = lib.mkIf (!(isNull c.ssl)) c.ssl.paths.key;
# Taken from https://github.com/authelia/authelia/issues/178
locations."/".extraConfig =
lib.optionalString (c.upstream != null) ''
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header X-Content-Type-Options nosniff;
add_header X-Frame-Options "SAMEORIGIN";
add_header X-XSS-Protection "1; mode=block";
add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
add_header X-Download-Options noopen;
add_header X-Permitted-Cross-Domain-Policies none;
locations."/".extraConfig = ''
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header X-Content-Type-Options nosniff;
add_header X-Frame-Options "SAMEORIGIN";
add_header X-XSS-Protection "1; mode=block";
add_header X-Robots-Tag "noindex, nofollow, nosnippet, noarchive";
add_header X-Download-Options noopen;
add_header X-Permitted-Cross-Domain-Policies none;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_cache_bypass $http_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_cache_bypass $http_upgrade;
proxy_pass ${c.upstream};
''
+ c.extraConfig
+ lib.optionalString (c.authEndpoint != null) ''
auth_request /authelia;
auth_request_set $user $upstream_http_remote_user;
auth_request_set $groups $upstream_http_remote_groups;
proxy_set_header X-Forwarded-User $user;
proxy_set_header X-Forwarded-Groups $groups;
# TODO: Are those needed?
# auth_request_set $name $upstream_http_remote_name;
# auth_request_set $email $upstream_http_remote_email;
# proxy_set_header Remote-Name $name;
# proxy_set_header Remote-Email $email;
# TODO: Would be nice to have this working, I think.
# set $new_cookie $http_cookie;
# if ($http_cookie ~ "(.*)(?:^|;)\s*example\.com\.session\.id=[^;]+(.*)") {
# set $new_cookie $1$2;
# }
# proxy_set_header Cookie $new_cookie;
proxy_pass ${c.upstream};
''
+ c.extraConfig
+ lib.optionalString (c.authEndpoint != null) ''
auth_request /authelia;
auth_request_set $user $upstream_http_remote_user;
auth_request_set $groups $upstream_http_remote_groups;
proxy_set_header X-Forwarded-User $user;
proxy_set_header X-Forwarded-Groups $groups;
# TODO: Are those needed?
# auth_request_set $name $upstream_http_remote_name;
# auth_request_set $email $upstream_http_remote_email;
# proxy_set_header Remote-Name $name;
# proxy_set_header Remote-Email $email;
# TODO: Would be nice to have this working, I think.
# set $new_cookie $http_cookie;
# if ($http_cookie ~ "(.*)(?:^|;)\s*example\.com\.session\.id=[^;]+(.*)") {
# set $new_cookie $1$2;
# }
# proxy_set_header Cookie $new_cookie;
auth_request_set $redirect $scheme://$http_host$request_uri;
error_page 401 =302 ${c.authEndpoint}?rd=$redirect;
error_page 403 = ${c.authEndpoint}/error/403;
'';
locations."~ \\.php$".extraConfig = lib.mkIf (c.phpForwardAuth) ''
fastcgi_param HTTP_X_FORWARDED_USER $user;
fastcgi_param HTTP_X_FORWARDED_GROUPS $groups;
auth_request_set $redirect $scheme://$http_host$request_uri;
error_page 401 =302 ${c.authEndpoint}?rd=$redirect;
error_page 403 = ${c.authEndpoint}/error/403;
'';
# Virtual endpoint created by nginx to forward auth requests.
locations."/authelia".extraConfig = lib.mkIf (c.authEndpoint != null) ''
locations."/authelia".extraConfig = lib.mkIf (!(isNull c.authEndpoint)) ''
internal;
proxy_pass ${c.authEndpoint}/api/verify;

View file

@ -1,211 +0,0 @@
# Nginx Block {#blocks-nginx}
Defined in [`/modules/blocks/nginx.nix`](@REPO@/modules/blocks/nginx.nix).
This block sets up a [Nginx](https://nginx.org/) instance.
It complements the upstream nixpkgs with some authentication and debugging improvements as shows in the Usage section.
## Usage {#blocks-nginx-usage}
### Access Logging {#blocks-nginx-usage-accesslog}
JSON access logging is enabled with the [`shb.nginx.accessLog`](#blocks-nginx-options-shb.nginx.accessLog) option:
```nix
{
shb.nginx.accessLog = true;
}
```
Looking at the systemd logs (`journalctl -fu nginx`) will show for example:
```json
nginx[969]: server nginx:
{
"remote_addr":"192.168.1.1",
"remote_user":"-",
"time_local":"29/Dec/2025:14:22:41 +0000",
"request":"POST /api/firstfactor HTTP/2.0",
"request_length":"264",
"server_name":"auth_example_com",
"status":"200",
"bytes_sent":"855",
"body_bytes_sent":"60",
"referrer":"-",
"user_agent":"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
"gzip_ration":"-",
"post":"{\x22username\x22:\x22charlie\x22,\x22password\x22:\x22CharliePassword\x22,\x22keepMeLoggedIn\x22:false,\x22targetURL\x22:\x22https://f.example.com/\x22,\x22requestMethod\x22:null}",
"upstream_addr":"127.0.0.1:9091",
"upstream_status":"200",
"request_time":"0.873",
"upstream_response_time":"0.873",
"upstream_connect_time":"0.001",
"upstream_header_time":"0.872"
}
```
This _will_ log the body of POST queries so it should only be enabled for debug logging.
### Debug Logging {#blocks-nginx-usage-debuglog}
Debug logging is enabled with the [`shb.nginx.debugLog`](#blocks-nginx-options-shb.nginx.debugLog) option:
```nix
{
shb.nginx.debugLog = true;
}
```
If enabled, it sets:
```
error_log stderr warn;
```
### Virtual Host Upstream Proxy {#blocks-nginx-usage-upstream}
Easy upstream proxy setup is done with the [`shb.nginx.vhosts.*.upstream`](#blocks-nginx-options-shb.nginx.vhosts._.upstream) option:
```nix
{
shb.nginx.vhosts = [
{
domain = "example.com";
subdomain = "mysubdomain";
upstream = "http://127.0.0.1:9090";
}
];
}
```
This will set also a few headers.
Some are shown here and others please see in the [nginx](@REPO@/modules/blocks/nginx.nix) module:
- `Host` = `$host`;
- `X-Real-IP` = `$remote_addr`;
- `X-Forwarded-For` = `$proxy_add_x_forwarded_for`;
- `X-Forwarded-Proto` = `$scheme`;
### Virtual Host SSL Generator Contract Integration {#blocks-nginx-usage-ssl}
This module integrates with the [SSL Generator Contract](./contracts-ssl.html)
to setup HTTPs with the [`shb.nginx.vhosts.*.ssl`](#blocks-nginx-options-shb.nginx.vhosts._.ssl) option:
```nix
{
shb.nginx.vhosts = [
{
domain = "example.com";
subdomain = "mysubdomain";
ssl = config.shb.certs.certs.letsencrypt.${domain};;
}
];
shb.certs.certs.letsencrypt.${domain} = {
inherit domain;
};
}
```
### Virtual Host SHB Forward Authentication {#blocks-nginx-usage-shbforwardauth}
For services provided by SelfHostBlocks that do not handle [OIDC integration][OIDC],
this block can provide [forward authentication][] which still allows the service
to still be protected by an SSO server.
[OIDC]: blocks-authelia.html#blocks-authelia-shb-oidc
The user could still be required to authenticate to the service itself,
although some services can automatically users authorized by Authelia.
[forward authentication]: https://doc.traefik.io/traefik/middlewares/http/forwardauth/
Integrating with this block is done with the following code:
```nix
shb.<services>.authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
```
### Virtual Host Forward Authentication {#blocks-nginx-usage-forwardauth}
Forward authentication is when Nginx talks with the SSO service directly
and the user is authenticated before reaching the upstream application.
The SSO service responds with the username, group and more information about the user.
This is then forwarded to the upstream application by Nginx.
Note that _every_ request is authenticated this way with the SSO server
so it involves more hops than a direct [OIDC integration](blocks-authelia.html#blocks-authelia-shb-oidc).
```nix
{
shb.nginx.vhosts = [
{
domain = "example.com";
subdomain = "mysubdomain";
authEndpoint = "authelia.example.com";
autheliaRules = [
[
# Protect /admin endpoint with 2FA
# and only allow access to admin users.
{
domain = "myapp.example.com";
policy = "two_factor";
subject = [ "group:service_admin" ];
resources = [
"^/admin"
];
}
# Leave /api endpoint open - assumes an API key is used to protect it.
{
domain = "myapp.example.com";
policy = "bypass";
resources = [
"^/api"
];
},
# Protect rest of app with 1FA
# and allow access to normal and admin users.
{
domain = "myapp.example.com";
policy = "one_factor";
subject = ["group:service_user"];
},
]
];
}
];
}
```
If PHP is used with fastCGI,
extra headers must be added by enabling the [`shb.nginx.vhosts.*.phpForwardAuth`](#blocks-nginx-options-shb.nginx.vhosts._.phpForwardAuth) option.
### Virtual Host Extra Config {#blocks-nginx-usage-extraconfig}
To add extra configuration to a virtual host,
use the [`shb.nginx.vhosts.*.extraConfig`](#blocks-nginx-options-shb.nginx.vhosts._.extraConfig) option.
This can be used to add headers, for example:
```nix
{
shb.nginx.vhosts = [
{
domain = "example.com";
subdomain = "mysubdomain";
extraConfig = ''
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
'';
}
];
}
```
## Options Reference {#blocks-nginx-options}
```{=include=} options
id-prefix: blocks-nginx-options-
list-id: selfhostblocks-block-nginx-options
source: @OPTIONS_JSON@
```

View file

@ -73,7 +73,7 @@ in
backupName = "postgres.sql";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
'';
restoreCmd = ''
@ -162,7 +162,7 @@ in
{ username, passwordFile, ... }:
''
password := trim(both from replace(pg_read_file('${passwordFile}'), E'\n', '''));
EXECUTE format('ALTER ROLE "${username}" WITH PASSWORD '''%s''';', password);
EXECUTE format('ALTER ROLE ${username} WITH PASSWORD '''%s''';', password);
'';
cfgsWithPasswords = builtins.filter (cfg: cfg.passwordFile != null) ensureCfgs;
in

View file

@ -6,46 +6,17 @@ This block sets up a [PostgreSQL][] database.
[postgresql]: https://www.postgresql.org/
Compared to the upstream nixpkgs module, this module also sets up:
## Tests {#blocks-postgresql-tests}
- Enabling TCP/IP login and also accepting password authentication from localhost with [`shb.postgresql.enableTCPIP`](#blocks-postgresql-options-shb.postgresql.enableTCPIP).
- Enhance the `ensure*` upstream option by setting up a database's password from a password file with [`shb.postgresql.ensures`](#blocks-postgresql-options-shb.postgresql.ensures).
- Debug logging with `auto_explain` and `pg_stat_statements` with [`shb.postgresql.debug`](#blocks-postgresql-options-shb.postgresql.debug).
Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
## Usage {#blocks-postgresql-usage}
### Ensure User and Database {#blocks-postgresql-ensures}
Ensure a database and user exists:
```nix
shb.postgresql.ensures = [
{
username = "firefly-iii";
database = "firefly-iii";
}
];
```
Also set up the database password from a file path:
```nix
shb.postgresql.ensures = [
{
username = "firefly-iii";
database = "firefly-iii";
passwordFile = "/run/secrets/firefly-iii_db_password";
}
];
```
### Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
## Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
This block can be backed up using the [database backup](contracts-databasebackup.html) contract.
Contract integration tests are defined in [`/test/contracts/databasebackup.nix`](@REPO@/test/contracts/databasebackup.nix).
#### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
```nix
{
@ -59,10 +30,6 @@ Contract integration tests are defined in [`/test/contracts/databasebackup.nix`]
}
```
## Tests {#blocks-postgresql-tests}
Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
## Options Reference {#blocks-postgresql-options}
```{=include=} options

View file

@ -63,7 +63,7 @@ let
mode = "0400";
owner = config.request.user;
ownerText = "[shb.restic.${prefix}.<name>.request.user](#blocks-restic-options-shb.restic.${prefix}._name_.request.user)";
restartUnits = [ "${fullName name config.settings.repository}.service" ];
restartUnits = [ (fullName name config.settings.repository) ];
restartUnitsText = "[ [shb.restic.${prefix}.<name>.settings.repository](#blocks-restic-options-shb.restic.${prefix}._name_.settings.repository) ]";
};
};
@ -102,7 +102,7 @@ let
OnCalendar = "daily";
Persistent = true;
};
description = "When to run the backup. See {manpage}`systemd.timer(5)` for details.";
description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
example = {
OnCalendar = "00:05";
RandomizedDelaySec = "5h";
@ -149,14 +149,9 @@ in
{
imports = [
../../lib/module.nix
../blocks/monitoring.nix
];
options.shb.restic = {
enableDashboard = lib.mkEnableOption "the Backups SHB dashboard" // {
default = true;
};
instances = mkOption {
description = "Files to backup following the [backup contract](./shb.contracts-backup.html).";
default = { };
@ -413,13 +408,7 @@ in
nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
enable = true;
wantedBy = [ "multi-user.target" ];
# Purposely not a oneshot systemd service otherwise
# the service waits on the completion the backup before deactivating.
# This seems like a nice property at first but there is one annoying
# edge case when deploying. If a backup job somehow is started when
# the deploy happens, the deploy will wait on the service to finish
# before considering the deploy done. And worse, it will consider the
# deploy as failed if the backup fails, which is not what you want.
serviceConfig.Type = "oneshot";
script = (
shb.replaceSecrets {
userConfig = instance.settings.repository.secrets // {
@ -440,16 +429,38 @@ in
let
mkResticBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"RESTIC_REPOSITORY"
"RESTIC_PASSWORD_FILE"
];
secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
set +a
echo "Will restore archive 'latest'"
sudocmd ${pkgs.restic}/bin/restic restore latest --target /
'';
};
in
flatten (mapAttrsToList mkResticBinary cfg.instances);
@ -459,26 +470,42 @@ in
let
mkResticBinary =
name: instance:
shb.contracts.backup.mkRestoreScript {
pkgs.writeShellApplication {
name = fullName name instance.settings.repository;
user = instance.request.user;
sudoPreserveEnvs = [
"RESTIC_REPOSITORY"
"RESTIC_PASSWORD_FILE"
];
secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
set +a
echo "Will restore archive 'latest'"
sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}"
'';
};
in
flatten (mapAttrsToList mkResticBinary cfg.databases);
}
(lib.mkIf (cfg.enableDashboard && (cfg.instances != { } || cfg.databases != { })) {
shb.monitoring.dashboards = [
./backup/dashboard/Backups.json
];
})
]
);
}

View file

@ -3,7 +3,6 @@
Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
This block sets up a backup job using [Restic][].
It is heavily based on the nixpkgs Restic module.
[restic]: https://restic.net/
@ -51,7 +50,7 @@ shb.restic.instances."myservice" = {
settings = {
enable = true;
passphrase.result = config.shb.sops.secret."passphrase".result;
passphrase.result = shb.sops.secret."passphrase".result;
repository = {
path = "/srv/backups/myservice";
@ -72,7 +71,7 @@ shb.restic.instances."myservice" = {
};
shb.sops.secret."passphrase".request =
config.shb.restic.instances."myservice".settings.passphrase.request;
shb.restic.instances."myservice".settings.passphrase.request;
```
### One folder backed up with contract {#blocks-restic-usage-provider-contract}
@ -83,12 +82,12 @@ the snippet above becomes:
```nix
shb.restic.instances."myservice" = {
request = config.myservice.backup.request;
request = config.myservice.backup;
settings = {
enable = true;
passphrase.result = config.shb.sops.secret."passphrase".result;
passphrase.result = shb.sops.secret."passphrase".result;
repository = {
path = "/srv/backups/myservice";
@ -109,7 +108,7 @@ shb.restic.instances."myservice" = {
};
shb.sops.secret."passphrase".request =
config.shb.restic.instances."myservice".settings.passphrase.request;
shb.restic.instances."myservice".settings.passphrase.request;
```
### One folder backed up to S3 {#blocks-restic-usage-provider-remote}
@ -230,44 +229,11 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
## Maintenance {#blocks-restic-maintenance}
### Manual Backup {#blocks-restic-maintenance-manuql}
To launch a backup manually, just run:
```bash
systemctl start <systemd-service-name>
```
You can easily discover the systemd service name you need by either listing the units:
```bash
systemctl list-units 'restic*'
```
Or by autocompleting the unit name with `<TAB>`:
```bash
systemctl start restic<TAB><TAB>
```
Note that the systemd services are of `Type=simple` which means the systemd service
will not wait for the backup completion to terminate.
If you want instead to wait for the backup to complete, use the `--wait` flag:
```bash
systemctl start --wait <systemd-service-name>
```
### Restore {#blocks-restic-maintenance-restore}
One command-line helper is provided per backup instance and repository pair which allows to:
- list snapshots: `<script> snapshots`
- to restore a snapshot: `<script> restore <snapshot>`
One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
The restore script has all the secrets needed to access the repo,
it will run `sudo` automatically
and the user running it needs to have correct permissions for privilege escalation.
and the user running it needs to have correct permissions for privilege escalation
In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:

View file

@ -1,164 +0,0 @@
{
config,
lib,
pkgs,
shb,
utils,
...
}:
let
cfg = config.shb.sanoid;
restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore";
backupScriptBase = "sanoid";
restoreScript =
name:
pkgs.writers.writePython3Bin (restoreScriptName name)
{
flakeIgnore = [ "E501" ];
}
''
import argparse
import subprocess
import sys
dataset = "${name}"
class ZFSError(Exception):
pass
def run_command(cmd: list[str]) -> str:
try:
result = subprocess.run(
cmd,
check=True,
text=True,
capture_output=True,
)
return result.stdout.strip()
except subprocess.CalledProcessError as e:
raise ZFSError(
f"Command failed: {' '.join(cmd)}\n"
f"Exit code: {e.returncode}\n"
f"stderr: {e.stderr.strip()}"
) from e
def list_snapshots() -> None:
"""List all ZFS snapshots."""
output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset])
if not output:
return []
return output.splitlines()
def restore_snapshot(snapshot: str) -> None:
"""Rollback to a given snapshot."""
if not snapshot:
raise ValueError("Snapshot name must not be empty")
print(f"Rolling back to snapshot: {snapshot}")
run_command(["zfs", "rollback", "-r", snapshot])
print("Rollback completed successfully.")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=f"Restore script for {dataset}")
subparsers = parser.add_subparsers(dest="command", required=True)
subparsers.add_parser(
"snapshots",
help="List all ZFS snapshots",
)
restore_parser = subparsers.add_parser(
"restore",
help="Rollback to a specific snapshot",
)
restore_parser.add_argument(
"snapshot",
help="Snapshot name (e.g. pool/dataset@snapname)",
)
return parser
def main():
parser = build_parser()
args = parser.parse_args()
try:
if args.command == "snapshots":
snapshots = list_snapshots()
for s in snapshots:
print(s)
elif args.command == "restore":
restore_snapshot(args.snapshot)
else:
parser.print_help()
sys.exit(1)
except ZFSError as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"Unexpected error: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()
'';
in
{
imports = [
../../lib/module.nix
];
options.shb.sanoid.backup = lib.mkOption {
description = "Sanoid prodiver for file backup contract";
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = shb.contracts.backup.mkProvider {
resultCfg = {
restoreScript = restoreScriptName name;
restoreScriptText = restoreScriptName "<name>";
backupService = "${backupScriptBase}.service";
backupServiceText = "${backupScriptBase}.service";
};
settings = lib.mkOption {
description = "Options passed to the `services.sanoid.datasets.<name>` option.";
default = { };
type = lib.types.attrsOf lib.types.anything;
};
};
}
)
);
};
config = lib.mkIf (cfg.backup != { }) {
services.sanoid.enable = true;
services.sanoid.datasets =
let
mkDataset = name: cfg': {
inherit name;
value = cfg'.settings;
};
in
lib.mapAttrs' mkDataset cfg.backup;
environment.systemPackages = map restoreScript (lib.attrNames cfg.backup);
};
}

View file

@ -1,97 +0,0 @@
# Sanoid Block {#blocks-sanoid}
Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix):
```nix
{
imports = [
inputs.selfhostblocks.nixosModules.sanoid
];
}
```
## Provider Contracts {#blocks-sanoid-contract-provider}
This block provides the following contracts:
- [dataset backup contract](contracts-datasetbackup.html) under the [`shb.sanoid.backup`][backup] option.
It is tested with the [generic contract tests][backup contract tests].
[backup]: #blocks-sanoid-options-shb.sanoid.backup
[backup contract tests]: @REPO@/test/contracts/backup.nix
## Usage {#blocks-sanoid-usage}
Sanoid uses templates to know when snapshots should be kept or pruned.
### Default Template {#blocks-sanoid-usage-default-template}
Backup a dataset using the default Sanoid template:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.sanoid.backup."root/home" = {
request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
};
}
```
This uses the dataset backup contract which is exposed through the ZFS module's [`shb.zfs.pools.<name>.datasets.<name>.datasetBackup`](blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup) option.
### Custom Template {#blocks-sanoid-usage-custom-template}
Create a custom template and use it:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
services.sanoid.templates."yearly" = {
hourly = 10;
daily = 3;
monthly = 3;
yearly = 2;
};
shb.sanoid.backup."root/home" = {
request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
template = "yearly";
};
}
```
Note we use the upstream `services.sanoid.templates` option to define the templates.
### Without contract {#blocks-sanoid-usage-without-contract}
To backup a dataset that does not provide the dataset backup contract,
we can just set the request manually:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.sanoid.backup."root/home" = {
request.dataset = "root/home";
};
}
```
Note the attr name under the `shb.sanoid.backup` option does not
set the dataset name.
## Options Reference {#blocks-sanoid-options}
```{=include=} options
id-prefix: blocks-sanoid-options-
list-id: selfhostblocks-blocks-sanoid-options
source: @OPTIONS_JSON@
```

View file

@ -13,7 +13,7 @@ to adapt it to the [secret contract](./contracts-secret.html).
This block provides the following contracts:
- [secret contract][] under the [`shb.sops.secret`][secret] option.
- [secret contract][] under the [`shb.sops.secrets`][secret] option.
It is not yet tested with [contract tests][secret contract tests] but it is used extensively on several machines.
[secret]: #blocks-sops-options-shb.sops.secret
@ -21,7 +21,7 @@ This block provides the following contracts:
[secret contract tests]: @REPO@/test/contracts/secret.nix
As requested by the contract, when asking for a secret with the `shb.sops` module,
the path where the secret will be located can be found under the [`shb.sops.secret.<name>.result`][result] option.
the path where the secret will be located can be found under the [`shb.sops.secrets.<name>.result`][result] option.
[result]: #blocks-sops-options-shb.sops.secret._name_.result

View file

@ -21,7 +21,6 @@ in
{
imports = [
../../lib/module.nix
./monitoring.nix
];
options.shb.certs = {
@ -32,9 +31,6 @@ in
type = lib.types.str;
default = "shb-ca-bundle.service";
};
enableDashboard = lib.mkEnableOption "the SSL SHB dashboard" // {
default = true;
};
cas.selfsigned = lib.mkOption {
description = "Generate a self-signed Certificate Authority.";
default = { };
@ -399,25 +395,21 @@ in
crl_signing_key
EOF
keyfile="${caCfg.paths.key}"
keydir="$(dirname -- "$keyfile")"
mkdir -p "$keydir"
[ -f "$keyfile" ] || ${pkgs.gnutls}/bin/certtool \
--generate-privkey \
--key-type rsa \
--sec-param High \
--outfile "$keyfile"
chmod 666 "$keyfile"
mkdir -p "$(dirname -- "${caCfg.paths.key}")"
${pkgs.gnutls}/bin/certtool \
--generate-privkey \
--key-type rsa \
--sec-param High \
--outfile ${caCfg.paths.key}
chmod 666 ${caCfg.paths.key}
certfile="${caCfg.paths.cert}"
certdir="$(dirname -- "$certfile")"
mkdir -p "$certdir"
[ -f "$certfile" ] || ${pkgs.gnutls}/bin/certtool \
--generate-self-signed \
--load-privkey "$keyfile" \
--template ca.template \
--outfile "$certfile"
chmod 666 "$certfile"
mkdir -p "$(dirname -- "${caCfg.paths.cert}")"
${pkgs.gnutls}/bin/certtool \
--generate-self-signed \
--load-privkey ${caCfg.paths.key} \
--template ca.template \
--outfile ${caCfg.paths.cert}
chmod 666 ${caCfg.paths.cert}
'';
}
) cfg.cas.selfsigned;
@ -431,8 +423,6 @@ in
script = ''
mkdir -p /etc/ssl/certs
# This file is automatically idempotent since the source files used are idempotent.
rm -f /etc/ssl/certs/ca-bundle.crt
rm -f /etc/ssl/certs/ca-certificates.crt
@ -462,8 +452,8 @@ in
let
extraDnsNames = lib.strings.concatStringsSep "\n" (map (n: "dns_name = ${n}") certCfg.extraDomains);
chmod = cert: ''
chown root:${certCfg.group} "${cert}"
chmod 640 "${cert}"
chown root:${certCfg.group} ${cert}
chmod 640 ${cert}
'';
in
''
@ -480,27 +470,23 @@ in
signing_key
EOF
keyfile="${certCfg.paths.key}"
keydir="$(dirname -- "$keyfile")"
mkdir -p "$keydir"
[ -f "$keyfile" ] || ${pkgs.gnutls}/bin/certtool \
--generate-privkey \
--key-type rsa \
--sec-param High \
--outfile "$keyfile"
${chmod "$keyfile"}
mkdir -p "$(dirname -- "${certCfg.paths.key}")"
${pkgs.gnutls}/bin/certtool \
--generate-privkey \
--key-type rsa \
--sec-param High \
--outfile ${certCfg.paths.key}
${chmod certCfg.paths.key}
certfile="${certCfg.paths.cert}"
certdir="$(dirname -- "$certfile")"
mkdir -p "$certdir"
[ -f "$certfile" ] || ${pkgs.gnutls}/bin/certtool \
--generate-certificate \
--load-privkey "$keyfile" \
--load-ca-privkey "${certCfg.ca.paths.key}" \
--load-ca-certificate "${certCfg.ca.paths.cert}" \
--template server.template \
--outfile "$certfile"
${chmod "$certfile"}
mkdir -p "$(dirname -- "${certCfg.paths.cert}")"
${pkgs.gnutls}/bin/certtool \
--generate-certificate \
--load-privkey ${certCfg.paths.key} \
--load-ca-privkey ${certCfg.ca.paths.key} \
--load-ca-certificate ${certCfg.ca.paths.cert} \
--template server.template \
--outfile ${certCfg.paths.cert}
${chmod certCfg.paths.cert}
'';
postStart = lib.optionalString (certCfg.reloadServices != [ ]) ''
@ -551,7 +537,7 @@ in
// lib.optionalAttrs (certCfg.dnsProvider != null) {
inherit (certCfg) dnsProvider dnsResolver;
inherit (certCfg) group reloadServices;
environmentFile = certCfg.credentialsFile;
credentialsFile = certCfg.credentialsFile;
};
}
]
@ -676,10 +662,5 @@ in
in
optionals (cfg.certs.letsencrypt != { }) (flatten (mapAttrsToList scrapeCfg cfg.certs.letsencrypt));
}
(lib.mkIf (cfg.enableDashboard && (cfg.certs.selfsigned != { } || cfg.certs.letsencrypt != { })) {
shb.monitoring.dashboards = [
./ssl/dashboard/SSL.json
];
})
];
}

View file

@ -2,258 +2,83 @@
config,
pkgs,
lib,
shb,
utils,
...
}:
let
cfg = config.shb.zfs;
datasets =
if cfg.snapshotBeforeActivation.datasets != null then
cfg.snapshotBeforeActivation.datasets
else
config.boot.zfs.extraPools;
in
{
imports = [
../../lib/module.nix
];
options.shb.zfs = {
pools = lib.mkOption {
defaultPoolName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "ZFS pool name datasets should be created on if no pool name is given in the dataset.";
};
datasets = lib.mkOption {
description = ''
Attrset of ZFS pools under which datasets will be created.
ZFS Datasets.
The ZFS pools are not managed by this module, they should already exist.
Each entry in the attrset will be created and mounted in the given path.
The attrset name is the dataset name.
Each pool named here will be added to the [`boot.zfs.extraPools`](https://search.nixos.org/options?channel=unstable&include_modular_service_options=0&include_nixos_options=1&query=boot.zfs.extrapools&show=option:boot.zfs.extraPools) option.
This block implements the following contracts:
- mount
'';
default = { };
example = lib.literalExpression ''
shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
'';
type = lib.types.attrsOf (
lib.types.submodule {
options = {
datasets = lib.mkOption {
description = ''
ZFS Datasets.
enable = lib.mkEnableOption "shb.zfs.datasets";
Each entry in the attrset will be created and mounted in the given path.
The attrset name is the dataset name.
poolName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "ZFS pool name this dataset should be created on. Overrides the defaultPoolName.";
};
This block implements the following contracts:
- mount
'';
default = { };
example = lib.literalExpression ''
shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
'';
type = lib.types.attrsOf (
lib.types.submodule (
{ config, name, ... }:
{
options = {
enable = lib.mkEnableOption "shb.zfs.datasets";
path = lib.mkOption {
type = lib.types.str;
description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
};
mode = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix mode to apply to the dataset root folder.";
default = null;
example = "ug=rwx,g+s";
};
owner = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix user to apply to the dataset root folder.";
default = null;
example = "syncthing";
};
group = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "If non null, unix group to apply to the dataset root folder.";
default = null;
example = "syncthing";
};
defaultACLs = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = ''
If non null, default ACL to set on the dataset root folder.
Executes "setfacl -d -m $acl $path"
'';
default = null;
example = "g:syncthing:rwX";
};
after = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
Order creating this dataset after the mentioned ones.
This only works with datasets managed by this module.
Use the name of the dataset without the pool name.
'';
default = [ ];
example = lib.literalExpression ''
[
"backup"
]
'';
};
backup = lib.mkOption {
description = ''
Backup contract consumer configuration.
This contract will backup the files inside the dataset.
'';
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = if config.owner == null then "root" else config.owner;
sourceDirectories = [
config.path
];
sourceDirectoriesText = ''
[
shb.zfs.pools.<name>.datasets.<name>.path
]
'';
};
};
};
datasetbackup = lib.mkOption {
description = ''
ZFS dataset backup contract configuration.
This contract will take snaphots of the dataset.
'';
type = lib.types.submodule {
options = shb.contracts.datasetbackup.mkRequester {
dataset = name;
};
};
};
};
}
)
);
path = lib.mkOption {
type = lib.types.str;
description = "Path this dataset should be mounted on.";
};
};
}
);
};
snapshotBeforeActivation = {
enable = lib.mkOption {
type = lib.types.bool;
description = "Take a snapshot of all datasets before activation";
default = false;
};
template = lib.mkOption {
type = lib.types.str;
description = "Bash command to generate the snapshot name. $1 is the path to the new generation.";
default = ''pre-$(date --utc '+%y%m%dT%H%M%S')-$(basename "$1")'';
};
datasets = lib.mkOption {
type = lib.types.nullOr (lib.types.listOf lib.types.str);
description = ''
Defines all datasets to take a snapshot of.
If set to `null`, the default, take the list of datasets from `config.boot.zfs.extraPools`.
The snapshots are not recursive unless specificed in the `recursive` option.
'';
};
recursive = lib.mkOption {
type = lib.types.bool;
description = "If true, take snapshots recurisevly on the given datasets.";
default = false;
};
};
};
# The implementation is greatly inspired by https://discourse.nixos.org/t/configure-zfs-filesystems-after-install/48633/3
config = {
boot.zfs.extraPools = lib.uniqueStrings (builtins.attrNames cfg.pools);
assertions = [
{
assertion =
lib.any (x: x.poolName == null) (lib.mapAttrsToList (n: v: v) cfg.datasets)
-> cfg.defaultPoolName != null;
message = "Cannot have both datasets.poolName and defaultPoolName set to null";
}
];
systemd.services =
system.activationScripts = lib.mapAttrs' (
name: cfg':
let
mkPool =
poolName: poolCfg: lib.listToAttrs (lib.mapAttrsToList (mkDataset poolName) poolCfg.datasets);
mkDataset =
poolName: name: cfg':
let
dataset = poolName + "/" + name;
in
lib.attrsets.nameValuePair "zfs-create-${utils.escapeSystemdPath dataset}" {
# oneshot is used to make the systemd service wait on completion of the script.
serviceConfig.Type = "oneshot";
unitConfig.DefaultDependencies = false;
requiredBy = [ "local-fs.target" ];
before = [ "local-fs.target" ];
after = [
"zfs-import-${poolName}.service"
"zfs-mount.service"
]
++ map (n: "zfs-create-${poolName}-${n}.service") cfg'.after;
unitConfig.ConditionPathIsMountPoint = lib.mkIf (cfg'.path != "none") "!${cfg'.path}";
script = ''
${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
|| ${pkgs.zfs}/bin/zfs create \
-o mountpoint=none \
${dataset} || :
[ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
|| ${pkgs.zfs}/bin/zfs set \
mountpoint="${cfg'.path}" \
${dataset}
''
+ lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
chmod "${cfg'.mode}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
chown "${cfg'.owner}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
chown :"${cfg'.group}" "${cfg'.path}"
''
+ lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
'';
};
mergeAttrs = lib.foldl lib.mergeAttrs { };
dataset = (if cfg'.poolName != null then cfg'.poolName else cfg.defaultPoolName) + "/" + name;
in
mergeAttrs (lib.mapAttrsToList mkPool cfg.pools);
lib.attrsets.nameValuePair "zfsCreate-${name}" {
text = ''
${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
|| ${pkgs.zfs}/bin/zfs create \
-o mountpoint=none \
${dataset} || :
system.preSwitchChecks."shb-zfs-snapshot" = lib.mkIf cfg.snapshotBeforeActivation.enable (
''
name="${cfg.snapshotBeforeActivation.template}"
''
+ (
let
recursiveFlag = lib.optionalString cfg.snapshotBeforeActivation.recursive "-r";
in
lib.concatMapStringsSep "\n" (ds: ''
echo "Taking ZFS snapshot of ${ds}@$name"
zfs snapshot ${recursiveFlag} ${ds}@"$name"
'') (lib.uniqueStrings datasets)
)
);
[ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
|| ${pkgs.zfs}/bin/zfs set \
mountpoint=${cfg'.path} \
${dataset}
'';
}
) cfg.datasets;
};
}

View file

@ -1,138 +0,0 @@
# ZFS Block {#blocks-zfs}
Defined in [`/modules/blocks/zfs.nix`](@REPO@/modules/blocks/zfs.nix):
```nix
{
inputs,
...
}:
{
imports = [
inputs.selfhostblocks.nixosModules.zfs
];
}
```
This block creates ZFS datasets, optionally mounts them and sets permissions on the mount point.
It also enables taking snapshots on activation.
## Features {#blocks-zfs-features}
- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
- Backup of the files in the dataset [`shb.zfs.<name>.backup`][backup] through the [backup contract](./contracts-backup.html).
- Backup of the dataset itself [`shb.zfs.<name>.datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html).
- Take a snapshot [before activating or deploying][activationsnapshot] allowing a safe rollback.
[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup
[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup
[activationsnapshot]: #blocks-zfs-options-shb.zfs.snapshotBeforeActivation.enable
## Usage {#blocks-zfs-usage}
Create a dataset at `root/safe/users` mounted on `/var/lib/nixos`:
```nix
shb.zfs.pools.root.datasets."safe/users".path = "/var/lib/nixos";
```
Create a dataset at `backup/syncoid` but do not mount it:
```nix
shb.zfs.pools.backup.datasets."syncoid".path = "none";
```
Create a dataset at `root/syncthing` and set custom permissions and ACL.
Permission and ACL are only enforced for the mount point.
```nix
shb.zfs.pools.root.datasets."syncthing" = {
path = "/srv/syncthing";
mode = "ug=rwx,g+s,o=";
owner = "syncthing";
group = "syncthing";
defaultACLs = "g:syncthing:rwX";
};
```
### Backup dataset {#blocks-zfs-usage-backup-dataset}
To backup the dataset directly, use the [dataset backup contract](contracts-datasetbackup.html).
For example, with the sanoid module as the dataset backup contract provider:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.sanoid.backup."root/home" = {
request = shb.zfs.pools.root.datasets.home.datasetBackup.request;
template = "yearly";
};
}
```
See the [Sanoid block](blocks-sanoid.html) for more examples.
### Backup files {#blocks-zfs-usage-backup-files}
To backup the files in the dataset, use the [file backup contract](contracts-backup.html).
For example, with the restic module as the file backup contract provider:
```nix
{
shb.zfs.pools.root.datasets.home = {
path = "/home";
};
shb.restic.instances."myservice" = {
request = shb.zfs.pools.root.datasets.home.backup.request;
};
}
```
See the [Restic block](blocks-restic.html) for more examples.
### Snapshot before activation {#blocks-zfs-usage-snapshot-before-activation}
To take a snapshot of all datasets before activating a new configuration:
```nix
{
shb.zfs.snapshotBeforeActivation.enable = true;
}
```
This does not take a recursive snapshot by default, to do it recursively, do:
```nix
{
shb.zfs.snapshotBeforeActivation = {
enable = true;
recursive = true;
};
}
```
We did not specify datasets manually, which means the datasets list comes from `boot.zfs.extraPools`.
To specify the datasets manually, you can use:
```nix
{
shb.zfs.snapshotBeforeActivation = {
enable = true;
datasets = [ "root" "root/var" ];
};
}
```
## Options Reference {#blocks-zfs-options}
```{=include=} options
id-prefix: blocks-zfs-options-
list-id: selfhostblocks-block-zfs-options
source: @OPTIONS_JSON@
```

View file

@ -1,9 +1,4 @@
{
lib,
shb,
pkgs,
...
}:
{ lib, shb, ... }:
let
inherit (lib)
concatStringsSep
@ -192,14 +187,12 @@ in
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
<snapshot 1> <metadata>
<snapshot 2> <metadata>
```
And restore the database with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
```
'';
type = str;
@ -229,56 +222,4 @@ in
};
};
};
passthru.mkRestoreScript =
{
name,
user,
sudoPreserveEnvs ? [ ],
secretsFile,
restoreCmd,
listCmd,
}:
pkgs.writeShellApplication {
inherit name;
text = ''
usage() {
echo "$0 snapshots"
echo "$0 restore <snapshot>"
}
sudocmd() {
sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
}
loadenv() {
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "${secretsFile}")
set +a
}
if [ "$1" = "restore" ]; then
shift
snapshot="$1"
loadenv
echo "Will restore snapshot '$snapshot'"
sudocmd sh -c "${restoreCmd}"
elif [ "$1" = "snapshots" ]; then
shift
loadenv
sudocmd sh -c "${listCmd}"
else
usage
exit 1
fi
'';
};
}

View file

@ -20,22 +20,25 @@ source: @OPTIONS_JSON@
## Usage {#contract-backup-usage}
A service that can be backed up will provide a `backup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
Here is an example module defining such a `backup` option,
which defines what directories to backup (`sourceDirectories`)
and the user to backup with (`user`).
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define what directories to backup,
the user to backup with
and possibly hooks to run before or after the backup job runs.
Here is an example module defining such a `backup` option:
```nix
{
options = {
myservice.backup = mkOption {
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = "nextcloud";
sourceDirectories = [
"/var/lib/nextcloud"
];
};
type = contracts.backup.request;
default = {
user = "myservice";
sourceDirectories = [
"/var/lib/myservice"
];
};
};
};
@ -45,13 +48,13 @@ and the user to backup with (`user`).
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `backupService` option
and that one can create multiple backup instances under `backupService.instances`.
Let's assume such a module is available under the `backupservice` option
and that one can create multiple backup instances under `backupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
backupService.instances.myservice = {
request = myservice.backup.request;
backupservice.instances.myservice = {
request = myservice.backup;
settings = {
enable = true;
@ -60,17 +63,17 @@ backupService.instances.myservice = {
path = "/srv/backup/myservice";
};
# ... Other options specific to backupService like scheduling.
# ... Other options specific to backupservice like scheduling.
};
};
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `backupService`:
Thanks to using contracts, this can be made easily either with the same `backupservice`:
```nix
backupService.instances.myservice_2 = {
request = myservice.backup.request;
backupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
@ -82,13 +85,12 @@ backupService.instances.myservice_2 = {
};
```
Or with another module `backupService_2`!
Or with another module `backupservice_2`!
## Providers of the Backup Contract {#contract-backup-providers}
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html).
- [ZFS block](blocks-zfs.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
## Requester Blocks and Services {#contract-backup-requesters}

View file

@ -21,8 +21,8 @@ in
"/opt/files/A"
"/opt/files/B"
],
settings ? { ... }: { }, # { filesRoot, config } -> attrset
extraConfig ? null, # { filesRoot, username, config } -> attrset
settings, # { repository, config } -> attrset
extraConfig ? null, # { username, config } -> attrset
}:
shb.test.runNixOSTest {
inherit name;
@ -40,7 +40,7 @@ shb.test.runNixOSTest {
};
settings = settings {
inherit config;
filesRoot = "/opt/files";
repository = "/opt/repos/${name}";
};
})
(mkIf (username != "root") {
@ -52,7 +52,6 @@ shb.test.runNixOSTest {
})
(optionalAttrs (extraConfig != null) (extraConfig {
inherit username config;
filesRoot = "/opt/files";
}))
];
};
@ -66,9 +65,7 @@ shb.test.runNixOSTest {
provider = (getAttrFromPath providerRoot nodes.machine).result;
in
''
from datetime import datetime, timedelta
from dictdiffer import diff
import re
username = "${username}"
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
@ -106,25 +103,9 @@ shb.test.runNixOSTest {
f'{path}/fileB': 'repo_fileB_1',
})
with subtest("Initial snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("First backup in repo"):
print(machine.succeed("systemctl cat ${provider.backupService}"))
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
# To accomodate for snapshot orchestrators which keep only a given amount
# of snapshots per unit of time, we set the time to now + 2h.
new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
machine.succeed(f"timedatectl set-time '{new_date}'")
machine.succeed("systemctl start ${provider.backupService}")
with subtest("New content"):
for path in sourceDirectories:
@ -138,37 +119,14 @@ shb.test.runNixOSTest {
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Second backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
with subtest("Delete content"):
for path in sourceDirectories:
machine.succeed(f"""rm -r {path}/*""")
assert_files(path, {})
with subtest("Restore second backup"):
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Restore first backup"):
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
with subtest("Restore initial content from repo"):
machine.succeed("""${provider.restoreScript} restore latest""")
for path in sourceDirectories:
assert_files(path, {

View file

@ -1,77 +0,0 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib.types)
nullOr
submodule
str
;
in
{
mkRequest =
{
serviceName ? "",
externalUrl ? "",
externalUrlText ? null,
internalUrl ? null,
internalUrlText ? null,
apiKey ? null,
}:
mkOption {
description = ''
Request part of the dashboard contract.
'';
default = { };
type = submodule {
options = {
externalUrl =
mkOption {
description = ''
URL at which the service can be accessed.
This URL should go through the reverse proxy.
'';
type = str;
default = externalUrl;
example = "https://jellyfin.example.com";
}
// (lib.optionalAttrs (externalUrlText != null) {
defaultText = externalUrlText;
});
internalUrl =
mkOption {
description = ''
URL at which the service can be accessed directly.
This URL should bypass the reverse proxy.
It can be used for example to ping the service
and making sure it is up and running correctly.
'';
type = nullOr str;
default = internalUrl;
example = "http://127.0.0.1:8081";
}
// (lib.optionalAttrs (internalUrlText != null) {
defaultText = internalUrlText;
});
};
};
};
mkResult =
{
}:
mkOption {
description = ''
Result part of the dashboard contract.
No option is provided here.
'';
default = { };
type = submodule {
options = {
};
};
};
}

View file

@ -1,65 +0,0 @@
# Dashboard Contract {#contract-dashboard}
This NixOS contract is used for user-facing services
that want to be displayed on a dashboard.
It is a contract between a service that can be accessed through an URL
and a service that wants to show a list of those services.
## Providers {#contract-dashboard-providers}
The providers of this contract in SHB are:
<!-- Somehow generate this list -->
- The homepage service under its [shb.homepage.servicesGroups.<name>.services.<name>.dashboard](#services-homepage-options-shb.homepage.servicesGroups._name_.services._name_.dashboard) option.
## Usage {#contracts-dashboard-usage}
A service that can be shown on a dashboard will provide a `dashboard` option.
Here is an example module defining such a requester option for this dashboard contract:
```nix
{
options = {
myservice.dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${config.myservice.subdomain}.${config.myservice.domain}";
internalUrl = "http://127.0.0.1:${config.myservice.port}";
};
};
};
};
};
```
Then, plug both consumer and provider together in the `config`:
```nix
{
config = {
<provider-module> = {
dashboard.request = config.myservice.dashboard.request;
};
};
}
```
And that's it for the contract part.
For more specific details on each provider, go to their respective manual pages.
## Contract Reference {#contract-dashboard-options}
These are all the options that are expected to exist for this contract to be respected.
```{=include=} options
id-prefix: contracts-dashboard-options-
list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```

View file

@ -1,30 +0,0 @@
{ lib, shb, ... }:
let
inherit (lib) mkOption;
inherit (lib.types) submodule;
in
{
imports = [
../../../lib/module.nix
];
options.shb.contracts.dashboard = mkOption {
description = ''
Contract for user-facing services that want to
be displayed on a dashboard.
The requester communicates to the provider
how to access the service
through the `request` options.
The provider reads from the `request` options
and configures what is necessary on its side
to show the service and check its availability.
It does not communicate back to the requester.
'';
type = submodule {
options = shb.contracts.dashboard.contract;
};
};
}

View file

@ -20,21 +20,21 @@ source: @OPTIONS_JSON@
## Usage {#contract-databasebackup-usage}
A database that can be backed up will provide a `databasebackup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define how to create a database dump,
the user to backup with
and how to restore from a database dump.
Here is an example module defining such a `databasebackup` option,
which defines the user to backup with (`user`)
and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
Here is an example module defining such a `databasebackup` option:
```nix
{
options = {
myservice.databasebackup = mkOption {
type = shb.contracts.databasebackup.mkRequester = {
type = contracts.databasebackup.request;
default = {
user = "myservice";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
@ -48,16 +48,16 @@ and how to backup (`backupCmd`) and restore (`restoreCmd`) the database:
};
```
Now, on the other side we have a service that uses this `databasebackup` option and actually backs up files.
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `databaseBackupService` option
and that one can create multiple backup instances under `databaseBackupService.instances`.
Let's assume such a module is available under the `databasebackupservice` option
and that one can create multiple backup instances under `databasebackupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
databaseBackupService.instances.myservice = {
request = myservice.databasebackup.request;
databasebackupservice.instances.myservice = {
request = myservice.databasebackup;
settings = {
enable = true;
@ -72,11 +72,11 @@ databaseBackupService.instances.myservice = {
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `databaseBackupService`:
Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
```nix
databaseBackupService.instances.myservice_2 = {
request = myservice.databasebackup.request;
databasebackupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
@ -88,12 +88,12 @@ databaseBackupService.instances.myservice_2 = {
};
```
Or with another module `databaseBackupService_2`!
Or with another module `databasebackupservice_2`!
## Providers of the Database Backup Contract {#contract-databasebackup-providers}
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
## Requester Blocks and Services {#contract-databasebackup-requesters}

View file

@ -51,18 +51,16 @@ shb.test.runNixOSTest {
testScript =
{ nodes, ... }:
let
provider = (getAttrFromPath providerRoot nodes.machine).result;
provider = getAttrFromPath providerRoot nodes.machine;
in
''
import csv
import re
start_all()
machine.wait_for_unit("postgresql.service")
machine.wait_for_unit("postgresql-setup.service")
machine.wait_for_open_port(5432)
def peer_cmd(cmd, db="${database}"):
def peer_cmd(cmd, db="me"):
return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
def query(query):
@ -75,68 +73,30 @@ shb.test.runNixOSTest {
if len(diff) > 0:
raise Exception(i, diff)
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
with subtest("create fixture"):
machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}]
cmp_tables(res, table)
with subtest("Initial snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("backup"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("New content"):
machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
cmp_tables(res, table)
with subtest("backup"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("Two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
print(machine.succeed("systemctl cat ${provider.result.backupService}"))
print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
machine.succeed("systemctl start ${provider.result.backupService}")
with subtest("drop database"):
machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
machine.fail(peer_cmd("SELECT * FROM test"))
with subtest("restore second snapshot"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
with subtest("restore"):
print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
machine.succeed("${provider.result.restoreScript} restore latest ")
with subtest("check restoration"):
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
cmp_tables(res, table)
with subtest("restore first snapshot"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}]
cmp_tables(res, table)
'';
}

View file

@ -1,113 +0,0 @@
{ lib, ... }:
let
inherit (lib)
literalMD
mkOption
optionalAttrs
;
inherit (lib.types)
submodule
str
;
in
{
mkRequest =
{
dataset ? "",
datasetText ? null,
}:
mkOption {
description = ''
Request part of the backup contract.
Options set by the requester module
enforcing how to backup files.
'';
default = { };
type = submodule {
options = {
dataset =
mkOption {
description = ''
Dataset to backup, including the pool name.
'';
type = str;
example = "root/home";
default = dataset;
}
// optionalAttrs (datasetText != null) {
defaultText = literalMD datasetText;
};
};
};
};
mkResult =
{
restoreScript ? "restore",
restoreScriptText ? null,
backupService ? "backup.service",
backupServiceText ? null,
}:
mkOption {
description = ''
Result part of the backup contract.
Options set by the provider module that indicates the name of the backup and restore scripts.
'';
default = { };
type = submodule {
options = {
restoreScript =
mkOption {
description = ''
Name of script that can restore the database.
One can then list snapshots with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
<snapshot 1> <metadata>
<snapshot 2> <metadata>
```
And restore the database with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
```
It is not garanteed to be able to restore back to a snapshot in the future.
With the above example, it may not be possible to restore `<snapshot 2>`
after having restored `<snapshot 1>`.
'';
type = str;
default = restoreScript;
}
// optionalAttrs (restoreScriptText != null) {
defaultText = literalMD restoreScriptText;
};
backupService =
mkOption {
description = ''
Name of service backing up the database.
This script can be ran manually to backup the database:
```bash
$ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
```
'';
type = str;
default = backupService;
}
// optionalAttrs (backupServiceText != null) {
defaultText = literalMD backupServiceText;
};
};
};
};
}

View file

@ -1,85 +0,0 @@
# Dataset Backup Contract {#contract-dataset-backup}
This NixOS contract represents a backup job
that will backup one ZFS dataset.
It is a contract between a service that manages ZFS datasets
and a service that backs up ZFS datasets.
## Contract Reference {#contract-datatsetbackup-options}
These are all the options that are expected to exist for this contract to be respected.
```{=include=} options
id-prefix: contracts-datasetbackup-options-
list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```
## Usage {#contract-datasetbackup-usage}
A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option.
Here is an example module defining such a `backup` option,
which defines what directories to backup (`sourceDirectories`)
and the user to backup with (`user`).
```nix
{
options = {
myservice.datasetbackup = mkOption {
type = lib.types.submodule {
options = shb.contracts.datasetbackup.mkRequester {
dataset = "root/test";
};
};
};
};
};
```
Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `backupService` option
and that one can create multiple backup instances under `backupService.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
backupService.instances.myservice = {
request = myservice.datasetbackup.request;
settings = {
enable = true;
targetDataset = "backup/myservice";
# ... Other options specific to backupService like scheduling.
};
};
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `backupService`:
```nix
backupService.instances.myservice_2 = {
request = myservice.datasetbackup.request;
settings = {
enable = true;
targetDataset = "backup2/myservice";
};
};
```
Or with another module `backupService_2`!
## Providers of the Backup Contract {#contract-datasetbackup-providers}
- [Sanoid block](blocks-sanoid.html).
## Requester Blocks and Services {#contract-datasetbackup-requesters}
- [ZFS](blocks-zfs.html#blocks-zfs-backup).

View file

@ -1,30 +0,0 @@
{ lib, shb, ... }:
let
inherit (lib) mkOption;
inherit (lib.types) submodule;
in
{
imports = [
../../../lib/module.nix
];
options.shb.contracts.datasetbackup = mkOption {
description = ''
Contract for backing up ZFS datasets.
The requester communicates to the provider
the dataset to backup
through the `request` options.
The provider reads from the `request` options
and backs up the requested dataset.
It communicates to the requester what script is used
to backup and restore the files
through the `result` options.
'';
type = submodule {
options = shb.contracts.datasetbackup.contract;
};
};
}

View file

@ -1,164 +0,0 @@
{
pkgs,
lib,
shb,
}:
let
inherit (lib)
concatMapStringsSep
getAttrFromPath
setAttrByPath
;
in
{
name,
providerRoot,
modules ? [ ],
dataset ? "root/test",
sourceDirectories ? [
"/opt/files/A"
"/opt/files/B"
],
settings ? { ... }: { }, # { filesRoot, config } -> attrset
extraConfig ? null, # { filesRoot, config } -> attrset
}:
shb.test.runNixOSTest {
inherit name;
nodes.machine =
{ config, ... }:
{
imports = [ shb.test.baseImports ] ++ modules;
config = lib.mkMerge [
(setAttrByPath providerRoot {
request = {
inherit dataset;
};
settings = settings {
inherit config;
filesRoot = "/opt/files";
};
})
(lib.optionalAttrs (extraConfig != null) (extraConfig {
inherit config;
filesRoot = "/opt/files";
}))
];
};
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
testScript =
{ nodes, ... }:
let
provider = (getAttrFromPath providerRoot nodes.machine).result;
in
''
from datetime import datetime, timedelta
from dictdiffer import diff
import re
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
def list_files(dir):
files_and_content = {}
files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
for f in files:
content = machine.succeed(f"""cat {f}""").strip()
files_and_content[f] = content
return files_and_content
def assert_files(dir, files):
result = list(diff(list_files(dir), files))
if len(result) > 0:
raise Exception("Unexpected files:", result)
with subtest("Create initial content"):
for path in sourceDirectories:
machine.succeed(f"""
mkdir -p {path}
echo repo_fileA_1 > {path}/fileA
echo repo_fileB_1 > {path}/fileB
""")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_1',
f'{path}/fileB': 'repo_fileB_1',
})
with subtest("Initial snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("First backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
# To accomodate for snapshot orchestrators which keep only a given amount
# of snapshots per unit of time, we set the time to now + 2h.
new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
machine.succeed(f"timedatectl set-time '{new_date}'")
with subtest("New content"):
for path in sourceDirectories:
machine.succeed(f"""
echo repo_fileA_2 > {path}/fileA
echo repo_fileB_2 > {path}/fileB
""")
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Second backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
with subtest("Delete content"):
for path in sourceDirectories:
machine.succeed(f"""rm -r {path}/*""")
assert_files(path, {})
with subtest("Restore second backup"):
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Restore first backup"):
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_1',
f'{path}/fileB': 'repo_fileB_1',
})
'';
}

View file

@ -48,31 +48,21 @@ let
importContract =
module:
let
importedModule = pkgs.callPackage module {
shb = shb // {
inherit contracts;
};
};
importedModule = pkgs.callPackage module { inherit shb; };
in
(mkContractFunctions {
mkContractFunctions {
inherit (importedModule) mkRequest mkResult;
})
// (importedModule.passthru or { });
contracts = {
databasebackup = importContract ./databasebackup.nix;
datasetbackup = importContract ./datasetbackup.nix;
dashboard = importContract ./dashboard.nix;
backup = importContract ./backup.nix;
mount = pkgs.callPackage ./mount.nix { };
secret = importContract ./secret.nix;
ssl = pkgs.callPackage ./ssl.nix { };
test = {
secret = pkgs.callPackage ./secret/test.nix { inherit shb; };
databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; };
datasetbackup = pkgs.callPackage ./datasetbackup/test.nix { inherit shb; };
backup = pkgs.callPackage ./backup/test.nix { inherit shb; };
};
};
in
contracts
{
databasebackup = importContract ./databasebackup.nix;
backup = importContract ./backup.nix;
mount = pkgs.callPackage ./mount.nix { };
secret = importContract ./secret.nix;
ssl = pkgs.callPackage ./ssl.nix { };
test = {
secret = pkgs.callPackage ./secret/test.nix { inherit shb; };
databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; };
backup = pkgs.callPackage ./backup/test.nix { inherit shb; };
};
}

View file

@ -42,12 +42,12 @@ Now, with this contract, a layer on top of `sops` is added which is found under
The configuration then becomes:
```nix
shb.sops.secret."ldap/user_password" = {
shb.sops.secrets."ldap/user_password" = {
request = config.shb.lldap.userPassword.request;
settings.sopsFile = ./secrets.yaml;
};
shb.lldap.userPassword.result = config.shb.sops.secret."ldap/user_password".result;
shb.lldap.userPassword.result = config.shb.sops.secrets."ldap/user_password".result;
```
The issue is now gone as the responsibility falls
@ -63,9 +63,9 @@ sops.defaultSopsFile = ./secrets.yaml;
Then the snippet above is even more simplified:
```nix
shb.sops.secret."ldap/user_password".request = config.shb.lldap.userPassword.request;
shb.sops.secrets."ldap/user_password".request = config.shb.lldap.userPassword.request;
shb.lldap.userPassword.result = config.shb.sops.secret."ldap/user_password".result;
shb.lldap.userPassword.result = config.shb.sops.secrets."ldap/user_password".result;
```
## Contract Reference {#contract-secret-options}

View file

@ -144,10 +144,6 @@ let
description = "Log level.";
default = "info";
};
ApiKey = lib.mkOption {
type = shb.secretFileType;
description = "Path to api key secret file.";
};
Port = lib.mkOption {
type = lib.types.port;
description = "Port on which bazarr listens to incoming requests.";
@ -176,10 +172,6 @@ let
description = "Log level.";
default = "info";
};
ApiKey = lib.mkOption {
type = shb.secretFileType;
description = "Path to api key secret file.";
};
Port = lib.mkOption {
type = lib.types.port;
description = "Port on which readarr listens to incoming requests.";
@ -207,10 +199,6 @@ let
description = "Log level.";
default = "info";
};
ApiKey = lib.mkOption {
type = shb.secretFileType;
description = "Path to api key secret file.";
};
Port = lib.mkOption {
type = lib.types.port;
description = "Port on which lidarr listens to incoming requests.";
@ -317,7 +305,7 @@ let
{
domain = "${c.subdomain}.${c.domain}";
policy = "two_factor";
subject = [ "group:${c.ldapUserGroup}" ];
subject = [ "group:arr_user" ];
}
];
};
@ -356,16 +344,6 @@ let
default = null;
};
ldapUserGroup = lib.mkOption {
description = ''
LDAP group a user must belong to be able to login.
Note that all users are admins too.
'';
type = lib.types.str;
default = "arr_user";
};
authEndpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@ -392,20 +370,6 @@ let
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.${name}.subdomain}.${cfg.${name}.domain}";
externalUrlText = "https://\${config.shb.arr.${name}.subdomain}.\${config.shb.arr.${name}.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.${name}.settings.Port}";
};
};
};
}
// (c.moreOptions or { });
};
@ -416,7 +380,6 @@ in
imports = [
../../lib/module.nix
../blocks/nginx.nix
../blocks/lldap.nix
];
options.shb.arr = lib.listToAttrs (lib.mapAttrsToList appOption apps);
@ -432,7 +395,7 @@ in
services.radarr = {
enable = true;
dataDir = cfg'.dataDir;
dataDir = "/var/lib/radarr";
};
systemd.services.radarr.preStart = shb.replaceSecrets {
@ -442,15 +405,11 @@ in
AuthenticationRequired = "DisabledForLocalAddresses";
AuthenticationMethod = "External";
});
resultPath = "${cfg'.dataDir}/config.xml";
resultPath = "${config.services.radarr.dataDir}/config.xml";
generator = shb.replaceSecretsFormatAdapter apps.radarr.settingsFormat;
};
shb.nginx.vhosts = [ (vhosts { } cfg') ];
shb.lldap.ensureGroups = {
${cfg'.ldapUserGroup} = { };
};
}
))
@ -460,15 +419,11 @@ in
isSSOEnabled = !(isNull cfg'.authEndpoint);
in
{
systemd.tmpfiles.rules = [
"d ${cfg'.dataDir} 0700 ${config.services.sonarr.user} ${config.services.sonarr.user}"
];
services.nginx.enable = true;
services.sonarr = {
enable = true;
dataDir = cfg'.dataDir;
dataDir = "/var/lib/sonarr";
};
users.users.sonarr = {
extraGroups = [ "media" ];
@ -481,15 +436,11 @@ in
AuthenticationRequired = "DisabledForLocalAddresses";
AuthenticationMethod = "External";
});
resultPath = "${cfg'.dataDir}/config.xml";
resultPath = "${config.services.sonarr.dataDir}/config.xml";
generator = apps.sonarr.settingsFormat.generate;
};
shb.nginx.vhosts = [ (vhosts { } cfg') ];
shb.lldap.ensureGroups = {
${cfg'.ldapUserGroup} = { };
};
}
))
@ -501,64 +452,45 @@ in
{
services.bazarr = {
enable = true;
dataDir = cfg'.dataDir;
listenPort = cfg'.settings.Port;
};
users.users.bazarr = {
extraGroups = [ "media" ];
};
# This is actually not working. Bazarr uses a config file in dataDir/config/config.yaml
# which includes all configuration so we must somehow merge our declarative config with it.
# It's doable but will take some time. Help is welcomed.
#
# systemd.services.bazarr.preStart = shb.replaceSecrets {
# userConfig =
# cfg'.settings
# // (lib.optionalAttrs isSSOEnabled {
# AuthenticationRequired = "DisabledForLocalAddresses";
# AuthenticationMethod = "External";
# });
# resultPath = "${cfg'.dataDir}/config.xml";
# generator = apps.bazarr.settingsFormat.generate;
# };
shb.nginx.vhosts = [ (vhosts { } cfg') ];
shb.lldap.ensureGroups = {
${cfg'.ldapUserGroup} = { };
};
}
))
(lib.mkIf cfg.readarr.enable (
let
cfg' = cfg.readarr;
isSSOEnabled = !(isNull cfg'.authEndpoint);
in
{
services.readarr = {
enable = true;
dataDir = cfg'.dataDir;
};
users.users.readarr = {
extraGroups = [ "media" ];
};
systemd.services.readarr.preStart = shb.replaceSecrets {
systemd.services.bazarr.preStart = shb.replaceSecrets {
userConfig =
cfg'.settings
// (lib.optionalAttrs isSSOEnabled {
AuthenticationRequired = "DisabledForLocalAddresses";
AuthenticationMethod = "External";
});
resultPath = "${cfg'.dataDir}/config.xml";
resultPath = "/var/lib/bazarr/config.xml";
generator = apps.bazarr.settingsFormat.generate;
};
shb.nginx.vhosts = [ (vhosts { } cfg') ];
}
))
(lib.mkIf cfg.readarr.enable (
let
cfg' = cfg.readarr;
in
{
services.readarr = {
enable = true;
dataDir = "/var/lib/readarr";
};
users.users.readarr = {
extraGroups = [ "media" ];
};
systemd.services.readarr.preStart = shb.replaceSecrets {
userConfig = cfg'.settings;
resultPath = "${config.services.readarr.dataDir}/config.xml";
generator = apps.readarr.settingsFormat.generate;
};
shb.nginx.vhosts = [ (vhosts { } cfg') ];
shb.lldap.ensureGroups = {
${cfg'.ldapUserGroup} = { };
};
}
))
@ -570,7 +502,7 @@ in
{
services.lidarr = {
enable = true;
dataDir = cfg'.dataDir;
dataDir = "/var/lib/lidarr";
};
users.users.lidarr = {
extraGroups = [ "media" ];
@ -582,15 +514,11 @@ in
AuthenticationRequired = "DisabledForLocalAddresses";
AuthenticationMethod = "External";
});
resultPath = "${cfg'.dataDir}/config.xml";
resultPath = "${config.services.lidarr.dataDir}/config.xml";
generator = apps.lidarr.settingsFormat.generate;
};
shb.nginx.vhosts = [ (vhosts { } cfg') ];
shb.lldap.ensureGroups = {
${cfg'.ldapUserGroup} = { };
};
}
))
@ -601,7 +529,7 @@ in
{
services.jackett = {
enable = true;
dataDir = cfg'.dataDir;
dataDir = "/var/lib/jackett";
};
# TODO: avoid implicitly relying on the media group
users.users.jackett = {
@ -609,7 +537,7 @@ in
};
systemd.services.jackett.preStart = shb.replaceSecrets {
userConfig = shb.renameAttrName cfg'.settings "ApiKey" "APIKey";
resultPath = "${cfg'.dataDir}/ServerConfig.json";
resultPath = "${config.services.jackett.dataDir}/ServerConfig.json";
generator = apps.jackett.settingsFormat.generate;
};
@ -618,10 +546,6 @@ in
extraBypassResources = [ "^/dl.*" ];
} cfg')
];
shb.lldap.ensureGroups = {
${cfg'.ldapUserGroup} = { };
};
}
))
];

View file

@ -3,242 +3,12 @@
Defined in [`/modules/services/arr.nix`](@REPO@/modules/services/arr.nix).
This NixOS module sets up multiple [Servarr](https://wiki.servarr.com/) services.
## Features {#services-arr-features}
Compared to the stock module from nixpkgs,
this one sets up, in a fully declarative manner
LDAP and SSO integration as well as the API key.
## Usage {#services-arr-usage}
### Initial Configuration {#services-arr-usage-configuration}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
- the [`shb.ssl` block](blocks-ssl.html#usage),
- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
- the [`shb.authelia` block](blocks-authelia.html#blocks-sso-global-setup).
```nix
{
shb.certs.certs.letsencrypt.${domain}.extraDomains = [
"moviesdl.${domain}"
"seriesdl.${domain}"
"subtitlesdl.${domain}"
"booksdl.${domain}"
"musicdl.${domain}"
"indexer.${domain}"
];
shb.arr = {
radarr = {
inherit domain;
enable = true;
ssl = config.shb.certs.certs.letsencrypt.${domain};
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
sonarr = {
inherit domain;
enable = true;
ssl = config.shb.certs.certs.letsencrypt."${domain}";
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
bazarr = {
inherit domain;
enable = true;
ssl = config.shb.certs.certs.letsencrypt."${domain}";
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
readarr = {
inherit domain;
enable = true;
ssl = config.shb.certs.certs.letsencrypt."${domain}";
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
lidarr = {
inherit domain;
enable = true;
ssl = config.shb.certs.certs.letsencrypt."${domain}";
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
jackett = {
inherit domain;
enable = true;
ssl = config.shb.certs.certs.letsencrypt."${domain}";
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
};
}
```
The user and admin LDAP groups are created automatically.
### API Keys {#services-arr-usage-apikeys}
The API keys for each arr service can be created declaratively.
First, generate one secret for each service with `nix run nixpkgs#openssl -- rand -hex 64`
and store it in your secrets file (for example the SOPS file).
Then, add the API key to each service:
```nix
{
shb.arr = {
radarr = {
settings = {
ApiKey.source = config.shb.sops.secret."radarr/apikey".result.path;
};
};
sonarr = {
settings = {
ApiKey.source = config.shb.sops.secret."sonarr/apikey".result.path;
};
};
bazarr = {
settings = {
ApiKey.source = config.shb.sops.secret."bazarr/apikey".result.path;
};
};
readarr = {
settings = {
ApiKey.source = config.shb.sops.secret."readarr/apikey".result.path;
};
};
lidarr = {
settings = {
ApiKey.source = config.shb.sops.secret."lidarr/apikey".result.path;
};
};
jackett = {
settings = {
ApiKey.source = config.shb.sops.secret."jackett/apikey".result.path;
};
};
};
shb.sops.secret."radarr/apikey".request = {
mode = "0440";
owner = "radarr";
group = "radarr";
restartUnits = [ "radarr.service" ];
};
shb.sops.secret."sonarr/apikey".request = {
mode = "0440";
owner = "sonarr";
group = "sonarr";
restartUnits = [ "sonarr.service" ];
};
shb.sops.secret."bazarr/apikey".request = {
mode = "0440";
owner = "bazarr";
group = "bazarr";
restartUnits = [ "bazarr.service" ];
};
shb.sops.secret."readarr/apikey".request = {
mode = "0440";
owner = "readarr";
group = "readarr";
restartUnits = [ "readarr.service" ];
};
shb.sops.secret."lidarr/apikey".request = {
mode = "0440";
owner = "lidarr";
group = "lidarr";
restartUnits = [ "lidarr.service" ];
};
shb.sops.secret."jackett/apikey".request = {
mode = "0440";
owner = "jackett";
group = "jackett";
restartUnits = [ "jackett.service" ];
};
}
```
### Application Dashboard {#services-arr-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the various dashboard options.
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Media.services.Radarr = {
sortOrder = 10;
dashboard.request = config.shb.arr.radarr.dashboard.request;
apiKey.result = config.shb.sops.secret."radarr/homepageApiKey".result;
};
shb.sops.secret."radarr/homepageApiKey" = {
settings.key = "radarr/apikey";
request = config.shb.homepage.servicesGroups.Media.services.Radarr.apiKey.request;
};
shb.homepage.servicesGroups.Media.services.Sonarr = {
sortOrder = 11;
dashboard.request = config.shb.arr.sonarr.dashboard.request;
apiKey.result = config.shb.sops.secret."sonarr/homepageApiKey".result;
};
shb.sops.secret."sonarr/homepageApiKey" = {
settings.key = "sonarr/apikey";
request = config.shb.homepage.servicesGroups.Media.services.Sonarr.apiKey.request;
};
shb.homepage.servicesGroups.Media.services.Bazarr = {
sortOrder = 12;
dashboard.request = config.shb.arr.bazarr.dashboard.request;
apiKey.result = config.shb.sops.secret."bazarr/homepageApiKey".result;
};
shb.sops.secret."bazarr/homepageApiKey" = {
settings.key = "bazarr/apikey";
request = config.shb.homepage.servicesGroups.Media.services.Bazarr.apiKey.request;
};
shb.homepage.servicesGroups.Media.services.Readarr = {
sortOrder = 13;
dashboard.request = config.shb.arr.readarr.dashboard.request;
apiKey.result = config.shb.sops.secret."readarr/homepageApiKey".result;
};
shb.sops.secret."readarr/homepageApiKey" = {
settings.key = "readarr/apikey";
request = config.shb.homepage.servicesGroups.Media.services.Readarr.apiKey.request;
};
shb.homepage.servicesGroups.Media.services.Lidarr = {
sortOrder = 14;
dashboard.request = config.shb.arr.lidarr.dashboard.request;
apiKey.result = config.shb.sops.secret."lidarr/homepageApiKey".result;
};
shb.sops.secret."lidarr/homepageApiKey" = {
settings.key = "lidarr/apikey";
request = config.shb.homepage.servicesGroups.Media.services.Lidarr.apiKey.request;
};
shb.homepage.servicesGroups.Media.services.Jackett = {
sortOrder = 15;
dashboard.request = config.shb.arr.jackett.dashboard.request;
apiKey.result = config.shb.sops.secret."jackett/homepageApiKey".result;
};
shb.sops.secret."jackett/homepageApiKey" = {
settings.key = "jackett/apikey";
request = config.shb.homepage.servicesGroups.Media.services.Jackett.apiKey.request;
};
}
```
This example reuses the API keys generated declaratively from the previous section.
### Jackett Proxy {#services-arr-usage-jackett-proxy}
The Jackett service can be made to use a proxy with:
```nix
{
shb.arr.jackett = {
settings = {
ProxyType = "0";
ProxyUrl = "127.0.0.1:1234";
};
};
};
```
This manual page is under construction.
## Options Reference {#services-arr-options}

View file

@ -152,20 +152,6 @@ in
default = false;
example = true;
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.audiobookshelf.subdomain}.\${config.shb.audiobookshelf.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.webPort}";
};
};
};
};
config = lib.mkIf cfg.enable (

View file

@ -30,15 +30,10 @@ in
imports = [
../../lib/module.nix
../blocks/nginx.nix
../blocks/monitoring.nix
];
options.shb.deluge = {
enable = lib.mkEnableOption "the SHB Deluge service";
enableDashboard = lib.mkEnableOption "the Torrents SHB monitoring dashboard" // {
default = true;
};
enable = lib.mkEnableOption "selfhostblocks.deluge";
subdomain = lib.mkOption {
type = lib.types.str;
@ -301,20 +296,6 @@ in
default = null;
example = "info";
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${fqdn}";
externalUrlText = "https://\${config.shb.deluge.subdomain}.\${config.shb.deluge.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.webPort}";
};
};
};
};
config = lib.mkIf cfg.enable (
@ -471,12 +452,6 @@ in
}
];
})
(lib.mkIf (cfg.enable && cfg.enableDashboard) {
shb.monitoring.dashboards = [
./deluge/dashboard/Torrents.json
];
})
]
);
}

View file

@ -1,457 +0,0 @@
{
config,
lib,
shb,
...
}:
let
cfg = config.shb.firefly-iii;
in
{
imports = [
../blocks/nginx.nix
../blocks/lldap.nix
../../lib/module.nix
];
options.shb.firefly-iii = {
enable = lib.mkEnableOption "SHB's firefly-iii module";
subdomain = lib.mkOption {
type = lib.types.str;
description = ''
Subdomain under which firefly-iii will be served.
```
<subdomain>.<domain>
```
'';
example = "firefly-iii";
};
domain = lib.mkOption {
description = ''
Domain under which firefly-iii is served.
```
<subdomain>.<domain>[:<port>]
```
'';
type = lib.types.str;
example = "domain.com";
};
ssl = lib.mkOption {
description = "Path to SSL files";
type = lib.types.nullOr shb.contracts.ssl.certs;
default = null;
};
siteOwnerEmail = lib.mkOption {
description = "Email of the site owner.";
type = lib.types.str;
example = "mail@example.com";
};
impermanence = lib.mkOption {
description = ''
Path to save when using impermanence setup.
'';
type = lib.types.str;
default = config.services.firefly-iii.dataDir;
defaultText = "services.firefly-iii.dataDir";
};
backup = lib.mkOption {
description = ''
Backup configuration.
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = config.services.firefly-iii.user;
userText = "services.firefly-iii.user";
sourceDirectories = [
config.services.firefly-iii.dataDir
];
sourceDirectoriesText = ''
[
config.services.firefly-iii.dataDir
]
'';
};
};
};
appKey = lib.mkOption {
description = "Encryption key used for sessions. Must be 32 characters long exactly.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.services.firefly-iii.user;
ownerText = "services.firefly-iii.user";
restartUnits = [ "firefly-iii-setup.service" ];
};
};
};
dbPassword = lib.mkOption {
description = "DB password.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0440";
owner = config.services.firefly-iii.user;
ownerText = "services.firefly-iii.user";
group = "postgres";
restartUnits = [
"postgresql.service"
"firefly-iii-setup.service"
];
};
};
};
ldap = lib.mkOption {
description = ''
LDAP Integration
'';
default = { };
type = lib.types.submodule {
options = {
userGroup = lib.mkOption {
type = lib.types.str;
description = "Group users must belong to to be able to login to Firefly-iii.";
default = "firefly-iii_user";
};
adminGroup = lib.mkOption {
type = lib.types.str;
description = "Group users must belong to to be able to import data user the Firefly-iii data importer.";
default = "firefly-iii_admin";
};
};
};
};
sso = lib.mkOption {
description = ''
SSO Integration
'';
default = { };
type = lib.types.submodule {
options = {
enable = lib.mkEnableOption "SSO integration.";
authEndpoint = lib.mkOption {
type = lib.types.str;
description = "OIDC endpoint for SSO.";
example = "https://authelia.example.com";
};
port = lib.mkOption {
description = "If given, adds a port to the endpoint.";
type = lib.types.nullOr lib.types.port;
default = null;
};
provider = lib.mkOption {
type = lib.types.enum [ "Authelia" ];
description = "OIDC provider name, used for display.";
default = "Authelia";
};
clientID = lib.mkOption {
type = lib.types.str;
description = "Client ID for the OIDC endpoint.";
default = "firefly-iii";
};
authorization_policy = lib.mkOption {
type = lib.types.enum [
"one_factor"
"two_factor"
];
description = "Require one factor (password) or two factor (device) authentication.";
default = "one_factor";
};
adminGroup = lib.mkOption {
type = lib.types.str;
description = "Group admins must belong to to be able to login to Firefly-iii.";
default = "firefly-iii_admin";
};
secret = lib.mkOption {
description = "OIDC shared secret.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = "firefly-iii";
restartUnits = [ "firefly-iii-setup.service" ];
};
};
};
secretForAuthelia = lib.mkOption {
description = "OIDC shared secret. Content must be the same as `secretFile` option.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = "authelia";
};
};
};
};
};
};
smtp = lib.mkOption {
description = ''
If set, send notifications through smtp.
https://docs.firefly-iii.org/how-to/firefly-iii/advanced/notifications/
'';
default = null;
type = lib.types.nullOr (
lib.types.submodule {
options = {
from_address = lib.mkOption {
type = lib.types.str;
description = "SMTP address from which the emails originate.";
example = "authelia@mydomain.com";
};
host = lib.mkOption {
type = lib.types.str;
description = "SMTP host to send the emails to.";
};
port = lib.mkOption {
type = lib.types.port;
description = "SMTP port to send the emails to.";
default = 25;
};
username = lib.mkOption {
type = lib.types.str;
description = "Username to connect to the SMTP host.";
};
password = lib.mkOption {
description = "File containing the password to connect to the SMTP host.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.services.firefly-iii.user;
ownerText = "services.firefly-iii.user";
restartUnits = [ "firefly-iii-setup.service" ];
};
};
};
};
}
);
};
debug = lib.mkOption {
type = lib.types.bool;
description = "Enable more verbose logging.";
default = false;
example = true;
};
importer = lib.mkOption {
description = ''
Configuration for Firefly-iii data importer.
'';
default = { };
type = lib.types.submodule {
options = {
enable = lib.mkEnableOption "Firefly-iii Data Importer." // {
default = true;
};
subdomain = lib.mkOption {
type = lib.types.str;
description = ''
Subdomain under which the firefly-iii data importer will be served.
'';
default = "${cfg.subdomain}-importer";
defaultText = lib.literalExpression "\${shb.firefly-iii.subdomain}-importer";
};
firefly-iii-accessToken = lib.mkOption {
type = lib.types.nullOr (
lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.services.firefly-iii-data-importer.user;
ownerText = "services.firefly-iii-data-importer.user";
restartUnits = [ "firefly-iii-data-importer-setup.service" ];
};
}
);
description = ''
Create a Personal Access Token then set then token in this option.
'';
default = null;
};
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.firefly-iii.subdomain}.\${config.shb.firefly-iii.domain}";
# This works thanks to the Personal Access Token.
internalUrl = "https://${cfg.subdomain}.${cfg.domain}";
internalUrlText = "https://\${config.shb.firefly-iii.subdomain}.\${config.shb.firefly-iii.domain}";
};
};
};
};
config = lib.mkIf cfg.enable (
lib.mkMerge [
{
services.firefly-iii = {
enable = true;
group = "nginx";
virtualHost = "${cfg.subdomain}.${cfg.domain}";
# https://github.com/firefly-iii/firefly-iii/blob/main/.env.example
settings = {
APP_ENV = "production";
APP_URL = "https://${cfg.subdomain}.${cfg.domain}";
APP_DEBUG = cfg.debug;
APP_LOG_LEVEL = if cfg.debug then "debug" else "notice";
LOG_CHANNEL = "stdout";
APP_KEY_FILE = cfg.appKey.result.path;
SITE_OWNER = cfg.siteOwnerEmail;
DB_CONNECTION = "pgsql";
DB_HOST = "localhost";
DB_PORT = config.services.postgresql.settings.port;
DB_DATABASE = "firefly-iii";
DB_USERNAME = "firefly-iii";
DB_PASSWORD_FILE = cfg.dbPassword.result.path;
# MAP_DEFAULT_LAT = "51.983333";
# MAP_DEFAULT_LONG = "5.916667";
# MAP_DEFAULT_ZOOM = "6";
};
};
shb.postgresql.enableTCPIP = true;
shb.postgresql.ensures = [
{
username = "firefly-iii";
database = "firefly-iii";
passwordFile = cfg.dbPassword.result.path;
}
];
# This should be using a contract instead of setting the option directly.
shb.lldap = lib.mkIf config.shb.lldap.enable {
ensureGroups = {
${cfg.ldap.userGroup} = { };
${cfg.ldap.adminGroup} = { };
};
};
# We enable the firefly-iii nginx integration and merge it with SHB's nginx configuration.
services.firefly-iii.enableNginx = true;
shb.nginx.vhosts = [
{
inherit (cfg) subdomain domain ssl;
}
];
}
(lib.mkIf cfg.importer.enable {
services.firefly-iii-data-importer = {
enable = true;
virtualHost = "${cfg.importer.subdomain}.${cfg.domain}";
settings = {
FIREFLY_III_URL = "https://${config.services.firefly-iii.virtualHost}";
}
// lib.optionalAttrs (cfg.importer.firefly-iii-accessToken != null) {
FIREFLY_III_ACCESS_TOKEN_FILE = cfg.importer.firefly-iii-accessToken.result.path;
};
};
# We enable the firefly-iii-data-importer nginx integration and merge it with SHB's nginx configuration.
services.firefly-iii-data-importer.enableNginx = true;
shb.nginx.vhosts = [
{
inherit (cfg) domain ssl;
subdomain = cfg.importer.subdomain;
}
];
})
(lib.mkIf (cfg.smtp != null) {
services.firefly-iii.settings = {
MAIL_MAILER = "smtp";
MAIL_HOST = cfg.smtp.host;
MAIL_PORT = cfg.smtp.port;
MAIL_FROM = cfg.smtp.from_address;
MAIL_USERNAME = cfg.smtp.username;
MAIL_PASSWORD_FILE = cfg.smtp.password.result.path;
MAIL_ENCRYPTION = "tls";
};
})
(lib.mkIf cfg.sso.enable {
services.firefly-iii.settings = {
AUTHENTICATION_GUARD = "remote_user_guard";
AUTHENTICATION_GUARD_HEADER = "HTTP_X_FORWARDED_USER";
};
shb.nginx.vhosts = [
{
inherit (cfg) subdomain domain ssl;
inherit (cfg.sso) authEndpoint;
phpForwardAuth = true;
autheliaRules = [
{
domain = "${cfg.subdomain}.${cfg.domain}";
policy = "bypass";
resources = [ "^/api" ];
}
{
domain = "${cfg.subdomain}.${cfg.domain}";
policy = cfg.sso.authorization_policy;
subject = [
"group:${cfg.ldap.userGroup}"
"group:${cfg.ldap.adminGroup}"
];
}
];
}
];
})
(lib.mkIf (cfg.sso.enable && cfg.importer.enable) {
shb.nginx.vhosts = [
{
inherit (cfg.importer) subdomain;
inherit (cfg) domain ssl;
inherit (cfg.sso) authEndpoint;
autheliaRules = [
{
domain = "${cfg.importer.subdomain}.${cfg.domain}";
policy = cfg.sso.authorization_policy;
subject = [ "group:${cfg.ldap.adminGroup}" ];
}
];
}
];
})
]
);
}

View file

@ -1,193 +0,0 @@
# Firefly-iii Service {#services-firefly-iii}
Defined in [`/modules/services/firefly-iii.nix`](@REPO@/modules/services/firefly-iii.nix).
This NixOS module is a service that sets up a [Firefly-iii](https://www.firefly-iii.org/) instance.
Compared to the stock module from nixpkgs,
this one sets up, in a fully declarative manner,
LDAP and SSO integration
and has a nicer option for secrets.
It also sets up the Firefly-iii data importer service
and nearly automatically links it to the Firefly-iii instance using a Personal Account Token.
Instructions on how to do so is given in the next section.
## Features {#services-firefly-iii-features}
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-firefly-iii-usage-applicationdashboard)
## Usage {#services-firefly-iii-usage}
### Initial Configuration {#services-firefly-iii-usage-configuration}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
- the [`shb.ssl` block](blocks-ssl.html#usage),
- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
- the [`shb.authelia` block](blocks-authelia.html#blocks-sso-global-setup).
```nix
shb.firefly-iii = {
enable = true;
debug = false;
appKey.result = config.shb.sops.secret."firefly-iii/appKey".result;
dbPassword.result = config.shb.sops.secret."firefly-iii/dbPassword".result;
domain = "example.com";
subdomain = "firefly-iii";
siteOwnerEmail = "mail@example.com";
ssl = config.shb.certs.certs.letsencrypt.${domain};
smtp = {
host = "smtp.eu.mailgun.org";
port = 587;
username = "postmaster@mg.example.com";
from_address = "firefly-iii@example.com";
password.result = config.shb.sops.secret."firefly-iii/smtpPassword".result;
};
sso = {
enable = true;
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
importer = {
# See note hereunder.
# firefly-iii-accessToken.result = config.shb.sops.secret."firefly-iii/importerAccessToken".result;
};
};
shb.sops.secret."firefly-iii/appKey".request = config.shb.firefly-iii.appKey.request;
shb.sops.secret."firefly-iii/dbPassword".request = config.shb.firefly-iii.dbPassword.request;
shb.sops.secret."firefly-iii/smtpPassword".request = config.shb.firefly-iii.smtp.password.request;
# See not hereunder.
# shb.sops.secret."firefly-iii/importerAccessToken".request = config.shb.firefly-iii.importer.firefly-iii-accessToken.request;
```
Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.
Note that for `appKey`, the secret length must be exactly 32 characters.
The [user](#services-firefly-iii-options-shb.firefly-iii.ldap.userGroup)
and [admin](#services-firefly-iii-options-shb.firefly-iii.ldap.adminGroup)
LDAP groups are created automatically.
Only admin users have access to the Firefly-iii data importer.
On the Firefly-iii web UI, the first user to login will be the admin.
We cannot yet create multiple admins in the Firefly-iii web UI.
On first start, leave the `shb.firefly-iii.importer.firefly-iii-accessToken` option empty.
To fill it out and connect the data importer to the Firefly-iii instance,
you must first create a personal access token then fill that option and redeploy.
### Backup {#services-firefly-iii-usage-backup}
Backing up Firefly-iii using the [Restic block](blocks-restic.html) is done like so:
```nix
shb.restic.instances."firefly-iii" = {
request = config.shb.firefly-iii.backup;
settings = {
enable = true;
};
};
```
The name `"firefly-iii"` in the `instances` can be anything.
The `config.shb.firefly-iii.backup` option provides what directories to backup.
You can define any number of Restic instances to backup Firefly-iii multiple times.
You will then need to configure more options like the `repository`,
as explained in the [restic](blocks-restic.html) documentation.
### Certificates {#services-firefly-iii-certs}
For Let's Encrypt certificates, add:
```nix
{
shb.certs.certs.letsencrypt.${domain}.extraDomains = [
"${config.shb.firefly-iii.subdomain}.${config.shb.firefly-iii.domain}"
"${config.shb.firefly-iii.importer.subdomain}.${config.shb.firefly-iii.domain}"
];
}
```
### Impermanence {#services-firefly-iii-impermanence}
To save the data folder in an impermanence setup, add:
```nix
{
shb.zfs.datasets."safe/firefly-iii".path = config.shb.firefly-iii.impermanence;
}
```
### Declarative LDAP {#services-firefly-iii-declarative-ldap}
To add a user `USERNAME` to the user and admin groups for Firefly-iii, add:
```nix
shb.lldap.ensureUsers.USERNAME.groups = [
config.shb.firefly-iii.ldap.userGroup
config.shb.firefly-iii.ldap.adminGroup
];
```
### Application Dashboard {#services-firefly-iii-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-firefly-iii-options-shb.firefly-iii.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Finance.services.Firefly-iii = {
sortOrder = 1;
dashboard.request = config.shb.firefly-iii.dashboard.request;
settings.widget.type = "firefly";
};
}
```
The widget type needs to be set manually otherwise it is not displayed correctly.
An API key can be set to show extra info:
```nix
{
shb.homepage.servicesGroups.Finance.services.Firefly-iii = {
apiKey.result = config.shb.sops.secret."firefly-iii/homepageApiKey".result;
};
shb.sops.secret."firefly-iii/homepageApiKey".request =
config.shb.homepage.servicesGroups.Finance.services.Firefly-iii.apiKey.request;
}
```
## Database Inspection {#services-firefly-iii-database-inspection}
Access the database with:
```nix
sudo -u firefly-iii psql
```
Dump the database with:
```nix
sudo -u firefly-iii pg_dump --data-only --inserts firefly-iii > dump
```
## Mobile Apps {#services-firefly-iii-mobile}
This module was tested with the [Abacus iOS](https://github.com/victorbalssa/abacus) mobile app
using a Personal Account Token.
## Options Reference {#services-firefly-iii-options}
```{=include=} options
id-prefix: services-firefly-iii-options-
list-id: selfhostblocks-service-firefly-iii-options
source: @OPTIONS_JSON@
```

View file

@ -44,18 +44,16 @@ in
{
imports = [
../blocks/nginx.nix
../blocks/lldap.nix
(lib.mkRemovedOptionModule [ "shb" "forgejo" "adminPassword" ] ''
Instead, define an admin user in shb.forgejo.users and give it the same password, like so:
shb.forgejo.users = {
"forgejoadmin" = {
isAdmin = true;
email = "forgejoadmin@example.com";
password.result = <path/to/password>;
shb.forgejo.users = {
"forgejoadmin" = {
isAdmin = true;
email = "forgejoadmin@example.com";
password.result = <path/to/password>;
};
};
};
'')
];
@ -235,7 +233,6 @@ in
users = mkOption {
description = "Users managed declaratively.";
default = { };
type = attrsOf (submodule {
options = {
isAdmin = mkOption {
@ -248,7 +245,7 @@ in
description = ''
Email of user.
This is only set when the user is created, changing this later on will have no effect.
This is only set when the user is created, changing this later on will have no effect.
'';
type = str;
};
@ -334,7 +331,7 @@ in
options = shb.contracts.backup.mkRequester {
user = options.services.forgejo.user.value;
sourceDirectories = [
config.services.forgejo.dump.backupDir
options.services.forgejo.dump.backupDir.value
]
++ optionals (cfg.repositoryRoot != null) [
cfg.repositoryRoot
@ -408,21 +405,6 @@ in
type = bool;
default = false;
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.forgejo.subdomain}.\${config.shb.forgejo.domain}";
internalUrl = "https://${cfg.subdomain}.${cfg.domain}";
internalUrlText = "https://\${config.shb.forgejo.subdomain}.\${config.shb.forgejo.domain}";
};
};
};
};
config = mkMerge [
@ -486,12 +468,6 @@ in
(mkIf (cfg.enable && cfg.ldap.enable != false) {
systemd.services.forgejo.wants = cfg.ldap.waitForSystemdServices;
systemd.services.forgejo.after = cfg.ldap.waitForSystemdServices;
shb.lldap.ensureGroups = {
${cfg.ldap.adminGroup} = { };
${cfg.ldap.userGroup} = { };
};
# The delimiter in the `cut` command is a TAB!
systemd.services.forgejo.preStart =
let
@ -517,7 +493,7 @@ in
--bind-password $(tr -d '\n' < ${cfg.ldap.adminPassword.result.path}) \
--security-protocol Unencrypted \
--user-search-base ou=people,${cfg.ldap.dcdomain} \
--user-filter '(&(|(memberof=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain})(memberof=cn=${cfg.ldap.adminGroup},ou=groups,${cfg.ldap.dcdomain}))(|(uid=%[1]s)(mail=%[1]s)))' \
--user-filter '(&(memberof=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain})(|(uid=%[1]s)(mail=%[1]s)))' \
--admin-filter '(memberof=cn=${cfg.ldap.adminGroup},ou=groups,${cfg.ldap.dcdomain})' \
--username-attribute uid \
--firstname-attribute givenName \
@ -536,7 +512,7 @@ in
--bind-password $(tr -d '\n' < ${cfg.ldap.adminPassword.result.path}) \
--security-protocol Unencrypted \
--user-search-base ou=people,${cfg.ldap.dcdomain} \
--user-filter '(&(|(memberof=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain})(memberof=cn=${cfg.ldap.adminGroup},ou=groups,${cfg.ldap.dcdomain}))(|(uid=%[1]s)(mail=%[1]s)))' \
--user-filter '(&(memberof=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain})(|(uid=%[1]s)(mail=%[1]s)))' \
--admin-filter '(memberof=cn=${cfg.ldap.adminGroup},ou=groups,${cfg.ldap.dcdomain})' \
--username-attribute uid \
--firstname-attribute givenName \
@ -553,13 +529,6 @@ in
# For Forgejo config: https://forgejo.org/docs/latest/admin/config-cheat-sheet
# For cli info: https://docs.gitea.com/usage/command-line
(mkIf (cfg.enable && cfg.sso.enable != false) {
assertions = [
{
assertion = cfg.ldap.enable == true;
message = "'shb.forgejo.ldap.enable' must be set to true and ldap configured when 'shb.forgejo.sso.enable' is true. Otherwise you will never be able to register new accounts.";
}
];
services.forgejo.settings = {
oauth2 = {
ENABLED = true;
@ -667,7 +636,7 @@ in
};
services.gitea-actions-runner = mkIf cfg.localActionRunner {
package = pkgs.forgejo-runner;
package = pkgs.forgejo-actions-runner;
instances.local = {
enable = true;
name = "local";

View file

@ -17,7 +17,6 @@ LDAP and SSO integration as well as one local runner.
- Access through [subdomain](#services-forgejo-options-shb.forgejo.subdomain) using reverse proxy. [Manual](#services-forgejo-usage-configuration).
- Access through [HTTPS](#services-forgejo-options-shb.forgejo.ssl) using reverse proxy. [Manual](#services-forgejo-usage-configuration).
- [Backup](#services-forgejo-options-shb.forgejo.sso) through the [backup block](./blocks-backup.html). [Manual](#services-forgejo-usage-backup).
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-forgejo-usage-applicationdashboard)
## Usage {#services-forgejo-usage}
@ -35,20 +34,20 @@ shb.forgejo = {
"theadmin" = {
isAdmin = true;
email = "theadmin@example.com";
password.result = config.shb.sops.secret.forgejoAdminPassword.result;
password.result = config.shb.sops.secrets.forgejoAdminPassword.result;
};
"theuser" = {
email = "theuser@example.com";
password.result = config.shb.sops.secret.forgejoUserPassword.result;
password.result = config.shb.sops.secrets.forgejoUserPassword.result;
};
};
};
shb.sops.secret."forgejo/admin/password" = {
shb.sops.secrets."forgejo/admin/password" = {
request = config.shb.forgejo.users."theadmin".password.request;
};
shb.sops.secret."forgejo/user/password" = {
shb.sops.secrets."forgejo/user/password" = {
request = config.shb.forgejo.users."theuser".password.request;
};
```
@ -111,10 +110,10 @@ shb.forgejo.ldap = {
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secret."forgejo/ldap/adminPassword".result
adminPassword.result = config.shb.sops.secrets."forgejo/ldap/adminPassword".result
};
shb.sops.secret."forgejo/ldap/adminPassword" = {
shb.sops.secrets."forgejo/ldap/adminPassword" = {
request = config.shb.forgejo.ldap.adminPassword.request;
settings.key = "ldap/userPassword";
};
@ -207,22 +206,6 @@ The name `"forgejo"` in the `instances` can be anything.
The `config.shb.forgejo.backup` option provides what directories to backup.
You can define any number of Restic instances to backup Forgejo multiple times.
### Application Dashboard {#services-forgejo-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-forgejo-options-shb.forgejo.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Admin.services.Forgejo = {
sortOrder = 1;
dashboard.request = config.shb.forgejo.dashboard.request;
};
}
```
### Extra Settings {#services-forgejo-usage-extra-settings}
Other Forgejo settings can be accessed through the nixpkgs [stock service][].

View file

@ -111,21 +111,6 @@ in
default = false;
example = true;
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.grocy.subdomain}.\${config.shb.grocy.domain}";
internalUrl = "https://${cfg.subdomain}.${cfg.domain}";
internalUrlText = "https://\${config.shb.grocy.subdomain}.\${config.shb.grocy.domain}";
};
};
};
};
config = lib.mkIf cfg.enable (

View file

@ -82,21 +82,6 @@ in
default = [ "--forecast" ];
type = lib.types.listOf lib.types.str;
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.hledger.subdomain}.\${config.shb.hledger.domain}";
internalUrl = "http://127.0.0.1:${toString config.services.hledger-web.port}";
internalUrlText = "http://127.0.0.1:\${config.services.hledger-web.port}";
};
};
};
};
config = lib.mkIf cfg.enable {

View file

@ -11,13 +11,23 @@ let
fqdn = "${cfg.subdomain}.${cfg.domain}";
ldap_auth_script_repo = pkgs.fetchFromGitHub {
owner = "lldap";
repo = "lldap";
rev = "7d1f5abc137821c500de99c94f7579761fc949d8";
sha256 = "sha256-8D+7ww70Ja6Qwdfa+7MpjAAHewtCWNf/tuTAExoUrg0=";
};
ldap_auth_script = pkgs.writeShellScriptBin "ldap_auth.sh" ''
export PATH=${pkgs.gnused}/bin:${pkgs.curl}/bin:${pkgs.jq}/bin
exec ${pkgs.bash}/bin/bash ${ldap_auth_script_repo}/example_configs/lldap-ha-auth.sh $@
'';
# Filter secrets from config. Secrets are those of the form { source = <path>; }
secrets = lib.attrsets.filterAttrs (k: v: builtins.isAttrs v) cfg.config;
nonSecrets = (lib.attrsets.filterAttrs (k: v: !(builtins.isAttrs v)) cfg.config);
lldap_ha_auth = pkgs.callPackage ./home-assistant/lldap_ha_auth.nix { };
configWithSecretsIncludes = nonSecrets // (lib.attrsets.mapAttrs (k: v: "!secret ${k}") secrets);
in
{
@ -208,21 +218,6 @@ in
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.home-assistant.subdomain}.\${config.shb.home-assistant.domain}";
internalUrl = "http://127.0.0.1:${toString config.services.home-assistant.config.http.server_port}";
internalUrlText = "http://127.0.0.1:\${config.services.home-assistant.config.http.server_port}";
};
};
};
};
config = lib.mkIf cfg.enable {
@ -253,6 +248,7 @@ in
config = {
# Includes dependencies for a basic setup
# https://www.home-assistant.io/integrations/default_config/
default_config = { };
http = {
use_x_forwarded_for = true;
server_host = "127.0.0.1";
@ -270,10 +266,9 @@ in
}
])
++ (lib.optionals cfg.ldap.enable [
# https://www.home-assistant.io/docs/authentication/providers/#command-line
{
type = "command_line";
command = lldap_ha_auth + "/bin/lldap-ha-auth";
command = ldap_auth_script + "/bin/ldap_auth.sh";
args = [
"http://${cfg.ldap.host}:${toString cfg.ldap.port}"
cfg.ldap.userGroup

View file

@ -15,7 +15,6 @@ LDAP and SSO integration.
- Access through [subdomain](#services-home-assistant-options-shb.home-assistant.subdomain) using reverse proxy. [Manual](#services-home-assistant-usage-configuration).
- Access through [HTTPS](#services-home-assistant-options-shb.home-assistant.ssl) using reverse proxy. [Manual](#services-home-assistant-usage-configuration).
- [Backup](#services-home-assistant-options-shb.home-assistant.backup) through the [backup block](./blocks-backup.html). [Manual](#services-home-assistant-usage-backup).
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-home-assistant-usage-applicationdashboard)
- Not yet: declarative SSO.
@ -164,57 +163,6 @@ You can define any number of Restic instances to backup Home-Assistant multiple
You will then need to configure more options like the `repository`,
as explained in the [restic](blocks-restic.html) documentation.
### Application Dashboard {#services-home-assistant-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-home-assistant-options-shb.home-assistant.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Home.services.HomeAssistant = {
sortOrder = 1;
dashboard.request = config.shb.home-assistant.dashboard.request;
settings.icon = "si-homeassistant";
};
}
```
The icon needs to be set manually otherwise it is not displayed correctly.
An API key can be set to show extra info:
```nix
{
shb.homepage.servicesGroups.Home.services.HomeAssistant = {
apiKey.result = config.shb.sops.secret."home-assistant/homepageApiKey".result;
};
shb.sops.secret."home-assistant/homepageApiKey".request =
config.shb.homepage.servicesGroups.Home.services.HomeAssistant.apiKey.request;
}
```
Custom widgets can be set using Home Assistant templating:
```nix
{
shb.homepage.servicesGroups.Home.services.HomeAssistant = {
settings.widget.custom = [
{
template = "{{ states('sensor.power_consumption_power_consumption', with_unit=True, rounded=True) }}";
label = "energy now";
}
{
state = "sensor.power_consumption_daily_power_consumption";
label = "energy today";
}
];
};
}
```
### Extra Components {#services-home-assistant-usage-extra-components}
Packaged components can be found in the documentation of the corresponding option

View file

@ -1,37 +0,0 @@
{
lib,
pkgs,
stdenvNoCC,
}:
stdenvNoCC.mkDerivation {
name = "lldap-ha-auth";
src = pkgs.fetchFromGitHub {
owner = "lldap";
repo = "lldap";
rev = "7d1f5abc137821c500de99c94f7579761fc949d8";
sha256 = "sha256-8D+7ww70Ja6Qwdfa+7MpjAAHewtCWNf/tuTAExoUrg0=";
};
nativeBuildInputs = [
pkgs.makeWrapper
];
buildPhase = ''
mkdir -p $out/bin
cp example_configs/lldap-ha-auth.sh $out/bin/lldap-ha-auth
chmod a+x $out/bin/lldap-ha-auth
'';
installPhase = ''
wrapProgram $out/bin/lldap-ha-auth \
--prefix PATH : ${
lib.makeBinPath [
pkgs.gnused
pkgs.curl
pkgs.jq
]
}
'';
}

View file

@ -1,286 +0,0 @@
{
config,
lib,
shb,
...
}:
let
cfg = config.shb.homepage;
inherit (lib) types;
in
{
imports = [
../../lib/module.nix
../blocks/lldap.nix
../blocks/nginx.nix
];
options.shb.homepage = {
enable = lib.mkEnableOption "the SHB homepage service";
subdomain = lib.mkOption {
type = types.str;
description = ''
Subdomain under which homepage will be served.
```
<subdomain>.<domain>
```
'';
example = "homepage";
};
domain = lib.mkOption {
description = ''
Domain under which homepage is served.
```
<subdomain>.<domain>
```
'';
type = types.str;
example = "domain.com";
};
ssl = lib.mkOption {
description = "Path to SSL files";
type = types.nullOr shb.contracts.ssl.certs;
default = null;
};
servicesGroups = lib.mkOption {
description = "Group of services that should be showed on the dashboard.";
default = { };
type = types.attrsOf (
types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = types.str;
description = "Display name of the group. Defaults to the attr name.";
default = name;
};
sortOrder = lib.mkOption {
description = ''
Order in which groups will be shown.
The rules are:
- Lowest number is shown first.
- Two groups having the same number are shown in a consistent (same across multiple deploys) but undefined order.
- Default is null which means at the end.
'';
type = types.nullOr types.int;
default = null;
};
services = lib.mkOption {
description = "Services that should be showed in the group on the dashboard.";
default = { };
type = types.attrsOf (
types.submodule (
{ name, ... }:
{
options = {
name = lib.mkOption {
type = types.str;
description = "Display name of the service. Defaults to the attr name.";
default = name;
};
sortOrder = lib.mkOption {
type = types.nullOr types.int;
description = ''
Order in which groups will be shown.
The rules are:
- Lowest number is shown first.
- Two groups having the same number are shown in a consistent (same across multiple deploys) but undefined order.
- Default is null which means at the end.
'';
default = null;
};
dashboard = lib.mkOption {
description = ''
Provider of the dashboard contract.
By default:
- The `serviceName` option comes from the attr name.
- The `icon` option comes from applying `toLower` on the attr name.
- The `siteMonitor` option is set only if `internalUrl` is set.
'';
type = types.submodule {
options = shb.contracts.dashboard.mkProvider {
resultCfg = { };
};
};
};
apiKey = lib.mkOption {
description = ''
API key used to access the service.
This can be used to get data from the service.
'';
default = null;
type = types.nullOr (
lib.types.submodule {
options = shb.contracts.secret.mkRequester {
owner = "root";
restartUnits = [ "homepage-dashboard.service" ];
};
}
);
};
settings = lib.mkOption {
description = ''
Extra options to pass to the homepage service.
Check https://gethomepage.dev/configs/services/#icons
if the default icon is not correct.
And check https://gethomepage.dev/widgets
if the default widget type is not correct.
'';
default = { };
type = types.attrsOf types.anything;
example = lib.literalExpression ''
{
icon = "si-homeassistant";
widget.type = "firefly";
widget.custom = [
{
template = "{{ states('sensor.total_power', with_unit=True, rounded=True) }}";
label = "energy now";
}
{
state = "sensor.total_power_today";
label = "energy today";
}
];
}
'';
};
};
}
)
);
};
};
}
)
);
};
ldap = lib.mkOption {
description = ''
Setup LDAP integration.
'';
default = { };
type = types.submodule {
options = {
userGroup = lib.mkOption {
type = types.str;
description = "Group users must belong to be able to login.";
default = "homepage_user";
};
};
};
};
sso = lib.mkOption {
description = ''
Setup SSO integration.
'';
default = { };
type = types.submodule {
options = {
enable = lib.mkEnableOption "SSO integration.";
authEndpoint = lib.mkOption {
type = lib.types.str;
description = "Endpoint to the SSO provider.";
example = "https://authelia.example.com";
};
authorization_policy = lib.mkOption {
type = types.enum [
"one_factor"
"two_factor"
];
description = "Require one factor (password) or two factor (device) authentication.";
default = "one_factor";
};
};
};
};
};
config = lib.mkIf cfg.enable {
services.homepage-dashboard = {
enable = true;
allowedHosts = "${cfg.subdomain}.${cfg.domain}";
settings = {
baseUrl = "https://${cfg.subdomain}.${cfg.domain}";
startUrl = "https://${cfg.subdomain}.${cfg.domain}";
disableUpdateCheck = true;
};
bookmarks = [ ];
services = shb.homepage.asServiceGroup cfg.servicesGroups;
widgets = [ ];
};
systemd.services.homepage-dashboard.serviceConfig =
let
keys = shb.homepage.allKeys cfg.servicesGroups;
in
{
# LoadCredential = [
# "Media_Jellyfin:/path"
# ];
LoadCredential = lib.mapAttrsToList (name: path: "${name}:${path}") keys;
# Environment = [
# "HOMEPAGE_FILE_Media_Jellyfin=%d/Media_Jellyfin"
# ];
Environment = lib.mapAttrsToList (name: path: "HOMEPAGE_FILE_${name}=%d/${name}") keys;
};
# This should be using a contract instead of setting the option directly.
shb.lldap = lib.mkIf config.shb.lldap.enable {
ensureGroups = {
${cfg.ldap.userGroup} = { };
};
};
shb.nginx.vhosts = [
(
{
inherit (cfg) subdomain domain ssl;
upstream = "http://127.0.0.1:${toString config.services.homepage-dashboard.listenPort}/";
extraConfig = ''
proxy_read_timeout 300s;
proxy_send_timeout 300s;
'';
autheliaRules = lib.optionals (cfg.sso.enable) [
{
domain = "${cfg.subdomain}.${cfg.domain}";
policy = cfg.sso.authorization_policy;
subject = [ "group:${cfg.ldap.userGroup}" ];
}
];
}
// lib.optionalAttrs cfg.sso.enable {
inherit (cfg.sso) authEndpoint;
}
)
];
};
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 128 KiB

View file

@ -1,193 +0,0 @@
# Homepage Service {#services-homepage}
Defined in [`/modules/services/homepage.nix`](@REPO@/modules/services/homepage.nix),
found in the `selfhostblocks.nixosModules.homepage` module.
See [the manual](usage.html#usage-flake) for how to import the module in your code.
This service sets up [Homepage Dashboard][] which provides
a highly customizable homepage Docker and service API integrations.
![](./Screenshot.png)
[Homepage Dashboard]: https://github.com/gethomepage/homepage
## Features {#services-homepage-features}
- Declarative SSO login through forward authentication.
Only users of the [Homepage LDAP user group][] can access the web UI.
This is enforced using the [Authelia block][] which integrates with the LLDAP block.
- Access through [subdomain][] using the reverse proxy.
It is implemented with the [Nginx block][].
- Access through [HTTPS][] using the reverse proxy.
It is implemented with the [SSL block][].
- Integration with [secrets contract][] to set the API key for a widget.
[Homepage LDAP user group]: #services-homepage-options-shb.homepage.ldap.userGroup
[Authelia block]: blocks-authelia.html
[subdomain]: #services-open-webui-options-shb.open-webui.subdomain
[HTTPS]: #services-open-webui-options-shb.open-webui.ssl
[Nginx block]: blocks-nginx.html
[SSL block]: blocks-ssl.html
[secrets contract]: contracts-secret.html
::: {.note}
The service does not use state so no backup or impermanence integration is provided.
:::
## Usage {#services-homepage-usage}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
- the [`shb.ssl` block](blocks-ssl.html#usage),
- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
- the [`shb.authelia` block](blocks-authelia.html#blocks-sso-global-setup).
::: {.note}
Part of the configuration is done through the `shb.homepage` option described here
and the rest is done through the upstream [`services.homepage-dashboard`][] option.
:::
[`services.homepage-dashboard`]: https://search.nixos.org/options?query=services.homepage-dashboard
### Main service configuration {#services-homepage-usage-main}
This part sets up the web UI and its integration with the other SHB services.
It also creates the various service groups which will hold each service.
The names are arbitrary and you can order them as you wish through the `sortOrder` option.
```nix
{
shb.certs.certs.letsencrypt.${domain}.extraDomains = [
"${config.shb.homepage.subdomain}.${config.shb.homepage.domain}"
];
shb.homepage = {
enable = true;
subdomain = "home";
inherit domain;
ssl = config.shb.certs.certs.letsencrypt.${domain};
sso = {
enable = true;
authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
};
servicesGroups = {
Home.sortOrder = 1;
Documents.sortOrder = 2;
Finance.sortOrder = 3;
Media.sortOrder = 4;
Admin.sortOrder = 5;
};
};
services.homepage-dashboard = {
settings = {
statusStyle = "dot";
disableIndexing = true;
};
widgets = [
{
datetime = {
locale = "fr";
format = {
dateStyle = "long";
timeStyle = "long";
};
};
}
];
};
}
```
The [Homepage LDAP user group][] is created automatically and users can be added declaratively to the group with:.
```nix
{
shb.lldap.ensureUsers.${user}.groups = [
config.shb.homepage.ldap.userGroup
];
}
```
### Display SHB service {#services-homepage-usage-service}
A service consumer of the dashboard contract provides a `dashboard` option that can be used like so:
```nix
{
shb.homepage.servicesGroups.Media.services.Jellyfin = {
sortOrder = 2;
dashboard.request = config.shb.jellyfin.dashboard.request;
};
}
```
By default:
- The `serviceName` option comes from the attr name, here `Jellyfin`.
- The `icon` option comes from applying `toLower` on the attr name.
- The `siteMonitor` option is set only if `internalUrl` is set.
They can be overridden by setting them in the [settings](#services-homepage-options-shb.homepage.servicesGroups._name_.services._name_.settings option) (see option documentation for examples):
```nix
{
shb.homepage.servicesGroups.Media.services.Jellyfin = {
sortOrder = 2;
dashboard.request = config.shb.<service>.dashboard.request;
settings = {
// custom options here.
};
};
}
```
Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.
### Display custom service {#services-homepage-usage-custom}
To display a service that does not provide a `dashboard` option like in the previous section, set the values of the request manually:
```nix
{
shb.homepage.servicesGroups.Media.services.Jellyfin = {
sortOrder = 2;
dashboard.request = {
externalUrl = "https://jellyfin.example.com";
internalUrl = "http://127.0.0.1:8081";
};
};
}
```
### Add API key for widget {#services-homepage-usage-widget}
For services [supporting a widget](https://gethomepage.dev/widgets/),
create an API key through the service's web UI if available
then store it securely (using SOPS for example) and provide it through the
`apiKey` option:
```nix
{
shb.homepage.servicesGroups.Media.services.Jellyfin = {
sortOrder = 1;
dashboard.request = config.shb.jellyfin.dashboard.request;
apiKey.result = config.shb.sops.secret."jellyfin/homepageApiKey".result;
};
shb.sops.secret."jellyfin/homepageApiKey".request =
config.shb.homepage.servicesGroups.Media.services.Jellyfin.apiKey.request;
}
```
Unfortunately creating API keys declaratively is rarely supported by upstream services.
## Options Reference {#services-homepage-options}
```{=include=} options
id-prefix: services-homepage-options-
list-id: selfhostblocks-service-homepage-options
source: @OPTIONS_JSON@
```

View file

@ -144,22 +144,6 @@ in
default = 2283;
};
publicProxyEnable = mkOption {
description = ''
Enable Immich Public Proxy service for sharing media publically.
'';
type = bool;
default = false;
};
publicProxyPort = mkOption {
description = ''
Port under which Immich Public Proxy will listen.
'';
type = port;
default = 2284;
};
ssl = mkOption {
description = "Path to SSL files";
type = nullOr shb.contracts.ssl.certs;
@ -465,20 +449,6 @@ in
default = false;
example = true;
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${fqdn}";
externalUrlText = "https://\${config.shb.immich.subdomain}.\${config.shb.immich.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.port}";
};
};
};
};
config = mkIf cfg.enable {
@ -505,6 +475,12 @@ in
# Database configuration defaults to Unix socket /run/postgresql
# Database configuration
database = {
# Disable pgvecto.rs, as it was deprecated before SHB integration
enableVectors = false;
};
# Machine learning configuration
machine-learning = mkIf cfg.machineLearning.enable {
enable = true;
@ -526,12 +502,6 @@ in
};
};
services.immich-public-proxy = mkIf (cfg.publicProxyEnable) {
enable = true;
port = cfg.publicProxyPort;
immichUrl = "https://${fqdn}";
};
# Create basic directories for Immich
systemd.tmpfiles.rules = [
"d /var/lib/immich 0700 immich immich"
@ -580,8 +550,6 @@ in
resources = [
"^/api.*"
"^/.well-known/immich"
"^/share.*"
"^/_app/immutable/.*"
];
}
{
@ -604,15 +572,9 @@ in
];
# Allow large uploads from mobile app
services.nginx.virtualHosts."${fqdn}" = {
extraConfig = ''
client_max_body_size 50G;
'';
locations."^~ /share" = {
recommendedProxySettings = true;
proxyPass = "http://127.0.0.1:${toString cfg.publicProxyPort}";
};
};
services.nginx.virtualHosts."${fqdn}".extraConfig = ''
client_max_body_size 50G;
'';
# Ensure services start in correct order
systemd.services.immich-server = {

View file

@ -13,38 +13,31 @@ let
fqdn = "${cfg.subdomain}.${cfg.domain}";
jellyfin = pkgs.buildDotnetModule rec {
pname = "jellyfin";
version = "10.11.6";
jellyfin-cli = pkgs.buildDotnetModule rec {
pname = "jellyfin-cli";
version = "10.10.7";
src = pkgs.fetchFromGitHub {
owner = "ibizaman";
repo = "jellyfin";
rev = "c58ca41d9ee76d137be788cd6f2d089e288ad561";
hash = "sha256-gTHsz5qRT+9FjAqBb4hDBkHChYDU52snBWu6cQb10i4=";
rev = "0b1a5d929960f852dba90c1fc36f3a19dc094f8d";
hash = "sha256-H9V65+886EYMn/xDEgmxvoEOrbZaI1wSfmkN9vAzGhw=";
};
propagatedBuildInputs = [ pkgs.sqlite ];
projectFile = "Jellyfin.Server/Jellyfin.Server.csproj";
executables = [ "jellyfin" ];
projectFile = "Jellyfin.Cli/Jellyfin.Cli.csproj";
executables = [ "jellyfin-cli" ];
nugetDeps = "${pkgs.path}/pkgs/by-name/je/jellyfin/nuget-deps.json";
runtimeDeps = [
pkgs.jellyfin-ffmpeg
pkgs.fontconfig
pkgs.freetype
];
dotnet-sdk = pkgs.dotnetCorePackages.sdk_9_0;
dotnet-runtime = pkgs.dotnetCorePackages.aspnetcore_9_0;
dotnet-sdk = pkgs.dotnetCorePackages.sdk_8_0;
dotnet-runtime = pkgs.dotnetCorePackages.aspnetcore_8_0;
dotnetBuildFlags = [ "--no-self-contained" ];
makeWrapperArgs = [
"--append-flags"
"--ffmpeg=${pkgs.jellyfin-ffmpeg}/bin/ffmpeg"
"--append-flags"
"--webdir=${pkgs.jellyfin-web}/share/jellyfin-web"
];
passthru.tests = {
smoke-test = pkgs.nixosTests.jellyfin;
};
@ -60,24 +53,10 @@ let
purcell
jojosch
];
mainProgram = "jellyfin";
mainProgram = "jellyfin-cli";
platforms = dotnet-runtime.meta.platforms;
};
};
pluginName =
src:
let
meta = builtins.fromJSON (builtins.readFile "${src}/meta.json");
in
"${meta.name}_${meta.version}";
pluginNamePrefix =
src:
let
meta = builtins.fromJSON (builtins.readFile "${src}/meta.json");
in
"${meta.name}";
in
{
options.shb.jellyfin = {
@ -140,30 +119,6 @@ in
);
};
plugins = lib.mkOption {
description = ''
Install plugins declaratively.
The LDAP and SSO plugins will be added if their respective
shb.jellyfin.ldap.enable and shb.jellyfin.sso.enable options are set to true.
The interface for plugin creation is WIP.
Feel free to add yours following the examples from the LDAP and SSO plugins
but know that they may require some tweaks later on.
Notably, configuration is not yet handled by this option
so that will be added in the future.
Each plugin's meta.json must be writeable because Jellyfin appends some information
upon installing the plugin, like its active or disabled status.
SHB automatically enables the plugin
and deletes any plugin with the same prefix but other versions.
Note that SHB does not attempt to find which version is latest.
If twice the same plugin is added, the last one in the "plugins" list wins.
'';
default = [ ];
type = types.listOf types.package;
};
ldap = lib.mkOption {
description = "LDAP configuration.";
default = { };
@ -171,17 +126,6 @@ in
options = {
enable = lib.mkEnableOption "LDAP";
plugin = lib.mkOption {
type = lib.types.package;
description = "Pluging used for LDAP authentication.";
default = shb.mkJellyfinPlugin (rec {
pname = "jellyfin-plugin-ldapauth";
version = "22";
url = "https://github.com/jellyfin/${pname}/releases/download/v${version}/ldap-authentication_${version}.0.0.0.zip";
hash = "sha256-m2oD9woEuoSRiV9OeifAxZN7XQULMKS0Yq4TF+LjjpI=";
});
};
host = lib.mkOption {
type = types.str;
description = "Host serving the LDAP server.";
@ -234,17 +178,6 @@ in
options = {
enable = lib.mkEnableOption "SSO";
plugin = lib.mkOption {
type = lib.types.package;
description = "Pluging used for SSO authentication.";
default = shb.mkJellyfinPlugin (rec {
pname = "jellyfin-plugin-sso";
version = "4.0.0.3";
url = "https://github.com/9p4/${pname}/releases/download/v${version}/sso-authentication_${version}.zip";
hash = "sha256-Jkuc+Ua7934iSutf/zTY1phTxaltUkfiujOkCi7BW8w=";
});
};
provider = lib.mkOption {
type = types.str;
description = "OIDC provider name";
@ -263,6 +196,18 @@ in
default = "jellyfin";
};
adminUserGroup = lib.mkOption {
type = types.str;
description = "OIDC admin group";
default = "jellyfin_admin";
};
userGroup = lib.mkOption {
type = types.str;
description = "OIDC user group";
default = "jellyfin_user";
};
authorization_policy = lib.mkOption {
type = types.enum [
"one_factor"
@ -311,23 +256,8 @@ in
];
sourceDirectoriesText = ''
[
"services.jellyfin.dataDir"
]
'';
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${fqdn}";
externalUrlText = "https://\${config.shb.jellyfin.subdomain}.\${config.shb.jellyfin.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.port}";
"services.jellyfin.dataDir"
]'';
};
};
};
@ -340,16 +270,6 @@ in
[ "shb" "jellyfin" "adminPassword" ]
[ "shb" "jellyfin" "admin" "password" ]
)
# (lib.mkRenamedOptionModule
# [ "shb" "jellyfin" "sso" "userGroup" ]
# [ "shb" "jellyfin" "ldap" "userGroup" ]
# )
# (lib.mkRenamedOptionModule
# [ "shb" "jellyfin" "sso" "adminUserGroup" ]
# [ "shb" "jellyfin" "ldap" "adminGroup" ]
# )
];
config = lib.mkIf cfg.enable {
@ -361,7 +281,6 @@ in
];
services.jellyfin.enable = true;
services.jellyfin.package = jellyfin;
networking.firewall = {
# from https://jellyfin.org/docs/general/networking/index.html, for auto-discovery
@ -552,10 +471,10 @@ in
<EnableAllFolders>true</EnableAllFolders>
<EnabledFolders />
<AdminRoles>
<string>${cfg.ldap.adminGroup}</string>
<string>${cfg.sso.adminUserGroup}</string>
</AdminRoles>
<Roles>
<string>${cfg.ldap.userGroup}</string>
<string>${cfg.sso.userGroup}</string>
</Roles>
<EnableFolderRoles>false</EnableFolderRoles>
<FolderRoleMappings />
@ -678,7 +597,7 @@ in
];
})
+ lib.strings.optionalString cfg.ldap.enable (
(shb.replaceSecretsScript {
shb.replaceSecretsScript {
file = ldapConfig;
resultPath = "${config.services.jellyfin.dataDir}/plugins/configurations/LDAP-Auth.xml";
replacements = [
@ -687,7 +606,7 @@ in
source = cfg.ldap.adminPassword.result.path;
}
];
})
}
)
+ lib.strings.optionalString cfg.sso.enable (
shb.replaceSecretsScript {
@ -708,52 +627,12 @@ in
replacements = [
];
}
)
+ (
let
pluginInstallScript = p: ''
pluginDir="${config.services.jellyfin.dataDir}/plugins/${pluginName p}"
mkdir -p "$pluginDir"
for f in "${p}"/*; do
ln -sf "$f" "$pluginDir"
done
rm "$pluginDir/meta.json"
${pkgs.jq}/bin/jq ". + {
status: \"Active\",
autoUpdate: false,
assemblies: []
}" "${p}/meta.json" > "$pluginDir/meta.json"
echo "Disabling other versions of plugin ${pluginName p}"
for p in "${config.services.jellyfin.dataDir}/plugins/${pluginNamePrefix p}"*; do
if [ "$p" = "$pluginDir" ]; then
continue
fi
echo "Marking plugin $p as disabled"
${pkgs.jq}/bin/jq ". + {
status: \"Disabled\",
}" "$p/meta.json" > "$p/meta.json.new"
mv "$p/meta.json.new" "$p/meta.json"
done
'';
in
lib.concatMapStringsSep "\n" pluginInstallScript cfg.plugins
);
shb.jellyfin.plugins =
lib.optionals cfg.ldap.enable [ cfg.ldap.plugin ]
++ lib.optionals cfg.sso.enable [ cfg.sso.plugin ];
systemd.tmpfiles.rules = lib.optionals cfg.ldap.enable [
"d '${config.services.jellyfin.dataDir}/plugins' 0750 jellyfin jellyfin - -"
];
systemd.services.jellyfin.serviceConfig.ExecStartPost =
let
# We must always wait for the service to be fully initialized,
# even if we're planning on changing the config and restarting.
# And the service is not initialized until this URL returns a 200 and not a 503.
waitForCurl = pkgs.writeShellApplication {
name = "waitForCurl";
runtimeInputs = [ pkgs.curl ];
@ -794,14 +673,14 @@ in
#
# If the file does not exist, write the config, create the file then restart.
# If the file exists, do nothing and remove the file, resetting the state for the next time.
restartedFile = "${config.services.jellyfin.dataDir}/shb-jellyfin-restarted";
restartedFile = "${config.services.jellyfin.dataDir}/.jellyfin-restarted";
writeConfig = pkgs.writeShellApplication {
name = "writeConfig";
runtimeInputs = [ pkgs.systemd ];
text = ''
if ! [ -f "${restartedFile}" ]; then
${lib.getExe config.services.jellyfin.package} config \
${lib.getExe jellyfin-cli} wizard \
--datadir='${config.services.jellyfin.dataDir}' \
--configdir='${config.services.jellyfin.configDir}' \
--cachedir='${config.services.jellyfin.cacheDir}' \
@ -823,7 +702,7 @@ in
rm "${restartedFile}"
else
echo "Restarting jellyfin.service"
echo "This file is used by SelfHostBlocks to know when to restart jellyfin" > "${restartedFile}"
touch "${restartedFile}"
systemctl reload-or-restart jellyfin.service
fi
'';
@ -840,7 +719,7 @@ in
systemd.services.jellyfin.serviceConfig.TimeoutStartSec = 300;
shb.authelia.oidcClients = lib.optionals cfg.sso.enable [
shb.authelia.oidcClients = lib.lists.optionals (!(isNull cfg.sso)) [
{
client_id = cfg.sso.clientID;
client_name = "Jellyfin";

View file

@ -5,33 +5,25 @@ Defined in [`/modules/services/jellyfin.nix`](@REPO@/modules/services/jellyfin.n
This NixOS module is a service that sets up a [Jellyfin](https://jellyfin.org/) instance.
Compared to the stock module from nixpkgs,
this one sets up, in a fully declarative manner:
- the initial wizard with an admin user thanks to a custom Jellyfin CLI
and a custom restart logic to apply the changes from the CLI.
- LDAP and SSO integration thanks to a custom declarative installation of plugins.
this one sets up, in a fully declarative manner
the initial wizard with an admin user
and LDAP and SSO integration.
## Features {#services-jellyfin-features}
- Declarative creation of admin user.
- Declarative selection of listening port.
- Access through [subdomain](#services-jellyfin-options-shb.jellyfin.subdomain)
and [HTTPS](#services-jellyfin-options-shb.jellyfin.ssl) using reverse proxy. [Manual](#services-jellyfin-usage).
- Declarative plugin installation. [Manual](#services-jellyfin-options-shb.jellyfin.plugins).
- Declarative [LDAP](#services-jellyfin-options-shb.jellyfin.ldap) configuration.
- Declarative [SSO](#services-jellyfin-options-shb.jellyfin.sso) configuration.
- Access through [subdomain](#services-jellyfin-options-shb.jellyfin.subdomain) using reverse proxy. [Manual](#services-jellyfin-usage-configuration).
- Access through [HTTPS](#services-jellyfin-options-shb.jellyfin.ssl) using reverse proxy. [Manual](#services-jellyfin-usage-https).
- Declarative [LDAP](#services-jellyfin-options-shb.jellyfin.ldap) configuration. [Manual](#services-jellyfin-usage-ldap).
- Declarative [SSO](#services-jellyfin-options-shb.jellyfin.sso) configuration. [Manual](#services-jellyfin-usage-sso).
- [Backup](#services-jellyfin-options-shb.jellyfin.backup) through the [backup block](./blocks-backup.html). [Manual](#services-jellyfin-usage-backup).
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-jellyfin-usage-applicationdashboard)
## Usage {#services-jellyfin-usage}
### Initial Configuration {#services-jellyfin-usage-configuration}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
- the [`shb.ssl` block](blocks-ssl.html#usage),
- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
- the [`shb.authelia` block](blocks-authelia.html#blocks-sso-global-setup).
The following snippet enables Jellyfin and makes it available under the `jellyfin.example.com` endpoint.
```nix
shb.jellyfin = {
@ -41,60 +33,122 @@ shb.jellyfin = {
admin = {
username = "admin";
password.result = config.shb.sops.secret."jellyfin/adminPassword".result;
};
ldap = {
enable = true;
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secret."jellyfin/ldap/adminPassword".result
};
sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secretFile = config.shb.sops.secret."jellyfin/sso_secret".result;
secretFileForAuthelia = config.shb.sops.secret."jellyfin/authelia/sso_secret".result;
password.result = config.shb.sops.secret.jellyfinAdminPassword.result;
};
};
shb.sops.secret."jellyfin/adminPassword".request = config.shb.jellyfin.admin.password.request;
shb.sops.secret.jellyfinAdminPassword.request = config.shb.jellyfin.admin.password.request;
```
shb.sops.secret."jellyfin/ldap/adminPassword".request = config.shb.jellyfin.ldap.adminPassword.request;
This assumes secrets are setup with SOPS
as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual.
shb.sops.secret."jellyfin/sso_secret".request = config.shb.jellyfin.sso.sharedSecret.request;
shb.sops.secret."jellyfin/authelia/sso_secret" = {
request = config.shb.jellyfin.sso.sharedSecretForAuthelia.request;
settings.key = "jellyfin/sso_secret";
### Jellyfin through HTTPS {#services-jellyfin-usage-https}
:::: {.note}
We will build upon the [Initial Configuration](#services-jellyfin-usage-configuration) section,
so please follow that first.
::::
If the `shb.ssl` block is used (see [manual](blocks-ssl.html#usage) on how to set it up),
the instance will be reachable at `https://jellyfin.example.com`.
Here is an example with Let's Encrypt certificates, validated using the HTTP method.
First, set the global configuration for your domain:
```nix
shb.certs.certs.letsencrypt."example.com" = {
domain = "example.com";
group = "nginx";
reloadServices = [ "nginx.service" ];
adminEmail = "myemail@mydomain.com";
};
```
Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.
Then you can tell Jellyfin to use those certificates.
The [user](#services-jellyfin-options-shb.jellyfin.ldap.userGroup)
and [admin](#services-jellyfin-options-shb.jellyfin.ldap.adminGroup)
LDAP groups are created automatically.
```nix
shb.certs.certs.letsencrypt."example.com".extraDomains = [ "jellyfin.example.com" ];
shb.jellyfin = {
ssl = config.shb.certs.certs.letsencrypt."example.com";
};
```
### With LDAP Support {#services-jellyfin-usage-ldap}
:::: {.note}
We will build upon the [HTTPS](#services-jellyfin-usage-https) section,
so please follow that first.
::::
We will use the [LLDAP block][] provided by Self Host Blocks.
Assuming it [has been set already][LLDAP block setup], add the following configuration:
[LLDAP block]: blocks-lldap.html
[LLDAP block setup]: blocks-lldap.html#blocks-lldap-global-setup
```nix
shb.jellyfin.ldap
enable = true;
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminPassword.result = config.shb.sops.secrets."jellyfin/ldap/adminPassword".result
};
shb.sops.secrets."jellyfin/ldap/adminPassword" = {
request = config.shb.jellyfin.ldap.adminPassword.request;
settings.key = "ldap/userPassword";
};
```
The `shb.jellyfin.ldap.adminPasswordFile` must be the same
as the `shb.lldap.ldapUserPasswordFile` which is achieved
with the `key` option.
The other secrets can be randomly generated with
`nix run nixpkgs#openssl -- rand -hex 64`.
And that's it.
Now, go to the LDAP server at `http://ldap.example.com`,
create the `jellyfin_user` and `jellyfin_admin` groups,
create a user and add it to one or both groups.
When that's done, go back to the Jellyfin server at
`http://jellyfin.example.com` and login with that user.
Work is in progress to make the creation of the LDAP user and group declarative too.
### With SSO Support {#services-jellyfin-usage-sso}
:::: {.note}
We will build upon the [LDAP](#services-jellyfin-usage-ldap) section,
so please follow that first.
::::
We will use the [SSO block][] provided by Self Host Blocks.
Assuming it [has been set already][SSO block setup], add the following configuration:
[SSO block]: blocks-sso.html
[SSO block setup]: blocks-sso.html#blocks-sso-global-setup
```nix
shb.jellyfin.sso = {
enable = true;
endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}";
secretFile = <path/to/oidcJellyfinSharedSecret>;
secretFileForAuthelia = <path/to/oidcJellyfinSharedSecret>;
};
```
Passing the `ssl` option will auto-configure nginx to force SSL connections with the given
certificate.
The `shb.jellyfin.sso.secretFile` and `shb.jellyfin.sso.secretFileForAuthelia` options
must have the same content. The former is a file that must be owned by the `jellyfin` user while
the latter must be owned by the `authelia` user. I want to avoid needing to define the same secret
twice with a future secrets SHB block.
### Certificates {#services-jellyfin-certs}
For Let's Encrypt certificates, add:
```nix
{
shb.certs.certs.letsencrypt.${domain}.extraDomains = [
"${config.shb.jellyfin.subdomain}.${config.shb.jellyfin.domain}"
];
}
```
### Backup {#services-jellyfin-usage-backup}
Backing up Jellyfin using the [Restic block](blocks-restic.html) is done like so:
@ -115,56 +169,6 @@ You can define any number of Restic instances to backup Jellyfin multiple times.
You will then need to configure more options like the `repository`,
as explained in the [restic](blocks-restic.html) documentation.
### Impermanence {#services-jellyfin-impermanence}
To save the data folder in an impermanence setup, add:
```nix
{
shb.zfs.datasets."safe/jellyfin".path = config.shb.jellyfin.impermanence;
}
```
### Declarative LDAP {#services-jellyfin-declarative-ldap}
To add a user `USERNAME` to the user and admin groups for jellyfin, add:
```nix
shb.lldap.ensureUsers.USERNAME.groups = [
config.shb.jellyfin.ldap.userGroup
config.shb.jellyfin.ldap.adminGroup
];
```
### Application Dashboard {#services-jellyfin-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-jellyfin-options-shb.jellyfin.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Media.services.Jellyfin = {
sortOrder = 1;
dashboard.request = config.shb.jellyfin.dashboard.request;
};
}
```
An API key can be set to show extra info:
```nix
{
shb.homepage.servicesGroups.Media.services.Jellyfin = {
apiKey.result = config.shb.sops.secret."jellyfin/homepageApiKey".result;
};
shb.sops.secret."jellyfin/homepageApiKey".request =
config.shb.homepage.servicesGroups.Media.services.Jellyfin.apiKey.request;
}
```
## Debug {#services-jellyfin-debug}
In case of an issue, check the logs for systemd service `jellyfin.service`.

View file

@ -175,20 +175,6 @@ in
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.karakeep.subdomain}.\${config.shb.karakeep.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.port}";
};
};
};
};
config = (

View file

@ -21,12 +21,9 @@ It integrates well with [Ollama][].
- Access through [subdomain](#services-karakeep-options-shb.karakeep.subdomain) using reverse proxy.
- Access through [HTTPS](#services-karakeep-options-shb.karakeep.ssl) using reverse proxy.
- [Backup](#services-karakeep-options-shb.karakeep.sso) through the [backup block](./blocks-backup.html).
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-karakeep-usage-applicationdashboard)
## Usage {#services-karakeep-usage}
### Initial Configuration {#services-karakeep-usage-configuration}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
@ -69,35 +66,6 @@ The [user](#services-open-webui-options-shb.open-webui.ldap.userGroup)
and [admin](#services-open-webui-options-shb.open-webui.ldap.adminGroup)
LDAP groups are created automatically.
### Application Dashboard {#services-karakeep-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-karakeep-options-shb.karakeep.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Documents.services.Karakeep = {
sortOrder = 3;
dashboard.request = config.shb.karakeep.dashboard.request;
};
}
```
An API key can be set to show extra info:
```nix
{
shb.homepage.servicesGroups.Documents.services.Karakeep = {
apiKey.result = config.shb.sops.secret."karakeep/homepageApiKey".result;
};
shb.sops.secret."karakeep/homepageApiKey".request =
config.shb.homepage.servicesGroups.Documents.services.Karakeep.apiKey.request;
}
```
## Integration with Ollama {#services-karakeep-ollama}
Assuming ollama is enabled, it will be available on port `config.services.ollama.port`.

View file

@ -1,792 +0,0 @@
{
config,
lib,
shb,
pkgs,
...
}:
let
cfg = config.shb.mailserver;
in
{
imports = [
(
builtins.fetchGit {
url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver.git";
ref = "master";
rev = "e33fbde199eaad513ef5d0746db19d5878150232";
}
+ "/default.nix"
)
../blocks/lldap.nix
];
options.shb.mailserver = {
enable = lib.mkEnableOption "SHB's nixos-mailserver module";
subdomain = lib.mkOption {
type = lib.types.str;
description = "Subdomain under which imap and smtp functions will be served.";
default = "imap";
};
domain = lib.mkOption {
type = lib.types.str;
description = "domain under which imap and smtp functions will be served.";
example = "mydomain.com";
};
ssl = lib.mkOption {
description = "Path to SSL files";
type = lib.types.nullOr shb.contracts.ssl.certs;
default = null;
};
adminUsername = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
Admin username.
postmaster will be made an alias of this user.
'';
example = "admin";
};
adminPassword = lib.mkOption {
description = "Admin user password.";
default = null;
type = lib.types.nullOr (
lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.services.postfix.user;
ownerText = "services.postfix.user";
restartUnits = [ "dovecot.service" ];
};
}
);
};
imapSync = lib.mkOption {
description = ''
Synchronize one or more email providers through IMAP
to your dovecot instance.
This allows you to backup that email provider
and centralize your accounts in this dovecot instance.
'';
default = null;
type = lib.types.nullOr (
lib.types.submodule {
options = {
syncTimer = lib.mkOption {
type = lib.types.str;
default = "5m";
description = ''
Systemd timer for when imap sync job should happen.
This timer is not scheduling the job at regular intervals.
After a job finishes, the given amount of time is waited then the next job is started.
The default is set deliberatily slow to not spam you when setting up your mailserver.
When everything works, you will want to reduce it to 10s or something like that.
'';
example = "10s";
};
debug = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Enable verbose mbsync logging.";
};
accounts = lib.mkOption {
description = ''
Accounts to sync emails from using IMAP.
Emails will be stored under `''${config.mailserver.storage.path}/''${name}/''${username}`
'';
type = lib.types.attrsOf (
lib.types.submodule {
options = {
host = lib.mkOption {
type = lib.types.str;
description = "Hostname of the email's provider IMAP server.";
example = "imap.fastmail.com";
};
port = lib.mkOption {
type = lib.types.port;
description = "Port of the email's provider IMAP server.";
default = 993;
};
username = lib.mkOption {
type = lib.types.str;
description = "Username used to login to the email's provider IMAP server.";
example = "userA@fastmail.com";
};
password = lib.mkOption {
description = ''
Password used to login to the email's provider IMAP server.
The password could be an "app password" like for [Fastmail](https://www.fastmail.help/hc/en-us/articles/360058752854-App-passwords)
'';
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.mailserver.storage.owner;
restartUnits = [ "mbsync.service" ];
};
};
};
sslType = lib.mkOption {
description = "Connection security method.";
type = lib.types.enum [
"IMAPS"
"STARTTLS"
];
default = "IMAPS";
};
timeout = lib.mkOption {
description = "Connect and data timeout.";
type = lib.types.int;
default = 120;
};
mapSpecialDrafts = lib.mkOption {
type = lib.types.str;
default = "Drafts";
description = ''
Drafts special folder name on far side.
You only need to change this if mbsync logs the following error:
Error: ... far side box Drafts cannot be opened
'';
};
mapSpecialSent = lib.mkOption {
type = lib.types.str;
default = "Sent";
description = ''
Sent special folder name on far side.
You only need to change this if mbsync logs the following error:
Error: ... far side box Sent cannot be opened
'';
};
mapSpecialTrash = lib.mkOption {
type = lib.types.str;
default = "Trash";
description = ''
Trash special folder name on far side.
You only need to change this if mbsync logs the following error:
Error: ... far side box Trash cannot be opened
'';
};
mapSpecialJunk = lib.mkOption {
type = lib.types.str;
default = "Junk";
description = ''
Junk special folder name on far side.
You only need to change this if mbsync logs the following error:
Error: ... far side box Junk cannot be opened
'';
example = "Spam";
};
};
}
);
};
};
}
);
};
smtpRelay = lib.mkOption {
description = ''
Proxy outgoing emails through an email provider.
In short, this can help you avoid having your outgoing emails marked as spam.
See the manual for a lengthier explanation.
'';
default = null;
type = lib.types.nullOr (
lib.types.submodule {
options = {
host = lib.mkOption {
type = lib.types.str;
description = "Hostname of the email's provider SMTP server.";
example = "smtp.fastmail.com";
};
port = lib.mkOption {
type = lib.types.port;
description = "Port of the email's provider SMTP server.";
default = 587;
};
username = lib.mkOption {
description = "Username used to login to the email's provider SMTP server.";
type = lib.types.str;
};
password = lib.mkOption {
description = ''
Password used to login to the email's provider IMAP server.
The password could be an "app password" like for [Fastmail](https://www.fastmail.help/hc/en-us/articles/360058752854-App-passwords)
'';
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = config.services.postfix.user;
ownerText = "services.postfix.user";
restartUnits = [ "postfix.service" ];
};
};
};
};
}
);
};
ldap = lib.mkOption {
description = ''
LDAP Integration.
Enabling this app will create a new LDAP configuration or update one that exists with
the given host.
'';
default = null;
type = lib.types.nullOr (
lib.types.submodule {
options = {
enable = lib.mkEnableOption "LDAP app.";
host = lib.mkOption {
type = lib.types.str;
description = ''
Host serving the LDAP server.
'';
default = "127.0.0.1";
};
port = lib.mkOption {
type = lib.types.port;
description = ''
Port of the service serving the LDAP server.
'';
default = 389;
};
dcdomain = lib.mkOption {
type = lib.types.str;
description = "dc domain for ldap.";
example = "dc=mydomain,dc=com";
};
account = lib.mkOption {
type = lib.types.str;
description = ''
Select one account from those defined in `shb.mailserver.imapSync.accounts`
to login with.
Using LDAP, you can only connect to one account.
This limitation could maybe be lifted, feel free to post an issue if you need this.
'';
};
adminName = lib.mkOption {
type = lib.types.str;
description = "Admin user of the LDAP server.";
default = "admin";
};
adminPassword = lib.mkOption {
description = "LDAP server admin password.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = "nextcloud";
restartUnits = [ "dovecot.service" ];
};
};
};
userGroup = lib.mkOption {
type = lib.types.str;
description = "Group users must belong to to be able to use mails.";
default = "mail_user";
};
};
}
);
};
stateVersion = lib.mkOption {
type = lib.types.ints.positive;
# SHB started at stateVersion 3.
default = 4;
description = ''
Tracking stateful version changes as an incrementing number.
When a new release comes out we may require manual migration steps to
be completed, before the new version can be put into production.
If your `stateVersion` is too low one or multiple assertions may
trigger to give you instructions on what migrations steps are required
to continue. Increase the `stateVersion` as instructed by the assertion
message.
See https://nixos-mailserver.readthedocs.io/en/latest/release-notes.html
and https://nixos-mailserver.readthedocs.io/en/latest/migrations.html
'';
};
backup = lib.mkOption {
description = ''
Backup emails, index and sieve.
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = config.mailserver.storage.owner;
sourceDirectories = builtins.filter (x: x != null) [
config.mailserver.indexDir
config.mailserver.storage.path
config.mailserver.sieveDirectory
];
sourceDirectoriesText = ''
[
config.mailserver.indexDir
config.mailserver.storage.path
config.mailserver.sieveDirectory
]
'';
};
};
};
backupDKIM = lib.mkOption {
description = ''
Backup dkim directory.
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.backup.mkRequester {
user = config.services.rspamd.user;
userText = "services.rspamd.user";
sourceDirectories = builtins.filter (x: x != null) [
config.mailserver.dkim.keyDirectory
];
sourceDirectoriesText = ''
[
config.mailserver.dkimKeyDirectory
]
'';
};
};
};
impermanence = lib.mkOption {
description = ''
Path to save when using impermanence setup.
'';
type = lib.types.attrsOf lib.types.str;
default = {
index = config.mailserver.indexDir;
mail = config.mailserver.storage.path;
sieve = config.mailserver.sieveDirectory;
dkim = config.mailserver.dkimKeyDirectory;
};
defaultText = lib.literalExpression ''
{
index = config.mailserver.indexDir;
mail = config.mailserver.storage.path;
sieve = config.mailserver.sieveDirectory;
dkim = config.mailserver.dkimKeyDirectory;
}
'';
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.mailserver.subdomain}.\${config.shb.mailserver.domain}";
};
};
};
};
config = lib.mkMerge [
(lib.mkIf cfg.enable {
mailserver = {
enable = true;
fqdn = "${cfg.subdomain}.${cfg.domain}";
inherit (cfg) stateVersion;
domains = [ cfg.domain ];
localDnsResolver = false;
enableImapSsl = true;
enableSubmissionSsl = true;
x509 = {
certificateFile = cfg.ssl.paths.cert;
privateKeyFile = cfg.ssl.paths.key;
};
# Using / is needed for iOS mail.
# Both following options are used to organize subfolders in subdirectories.
hierarchySeparator = "/";
storage = {
directoryLayout = "fs";
};
};
services.postfix.settings.main = {
smtpd_tls_security_level = lib.mkForce "encrypt";
};
# Is probably needed for iOS mail.
services.dovecot2.settings = {
ssl_min_protocol = "TLSv1.2";
# This conflicts with the default setting which seems much better than this one.
# So I'm leaving the default settings and we'll see if something breaks.
# ssl_cipher_list = "HIGH:!aNULL:!MD5";
};
services.nginx = {
enable = true;
virtualHosts."${cfg.domain}" =
let
announce = pkgs.writeTextDir "config-v1.1.xml" ''
<?xml version="1.0" encoding="UTF-8"?>
<clientConfig version="1.1">
<emailProvider id="${cfg.domain}">
<domain>${cfg.domain}</domain>
<displayName>${cfg.domain} Mailserver</displayName>
<!-- Incoming IMAP server -->
<incomingServer type="imap">
<hostname>${cfg.subdomain}.${cfg.domain}</hostname>
<port>993</port>
<socketType>SSL</socketType>
<authentication>password-cleartext</authentication>
<username>%EMAILADDRESS%</username>
</incomingServer>
<!-- Outgoing SMTP server -->
<outgoingServer type="smtp">
<hostname>${cfg.subdomain}.${cfg.domain}</hostname>
<port>465</port>
<socketType>SSL</socketType>
<authentication>password-cleartext</authentication>
<username>%EMAILADDRESS%</username>
</outgoingServer>
</emailProvider>
</clientConfig>
'';
in
{
forceSSL = true; # Redirect HTTP → HTTPS
sslCertificate = cfg.ssl.paths.cert;
sslCertificateKey = cfg.ssl.paths.key;
root = "/var/www"; # Dummy root
locations."/.well-known/autoconfig/mail/" = {
alias = "${announce}/";
extraConfig = ''
default_type application/xml;
'';
};
};
virtualHosts."${cfg.subdomain}.${cfg.domain}" =
let
landingPage = pkgs.writeTextDir "index.html" ''
<html><body>
<p>Configuration of the mailserver is done automatically thanks to
<a href="https://${cfg.domain}/.well-known/autoconfig/mail/config-v1.1.xml">${cfg.domain}/.well-known/autoconfig/mail/config-v1.1.xml</a>.</p>
</body></html>
'';
in
{
forceSSL = true; # Redirect HTTP → HTTPS
sslCertificate = cfg.ssl.paths.cert;
sslCertificateKey = cfg.ssl.paths.key;
root = "/var/www"; # Dummy root
locations."/" = {
alias = "${landingPage}/";
extraConfig = ''
default_type application/html;
'';
};
};
};
})
(lib.mkIf (cfg.enable && cfg.adminUsername != null) {
assertions = [
{
assertion = cfg.adminPassword != null;
message = "`shb.mailserver.adminPassword` must be not null if `shb.mailserver.adminUsername` is not null.";
}
];
mailserver = {
# To create the password hashes, use:
# nix run nixpkgs#mkpasswd -- --run 'mkpasswd -s'
loginAccounts = {
"${cfg.adminUsername}@${cfg.domain}" = {
hashedPasswordFile = cfg.adminPassword.result.path;
aliases = [ "postmaster@${cfg.domain}" ];
};
};
};
})
(lib.mkIf (cfg.enable && cfg.ldap != null) {
assertions = [
{
assertion = cfg.adminUsername == null;
message = "`shb.mailserver.adminUsername` must be null `shb.mailserver.ldap` integration is set.";
}
];
shb.lldap.ensureGroups = {
${cfg.ldap.userGroup} = { };
};
mailserver = {
ldap = {
enable = true;
uris = [
"ldap://${cfg.ldap.host}:${toString cfg.ldap.port}"
];
base = "ou=people,${cfg.ldap.dcdomain}";
scope = "sub";
bind = {
dn = "uid=${cfg.ldap.adminName},ou=people,${cfg.ldap.dcdomain}";
passwordFile = cfg.ldap.adminPassword.result.path;
};
dovecot =
let
filter = "(&(objectClass=inetOrgPerson)(mail=%{user})(memberOf=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain}))";
in
{
passFilter = filter;
userFilter = filter;
};
# username needs to be set to mail so postfix maps correctly the mail address.
# Otherwise we get error messages when sending
# Sender address rejected: not owned by user
attributes = {
username = "mail";
};
postfix = {
filter = "(&(objectClass=inetOrgPerson)(mail=%s)(memberOf=cn=${cfg.ldap.userGroup},ou=groups,${cfg.ldap.dcdomain}))";
};
};
};
services.dovecot2.settings = {
"userdb ldap" = {
fields.home = lib.mkForce "${config.mailserver.storage.path}/${cfg.ldap.account}/%{user}";
fields.mail_index_path = lib.mkForce "${config.mailserver.storage.path}/${cfg.ldap.account}/%{user}";
};
"passdb ldap" = {
# LLDAP does not understand ldap user password setup.
fields.password = lib.mkForce null;
# We use bind DN auth.
# https://doc.dovecot.org/2.3/configuration_manual/authentication/ldap_bind/#authentication-ldap-bind
bind = true;
};
};
})
(lib.mkIf (cfg.enable && cfg.imapSync != null) {
systemd.services.mbsync =
let
configFile =
let
mkAccount = name: acct: ''
# ${name} account
IMAPAccount ${name}
Host ${acct.host}
Port ${toString acct.port}
User ${acct.username}
PassCmd "cat ${acct.password.result.path}"
TLSType ${acct.sslType}
AuthMechs LOGIN
Timeout ${toString acct.timeout}
IMAPStore ${name}-remote
Account ${name}
MaildirStore ${name}-local
INBOX ${config.mailserver.storage.path}/${name}/${acct.username}/mail/
# Maps subfolders on far side to actual subfolders on disk.
# The other option is Maildir++ but then the mailserver.hierarchySeparator must be set to a dot '.'
SubFolders Verbatim
Path ${config.mailserver.storage.path}/${name}/${acct.username}/mail/
Channel ${name}-main
Far :${name}-remote:
Near :${name}-local:
Patterns * !Drafts !Sent !Trash !Junk !${acct.mapSpecialDrafts} !${acct.mapSpecialSent} !${acct.mapSpecialTrash} !${acct.mapSpecialJunk}
Create Both
Expunge Both
SyncState *
Sync All
CopyArrivalDate yes # Preserve date from incoming message.
Channel ${name}-drafts
Far :${name}-remote:"${acct.mapSpecialDrafts}"
Near :${name}-local:"Drafts"
Create Both
Expunge Both
SyncState *
Sync All
CopyArrivalDate yes # Preserve date from incoming message.
Channel ${name}-sent
Far :${name}-remote:"${acct.mapSpecialSent}"
Near :${name}-local:"Sent"
Create Both
Expunge Both
SyncState *
Sync All
CopyArrivalDate yes # Preserve date from incoming message.
Channel ${name}-trash
Far :${name}-remote:"${acct.mapSpecialTrash}"
Near :${name}-local:"Trash"
Create Both
Expunge Both
SyncState *
Sync All
CopyArrivalDate yes # Preserve date from incoming message.
Channel ${name}-junk
Far :${name}-remote:"${acct.mapSpecialJunk}"
Near :${name}-local:"Junk"
Create Both
Expunge Both
SyncState *
Sync All
CopyArrivalDate yes # Preserve date from incoming message.
Group ${name}
Channel ${name}-main
Channel ${name}-drafts
Channel ${name}-sent
Channel ${name}-trash
Channel ${name}-junk
# END ${name} account
'';
in
pkgs.writeText "mbsync.conf" (
lib.concatStringsSep "\n" (lib.mapAttrsToList mkAccount cfg.imapSync.accounts)
);
in
{
description = "Sync mailbox";
serviceConfig = {
Type = "oneshot";
User = config.mailserver.storage.owner;
};
script =
let
debug = if cfg.imapSync.debug then "-V" else "";
in
''
${pkgs.isync}/bin/mbsync --all ${debug} --config ${configFile}
'';
};
systemd.tmpfiles.rules =
let
mkAccount =
name: acct:
# The equal sign makes sure parent directories have the corret user and group too.
[
"d '${config.mailserver.storage.path}/${name}' 0750 ${config.mailserver.storage.owner} ${config.mailserver.storage.group} - -"
"d '${config.mailserver.storage.path}/${name}/${acct.username}' 0750 ${config.mailserver.storage.owner} ${config.mailserver.storage.group} - -"
];
in
lib.flatten (lib.mapAttrsToList mkAccount cfg.imapSync.accounts);
systemd.timers.mbsync = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = cfg.imapSync.syncTimer;
OnUnitActiveSec = cfg.imapSync.syncTimer;
};
};
})
(lib.mkIf (cfg.enable && cfg.smtpRelay != null) (
let
url = "[${cfg.smtpRelay.host}]:${toString cfg.smtpRelay.port}";
in
{
assertions = [
{
assertion = lib.hasAttr cfg.adminPassword != null;
message = "`shb.mailserver.adminPassword` must be not null if `shb.mailserver.adminUsername` is not null.";
}
];
# Inspiration from https://www.brull.me/postfix/debian/fastmail/2016/08/16/fastmail-smtp.html
services.postfix = {
settings.main = {
relayhost = [ url ];
smtp_sasl_auth_enable = "yes";
smtp_sasl_password_maps = "texthash:/run/secrets/postfix/postfix-smtp-relay-password";
smtp_sasl_security_options = "noanonymous";
smtp_use_tls = "yes";
};
};
systemd.services.postfix-pre = {
script = shb.replaceSecrets {
userConfig = {
inherit url;
inherit (cfg.smtpRelay) username;
password.source = cfg.smtpRelay.password.result.path;
};
generator =
name:
{
url,
username,
password,
}:
pkgs.writeText "postfix-smtp-relay-password" ''
${url} ${username}:${password}
'';
resultPath = "/run/secrets/postfix/postfix-smtp-relay-password";
user = config.services.postfix.user;
};
serviceConfig.Type = "oneshot";
wantedBy = [ "multi-user.target" ];
before = [ "postfix.service" ];
requiredBy = [ "postfix.service" ];
};
}
))
];
}

View file

@ -1,398 +0,0 @@
# Mailserver Service {#services-mailserver}
Defined in [`/modules/services/mailserver.nix`](@REPO@/modules/services/mailserver.nix).
This NixOS module is a service that sets up
the [NixOS Simple Mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver) project.
It integrates the upstream project
with the SHB modules like the SSL module, the contract for secrets and the LLDAP module.
It also exposes an XML file which allows some email clients to auto configure themselves.
Setting up a self-hosted email server in this age
can be quite time consuming because you need to maintain
a good IP hygiene to avoid being marked as spam from the big players.
To avoid needing to deal with this,
this module provides the means
to use an email provider (like Fastmail or ProtonMail) as a mere proxy.
If you also setup the email provider using your own custom domain,
this combination allows you to change email provider
without needing to change your clients or notify your email correspondents
and keep a backup of all your emails at the same time.
The setup looks like so:
```
Domain --[ DNS records ]-> Email Provider --[ mbsync ]-> SHB Server
Internet <---------------- Email Provider <-[ postfix ]-- SHB Server
```
Configuring your domain name to point to your email provider is out of scope here.
See the documentation for "custom domain" for you email provider,
like for [Fastmail](https://www.fastmail.com/features/domains/)
and [ProtonMail](https://proton.me/support/custom-domain)
To use an email provider as a proxy, use the
[shb.mailserver.imapSync](#services-mailserver-options-shb.mailserver.imapSync)
and [shb.mailserver.smtpRelay](#services-mailserver-options-shb.mailserver.smtpRelay),
options.
## Usage {#services-mailserver-usage}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
- the [`shb.ssl` block](blocks-ssl.html#usage),
- the [`shb.lldap` block](blocks-lldap.html#blocks-lldap-global-setup).
```nix
let
domain = "example.com";
username = "me@example.com";
in
{
imports = [
selfhostblocks.nixosModules.mailserver
];
shb.mailserver = {
enable = true;
inherit domain;
subdomain = "imap";
ssl = config.shb.certs.certs.letsencrypt."domain";
imapSync = {
syncTimer = "10s";
accounts.fastmail = {
host = "imap.fastmail.com";
port = 993;
inherit username;
password.result = config.shb.sops.secret."mailserver/imap/fastmail/password".result;
mapSpecialJunk = "Spam";
};
};
smtpRelay = {
host = "smtp.fastmail.com";
port = 587;
inherit username;
password.result = config.shb.sops.secret."mailserver/smtp/fastmail/password".result;
};
ldap = {
enable = true;
host = "127.0.0.1";
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminName = "admin";
adminPassword.result = config.shb.sops.secret."mailserver/ldap_admin_password".result;
account = "fastmail";
};
};
# Optionally add some mailboxes
mailserver.mailboxes = {
Drafts = {
auto = "subscribe";
specialUse = "Drafts";
};
Junk = {
auto = "subscribe";
specialUse = "Junk";
};
Sent = {
auto = "subscribe";
specialUse = "Sent";
};
Trash = {
auto = "subscribe";
specialUse = "Trash";
};
Archive = {
auto = "subscribe";
specialUse = "Archive";
};
};
shb.sops.secret."mailserver/smtp/fastmail/password".request =
config.shb.mailserver.smtpRelay.password.request;
shb.sops.secret."mailserver/imap/fastmail/password".request =
config.shb.mailserver.imapSync.accounts.fastmail.password.request;
shb.sops.secret."mailserver/ldap_admin_password" = {
request = config.shb.mailserver.ldap.adminPassword.request;
# This reuses the admin password set in the shb.lldap module.
settings.key = "lldap/user_password";
};
}
```
With the example above, the mails are stored under `/var/vmail/fastmail/<email address>`.
### Secrets {#services-mailserver-usage-secrets}
Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`.
### LDAP {#services-mailserver-usage-ldap}
The [user](#services-mailserver-options-shb.mailserver.ldap.userGroup)
LDAP group is created automatically.
### Disk Layout {#services-mailserver-usage-disk-layout}
The disk layout has been purposely set to use slashes `/` for subfolders.
By experience, this works better with iOS mail.
### Backup {#services-mailserver-usage-backup}
Backing up your emails using the [Restic block](blocks-restic.html) is done like so:
```nix
shb.restic.instances."mailserver" = {
request = config.shb.mailserver.backup;
settings = {
enable = true;
};
};
```
The name `"mailserver"` in the `instances` can be anything.
The `config.shb.mailserver.backup` option provides what directories to backup.
You can define any number of Restic instances to backup your emails multiple times.
You will then need to configure more options like the `repository`,
as explained in the [restic](blocks-restic.html) documentation.
### Certificates {#services-mailserver-certs}
For Let's Encrypt certificates, add:
```nix
let
domain = "example.com";
in
{
shb.certs.certs.letsencrypt.${domain}.extraDomains = [
"${config.shb.mailserver.subdomain}.${config.shb.mailserver.domain}"
];
}
```
### Impermanence {#services-mailserver-impermanence}
To save the data folder in an impermanence setup, add:
```nix
{
shb.zfs.datasets."safe/mailserver/index".path = config.shb.mailserver.impermanence.index;
shb.zfs.datasets."safe/mailserver/mail".path = config.shb.mailserver.impermanence.mail;
shb.zfs.datasets."safe/mailserver/sieve".path = config.shb.mailserver.impermanence.sieve;
shb.zfs.datasets."safe/mailserver/dkim".path = config.shb.mailserver.impermanence.dkim;
}
```
### Declarative LDAP {#services-mailserver-declarative-ldap}
To add a user `USERNAME` to the user group, add:
```nix
shb.lldap.ensureUsers.USERNAME.groups = [
config.shb.mailserver.ldap.userGroup
];
```
### Application Dashboard {#services-mailserver-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-mailserver-options-shb.mailserver.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Home.services.Mailserver = {
sortOrder = 1;
dashboard.request = config.shb.mailserver.dashboard.request;
};
}
```
## Debug {#services-mailserver-debug}
Debugging this will be certainly necessary.
The first issue you will encounter will probably be with `mbsync`
under the [shb.mailserver.imapSync](#services-mailserver-options-shb.mailserver.imapSync) option
with the folder name mapping.
### Systemd Services {#services-mailserver-debug-systemd}
The 3 systemd services setup by this module are:
- `mbsync.service`
- `dovecot.service`
- `postfix.service`
### Folders {#services-mailserver-debug-folders}
The 4 folders where state is stored are:
- `config.mailserver.indexDir` = `/var/lib/dovecot/indices`
- `config.mailserver.mailDirectory` = `/var/vmail`
- `config.mailserver.sieveDirectory` = `/var/sieve`
- `config.mailserver.dkimKeyDirectory` = `/var/dkim`
### Open Ports {#services-mailserver-debug-ports}
The ports opened by default in this module are:
- Submissions: 465
- Imap: 993
You will need to forward those ports on your router
if you want to access to your emails from the internet.
The complete list can be found in the [upstream repository](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/blob/5965fae920b6b97f39f94bdb6195631e274c93a5/mail-server/networking.nix).
### List Email Provider Folder Mapping {#services-mailserver-debug-folder-mapping}
Replace `$USER` and `$PASSWORD` by those used to connect to your email provider.
Yes, you will need to enter verbatim `a LOGIN ...` and `b LIST "" "*"`.
```
$ nix run nixpkgs#openssl -- s_client -connect imap.fastmail.com:993 -crlf -quiet
a LOGIN $USER $password
b LIST "" "*"
```
Example output will be:
```
* LIST (\HasNoChildren) "/" INBOX
* LIST (\HasNoChildren \Drafts) "/" Drafts
* LIST (\HasNoChildren \Sent) "/" Sent
* LIST (\Noinferiors \HasNoChildren \Junk) "/" Spam
...
```
Here you can see the special folder `\Junk` is actually named `Spam`.
To handle this, set the `.mapSpecial*` options:
```
{
shb.mailserver.imapSync.accounts.<account> = {
mapSpecialJunk = "Spam";
};
}
```
### List Local Folders {#services-mailserver-debug-local-folders}
Check the local folders to make sure the mapping is correct
and all folders are correctly downloaded.
For example, if the mapping above is wrong, you will see both a
`Junk` and `Spam` folder while if it is correct,
you will only see the `Junk` folder.
```
$ sudo doveadm mailbox list -u $USER
Junk
Trash
Drafts
Sent
INBOX
MyCustomFolder
```
The following command shows the number of messages in a folder:
```
$ sudo doveadm mailbox status -u $USER messages INBOX
INBOX messages=13591
```
If any folder is not appearing or has 0 message but should have some,
it could mean dovecot is not setup correctly and assumes an incorrect folder layout.
If that is the case, check the user config with:
```
$ sudo doveadm user $USER
field value
uid 5000
gid 5000
home /var/vmail/fastmail/$USER
mail maildir:~/mail:LAYOUT=fs
virtualMail
```
### Test Auth {#services-mailserver-debug-auth}
To test authentication to your dovecot instance, run:
```
$ nix run nixpkgs#openssl -- s_client -connect $SUBDOMAIN.$DOMAIN:993 -crlf -quiet
. LOGIN $USER $PASSWORD
```
You must here also enter the second line verbatim,
replacing your user and password with the real one.
On success, you will see:
```
. OK [CAPABILITY IMAP4rev1 ...] Logged in
```
Otherwise, either if the password is wrong or,
when using LDAP if the user is not part of the LDAP group, you will see:
```
. NO [AUTHENTICATIONFAILED] Authentication failed.
```
To test the postfix instance, run:
```
$ swaks \
--server $SUBDOMAIN.$DOMAIN \
--port 465 \
--tls-on-connect \
--auth LOGIN \
--auth-user $USER \
--auth-password '$PASSWORD' \
--from $USER \
--to $USER
```
Try once with a wrong password and once with a correct one.
The former should log:
```
<~* 535 5.7.8 Error: authentication failed: (reason unavailable)
```
## Mobile Apps {#services-mailserver-mobile}
This module was tested with:
- the iOS mail mobile app,
- Thunderbird on NixOS.
The iOS mail app is pretty finicky.
If downloading emails does not work,
make sure the certificate used includes the whole chain:
```bash
$ openssl s_client -connect $SUBDOMAIN.$DOMAIN:993 -showcerts
```
Normally, the other options are setup correctly but if it fails for you,
feel free to open an issue.
## Options Reference {#services-mailserver-options}
```{=include=} options
id-prefix: services-mailserver-options-
list-id: selfhostblocks-service-mailserver-options
source: @OPTIONS_JSON@
```

View file

@ -28,21 +28,10 @@ in
{
imports = [
../../lib/module.nix
../blocks/authelia.nix
../blocks/monitoring.nix
(lib.mkRenamedOptionModule
[ "shb" "nextcloud" "adminUser" ]
[ "shb" "nextcloud" "initialAdminUsername" ]
)
];
options.shb.nextcloud = {
enable = lib.mkEnableOption "the SHB Nextcloud service";
enableDashboard = lib.mkEnableOption "the Nextcloud SHB dashboard" // {
default = true;
};
enable = lib.mkEnableOption "selfhostblocks.nextcloud-server";
subdomain = lib.mkOption {
type = lib.types.str;
@ -96,10 +85,10 @@ in
version = lib.mkOption {
description = "Nextcloud version to choose from.";
type = lib.types.enum [
31
32
33
];
default = 32;
default = 31;
};
dataDir = lib.mkOption {
@ -115,9 +104,9 @@ in
example = lib.literalExpression ''["var.mount"]'';
};
initialAdminUsername = lib.mkOption {
adminUser = lib.mkOption {
type = lib.types.str;
description = "Initial username of the admin user. Once it is set, it cannot be changed!";
description = "Username of the initial admin user.";
default = "root";
};
@ -468,6 +457,9 @@ in
sso = lib.mkOption {
description = ''
SSO Integration App. [Manual](https://docs.nextcloud.com/server/latest/admin_manual/configuration_user/oidc_auth.html)
Enabling this app will create a new LDAP configuration or update one that exists with
the given host.
'';
default = { };
type = lib.types.submodule {
@ -509,12 +501,7 @@ in
adminGroup = lib.mkOption {
type = lib.types.str;
description = ''
Group admins must belong to to be able to login to Nextcloud.
This option is purposely not inside the LDAP app because only SSO allows
distinguising between users and admins.
'';
description = "Group admins must belong to to be able to login to Nextcloud.";
default = "nextcloud_admin";
};
@ -686,11 +673,6 @@ in
Upon starting the service, disable maintenance mode if set.
This is useful if a deploy failed and you try to redeploy.
Note that even if the disabling of maintenance mode fails,
SHB will still allow the startup to continue
because there are valid reasons for maintenance mode
to not be able to be lifted, like for example this is a brand new installation.
'';
};
@ -701,27 +683,9 @@ in
Run `occ maintenance:repair --include-expensive` on service start.
Larger instances should disable this and run the command at a convenient time
but SHB assumes that it will not be the case for most users.
Note that SHB will still allow the startup
even if the repair failed.
but Self Host Blocks assumes that it will not be the case for most users.
'';
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${fqdn}";
externalUrlText = "https://\${config.shb.nextcloud.subdomain}.\${config.shb.nextcloud.domain}";
internalUrl = "https://${fqdn}";
internalUrlText = "https://\${config.shb.nextcloud.subdomain}.\${config.shb.nextcloud.domain}";
};
};
};
};
config = lib.mkMerge [
@ -763,7 +727,7 @@ in
config = {
dbtype = "pgsql";
adminuser = cfg.initialAdminUsername;
adminuser = cfg.adminUser;
adminpassFile = cfg.adminPass.result.path;
};
database.createLocally = true;
@ -1061,7 +1025,6 @@ in
${occ} app:enable user_ldap
${occ} config:app:set user_ldap ${cID}ldap_configuration_active --value=0
${occ} config:app:set user_ldap configuration_prefixes --value '["${cID}"]'
# The following CLI commands follow
# https://github.com/lldap/lldap/blob/main/example_configs/nextcloud.md#nextcloud-config--the-cli-way
@ -1194,10 +1157,7 @@ in
groups = "groups";
is_admin = "is_nextcloud_admin";
};
oidc_login_allowed_groups = [
cfg.apps.ldap.userGroup
cfg.apps.sso.adminGroup
];
oidc_login_allowed_groups = [ cfg.apps.ldap.userGroup ];
oidc_login_default_group = "oidc";
oidc_login_use_external_storage = false;
oidc_login_scope = lib.concatStringsSep " " scopes;
@ -1267,7 +1227,7 @@ in
(lib.mkIf (cfg.enable && cfg.autoDisableMaintenanceModeOnStart) {
systemd.services.nextcloud-setup.preStart = lib.mkBefore ''
if [[ -e /var/lib/nextcloud/config/config.php ]]; then
${occ} maintenance:mode --no-interaction --quiet --off || true
${occ} maintenance:mode --no-interaction --quiet --off
fi
'';
})
@ -1275,7 +1235,7 @@ in
(lib.mkIf (cfg.enable && cfg.alwaysApplyExpensiveMigrations) {
systemd.services.nextcloud-setup.script = ''
if [[ -e /var/lib/nextcloud/config/config.php ]]; then
${occ} maintenance:repair --include-expensive || true
${occ} maintenance:repair --include-expensive
fi
'';
})
@ -1369,11 +1329,5 @@ in
'';
}
))
(lib.mkIf (cfg.enable && cfg.enableDashboard) {
shb.monitoring.dashboards = [
./nextcloud-server/dashboard/Nextcloud.json
];
})
];
}

View file

@ -11,10 +11,8 @@ It is based on the nixpkgs Nextcloud server and provides opinionated defaults.
to configure those with the UI.
- [LDAP](#services-nextcloudserver-usage-ldap) app:
enables app and sets up integration with an existing LDAP server, in this case LLDAP.
Note that the LDAP app cannot distinguish between normal users and admin users.
- [SSO](#services-nextcloudserver-usage-oidc) app:
enables app and sets up integration with an existing SSO server, in this case Authelia.
The SSO app can distinguish between normal users and admin users.
- [Preview Generator](#services-nextcloudserver-usage-previewgenerator) app:
enables app and sets up required cron job.
- [External Storage](#services-nextcloudserver-usage-externalstorage) app:
@ -35,8 +33,8 @@ It is based on the nixpkgs Nextcloud server and provides opinionated defaults.
- Forces PostgreSQL as the database.
- Forces Redis as the cache and sets good defaults.
- Backup of the [`shb.nextcloud.dataDir`][dataDir] through the [backup block](./blocks-backup.html).
- [Monitoring Dashboard](#services-nextcloudserver-dashboard) for monitoring of reverse proxy, PHP-FPM, and database backups through the [monitoring block](./blocks-monitoring.html).
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard.
- [Dashboard](#services-nextcloudserver-dashboard) for monitoring of reverse proxy, PHP-FPM, and database backups through the [monitoring
block](./blocks-monitoring.html).
- [Integration Tests](@REPO@/test/services/nextcloud.nix)
- Tests system cron job is setup correctly.
- Tests initial admin user and password are setup correctly.
@ -72,11 +70,10 @@ shb.nextcloud = {
domain = "example.com";
subdomain = "n";
defaultPhoneRegion = "US";
initialAdminUsername = "root";
adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result;
adminPass.result = config.shb.sops.secrets."nextcloud/adminpass".result;
};
shb.sops.secret."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
shb.sops.secrets."nextcloud/adminpass".request = config.shb.nextcloud.adminPass.request;
```
This assumes secrets are setup with SOPS as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual.
@ -86,7 +83,7 @@ Note though that Nextcloud will not be very happy to be accessed through HTTP,
it much prefers - rightfully - to be accessed through HTTPS.
We will set that up in the next section.
You can now login as the admin user using the username `root`
You can now login as the admin user using the username `admin`
and the password defined in `sops.secrets."nextcloud/adminpass"`.
### Nextcloud through HTTPS {#services-nextcloudserver-usage-https}
@ -166,11 +163,11 @@ shb.nextcloud.apps.ldap = {
port = config.shb.lldap.ldapPort;
dcdomain = config.shb.lldap.dcdomain;
adminName = "admin";
adminPassword.result = config.shb.sops.secret."nextcloud/ldap/adminPassword".result
adminPassword.result = config.shb.sops.secrets."nextcloud/ldap/adminPassword".result
userGroup = "nextcloud_user";
};
shb.sops.secret."nextcloud/ldap/adminPassword" = {
shb.sops.secrets."nextcloud/ldap/adminPassword" = {
request = config.shb.nextcloud.apps.ldap.adminPassword.request;
settings.key = "ldap/userPassword";
};
@ -217,8 +214,8 @@ shb.nextcloud.apps.sso = {
clientID = "nextcloud";
fallbackDefaultAuth = false;
secret.result = config.shb.sops.secret."nextcloud/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secret."nextcloud/sso/secretForAuthelia".result;
secret.result = config.shb.sops.secrets."nextcloud/sso/secret".result;
secretForAuthelia.result = config.shb.sops.secrets."nextcloud/sso/secretForAuthelia".result;
};
shb.sops.secret."nextcloud/sso/secret".request = config.shb.nextcloud.apps.sso.secret.request;
@ -345,22 +342,6 @@ Note that this will backup the whole PostgreSQL instance,
not just the Nextcloud database.
This limitation will be lifted in the future.
### Application Dashboard {#services-nextcloudserver-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-nextcloudserver-options-shb.nextcloud.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Documents.services.Nextcloud = {
sortOrder = 1;
dashboard.request = config.shb.nextcloud.dashboard.request;
};
}
```
### Enable Preview Generator App {#services-nextcloudserver-usage-previewgenerator}
The following snippet installs and enables the [Preview
@ -554,7 +535,7 @@ by issuing the command `nextcloud-occ config:system:delete instanceid`.
Head over to the [Nextcloud demo](demo-nextcloud-server.html) for a demo that installs Nextcloud with or
without LDAP integration on a VM with minimal manual steps.
## Monitoring Dashboard {#services-nextcloudserver-dashboard}
## Dashboard {#services-nextcloudserver-dashboard}
The dashboard is added to Grafana automatically under "Self Host Blocks > Nextcloud"
as long as the Nextcloud service is [enabled][]

View file

@ -59,12 +59,12 @@ in
WEBUI_NAME = "SelfHostBlocks";
OLLAMA_BASE_URL = "http://127.0.0.1:''${toString config.services.ollama.port}";
RAG_EMBEDDING_ENGINE = "ollama";
RAG_EMBEDDING_MODEL = "nomic-embed-text:v1.5";
ENABLE_OPENAI_API = "True";
OPENAI_API_BASE_URL = "http://127.0.0.1:''${toString config.services.llama-cpp.port}";
ENABLE_WEB_SEARCH = "True";
RAG_EMBEDDING_ENGINE = "openai";
}
'';
};
@ -160,20 +160,6 @@ in
};
};
};
dashboard = lib.mkOption {
description = ''
Dashboard contract consumer
'';
default = { };
type = lib.types.submodule {
options = shb.contracts.dashboard.mkRequester {
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
externalUrlText = "https://\${config.shb.open-webui.subdomain}.\${config.shb.open-webui.domain}";
internalUrl = "http://127.0.0.1:${toString cfg.port}";
};
};
};
};
config = (
@ -230,6 +216,7 @@ in
package = pkgs.open-webui.overrideAttrs (finalAttrs: {
patches = [
../../patches/0001-selfhostblocks-never-onboard.patch
../../patches/0002-selfhostblocks-do-not-allow-unauthorized-roles.patch
];
});
environment = {

View file

@ -21,12 +21,9 @@ This service sets up [Open WebUI][] which provides a frontend to various LLMs.
- Access through [subdomain](#services-open-webui-options-shb.open-webui.subdomain) using reverse proxy.
- Access through [HTTPS](#services-open-webui-options-shb.open-webui.ssl) using reverse proxy.
- [Backup](#services-open-webui-options-shb.open-webui.sso) through the [backup block](./blocks-backup.html).
- Integration with the [dashboard contract](contracts-dashboard.html) for displaying user facing application in a dashboard. [Manual](#services-open-webui-usage-applicationdashboard)
## Usage {#services-open-webui-usage}
### Initial Configuration {#services-open-webui-usage-configuration}
The following snippet assumes a few blocks have been setup already:
- the [secrets block](usage.html#usage-secrets) with SOPS,
@ -66,25 +63,6 @@ The [user](#services-open-webui-options-shb.open-webui.ldap.userGroup)
and [admin](#services-open-webui-options-shb.open-webui.ldap.adminGroup)
LDAP groups are created automatically.
### Application Dashboard {#services-open-webui-usage-applicationdashboard}
Integration with the [dashboard contract](contracts-dashboard.html) is provided
by the [dashboard option](#services-open-webui-options-shb.open-webui.dashboard).
For example using the [Homepage](services-homepage.html) service:
```nix
{
shb.homepage.servicesGroups.Documents.services.OpenWebUI = {
sortOrder = 1;
dashboard.request = config.shb.home-assistant.dashboard.request;
settings.icon = "sh-open-webui";
};
}
```
The icon needs to be set manually otherwise it is not displayed correctly.
## Integration with OLLAMA {#services-open-webui-ollama}
Assuming ollama is enabled, it will be available on port `config.services.ollama.port`.

Some files were not shown because too many files have changed in this diff Show more