From 0e9f068bfbfdb598fe7df2a65e5fce3fbc1616c0 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Fri, 31 Oct 2025 11:58:09 +0100 Subject: [PATCH] borgbackup: use backup and databasebackup contracts --- docs/blocks.md | 50 +- docs/redirects.json | 270 ++++++- docs/services.md | 64 +- flake.nix | 5 + modules/blocks/borgbackup.nix | 681 +++++++++++------- modules/blocks/borgbackup/docs/default.md | 270 +++++++ modules/blocks/restic.nix | 86 ++- modules/blocks/restic/docs/default.md | 22 +- ...-add-option-to-override-state-direct.patch | 78 ++ test/blocks/borgbackup.nix | 190 +++++ test/contracts/backup.nix | 70 ++ test/contracts/databasebackup.nix | 47 ++ 12 files changed, 1502 insertions(+), 331 deletions(-) create mode 100644 modules/blocks/borgbackup/docs/default.md create mode 100644 patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch create mode 100644 test/blocks/borgbackup.nix diff --git a/docs/blocks.md b/docs/blocks.md index ec7f4ee..eac0702 100644 --- a/docs/blocks.md +++ b/docs/blocks.md @@ -8,6 +8,8 @@ others. Not all blocks are documented yet. You can find all available blocks [in the repository](@REPO@/modules/blocks). +## Authentication {#blocks-category-authentication} + ```{=include=} chapters html:into-file=//blocks-authelia.html modules/blocks/authelia/docs/default.md ``` @@ -16,26 +18,40 @@ modules/blocks/authelia/docs/default.md modules/blocks/lldap/docs/default.md ``` -```{=include=} chapters html:into-file=//blocks-mitmdump.html -modules/blocks/mitmdump/docs/default.md -``` +## Backup {#blocks-category-backup} -```{=include=} chapters html:into-file=//blocks-monitoring.html -modules/blocks/monitoring/docs/default.md -``` - -```{=include=} chapters html:into-file=//blocks-postgresql.html -modules/blocks/postgresql/docs/default.md -``` - -```{=include=} chapters html:into-file=//blocks-sops.html -modules/blocks/sops/docs/default.md -``` - -```{=include=} chapters html:into-file=//blocks-ssl.html -modules/blocks/ssl/docs/default.md +```{=include=} chapters html:into-file=//blocks-borgbackup.html +modules/blocks/borgbackup/docs/default.md ``` ```{=include=} chapters html:into-file=//blocks-restic.html modules/blocks/restic/docs/default.md ``` + +## Database {#blocks-category-database} + +```{=include=} chapters html:into-file=//blocks-postgresql.html +modules/blocks/postgresql/docs/default.md +``` + +## Secrets {#blocks-category-secrets} + +```{=include=} chapters html:into-file=//blocks-sops.html +modules/blocks/sops/docs/default.md +``` + +## Network {#blocks-category-network} + +```{=include=} chapters html:into-file=//blocks-ssl.html +modules/blocks/ssl/docs/default.md +``` + +## Introspection {#blocks-category-introspection} + +```{=include=} chapters html:into-file=//blocks-monitoring.html +modules/blocks/monitoring/docs/default.md +``` + +```{=include=} chapters html:into-file=//blocks-mitmdump.html +modules/blocks/mitmdump/docs/default.md +``` diff --git a/docs/redirects.json b/docs/redirects.json index fe8f736..f2b8e5f 100644 --- a/docs/redirects.json +++ b/docs/redirects.json @@ -326,6 +326,255 @@ "blocks-authelia-troubleshooting": [ "blocks-authelia.html#blocks-authelia-troubleshooting" ], + "blocks-borgbackup": [ + "blocks-borgbackup.html#blocks-borgbackup" + ], + "blocks-borgbackup-contract-provider": [ + "blocks-borgbackup.html#blocks-borgbackup-contract-provider" + ], + "blocks-borgbackup-maintenance": [ + "blocks-borgbackup.html#blocks-borgbackup-maintenance" + ], + "blocks-borgbackup-maintenance-troubleshooting": [ + "blocks-borgbackup.html#blocks-borgbackup-maintenance-troubleshooting" + ], + "blocks-borgbackup-monitoring": [ + "blocks-borgbackup.html#blocks-borgbackup-monitoring" + ], + "blocks-borgbackup-options": [ + "blocks-borgbackup.html#blocks-borgbackup-options" + ], + "blocks-borgbackup-options-shb.borgbackup.borgServer": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.borgServer" + ], + "blocks-borgbackup-options-shb.borgbackup.databases": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.request": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.request" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.request.backupCmd": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.request.backupCmd" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.request.backupName": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.request.backupName" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.request.restoreCmd": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.request.restoreCmd" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.request.user": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.request.user" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.result": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.result" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.result.backupService": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.result.backupService" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.result.restoreScript": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.result.restoreScript" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.consistency": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.consistency" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.enable": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.enable" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.limitUploadKiBs": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.limitUploadKiBs" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.group": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.group" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.mode": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.mode" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.owner": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.owner" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.restartUnits": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.request.restartUnits" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.result": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.result" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.result.path": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.passphrase.result.path" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.path": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.path" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.secrets": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.secrets" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.secrets._name_.source": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.secrets._name_.source" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.secrets._name_.transform": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.secrets._name_.transform" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.timerConfig": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.repository.timerConfig" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.retention": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.retention" + ], + "blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.stateDir": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.databases._name_.settings.stateDir" + ], + "blocks-borgbackup-options-shb.borgbackup.instances": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request.excludePatterns": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request.excludePatterns" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request.hooks": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request.hooks" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request.hooks.afterBackup": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request.hooks.afterBackup" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request.hooks.beforeBackup": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request.hooks.beforeBackup" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request.sourceDirectories": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request.sourceDirectories" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.request.user": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.request.user" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.result": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.result" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.result.backupService": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.result.backupService" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.result.restoreScript": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.result.restoreScript" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.consistency": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.consistency" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.enable": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.enable" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.limitUploadKiBs": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.limitUploadKiBs" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.group": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.group" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.mode": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.mode" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.owner": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.owner" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.restartUnits": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.request.restartUnits" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.result": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.result" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.result.path": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.passphrase.result.path" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.path": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.path" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.secrets": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.secrets" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.secrets._name_.source": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.secrets._name_.source" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.secrets._name_.transform": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.secrets._name_.transform" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.timerConfig": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.repository.timerConfig" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.retention": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.retention" + ], + "blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.stateDir": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.instances._name_.settings.stateDir" + ], + "blocks-borgbackup-options-shb.borgbackup.performance": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.performance" + ], + "blocks-borgbackup-options-shb.borgbackup.performance.ioPriority": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.performance.ioPriority" + ], + "blocks-borgbackup-options-shb.borgbackup.performance.ioSchedulingClass": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.performance.ioSchedulingClass" + ], + "blocks-borgbackup-options-shb.borgbackup.performance.niceness": [ + "blocks-borgbackup.html#blocks-borgbackup-options-shb.borgbackup.performance.niceness" + ], + "blocks-borgbackup-tests": [ + "blocks-borgbackup.html#blocks-borgbackup-tests" + ], + "blocks-borgbackup-usage": [ + "blocks-borgbackup.html#blocks-borgbackup-usage" + ], + "blocks-borgbackup-usage-multiple": [ + "blocks-borgbackup.html#blocks-borgbackup-usage-multiple" + ], + "blocks-borgbackup-usage-provider-contract": [ + "blocks-borgbackup.html#blocks-borgbackup-usage-provider-contract" + ], + "blocks-borgbackup-usage-provider-manual": [ + "blocks-borgbackup.html#blocks-borgbackup-usage-provider-manual" + ], + "blocks-borgbackup-usage-provider-remote": [ + "blocks-borgbackup.html#blocks-borgbackup-usage-provider-remote" + ], + "blocks-category-authentication": [ + "blocks.html#blocks-category-authentication" + ], + "blocks-category-backup": [ + "blocks.html#blocks-category-backup" + ], + "blocks-category-database": [ + "blocks.html#blocks-category-database" + ], + "blocks-category-introspection": [ + "blocks.html#blocks-category-introspection" + ], + "blocks-category-network": [ + "blocks.html#blocks-category-network" + ], + "blocks-category-secrets": [ + "blocks.html#blocks-category-secrets" + ], "blocks-lldap": [ "blocks-lldap.html#blocks-lldap" ], @@ -860,9 +1109,6 @@ "blocks-restic-contract-provider": [ "blocks-restic.html#blocks-restic-contract-provider" ], - "blocks-restic-demo": [ - "blocks-restic.html#blocks-restic-demo" - ], "blocks-restic-maintenance": [ "blocks-restic.html#blocks-restic-maintenance" ], @@ -2216,6 +2462,24 @@ "services-arr-options-shb.arr.sonarr.subdomain": [ "services-arr.html#services-arr-options-shb.arr.sonarr.subdomain" ], + "services-category-ai": [ + "services.html#services-category-ai" + ], + "services-category-automation": [ + "services.html#services-category-automation" + ], + "services-category-code": [ + "services.html#services-category-code" + ], + "services-category-documents": [ + "services.html#services-category-documents" + ], + "services-category-media": [ + "services.html#services-category-media" + ], + "services-category-passwords": [ + "services.html#services-category-passwords" + ], "services-forgejo": [ "services-forgejo.html#services-forgejo" ], diff --git a/docs/services.md b/docs/services.md index ac20e26..afffa65 100644 --- a/docs/services.md +++ b/docs/services.md @@ -1,8 +1,10 @@ # Services {#services} -Services are usually web applications that SHB help you self-host. Configuration of those is -purposely made opinionated and require as few nix options as can make sense. That is possible thanks to the extensive use of blocks provided by SHB. +Services are usually web applications that SHB help you self-host. +Configuration of those is purposely made more opinionated than the upstream nixpkgs modules +in exchange for requiring less options to define. +That is possible thanks to the extensive use of blocks provided by SHB. ::: {.note} Not all services are yet documented. You can find all available services [in the repository](@REPO@/modules/services). @@ -42,38 +44,50 @@ Legend: **N**: no but WIP; **P**: partial; **Y**: yes [Pinchflat]: services-pinchflat.html [Vaultwarden]: services-vaultwarden.html -```{=include=} chapters html:into-file=//services-arr.html -modules/services/arr/docs/default.md -``` - -```{=include=} chapters html:into-file=//services-forgejo.html -modules/services/forgejo/docs/default.md -``` - -```{=include=} chapters html:into-file=//services-home-assistant.html -modules/services/home-assistant/docs/default.md -``` - -```{=include=} chapters html:into-file=//services-jellyfin.html -modules/services/jellyfin/docs/default.md -``` - -```{=include=} chapters html:into-file=//services-karakeep.html -modules/services/karakeep/docs/default.md -``` +## Documents {#services-category-documents} ```{=include=} chapters html:into-file=//services-nextcloud.html modules/services/nextcloud-server/docs/default.md ``` -```{=include=} chapters html:into-file=//services-pinchflat.html -modules/services/pinchflat/docs/default.md +## Passwords {#services-category-passwords} + +```{=include=} chapters html:into-file=//services-vaultwarden.html +modules/services/vaultwarden/docs/default.md +``` + +## Automation {#services-category-automation} + +```{=include=} chapters html:into-file=//services-home-assistant.html +modules/services/home-assistant/docs/default.md +``` + +## AI {#services-category-ai} + +```{=include=} chapters html:into-file=//services-karakeep.html +modules/services/karakeep/docs/default.md ``` ```{=include=} chapters html:into-file=//services-open-webui.html modules/services/open-webui/docs/default.md ``` -```{=include=} chapters html:into-file=//services-vaultwarden.html -modules/services/vaultwarden/docs/default.md +## Code {#services-category-code} + +```{=include=} chapters html:into-file=//services-forgejo.html +modules/services/forgejo/docs/default.md +``` + +## Media {#services-category-media} + +```{=include=} chapters html:into-file=//services-arr.html +modules/services/arr/docs/default.md +``` + +```{=include=} chapters html:into-file=//services-jellyfin.html +modules/services/jellyfin/docs/default.md +``` + +```{=include=} chapters html:into-file=//services-pinchflat.html +modules/services/pinchflat/docs/default.md ``` diff --git a/flake.nix b/flake.nix index 841d3fd..c4d3ede 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,7 @@ shbPatches = originPkgs.lib.optionals (system == "x86_64-linux") [ # Get rid of lldap patches when https://github.com/NixOS/nixpkgs/pull/425923 is merged. ./patches/lldap.patch + ./patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch # Leaving commented out as an example. # (originPkgs.fetchpatch { @@ -102,6 +103,7 @@ ]; modules = { "blocks/authelia" = ./modules/blocks/authelia.nix; + "blocks/borgbackup" = ./modules/blocks/borgbackup.nix; "blocks/lldap" = ./modules/blocks/lldap.nix; "blocks/ssl" = { module = ./modules/blocks/ssl.nix; @@ -294,6 +296,7 @@ // (vm_test "vaultwarden" ./test/services/vaultwarden.nix) // (vm_test "authelia" ./test/blocks/authelia.nix) + // (vm_test "borgbackup" ./test/blocks/borgbackup.nix) // (vm_test "lldap" ./test/blocks/lldap.nix) // (vm_test "lib" ./test/blocks/lib.nix) // (vm_test "mitmdump" ./test/blocks/mitmdump.nix) @@ -368,6 +371,7 @@ imports = [ # blocks self.nixosModules.authelia + self.nixosModules.borgbackup self.nixosModules.davfs self.nixosModules.hardcodedsecret self.nixosModules.lldap @@ -400,6 +404,7 @@ }; nixosModules.authelia = modules/blocks/authelia.nix; + nixosModules.borgbackup = modules/blocks/borgbackup.nix; nixosModules.davfs = modules/blocks/davfs.nix; nixosModules.hardcodedsecret = modules/blocks/hardcodedsecret.nix; nixosModules.lldap = modules/blocks/lldap.nix; diff --git a/modules/blocks/borgbackup.nix b/modules/blocks/borgbackup.nix index d1a4101..d96f85d 100644 --- a/modules/blocks/borgbackup.nix +++ b/modules/blocks/borgbackup.nix @@ -9,54 +9,92 @@ let cfg = config.shb.borgbackup; - instanceOptions = { - enable = lib.mkEnableOption "shb borgbackup"; + contracts = pkgs.callPackage ../contracts { }; - keySopsFile = lib.mkOption { - description = "Sops file that holds this instance's repository key and passphrase."; - type = lib.types.path; - example = "secrets/backup.yaml"; - }; + inherit (lib) + concatStringsSep + filterAttrs + flatten + literalExpression + optionals + listToAttrs + mapAttrsToList + mkOption + mkMerge + ; + inherit (lib) + mkIf + nameValuePair + optionalAttrs + removePrefix + ; + inherit (lib.types) + attrsOf + int + oneOf + nonEmptyStr + nullOr + str + submodule + ; - encryptionKeyFile = lib.mkOption { - description = "Encryption key for the backup."; - type = lib.types.path; - }; + commonOptions = + { + name, + prefix, + config, + ... + }: + { + enable = lib.mkEnableOption '' + SelfHostBlocks' BorgBackup block; - encryption_passcommand = "cat /run/secrets/borgmatic/passphrases/${ - if isNull instance.secretName then name else instance.secretName - }"; - borg_keys_directory = "/run/secrets/borgmatic/keys"; + A disabled instance will not backup data anymore + but still provides the helper tool to restore snapshots + ''; - sourceDirectories = lib.mkOption { - description = "Source directories."; - type = lib.types.nonEmptyListOf lib.types.str; - }; + passphrase = lib.mkOption { + description = "Encryption key for the backup repository."; + type = lib.types.submodule { + options = contracts.secret.mkRequester { + mode = "0400"; + owner = config.request.user; + ownerText = "[shb.borgbackup.${prefix}..request.user](#blocks-borgbackup-options-shb.borgbackup.${prefix}._name_.request.user)"; + restartUnits = [ (fullName name config.settings.repository) ]; + restartUnitsText = "[ [shb.borgbackup.${prefix}..settings.repository](#blocks-borgbackup-options-shb.borgbackup.${prefix}._name_.settings.repository) ]"; + }; + }; + }; - excludePatterns = lib.mkOption { - description = "Exclude patterns."; - type = lib.types.listOf lib.types.str; - default = [ ]; - }; - - secretName = lib.mkOption { - description = "Secret name, if null use the name of the backup instance."; - type = lib.types.nullOr lib.types.str; - default = null; - }; - - repositories = lib.mkOption { - description = "Repositories to back this instance to."; - type = lib.types.nonEmptyListOf ( - lib.types.submodule { + repository = lib.mkOption { + description = "Repository to send the backups to."; + type = submodule { options = { - path = lib.mkOption { - type = lib.types.str; + path = mkOption { + type = str; description = "Repository location"; }; - timerConfig = lib.mkOption { - type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption; + secrets = mkOption { + type = attrsOf lib.shb.secretFileType; + default = { }; + description = '' + Secrets needed to access the repository where the backups will be stored. + + See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example + and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets. + + ''; + example = literalExpression '' + { + AWS_ACCESS_KEY_ID.source = ; + AWS_SECRET_ACCESS_KEY.source = ; + } + ''; + }; + + timerConfig = mkOption { + type = attrsOf utils.systemdUtils.unitOptions.unitOption; default = { OnCalendar = "daily"; Persistent = true; @@ -69,95 +107,127 @@ let }; }; }; - } - ); - }; - - retention = lib.mkOption { - description = "Retention options."; - type = lib.types.attrsOf ( - lib.types.oneOf [ - lib.types.int - lib.types.nonEmptyStr - ] - ); - default = { - keep_within = "1d"; - keep_hourly = 24; - keep_daily = 7; - keep_weekly = 4; - keep_monthly = 6; - }; - }; - - consistency = lib.mkOption { - description = "Consistency frequency options."; - type = lib.types.attrsOf lib.types.nonEmptyStr; - default = { }; - example = { - repository = "2 weeks"; - archives = "1 month"; - }; - }; - - hooks = lib.mkOption { - description = "Hooks to run before or after the backup."; - default = { }; - type = lib.types.submodule { - options = { - beforeBackup = lib.mkOption { - description = "Hooks to run before backup"; - type = lib.types.listOf lib.types.str; - default = [ ]; - }; - - afterBackup = lib.mkOption { - description = "Hooks to run after backup"; - type = lib.types.listOf lib.types.str; - default = [ ]; - }; }; }; + + retention = lib.mkOption { + description = "Retention options. See {command}`borg help prune` for the available options."; + type = attrsOf (oneOf [ + int + nonEmptyStr + ]); + default = { + within = "1d"; + hourly = 24; + daily = 7; + weekly = 4; + monthly = 6; + }; + }; + + consistency = lib.mkOption { + description = "Consistency frequency options."; + type = lib.types.attrsOf lib.types.nonEmptyStr; + default = { }; + example = { + repository = "2 weeks"; + archives = "1 month"; + }; + }; + + limitUploadKiBs = mkOption { + type = nullOr int; + description = "Limit upload bandwidth to the given KiB/s amount."; + default = null; + example = 8000; + }; + + stateDir = mkOption { + type = nullOr lib.types.str; + description = '' + Override the directory in which {command}`borg` stores its + configuration and cache. By default it uses the user's + home directory but is some cases this can cause conflicts. + ''; + default = null; + }; }; - environmentFile = lib.mkOption { - type = lib.types.bool; - description = "Add environment file to be read by the systemd service."; - default = false; - example = true; - }; - }; - - repoSlugName = - name: builtins.replaceStrings [ "/" ":" ] [ "_" "_" ] (lib.strings.removePrefix "/" name); - + repoSlugName = name: builtins.replaceStrings [ "/" ":" ] [ "_" "_" ] (removePrefix "/" name); + fullName = name: repository: "borgbackup-job-${name}_${repoSlugName repository.path}"; in { options.shb.borgbackup = { - user = lib.mkOption { - description = "Unix user doing the backups."; - type = lib.types.str; - default = "backup"; - }; - - group = lib.mkOption { - description = "Unix group doing the backups."; - type = lib.types.str; - default = "backup"; - }; - - instances = lib.mkOption { - description = "Each instance is a backup setting"; + instances = mkOption { + description = "Files to backup following the [backup contract](./contracts-backup.html)."; default = { }; - type = lib.types.attrsOf ( - lib.types.submodule { - options = instanceOptions; - } + type = attrsOf ( + submodule ( + { name, config, ... }: + { + options = contracts.backup.mkProvider { + settings = mkOption { + description = '' + Settings specific to the BorgBackup provider. + ''; + + type = submodule { + options = commonOptions { + inherit name config; + prefix = "instances"; + }; + }; + }; + + resultCfg = { + restoreScript = fullName name config.settings.repository; + restoreScriptText = "${fullName "" { path = "path/to/repository"; }}"; + + backupService = "${fullName name config.settings.repository}.service"; + backupServiceText = "${fullName "" { path = "path/to/repository"; }}.service"; + }; + }; + } + ) + ); + }; + + databases = mkOption { + description = "Databases to backup following the [database backup contract](./contracts-databasebackup.html)."; + default = { }; + type = attrsOf ( + submodule ( + { name, config, ... }: + { + options = contracts.databasebackup.mkProvider { + settings = mkOption { + description = '' + Settings specific to the BorgBackup provider. + ''; + + type = submodule { + options = commonOptions { + inherit name config; + prefix = "databases"; + }; + }; + }; + + resultCfg = { + restoreScript = fullName name config.settings.repository; + restoreScriptText = "${fullName "" { path = "path/to/repository"; }}"; + + backupService = "${fullName name config.settings.repository}.service"; + backupServiceText = "${fullName "" { path = "path/to/repository"; }}.service"; + }; + }; + } + ) ); }; borgServer = lib.mkOption { - description = "Add borgbackup package so external backups can use this server as a remote."; + description = "Add borgbackup package to `environment.systemPackages` so external backups can use this server as a remote."; default = false; example = true; type = lib.types.bool; @@ -193,153 +263,262 @@ in }; }; - config = lib.mkIf (cfg.instances != { }) ( + config = lib.mkIf (cfg.instances != { } || cfg.databases != { }) ( let - enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances; + enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances; + enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases; in lib.mkMerge [ - # Secrets configuration { - users.users = { - ${cfg.user} = { - name = cfg.user; - group = cfg.group; - home = "/var/lib/backup"; - createHome = true; - isSystemUser = true; - extraGroups = [ "keys" ]; - }; - }; - users.groups = { - ${cfg.group} = { - name = cfg.group; - }; - }; - - sops.secrets = - let - mkSopsSecret = - name: instance: - ( - [ - { - "${instance.backend}/passphrases/${ - if isNull instance.secretName then name else instance.secretName - }" = - { - sopsFile = instance.keySopsFile; - mode = "0440"; - owner = cfg.user; - group = cfg.group; - }; - } - ] - ++ - lib.optional - ((lib.filter ({ path, ... }: lib.strings.hasPrefix "s3" path) instance.repositories) != [ ]) - { - "${instance.backend}/environmentfiles/${ - if isNull instance.secretName then name else instance.secretName - }" = - { - sopsFile = instance.keySopsFile; - mode = "0440"; - owner = cfg.user; - group = cfg.group; - }; - } - ++ lib.optionals (instance.backend == "borgmatic") ( - lib.flatten ( - map ( - { path, ... }: - { - "${instance.backend}/keys/${repoSlugName path}" = { - key = "${instance.backend}/keys/${ - if isNull instance.secretName then name else instance.secretName - }"; - sopsFile = instance.keySopsFile; - mode = "0440"; - owner = cfg.user; - group = cfg.group; - }; - } - ) instance.repositories - ) - ) - ); - in - lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSopsSecret enabledInstances)); - } - # Borgmatic configuration - { - systemd.timers.borgmatic = lib.mkIf (enabledInstances != { }) { - timerConfig = { - OnCalendar = "hourly"; - }; - }; - - systemd.services.borgmatic = lib.mkIf (enabledInstances != { }) { - serviceConfig = { - User = cfg.user; - Group = cfg.group; - ExecStartPre = [ "" ]; # Do not sleep before starting. - ExecStart = [ - "" - "${pkgs.borgmatic}/bin/borgmatic --verbosity -1 --syslog-verbosity 1" + environment.systemPackages = + optionals (cfg.borgServer || enabledInstances != { } || enabledDatabases != { }) + [ + pkgs.borgbackup ]; - # For borgmatic, since we have only one service, we need to merge all environmentFile - # from all instances. - EnvironmentFile = lib.mapAttrsToList (name: value: value.environmentFile) enabledInstances; - }; - }; - - systemd.packages = lib.mkIf (enabledInstances != { }) [ pkgs.borgmatic ]; - environment.systemPackages = ( - lib.optionals cfg.borgServer [ pkgs.borgbackup ] - ++ lib.optionals (enabledInstances != { }) [ - pkgs.borgbackup - pkgs.borgmatic - ] - ); - - environment.etc = + } + { + services.borgbackup.jobs = let - mkSettings = name: instance: { - "borgmatic.d/${name}.yaml".text = lib.generators.toYAML { } { - location = { - source_directories = instance.sourceDirectories; - repositories = map ({ path, ... }: path) instance.repositories; - } - // (lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) { - excludePatterns = instance.excludePatterns; - }); + mkJob = name: instance: { + "${name}_${repoSlugName instance.settings.repository.path}" = { + inherit (instance.request) user; - storage = { - encryption_passcommand = "cat ${instance.encryptionKeyFile}"; - borg_keys_directory = "/run/secrets/borgmatic/keys"; - }; + repo = instance.settings.repository.path; - retention = instance.retention; - consistency.checks = - let - mkCheck = name: frequency: { - inherit name frequency; - }; - in - lib.attrsets.mapAttrsToList mkCheck instance.consistency; + paths = instance.request.sourceDirectories; - # hooks = lib.mkMerge [ - # lib.optionalAttrs (builtins.length instance.hooks.beforeBackup > 0) { - # inherit (instance.hooks) beforeBackup; - # } - # lib.optionalAttrs (builtins.length instance.hooks.afterBackup > 0) { - # inherit (instance.hooks) afterBackup; - # } - # ]; + encryption.mode = "repokey-blake2"; + # We do not set encryption.passphrase here, we set BORG_PASSPHRASE_FD further down. + encryption.passCommand = "cat ${instance.settings.passphrase.result.path}"; + + doInit = true; + failOnWarnings = true; + stateDir = instance.settings.stateDir; + + persistentTimer = instance.settings.repository.timerConfig.Persistent or false; + startAt = ""; # Some non-empty string value tricks the upstream module in creating the systemd timer. + + prune.keep = instance.settings.retention; + + preHook = concatStringsSep "\n" instance.request.hooks.beforeBackup; + + postHook = concatStringsSep "\n" instance.request.hooks.afterBackup; + + extraArgs = ( + optionals (instance.settings.limitUploadKiBs != null) [ + "--upload-ratelimit=${toString instance.settings.limitUploadKiBs}" + ] + ); + + exclude = instance.request.excludePatterns; }; }; in - lib.mkMerge (lib.attrsets.mapAttrsToList mkSettings enabledInstances); + mkMerge (mapAttrsToList mkJob enabledInstances); + } + { + services.borgbackup.jobs = + let + mkJob = name: instance: { + "${name}_${repoSlugName instance.settings.repository.path}" = { + inherit (instance.request) user; + + repo = instance.settings.repository.path; + + dumpCommand = lib.getExe (pkgs.writeShellApplication { + name = "dump-command"; + text = instance.request.backupCmd; + }); + + encryption.mode = "repokey-blake2"; + # We do not set encryption.passphrase here, we set BORG_PASSPHRASE_FD further down. + encryption.passCommand = "cat ${instance.settings.passphrase.result.path}"; + + doInit = true; + failOnWarnings = true; + stateDir = instance.settings.stateDir; + + persistentTimer = instance.settings.repository.timerConfig.Persistent or false; + startAt = ""; # Some non-empty list value that tricks upstream in creating the systemd timer. + + prune.keep = instance.settings.retention; + + extraArgs = ( + optionals (instance.settings.limitUploadKiBs != null) [ + "--upload-ratelimit=${toString instance.settings.limitUploadKiBs}" + ] + ); + }; + }; + in + mkMerge (mapAttrsToList mkJob enabledDatabases); + } + { + systemd.timers = + let + mkTimer = name: instance: { + ${fullName name instance.settings.repository} = { + timerConfig = lib.mkForce instance.settings.repository.timerConfig; + }; + }; + in + mkMerge (mapAttrsToList mkTimer (enabledInstances // enabledDatabases)); + } + { + systemd.services = + let + mkSettings = + name: instance: + let + serviceName = fullName name instance.settings.repository; + in + { + ${serviceName} = mkMerge [ + { + serviceConfig = { + # Makes the systemd service wait for the backup to be done before changing state to inactive. + Type = "oneshot"; + Nice = lib.mkForce cfg.performance.niceness; + IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass; + IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority; + # BindReadOnlyPaths = instance.sourceDirectories; + }; + } + (optionalAttrs (instance.settings.repository.secrets != { }) { + serviceConfig.EnvironmentFile = [ + "/run/secrets_borgbackup/${serviceName}" + ]; + after = [ "${serviceName}-pre.service" ]; + requires = [ "${serviceName}-pre.service" ]; + }) + ]; + + "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != { }) ( + let + script = lib.shb.genConfigOutOfBandSystemd { + config = instance.settings.repository.secrets; + configLocation = "/run/secrets_borgbackup/${serviceName}"; + generator = lib.shb.toEnvVar; + user = instance.request.user; + }; + in + { + script = script.preStart; + serviceConfig.Type = "oneshot"; + serviceConfig.LoadCredential = script.loadCredentials; + } + ); + }; + in + mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases))); + } + { + systemd.services = + let + mkEnv = + name: instance: + nameValuePair "${fullName name instance.settings.repository}_restore_gen" { + enable = true; + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + script = ( + lib.shb.replaceSecrets { + userConfig = instance.settings.repository.secrets // { + BORG_PASSCOMMAND = ''"cat ${instance.settings.passphrase.result.path}"''; + BORG_REPO = instance.settings.repository.path; + }; + resultPath = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}"; + generator = lib.shb.toEnvVar; + user = instance.request.user; + } + ); + }; + in + listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases))); + } + { + environment.systemPackages = + let + mkBorgBackupBinary = + name: instance: + pkgs.writeShellApplication { + name = fullName name instance.settings.repository; + text = '' + usage() { + echo "$0 restore latest" + } + + if ! [ "$1" = "restore" ]; then + usage + exit 1 + fi + shift + + if ! [ "$1" = "latest" ]; then + usage + exit 1 + fi + shift + + sudocmd() { + sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@" + } + + set -a + # shellcheck disable=SC1090 + source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}") + set +a + + archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)" + echo "Will restore archive $archive" + + (cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive") + ''; + }; + in + flatten (mapAttrsToList mkBorgBackupBinary cfg.instances); + } + { + environment.systemPackages = + let + mkBorgBackupBinary = + name: instance: + pkgs.writeShellApplication { + name = fullName name instance.settings.repository; + text = '' + usage() { + echo "$0 restore latest" + } + + if ! [ "$1" = "restore" ]; then + usage + exit 1 + fi + shift + + if ! [ "$1" = "latest" ]; then + usage + exit 1 + fi + shift + + sudocmd() { + sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@" + } + + set -a + # shellcheck disable=SC1090 + source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}") + set +a + + archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)" + echo "Will restore archive $archive" + + sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}" + ''; + }; + in + flatten (mapAttrsToList mkBorgBackupBinary cfg.databases); } ] ); diff --git a/modules/blocks/borgbackup/docs/default.md b/modules/blocks/borgbackup/docs/default.md new file mode 100644 index 0000000..b7f449e --- /dev/null +++ b/modules/blocks/borgbackup/docs/default.md @@ -0,0 +1,270 @@ +# Borgbackup Block {#blocks-borgbackup} + +Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix). + +This block sets up a backup job using [BorgBackup][]. + +[borgbackup]: https://www.borgbackup.org/ + +## Provider Contracts {#blocks-borgbackup-contract-provider} + +This block provides the following contracts: + +- [backup contract](contracts-backup.html) under the [`shb.borgbackup.instances`][instances] option. + It is tested with [contract tests][backup contract tests]. +- [database backup contract](contracts-databasebackup.html) under the [`shb.borgbackup.databases`][databases] option. + It is tested with [contract tests][database backup contract tests]. + +[instances]: #blocks-borgbackup-options-shb.borgbackup.instances +[databases]: #blocks-borgbackup-options-shb.borgbackup.databases +[backup contract tests]: @REPO@/test/contracts/backup.nix +[database backup contract tests]: @REPO@/test/contracts/databasebackup.nix + +As requested by those two contracts, when setting up a backup with BorgBackup, +a backup Systemd service and a [restore script](#blocks-borgbackup-maintenance) are provided. + +## Usage {#blocks-borgbackup-usage} + +The following examples assume usage of the [sops block][] to provide secrets +although any blocks providing the [secrets contract][] works too. + +[sops block]: ./blocks-sops.html +[secrets contract]: ./contracts-secrets.html + +### One folder backed up manually {#blocks-borgbackup-usage-provider-manual} + +The following snippet shows how to configure +the backup of 1 folder to 1 repository. +We assume that the folder `/var/lib/myfolder` of the service `myservice` must be backed up. + +```nix +shb.borgbackup.instances."myservice" = { + request = { + user = "myservice"; + + sourceDirectories = [ + "/var/lib/myfolder" + ]; + }; + + settings = { + enable = true; + + passphrase.result = shb.sops.secret."passphrase".result; + + repository = { + path = "/srv/backups/myservice"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "3h"; + }; + }; + + retention = { + within = "1d"; + hourly = 24; + daily = 7; + weekly = 4; + monthly = 6; + }; + }; +}; + +shb.sops.secret."passphrase".request = + shb.borgbackup.instances."myservice".settings.passphrase.request; +``` + +### One folder backed up with contract {#blocks-borgbackup-usage-provider-contract} + +With the same example as before but assuming the `myservice` service +has a `myservice.backup` option that is a requester for the backup contract, +the snippet above becomes: + +```nix +shb.borgbackup.instances."myservice" = { + request = config.myservice.backup; + + settings = { + enable = true; + + passphrase.result = shb.sops.secret."passphrase".result; + + repository = { + path = "/srv/backups/myservice"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "3h"; + }; + }; + + retention = { + within = "1d"; + hourly = 24; + daily = 7; + weekly = 4; + monthly = 6; + }; + }; +}; + +shb.sops.secret."passphrase".request = + shb.borgbackup.instances."myservice".settings.passphrase.request; +``` + +### One folder backed up to S3 {#blocks-borgbackup-usage-provider-remote} + +Here we will only highlight the differences with the previous configuration. + +This assumes you have access to such a remote S3 store, for example by using [Backblaze](https://www.backblaze.com/). + +```diff + shb.backup.instances.myservice = { + + repository = { +- path = "/srv/pool1/backups/myfolder"; ++ path = "s3:s3.us-west-000.backblazeb2.com/backups/myfolder"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "3h"; + }; + ++ extraSecrets = { ++ AWS_ACCESS_KEY_ID.source=""; ++ AWS_SECRET_ACCESS_KEY.source=""; ++ }; + }; + } +``` + +### Multiple directories to multiple destinations {#blocks-borgbackup-usage-multiple} + +The following snippet shows how to configure backup of any number of folders to 3 repositories, +each happening at different times to avoid I/O contention. + +We will also make sure to be able to re-use as much as the configuration as possible. + +A few assumptions: +- 2 hard drive pools used for backup are mounted respectively on `/srv/pool1` and `/srv/pool2`. +- You have a backblaze account. + +First, let's define a variable to hold all the repositories we want to back up to: + +```nix +repos = [ + { + path = "/srv/pool1/backups"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "3h"; + }; + } + { + path = "/srv/pool2/backups"; + timerConfig = { + OnCalendar = "08:00:00"; + RandomizedDelaySec = "3h"; + }; + } + { + path = "s3:s3.us-west-000.backblazeb2.com/backups"; + timerConfig = { + OnCalendar = "16:00:00"; + RandomizedDelaySec = "3h"; + }; + } +]; +``` + +Compared to the previous examples, we do not include the name of what we will back up in the +repository paths. + +Now, let's define a function to create a backup configuration. It will take a list of repositories, +a name identifying the backup and a list of folders to back up. + +```nix +backupcfg = repositories: name: sourceDirectories { + enable = true; + + backend = "borgbackup"; + + keySopsFile = ../secrets/backup.yaml; + + repositories = builtins.map (r: { + path = "${r.path}/${name}"; + inherit (r) timerConfig; + }) repositories; + + inherit sourceDirectories; + + retention = { + within = "1d"; + hourly = 24; + daily = 7; + weekly = 4; + monthly = 6; + }; + + environmentFile = true; +}; +``` + +Now, we can define multiple backup jobs to backup different folders: + +```nix +shb.backup.instances.myfolder1 = backupcfg repos ["/var/lib/myfolder1"]; +shb.backup.instances.myfolder2 = backupcfg repos ["/var/lib/myfolder2"]; +``` + +The difference between the above snippet and putting all the folders into one configuration (shown +below) is the former splits the backups into sub-folders on the repositories. + +```nix +shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"]; +``` + +## Monitoring {#blocks-borgbackup-monitoring} + +[WIP](https://github.com/ibizaman/selfhostblocks/issues/151) + +## Maintenance {#blocks-borgbackup-maintenance} + +One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets. + +The restore script has all the secrets needed to access the repo, +it will run `sudo` automatically +and the user running it needs to have correct permissions for privilege escalation + +In the [multiple directories example](#blocks-borgbackup-usage-multiple) above, the following 6 helpers are provided in the `$PATH`: + +```bash +borgbackup-job-myfolder1_srv_pool1_backups +borgbackup-job-myfolder1_srv_pool2_backups +borgbackup-job-myfolder1_s3_s3.us-west-000.backblazeb2.com_backups +borgbackup-job-myfolder2_srv_pool1_backups +borgbackup-job-myfolder2_srv_pool2_backups +borgbackup-job-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups +``` + +Discovering those is easy thanks to tab-completion. + +One can then restore a backup from a given repository with: + +```bash +borgbackup-job-myfolder1_srv_pool1_backups restore latest +``` + +### Troubleshooting {#blocks-borgbackup-maintenance-troubleshooting} + +In case something bad happens with a backup, the [official documentation](https://borgbackup.readthedocs.io/en/stable/077_troubleshooting.html) has a lot of tips. + +## Tests {#blocks-borgbackup-tests} + +Specific integration tests are defined in [`/test/blocks/borgbackup.nix`](@REPO@/test/blocks/borgbackup.nix). + +## Options Reference {#blocks-borgbackup-options} + +```{=include=} options +id-prefix: blocks-borgbackup-options- +list-id: selfhostblocks-block-borgbackup-options +source: @OPTIONS_JSON@ +``` diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix index 4b78dce..5296725 100644 --- a/modules/blocks/restic.nix +++ b/modules/blocks/restic.nix @@ -51,7 +51,7 @@ let }: { enable = mkEnableOption '' - this backup intance. + SelfHostBlocks' Restic block A disabled instance will not backup data anymore but still provides the helper tool to restore snapshots @@ -426,19 +426,39 @@ in let mkResticBinary = name: instance: - pkgs.writeShellScriptBin (fullName name instance.settings.repository) '' - set -euo pipefail + pkgs.writeShellApplication { + name = fullName name instance.settings.repository; + text = '' + usage() { + echo "$0 restore latest" + } - export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \ - | xargs -d '\n') - - if ! [ "$1" = "restore" ]; then - sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@ - else + if ! [ "$1" = "restore" ]; then + usage + exit 1 + fi shift - sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic restore $@ --target /" - fi - ''; + + if ! [ "$1" = "latest" ]; then + usage + exit 1 + fi + shift + + sudocmd() { + sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@" + } + + set -a + # shellcheck disable=SC1090 + source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}") + set +a + + echo "Will restore archive 'latest'" + + sudocmd ${pkgs.restic}/bin/restic restore latest --target / + ''; + }; in flatten (mapAttrsToList mkResticBinary cfg.instances); } @@ -447,19 +467,39 @@ in let mkResticBinary = name: instance: - pkgs.writeShellScriptBin (fullName name instance.settings.repository) '' - set -euo pipefail + pkgs.writeShellApplication { + name = fullName name instance.settings.repository; + text = '' + usage() { + echo "$0 restore latest" + } - export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \ - | xargs -d '\n') - - if ! [ "$1" = "restore" ]; then - sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@ - else + if ! [ "$1" = "restore" ]; then + usage + exit 1 + fi shift - sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupName} | ${instance.request.restoreCmd}" - fi - ''; + + if ! [ "$1" = "latest" ]; then + usage + exit 1 + fi + shift + + sudocmd() { + sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@" + } + + set -a + # shellcheck disable=SC1090 + source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}") + set +a + + echo "Will restore archive 'latest'" + + sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}" + ''; + }; in flatten (mapAttrsToList mkResticBinary cfg.databases); } diff --git a/modules/blocks/restic/docs/default.md b/modules/blocks/restic/docs/default.md index 4520b07..36427ab 100644 --- a/modules/blocks/restic/docs/default.md +++ b/modules/blocks/restic/docs/default.md @@ -6,10 +6,6 @@ This block sets up a backup job using [Restic][]. [restic]: https://restic.net/ -## Tests {#blocks-restic-tests} - -Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix). - ## Provider Contracts {#blocks-restic-contract-provider} This block provides the following contracts: @@ -25,9 +21,7 @@ This block provides the following contracts: [database backup contract tests]: @REPO@/test/contracts/databasebackup.nix As requested by those two contracts, when setting up a backup with Restic, -a backup Systemd service and restore script are provided. -The restore script has all the secrets needed to access the repo, -the only requirement to run it is to be able to `sudo` in the expected user. +a backup Systemd service and a [restore script](#blocks-restic-maintenance) are provided. ## Usage {#blocks-restic-usage} @@ -228,18 +222,18 @@ below) is the former splits the backups into sub-folders on the repositories. shb.backup.instances.all = backupcfg repos ["/var/lib/myfolder1" "/var/lib/myfolder2"]; ``` -## Demo {#blocks-restic-demo} - -[WIP] - ## Monitoring {#blocks-restic-monitoring} -[WIP] +[WIP](https://github.com/ibizaman/selfhostblocks/issues/151) ## Maintenance {#blocks-restic-maintenance} One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets. +The restore script has all the secrets needed to access the repo, +it will run `sudo` automatically +and the user running it needs to have correct permissions for privilege escalation + In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`: ```bash @@ -263,6 +257,10 @@ restic-myfolder1_srv_pool1_backups restore latest In case something bad happens with a backup, the [official documentation](https://restic.readthedocs.io/en/stable/077_troubleshooting.html) has a lot of tips. +## Tests {#blocks-restic-tests} + +Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix). + ## Options Reference {#blocks-restic-options} ```{=include=} options diff --git a/patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch b/patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch new file mode 100644 index 0000000..e71e210 --- /dev/null +++ b/patches/0001-nixos-borgbackup-add-option-to-override-state-direct.patch @@ -0,0 +1,78 @@ +From dda895b551c7cf56476ac8892904e289a4d8b920 Mon Sep 17 00:00:00 2001 +From: ibizaman +Date: Sat, 1 Nov 2025 13:49:20 +0100 +Subject: [PATCH] nixos/borgbackup: add option to override state directory + +--- + nixos/modules/services/backup/borgbackup.nix | 23 +++++++++++++++----- + 1 file changed, 18 insertions(+), 5 deletions(-) + +diff --git a/nixos/modules/services/backup/borgbackup.nix b/nixos/modules/services/backup/borgbackup.nix +index adabb2ce0f8b..82baeb928398 100644 +--- a/nixos/modules/services/backup/borgbackup.nix ++++ b/nixos/modules/services/backup/borgbackup.nix +@@ -136,7 +136,7 @@ let + mkBackupService = + name: cfg: + let +- userHome = config.users.users.${cfg.user}.home; ++ userHome = if cfg.stateDir != null then cfg.stateDir else config.users.users.${cfg.user}.home; + backupJobName = "borgbackup-job-${name}"; + backupScript = mkBackupScript backupJobName cfg; + in +@@ -177,6 +177,7 @@ let + environment = { + BORG_REPO = cfg.repo; + } ++ // (lib.optionalAttrs (cfg.stateDir != null) { BORG_BASE_DIR = cfg.stateDir; }) + // (mkPassEnv cfg) + // cfg.environment; + }; +@@ -223,6 +224,7 @@ let + set = { + BORG_REPO = cfg.repo; + } ++ // (lib.optionalAttrs (cfg.stateDir != null) { BORG_BASE_DIR = cfg.stateDir; }) + // (mkPassEnv cfg) + // cfg.environment; + }); +@@ -232,14 +234,15 @@ let + name: cfg: + let + settings = { inherit (cfg) user group; }; ++ userHome = if cfg.stateDir != null then cfg.stateDir else config.users.users.${cfg.user}.home; + in + lib.nameValuePair "borgbackup-job-${name}" ( + { + # Create parent dirs separately, to ensure correct ownership. +- "${config.users.users."${cfg.user}".home}/.config".d = settings; +- "${config.users.users."${cfg.user}".home}/.cache".d = settings; +- "${config.users.users."${cfg.user}".home}/.config/borg".d = settings; +- "${config.users.users."${cfg.user}".home}/.cache/borg".d = settings; ++ "${userHome}/.config".d = settings; ++ "${userHome}/.cache".d = settings; ++ "${userHome}/.config/borg".d = settings; ++ "${userHome}/.cache/borg".d = settings; + } + // lib.optionalAttrs (isLocalPath cfg.repo && !cfg.removableDevice) { + "${cfg.repo}".d = settings; +@@ -487,6 +490,16 @@ in + default = "root"; + }; + ++ stateDir = lib.mkOption { ++ type = lib.types.nullOr lib.types.str; ++ description = '' ++ Override the directory in which {command}`borg` stores its ++ configuration and cache. By default it uses the user's ++ home directory but is some cases this can cause conflicts. ++ ''; ++ default = null; ++ }; ++ + wrapper = lib.mkOption { + type = with lib.types; nullOr str; + description = '' +-- +2.50.1 + diff --git a/test/blocks/borgbackup.nix b/test/blocks/borgbackup.nix new file mode 100644 index 0000000..6c1cb38 --- /dev/null +++ b/test/blocks/borgbackup.nix @@ -0,0 +1,190 @@ +{ pkgs, lib, ... }: +let + testLib = pkgs.callPackage ../common.nix { }; + + commonTest = + user: + lib.shb.runNixOSTest { + name = "borgbackup_backupAndRestore_${user}"; + + nodes.machine = + { config, ... }: + { + imports = [ + testLib.baseImports + + ../../modules/blocks/hardcodedsecret.nix + ../../modules/blocks/borgbackup.nix + ]; + + shb.hardcodedsecret.A = { + request = { + owner = "root"; + group = "keys"; + mode = "0440"; + }; + settings.content = "secretA"; + }; + shb.hardcodedsecret.B = { + request = { + owner = "root"; + group = "keys"; + mode = "0440"; + }; + settings.content = "secretB"; + }; + + shb.hardcodedsecret.passphrase = { + request = config.shb.borgbackup.instances."testinstance".settings.passphrase.request; + settings.content = "passphrase"; + }; + + shb.borgbackup.instances."testinstance" = { + settings = { + enable = true; + + passphrase.result = config.shb.hardcodedsecret.passphrase.result; + + repository = { + path = "/opt/repos/A"; + timerConfig = { + OnCalendar = "00:00:00"; + RandomizedDelaySec = "5h"; + }; + # Those are not needed by the repository but are still included + # so we can test them in the hooks section. + secrets = { + A.source = config.shb.hardcodedsecret.A.result.path; + B.source = config.shb.hardcodedsecret.B.result.path; + }; + }; + }; + + request = { + inherit user; + + sourceDirectories = [ + "/opt/files/A" + "/opt/files/B" + ]; + + hooks.beforeBackup = [ + '' + echo $RUNTIME_DIRECTORY + if [ "$RUNTIME_DIRECTORY" = /run/borgbackup-backups-testinstance_opt_repos_A ]; then + if ! [ -f /run/secrets_borgbackup/borgbackup-backups-testinstance_opt_repos_A ]; then + exit 10 + fi + if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then + echo "A:$A" + exit 11 + fi + if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then + echo "B:$B" + exit 12 + fi + fi + '' + ]; + }; + }; + }; + + extraPythonPackages = p: [ p.dictdiffer ]; + skipTypeCheck = true; + + testScript = + { nodes, ... }: + let + provider = nodes.machine.shb.borgbackup.instances."testinstance"; + backupService = provider.result.backupService; + restoreScript = provider.result.restoreScript; + in + '' + from dictdiffer import diff + + def list_files(dir): + files_and_content = {} + + files = machine.succeed(f""" + find {dir} -type f + """).split("\n")[:-1] + + for f in files: + content = machine.succeed(f""" + cat {f} + """).strip() + files_and_content[f] = content + + return files_and_content + + def assert_files(dir, files): + result = list(diff(list_files(dir), files)) + if len(result) > 0: + raise Exception("Unexpected files:", result) + + with subtest("Create initial content"): + machine.succeed(""" + mkdir -p /opt/files/A + mkdir -p /opt/files/B + + echo repoA_fileA_1 > /opt/files/A/fileA + echo repoA_fileB_1 > /opt/files/A/fileB + echo repoB_fileA_1 > /opt/files/B/fileA + echo repoB_fileB_1 > /opt/files/B/fileB + + chown ${user}: -R /opt/files + chmod go-rwx -R /opt/files + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_1', + '/opt/files/B/fileB': 'repoB_fileB_1', + '/opt/files/A/fileA': 'repoA_fileA_1', + '/opt/files/A/fileB': 'repoA_fileB_1', + }) + + with subtest("First backup in repo A"): + machine.succeed("systemctl start ${backupService}") + + with subtest("New content"): + machine.succeed(""" + echo repoA_fileA_2 > /opt/files/A/fileA + echo repoA_fileB_2 > /opt/files/A/fileB + echo repoB_fileA_2 > /opt/files/B/fileA + echo repoB_fileB_2 > /opt/files/B/fileB + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_2', + '/opt/files/B/fileB': 'repoB_fileB_2', + '/opt/files/A/fileA': 'repoA_fileA_2', + '/opt/files/A/fileB': 'repoA_fileB_2', + }) + + with subtest("Delete content"): + machine.succeed(""" + rm -r /opt/files/A /opt/files/B + """) + + assert_files("/opt/files", {}) + + with subtest("Restore initial content from repo A"): + machine.succeed(""" + ${restoreScript} restore latest + """) + + assert_files("/opt/files", { + '/opt/files/B/fileA': 'repoB_fileA_1', + '/opt/files/B/fileB': 'repoB_fileB_1', + '/opt/files/A/fileA': 'repoA_fileA_1', + '/opt/files/A/fileB': 'repoA_fileB_1', + }) + ''; + + }; +in +{ + backupAndRestoreRoot = commonTest "root"; + backupAndRestoreUser = commonTest "nobody"; +} diff --git a/test/contracts/backup.nix b/test/contracts/backup.nix index 2bb3d3c..c70abc3 100644 --- a/test/contracts/backup.nix +++ b/test/contracts/backup.nix @@ -72,4 +72,74 @@ in }; }; }; + + borgbackup_root = contracts.test.backup { + name = "borgbackup_root"; + username = "root"; + providerRoot = [ + "shb" + "borgbackup" + "instances" + "mytest" + ]; + modules = [ + ../../modules/blocks/borgbackup.nix + ../../modules/blocks/hardcodedsecret.nix + ]; + settings = + { repository, config, ... }: + { + enable = true; + passphrase.result = config.shb.hardcodedsecret.passphrase.result; + repository = { + path = repository; + timerConfig = { + OnCalendar = "00:00:00"; + }; + }; + }; + extraConfig = + { username, config, ... }: + { + shb.hardcodedsecret.passphrase = { + request = config.shb.borgbackup.instances."mytest".settings.passphrase.request; + settings.content = "passphrase"; + }; + }; + }; + + borgbackup_nonroot = contracts.test.backup { + name = "borgbackup_nonroot"; + username = "me"; + providerRoot = [ + "shb" + "borgbackup" + "instances" + "mytest" + ]; + modules = [ + ../../modules/blocks/borgbackup.nix + ../../modules/blocks/hardcodedsecret.nix + ]; + settings = + { repository, config, ... }: + { + enable = true; + passphrase.result = config.shb.hardcodedsecret.passphrase.result; + repository = { + path = repository; + timerConfig = { + OnCalendar = "00:00:00"; + }; + }; + }; + extraConfig = + { username, config, ... }: + { + shb.hardcodedsecret.passphrase = { + request = config.shb.borgbackup.instances."mytest".settings.passphrase.request; + settings.content = "passphrase"; + }; + }; + }; } diff --git a/test/contracts/databasebackup.nix b/test/contracts/databasebackup.nix index c5b490b..78b12c9 100644 --- a/test/contracts/databasebackup.nix +++ b/test/contracts/databasebackup.nix @@ -48,4 +48,51 @@ in }; }; }; + + borgbackup_postgres = contracts.test.databasebackup { + name = "borgbackup_postgres"; + requesterRoot = [ + "shb" + "postgresql" + "databasebackup" + ]; + providerRoot = [ + "shb" + "borgbackup" + "databases" + "postgresql" + ]; + modules = [ + ../../modules/blocks/postgresql.nix + ../../modules/blocks/borgbackup.nix + ../../modules/blocks/hardcodedsecret.nix + ]; + settings = + { repository, config, ... }: + { + enable = true; + stateDir = "/var/lib/borgbackup_postgres"; + passphrase.result = config.shb.hardcodedsecret.passphrase.result; + repository = { + path = repository; + timerConfig = { + OnCalendar = "00:00:00"; + }; + }; + }; + extraConfig = + { config, database, ... }: + { + shb.postgresql.ensures = [ + { + inherit database; + username = database; + } + ]; + shb.hardcodedsecret.passphrase = { + request = config.shb.borgbackup.databases.postgresql.settings.passphrase.request; + settings.content = "passphrase"; + }; + }; + }; }