danvics/selfhostblocks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Backup contracts for files and databases (#344)

Browse source 
This PR continues the work started in
https://github.com/ibizaman/selfhostblocks/pull/314

I had to create my own PR since I couldn't add commits on the fork.

---------

Co-authored-by: sivertism <10866270+sivertism@users.noreply.github.com>
This commit is contained in:
Pierre Penninckx 2024-11-12 21:40:52 +01:00 committed by GitHub
parent 7f2aa36d27
commit 431dd058c5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 1129 additions and 365 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CHANGELOG.md
View file

@ -1,5 +1,10 @@
# Upcoming Release
## New Features
- Backup:
- Add feature to backup databases with the database backup contract, implemented with `shb.restic.databases`.
## Breaking Changes
- Remove dependency on `sops-nix`.
@ -33,6 +38,8 @@
- `shb.forgejo.sso.secretFileForAuthelia` -> `shb.forgejo.ldap.sharedSecretForAuthelia.result.path`.
- `shb.forgejo.adminPasswordFile` -> `shb.forgejo.adminPassword.result.path`.
- `shb.forgejo.databasePasswordFile` -> `shb.forgejo.databasePassword.result.path`.
- Backup:
- `shb.restic.instances` options has been split between `shb.restic.instances.request` and `shb.restic.instances.settings`, matching better with contracts.
## User Facing Backwards Compatible Changes

4
docs/blocks.md
View file

@ -34,6 +34,10 @@ Not all blocks are yet documented. You can find all available blocks [in the rep
modules/blocks/ssl/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-postgresql.html
modules/blocks/postgresql/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-restic.html
modules/blocks/restic/docs/default.md
```

11
docs/contracts.md
View file

@ -20,10 +20,13 @@ as possible, reducing the quite thick layer that it is now.
Provided contracts are:
- [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
Two implementations are provided: self-signed and Let's Encrypt.
Two providers are implemented: self-signed and Let's Encrypt.
- [Backup contract](contracts-backup.html) to backup directories.
This contract allows to backup multiple times the same directories for extra protection.
One provider is implemented: Restic.
- [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
One provider is implemented: Restic.
- [Secret contract](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
One provider is implemented: SOPS.
```{=include=} chapters html:into-file=//contracts-ssl.html
modules/contracts/ssl/docs/default.md
@ -33,6 +36,10 @@ modules/contracts/ssl/docs/default.md
modules/contracts/backup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-databasebackup.html
modules/contracts/databasebackup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-secret.html
modules/contracts/secret/docs/default.md
```

10
docs/default.nix
View file

@ -132,6 +132,11 @@ in stdenv.mkDerivation {
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/blocks/ssl.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/blocks/postgresql/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/blocks/postgresql.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/blocks/restic/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
@ -160,6 +165,11 @@ in stdenv.mkDerivation {
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/contracts/backup/dummyModule.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/contracts/databasebackup/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/contracts/databasebackup/dummyModule.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/contracts/secret/docs/default.md \
--replace \
'@OPTIONS_JSON@' \

2
flake.nix
View file

@ -136,6 +136,8 @@
// (vm_test "restic" ./test/blocks/restic.nix)
// (vm_test "ssl" ./test/blocks/ssl.nix)
// (vm_test "contracts-backup" ./test/contracts/backup.nix)
// (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
// (vm_test "contracts-secret" ./test/contracts/secret.nix)
));
}

2
lib/default.nix
View file

@ -248,7 +248,7 @@ rec {
results = pkgs.lib.runTests tests;
in
if results != [ ] then
builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeqN 3 results)))
builtins.throw (concatStringsSep "\n" (map resultToString (lib.traceValSeq results)))
else
pkgs.runCommand "nix-flake-tests-success" { } "echo > $out";

11
modules/blocks/ldap.nix
View file

@ -95,7 +95,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -104,10 +104,11 @@ in
```
shb.restic.instances."lldap" = {
enable = true;
# Options specific to Restic.
} // config.shb.lldap.backup;
request = config.shb.lldap.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

35
modules/blocks/postgresql.nix
View file

@ -1,6 +1,7 @@
{ config, lib, pkgs, ... }:
let
cfg = config.shb.postgresql;
contracts = pkgs.callPackage ../contracts {};
upgrade-script = old: new:
let
@ -49,6 +50,40 @@ in
default = false;
};
databasebackup = lib.mkOption {
description = ''
Backup configuration. This is an output option.
Use it to initialize a block implementing the "backup" contract.
For example, with the restic block:
```
shb.restic.instances."postgresql" = {
request = config.shb.postgresl.backup;
settings = {
enable = true;
};
};
```
'';
type = contracts.databasebackup.requestType;
default = {
user = "postgres";
backupFile = "postgres.sql";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
'';
restoreCmd = ''
${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres
'';
};
};
ensures = lib.mkOption {
description = "List of username, database and/or passwords that should be created.";
type = lib.types.listOf (lib.types.submodule {

39
modules/blocks/postgresql/docs/default.md Normal file
View file

@ -0,0 +1,39 @@
# PostgreSQL Block {#blocks-postgresql}
Defined in [`/modules/blocks/postgresql.nix`](@REPO@/modules/blocks/postgresql.nix).
This block sets up a [PostgreSQL][] database.
[postgresql]: https://www.postgresql.org/
## Tests {#blocks-postgresql-tests}
Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
## Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
This block can be backed up using the [database backup](contracts-databasebackup.html) contract.
Contract integration tests are defined in [`/test/contracts/databasebackup.nix`](@REPO@/test/contracts/databasebackup.nix).
### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
```nix
{
my.backup.provider."postgresql" = {
request = config.shb.postgresql.databasebackup;
settings = {
// Specific options for the backup provider.
};
};
}
```
## Options Reference {#blocks-postgresql-options}
```{=include=} options
id-prefix: blocks-postgresql-options-
list-id: selfhostblocks-block-postgresql-options
source: @OPTIONS_JSON@
```

394
modules/blocks/restic.nix
View file

@ -4,44 +4,36 @@ let
cfg = config.shb.restic;
shblib = pkgs.callPackage ../../lib {};
contracts = pkgs.callPackage ../contracts {};
instanceOptions = {
enable = lib.mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots";
inherit (lib) concatStringsSep filterAttrs flatten literalExpression optionals optionalString listToAttrs mapAttrsToList mkEnableOption mkOption mkMerge;
inherit (lib) generators hasPrefix mkIf nameValuePair optionalAttrs removePrefix;
inherit (lib.types) attrsOf enum int ints listOf oneOf nonEmptyListOf nonEmptyStr nullOr path str submodule;
passphraseFile = lib.mkOption {
commonOptions = {
enable = mkEnableOption ''
this backup intance.
A disabled instance will not backup data anymore
but still provides the helper tool to restore snapshots
'';
passphraseFile = mkOption {
description = "Encryption key for the backups.";
type = lib.types.path;
type = path;
};
user = lib.mkOption {
description = ''
Unix user doing the backups. Must be the user owning the files to be backed up.
'';
type = lib.types.str;
};
sourceDirectories = lib.mkOption {
description = "Source directories.";
type = lib.types.nonEmptyListOf lib.types.str;
};
excludePatterns = lib.mkOption {
description = "Exclude patterns.";
type = lib.types.listOf lib.types.str;
default = [];
};
repositories = lib.mkOption {
repository = mkOption {
description = "Repositories to back this instance to.";
type = lib.types.nonEmptyListOf (lib.types.submodule {
type = submodule {
options = {
path = lib.mkOption {
type = lib.types.str;
path = mkOption {
type = str;
description = "Repository location";
};
secrets = lib.mkOption {
type = lib.types.attrsOf shblib.secretFileType;
secrets = mkOption {
type = attrsOf shblib.secretFileType;
default = {};
description = ''
Secrets needed to access the repository where the backups will be stored.
@ -49,17 +41,17 @@ let
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
'';
example = lib.literalExpression ''
'';
example = literalExpression ''
{
AWS_ACCESS_KEY_ID = <path/to/secret>;
AWS_SECRET_ACCESS_KEY = <path/to/secret>;
AWS_ACCESS_KEY_ID.source = <path/to/secret>;
AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
}
'';
'';
};
timerConfig = lib.mkOption {
type = lib.types.attrsOf utils.systemdUtils.unitOptions.unitOption;
timerConfig = mkOption {
type = attrsOf utils.systemdUtils.unitOptions.unitOption;
default = {
OnCalendar = "daily";
Persistent = true;
@ -72,12 +64,12 @@ let
};
};
};
});
};
};
retention = lib.mkOption {
retention = mkOption {
description = "For how long to keep backup files.";
type = lib.types.attrsOf (lib.types.oneOf [ lib.types.int lib.types.nonEmptyStr ]);
type = attrsOf (oneOf [ int nonEmptyStr ]);
default = {
keep_within = "1d";
keep_hourly = 24;
@ -87,73 +79,139 @@ let
};
};
hooks = lib.mkOption {
description = "Hooks to run before or after the backup.";
default = {};
type = lib.types.submodule {
options = {
before_backup = lib.mkOption {
description = "Hooks to run before backup";
type = lib.types.listOf lib.types.str;
default = [];
};
after_backup = lib.mkOption {
description = "Hooks to run after backup";
type = lib.types.listOf lib.types.str;
default = [];
};
};
};
};
limitUploadKiBs = lib.mkOption {
type = lib.types.nullOr lib.types.int;
limitUploadKiBs = mkOption {
type = nullOr int;
description = "Limit upload bandwidth to the given KiB/s amount.";
default = null;
example = 8000;
};
limitDownloadKiBs = lib.mkOption {
type = lib.types.nullOr lib.types.int;
limitDownloadKiBs = mkOption {
type = nullOr int;
description = "Limit download bandwidth to the given KiB/s amount.";
default = null;
example = 8000;
};
};
repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (lib.strings.removePrefix "/" name);
repoSlugName = name: builtins.replaceStrings ["/" ":"] ["_" "_"] (removePrefix "/" name);
backupName = name: repository: "${name}_${repoSlugName repository.path}";
fullName = name: repository: "restic-backups-${name}_${repoSlugName repository.path}";
in
{
options.shb.restic = {
instances = lib.mkOption {
description = "Each instance is a backup setting";
instances = mkOption {
description = "Each instance is backing up some directories to one repository.";
default = {};
type = lib.types.attrsOf (lib.types.submodule {
options = instanceOptions;
});
type = attrsOf (submodule ({ name, options, ... }: {
options = {
request = mkOption {
description = ''
Request part of the backup contract.
Accepts values from a requester.
'';
type = contracts.backup.request;
};
settings = mkOption {
description = ''
Settings specific to the Restic provider.
'';
type = submodule {
options = commonOptions;
};
};
result = mkOption {
description = ''
Result part of the backup contract.
Contains the output of the Restic provider.
'';
type = lib.types.anything; # contracts.databasebackup.result;
default = {
restoreScript = fullName name options.settings.value.repository;
backupService = "${fullName name options.settings.value.repository}.service";
};
defaultText = literalExpression ''
{
restoreScript = "${fullName "<name>" { path = "path/to/repository"; }}";
backupService = "${fullName "<name>" { path = "path/to/repository"; }}.service";
}
'';
};
};
}));
};
databases = mkOption {
description = "Databases to backup following the database backup contract.";
default = {};
type = attrsOf (submodule ({ name, options, ... }: {
options = {
request = mkOption {
description = ''
Request part of the backup contract.
Accepts values from a requester.
'';
type = contracts.databasebackup.requestType;
};
settings = mkOption {
description = ''
Settings specific to the Restic provider.
'';
type = submodule {
options = commonOptions;
};
};
result = mkOption {
description = ''
Result part of the backup contract.
Contains the output of the Restic provider.
'';
type = contracts.databasebackup.resultType;
default = {
restoreScript = fullName name options.settings.value.repository;
backupService = "${fullName name options.settings.value.repository}.service";
};
defaultText = literalExpression ''
{
restoreScript = "${fullName "<name>" { path = "path/to/repository"; }}";
backupService = "${fullName "<name>" { path = "path/to/repository"; }}.service";
}
'';
};
};
}));
};
# Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
performance = lib.mkOption {
performance = mkOption {
description = "Reduce performance impact of backup jobs.";
default = {};
type = lib.types.submodule {
type = submodule {
options = {
niceness = lib.mkOption {
type = lib.types.ints.between (-20) 19;
niceness = mkOption {
type = ints.between (-20) 19;
description = "nice priority adjustment, defaults to 15 for ~20% CPU time of normal-priority process";
default = 15;
};
ioSchedulingClass = lib.mkOption {
type = lib.types.enum [ "idle" "best-effort" "realtime" ];
ioSchedulingClass = mkOption {
type = enum [ "idle" "best-effort" "realtime" ];
description = "ionice scheduling class, defaults to best-effort IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
default = "best-effort";
};
ioPriority = lib.mkOption {
type = lib.types.nullOr (lib.types.ints.between 0 7);
ioPriority = mkOption {
type = nullOr (ints.between 0 7);
description = "ionice priority, defaults to 7 for lowest priority IO. Only used for `restic backup`, `restic forget` and `restic check` commands.";
default = 7;
};
@ -162,80 +220,121 @@ in
};
};
config = lib.mkIf (cfg.instances != {}) (
config = mkIf (cfg.instances != {} || cfg.databases != {}) (
let
enabledInstances = lib.attrsets.filterAttrs (k: i: i.enable) cfg.instances;
in lib.mkMerge [
enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances;
enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases;
in mkMerge [
{
environment.systemPackages = lib.optionals (enabledInstances != {}) [ pkgs.restic ];
environment.systemPackages = optionals (enabledInstances != {} || enabledDatabases != {}) [ pkgs.restic ];
}
{
# Create repository if it is a local path.
systemd.tmpfiles.rules =
let
mkRepositorySettings = name: instance: repository: lib.optionals (lib.hasPrefix "/" repository.path) [
"d '${repository.path}' 0750 ${instance.user} root - -"
mkSettings = name: instance: optionals (hasPrefix "/" instance.settings.repository.path) [
"d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -"
];
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
in
lib.flatten (lib.attrsets.mapAttrsToList mkSettings cfg.instances);
flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
}
{
services.restic.backups =
let
mkRepositorySettings = name: instance: repository: {
"${name}_${repoSlugName repository.path}" = {
inherit (instance) user;
mkSettings = name: instance: {
"${name}_${repoSlugName instance.settings.repository.path}" = {
inherit (instance.request) user;
repository = repository.path;
repository = instance.settings.repository.path;
paths = instance.sourceDirectories;
paths = instance.request.sourceDirectories;
passwordFile = toString instance.passphraseFile;
passwordFile = toString instance.settings.passphraseFile;
initialize = true;
inherit (repository) timerConfig;
inherit (instance.settings.repository) timerConfig;
pruneOpts = lib.mapAttrsToList (name: value:
pruneOpts = mapAttrsToList (name: value:
"--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
) instance.retention;
) instance.settings.retention;
backupPrepareCommand = lib.strings.concatStringsSep "\n" instance.hooks.before_backup;
backupPrepareCommand = concatStringsSep "\n" instance.request.hooks.before_backup;
backupCleanupCommand = lib.strings.concatStringsSep "\n" instance.hooks.after_backup;
} // lib.attrsets.optionalAttrs (builtins.length instance.excludePatterns > 0) {
exclude = instance.excludePatterns;
backupCleanupCommand = concatStringsSep "\n" instance.request.hooks.after_backup;
extraBackupArgs =
(lib.optionals (instance.limitUploadKiBs != null) [
"--limit-upload=${toString instance.limitUploadKiBs}"
(optionals (instance.settings.limitUploadKiBs != null) [
"--limit-upload=${toString instance.settings.limitUploadKiBs}"
])
++ (lib.optionals (instance.limitDownloadKiBs != null) [
"--limit-download=${toString instance.limitDownloadKiBs}"
++ (optionals (instance.settings.limitDownloadKiBs != null) [
"--limit-download=${toString instance.settings.limitDownloadKiBs}"
]);
} // optionalAttrs (builtins.length instance.request.excludePatterns > 0) {
exclude = instance.request.excludePatterns;
};
};
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
in
lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances));
mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
}
{
services.restic.backups =
let
mkSettings = name: instance: {
"${name}_${repoSlugName instance.settings.repository.path}" = {
inherit (instance.request) user;
repository = instance.settings.repository.path;
dynamicFilesFrom = "echo";
passwordFile = toString instance.settings.passphraseFile;
initialize = true;
inherit (instance.settings.repository) timerConfig;
pruneOpts = mapAttrsToList (name: value:
"--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
) instance.settings.retention;
extraBackupArgs =
(optionals (instance.settings.limitUploadKiBs != null) [
"--limit-upload=${toString instance.settings.limitUploadKiBs}"
])
++ (optionals (instance.settings.limitDownloadKiBs != null) [
"--limit-download=${toString instance.settings.limitDownloadKiBs}"
])
++
(let
cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd;
in
[
"--stdin-filename ${instance.request.backupFile} --stdin-from-command -- ${cmd}/bin/dump.sh"
]);
};
};
in
mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
}
{
systemd.services =
let
mkRepositorySettings = name: instance: repository:
mkSettings = name: instance:
let
serviceName = fullName name repository;
serviceName = fullName name instance.settings.repository;
in
{
${serviceName} = lib.mkMerge [
${serviceName} = mkMerge [
{
serviceConfig = {
Nice = cfg.performance.niceness;
IOSchedulingClass = cfg.performance.ioSchedulingClass;
IOSchedulingPriority = cfg.performance.ioPriority;
BindReadOnlyPaths = instance.sourceDirectories;
# BindReadOnlyPaths = instance.sourceDirectories;
};
}
(lib.attrsets.optionalAttrs (repository.secrets != {})
(optionalAttrs (instance.settings.repository.secrets != {})
{
serviceConfig.EnvironmentFile = [
"/run/secrets_restic/${serviceName}"
@ -245,13 +344,13 @@ in
})
];
"${serviceName}-pre" = lib.mkIf (repository.secrets != {})
"${serviceName}-pre" = mkIf (instance.settings.repository.secrets != {})
(let
script = shblib.genConfigOutOfBandSystemd {
config = repository.secrets;
config = instance.settings.repository.secrets;
configLocation = "/run/secrets_restic/${serviceName}";
generator = name: v: pkgs.writeText "template" (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
user = instance.user;
generator = name: v: pkgs.writeText "template" (generators.toINIWithGlobalSection {} { globalSection = v; });
user = instance.request.user;
};
in
{
@ -260,37 +359,62 @@ in
serviceConfig.LoadCredential = script.loadCredentials;
});
};
mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
in
lib.mkMerge (lib.flatten (lib.attrsets.mapAttrsToList mkSettings enabledInstances));
mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
}
{
system.activationScripts = let
mkEnv = name: instance: repository:
lib.nameValuePair "${fullName name repository}_gen"
mkEnv = name: instance:
nameValuePair "${fullName name instance.settings.repository}_gen"
(shblib.replaceSecrets {
userConfig = repository.secrets // {
RESTIC_PASSWORD_FILE = instance.passphraseFile;
RESTIC_REPOSITORY = repository.path;
userConfig = instance.settings.repository.secrets // {
RESTIC_PASSWORD_FILE = instance.settings.passphraseFile;
RESTIC_REPOSITORY = instance.settings.repository.path;
};
resultPath = "/run/secrets_restic_env/${fullName name repository}";
generator = name: v: pkgs.writeText (fullName name repository) (lib.generators.toINIWithGlobalSection {} { globalSection = v; });
user = instance.user;
resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
generator = name: v: pkgs.writeText (fullName name instance.settings.repository) (generators.toINIWithGlobalSection {} { globalSection = v; });
user = instance.request.user;
});
mkSettings = name: instance: builtins.map (mkEnv name instance) instance.repositories;
in
lib.listToAttrs (lib.flatten (lib.attrsets.mapAttrsToList mkSettings cfg.instances));
listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases)));
}
{
environment.systemPackages = let
mkResticBinary = name: instance: repository:
pkgs.writeShellScriptBin (fullName name repository) ''
export $(grep -v '^#' "/run/secrets_restic_env/${fullName name repository}" \
mkResticBinary = name: instance:
pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
set -euo pipefail
export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
| xargs -d '\n')
${pkgs.restic}/bin/restic $@
if ! [ "$1" = "restore" ]; then
sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
else
shift
sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic restore $@ --target /"
fi
'';
mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories;
in
lib.flatten (lib.attrsets.mapAttrsToList mkSettings cfg.instances);
flatten (mapAttrsToList mkResticBinary cfg.instances);
}
{
environment.systemPackages = let
mkResticBinary = name: instance:
pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
set -euo pipefail
export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
| xargs -d '\n')
if ! [ "$1" = "restore" ]; then
sudo --preserve-env -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
else
shift
sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupFile} | ${instance.request.restoreCmd}"
fi
'';
in
flatten (mapAttrsToList mkResticBinary cfg.databases);
}
]);
}

88
modules/blocks/restic/docs/default.md
View file

@ -6,15 +6,17 @@ This block sets up a backup job using [Restic][restic].
[restic]: https://restic.net/
## Contract {#blocks-restic-features}
## Tests {#blocks-restic-tests}
This block implements the [backup](contracts-backup.html) contract.
Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
Integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
## Provider Contracts {#blocks-restic-contract-provider}
## Usage {#blocks-backup-usage}
This block implements the [backup](contracts-backup.html) and [database backup](contracts-databasebackup.html) contracts.
### One folder backed up to mounted hard drives {#blocks-backup-usage-one}
Contract integration tests are defined in [`/test/contracts/backup.nix`](@REPO@/test/contracts/backup.nix).
### One folder backed up to mounted hard drives {#blocks-restic-contract-provider-one}
The following snippet shows how to configure
the backup of 1 folder to 1 repository.
@ -22,35 +24,39 @@ We assume that the folder is used by the `myservice` service and is owned by a u
```nix
shb.restic.instances.myservice = {
enable = true;
request = {
user = "myservice";
user = "myservice";
sourceDirectories = [
"/var/lib/myfolder"
];
};
passphraseFile = "<path/to/passphrase>";
settings = {
enable = true;
repositories = [{
path = "/srv/backups/myservice";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "3h";
passphraseFile = "<path/to/passphrase>";
repository = {
path = "/srv/backups/myservice";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "3h";
};
};
}];
sourceDirectories = [
"/var/lib/myfolder"
];
retention = {
keep_within = "1d";
keep_hourly = 24;
keep_daily = 7;
keep_weekly = 4;
keep_monthly = 6;
retention = {
keep_within = "1d";
keep_hourly = 24;
keep_daily = 7;
keep_weekly = 4;
keep_monthly = 6;
};
};
};
```
### One folder backed up to S3 {#blocks-restic-usage-remote}
### One folder backed up to S3 {#blocks-restic-contract-provider-remote}
Here we will only highlight the differences with the previous configuration.
@ -59,7 +65,7 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
```diff
shb.backup.instances.myservice = {
repositories = [{
repository = {
- path = "/srv/pool1/backups/myfolder";
+ path = "s3:s3.us-west-000.backblazeb2.com/backups/myfolder";
timerConfig = {
@ -71,11 +77,11 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
+ AWS_ACCESS_KEY_ID.source="<path/to/access_key_id>";
+ AWS_SECRET_ACCESS_KEY.source="<path/to/secret_access_key>";
+ };
}];
};
}
```
### Secrets {#blocks-restic-secrets}
## Secrets {#blocks-restic-secrets}
To be secure, the secrets should deployed out of band, otherwise they will be world-readable in the nix store.
@ -84,15 +90,13 @@ The code to backup to Backblaze with secrets stored in Sops would look like so:
```nix
shb.restic.instances.myfolder.passphraseFile = config.sops.secrets."myservice/backup/passphrase".path;
shb.restic.instances.myfolder.repositories = [
{
path = "s3:s3.us-west-000.backblazeb2.com/<mybucket>";
secrets = {
AWS_ACCESS_KEY_ID.source = config.sops.secrets."backup/b2/access_key_id".path;
AWS_SECRET_ACCESS_KEY.source = config.sops.secrets."backup/b2/secret_access_key".path;
};
}
];
shb.restic.instances.myfolder.repository = {
path = "s3:s3.us-west-000.backblazeb2.com/<mybucket>";
secrets = {
AWS_ACCESS_KEY_ID.source = config.sops.secrets."backup/b2/access_key_id".path;
AWS_SECRET_ACCESS_KEY.source = config.sops.secrets."backup/b2/secret_access_key".path;
};
};
sops.secrets."myservice/backup/passphrase" = {
sopsFile = ./secrets.yaml;
@ -117,7 +121,7 @@ sops.secrets."backup/b2/secret_access_key" = {
Pay attention that the owner must be the `myservice` user, the one owning the files to be backed up.
A `secrets` contract is in progress that will allow one to not care about such details.
### Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
## Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
The following snippet shows how to configure backup of any number of folders to 3 repositories,
each happening at different times to avoid I/O contention.
@ -228,10 +232,10 @@ restic-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups
Discovering those is easy thanks to tab-completion.
One can then restore a backup with:
One can then restore a backup from a given repository with:
```bash
restic-myfolder1_srv_pool1_backups restore latest -t /
restic-myfolder1_srv_pool1_backups restore latest
```
### Troubleshooting {#blocks-restic-maintenance-troubleshooting}
@ -241,7 +245,7 @@ In case something bad happens with a backup, the [official documentation](https:
## Options Reference {#blocks-restic-options}
```{=include=} options
id-prefix: blocks-backup-options-
list-id: selfhostblocks-block-backup-options
id-prefix: blocks-restic-options-
list-id: selfhostblocks-block-restic-options
source: @OPTIONS_JSON@
```

3
modules/blocks/restic/dummyModule.nix Normal file
View file

@ -0,0 +1,3 @@
{ lib, ... }:
{
}

80
modules/contracts/backup.nix
View file

@ -1,42 +1,60 @@
{ lib, ... }:
lib.types.submodule {
freeformType = lib.types.anything;
let
inherit (lib) mkOption;
inherit (lib.types) anything listOf nonEmptyListOf submodule str;
in
{
request = submodule {
options = {
user = mkOption {
description = "Unix user doing the backups.";
type = str;
};
options = {
user = lib.mkOption {
description = "Unix user doing the backups.";
type = lib.types.str;
};
sourceDirectories = mkOption {
description = "Directories to backup.";
type = nonEmptyListOf str;
};
sourceDirectories = lib.mkOption {
description = "Directories to backup.";
type = lib.types.nonEmptyListOf lib.types.str;
};
excludePatterns = mkOption {
description = "Patterns to exclude.";
type = listOf str;
default = [];
};
excludePatterns = lib.mkOption {
description = "Patterns to exclude.";
type = lib.types.listOf lib.types.str;
default = [];
};
hooks = mkOption {
description = "Hooks to run around the backup.";
default = {};
type = submodule {
options = {
before_backup = mkOption {
description = "Hooks to run before backup";
type = listOf str;
default = [];
};
hooks = lib.mkOption {
description = "Hooks to run around the backup.";
default = {};
type = lib.types.submodule {
options = {
before_backup = lib.mkOption {
description = "Hooks to run before backup";
type = lib.types.listOf lib.types.str;
default = [];
};
after_backup = lib.mkOption {
description = "Hooks to run after backup";
type = lib.types.listOf lib.types.str;
default = [];
after_backup = mkOption {
description = "Hooks to run after backup";
type = listOf str;
default = [];
};
};
};
};
};
};
result = submodule {
options = {
restoreScript = mkOption {
description = "Name of script that can restore the database.";
type = str;
};
backupService = mkOption {
description = "Name of service backing up the database.";
type = str;
};
};
};
}

63
modules/contracts/backup/docs/default.md
View file

@ -1,15 +1,13 @@
# Backup Contract {#backup-contract}
# Backup Contract {#contract-backup}
This NixOS contract represents a backup job
that will backup one or more files or directories
at a regular schedule.
on a regular schedule.
It is a contract between a service that has files to be backed up
and a service that backs up files.
All options in this contract should be set by the former.
The latter will then use the values of those options to know what to backup.
## Contract Reference {#backup-contract-options}
## Contract Reference {#contract-backup-options}
These are all the options that are expected to exist for this contract to be respected.
@ -19,9 +17,11 @@ list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```
## Usage {#backup-contract-usage}
## Usage {#contract-backup-usage}
A service that can be backed up will provide a `backup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define what directories to backup,
the user to backup with
@ -32,9 +32,8 @@ Here is an example module defining such a `backup` option:
```nix
{
options = {
myservice.backup = lib.mkOption {
type = contracts.backup;
readOnly = true;
myservice.backup = mkOption {
type = contracts.backup.request;
default = {
user = "myservice";
sourceDirectories = [
@ -46,23 +45,26 @@ Here is an example module defining such a `backup` option:
};
```
As you can see, NixOS modules are a bit abused to make contracts work.
Default values are set as well as the `readOnly` attribute to ensure those values stay as defined.
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `backupservice` option
and that one can create multiple backup instances under `backupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
backupservice.instances.myservice = myservice.backup // {
enable = true;
backupservice.instances.myservice = {
request = myservice.backup;
settings = {
enable = true;
repository = {
path = "/srv/backup/myservice";
repository = {
path = "/srv/backup/myservice";
};
# ... Other options specific to backupservice like scheduling.
};
# ... Other options specific to backupservice like scheduling.
};
```
@ -70,28 +72,27 @@ It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `backupservice`:
```nix
backupservice.instances.myservice_2 = myservice.backup // {
enable = true;
repository = {
path = "<remote path>";
backupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
repository = {
path = "<remote path>";
};
};
};
```
Or with another module `backupservice_2`!
## Provided Implementations {#backup-contract-impl}
## Providers of the Backup Contract {#contract-backup-providers}
An implementation here is a service that understands the `backup` contract
and will backup the files accordingly.
One implementation is provided out of the box:
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
A second one based on `borgbackup` is in progress.
## Services Providing `backup` Option {#backup-contract-services}
## Requester Blocks and Services {#contract-backup-requesters}
- <!-- [ -->Audiobookshelf<!-- ](services-audiobookshelf.html). --> (no manual yet)
- <!-- [ -->Deluge<!--](services-deluge.html). --> (no manual yet)

6
modules/contracts/backup/dummyModule.nix
View file

@ -1,10 +1,12 @@
{ pkgs, lib, ... }:
let
contracts = pkgs.callPackage ../. {};
inherit (lib) mkOption;
in
{
options.shb.contracts.backup = lib.mkOption {
options.shb.contracts.backup = mkOption {
description = "Contract for backups.";
type = contracts.backup;
type = contracts.backup.request;
};
}

121
modules/contracts/backup/test.nix Normal file
View file

@ -0,0 +1,121 @@
{ pkgs, lib, ... }:
let
pkgs' = pkgs;
testLib = pkgs.callPackage ../../../test/common.nix {};
inherit (lib) concatStringsSep concatMapStringsSep getAttrFromPath mkIf optionalAttrs setAttrByPath;
inherit (testLib) indent;
in
{ name,
providerRoot,
modules ? [],
username ? "me",
sourceDirectories ? [
"/opt/files/A"
"/opt/files/B"
],
settings, # repository -> attrset
}: pkgs.testers.runNixOSTest {
inherit name;
nodes.machine = { config, ... }: {
imports = ( testLib.baseImports pkgs' ) ++ modules;
config = lib.mkMerge [
(setAttrByPath providerRoot {
request = {
inherit sourceDirectories;
user = username;
};
settings = settings "/opt/repos/${name}";
})
(mkIf (username != "root") {
users.users.${username} = {
isSystemUser = true;
extraGroups = [ "sudoers" ];
group = "root";
};
})
];
};
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
testScript = { nodes, ... }: let
provider = getAttrFromPath providerRoot nodes.machine;
backupService = provider.result.backupService;
restoreScript = provider.result.restoreScript;
onAllSourceDirectories = f: concatMapStringsSep "\n" (path: indent 4 (f path)) sourceDirectories;
in ''
from dictdiffer import diff
username = "${username}"
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
def list_files(dir):
files_and_content = {}
files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
for f in files:
content = machine.succeed(f"""cat {f}""").strip()
files_and_content[f] = content
return files_and_content
def assert_files(dir, files):
result = list(diff(list_files(dir), files))
if len(result) > 0:
raise Exception("Unexpected files:", result)
with subtest("Create initial content"):
for path in sourceDirectories:
machine.succeed(f"""
mkdir -p {path}
echo repo_fileA_1 > {path}/fileA
echo repo_fileB_1 > {path}/fileB
chown {username}: -R {path}
chmod go-rwx -R {path}
""")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_1',
f'{path}/fileB': 'repo_fileB_1',
})
with subtest("First backup in repo"):
print(machine.succeed("systemctl cat ${backupService}"))
machine.succeed("systemctl start ${backupService}")
with subtest("New content"):
for path in sourceDirectories:
machine.succeed(f"""
echo repo_fileA_2 > {path}/fileA
echo repo_fileB_2 > {path}/fileB
""")
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Delete content"):
for path in sourceDirectories:
machine.succeed(f"""rm -r {path}/*""")
assert_files(path, {})
with subtest("Restore initial content from repo"):
machine.succeed("""${restoreScript} restore latest""")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_1',
f'{path}/fileB': 'repo_fileB_1',
})
'';
}

77
modules/contracts/databasebackup.nix Normal file
View file

@ -0,0 +1,77 @@
{ lib, ... }:
let
inherit (lib) mkIf mkOption literalExpression;
inherit (lib.types) anything submodule str;
in
{
requestType = submodule {
options = {
user = mkOption {
description = "Unix user doing the backups.";
type = str;
example = "postgres";
};
backupFile = mkOption {
description = "Filename of the backup.";
type = str;
default = "dump";
example = "postgresql.sql";
};
backupCmd = mkOption {
description = "Command that produces the database dump on stdout.";
type = str;
example = literalExpression ''
''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable
'';
};
restoreCmd = mkOption {
description = "Command that reads the database dump on stdin and restores the database.";
type = str;
example = literalExpression ''
''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres
'';
};
};
};
resultType = submodule {
options = {
restoreScript = mkOption {
description = ''
Name of script that can restore the database.
One can then list snapshots with:
```bash
$ my_restore_script snapshots
```
And restore the database with:
```bash
$ my_restore_script restore latest
```
'';
type = str;
example = "my_restore_script";
};
backupService = mkOption {
description = ''
Name of service backing up the database.
This script can be ran manually to backup the database:
```bash
$ systemctl start my_backup_service
```
'';
type = str;
example = "my_backup_service.service";
};
};
};
}

100
modules/contracts/databasebackup/docs/default.md Normal file
View file

@ -0,0 +1,100 @@
# Database Backup Contract {#contract-databasebackup}
This NixOS contract represents a backup job
that will backup everything in one database
on a regular schedule.
It is a contract between a service that has database dumps to be backed up
and a service that backs up databases dumps.
## Contract Reference {#contract-databasebackup-options}
These are all the options that are expected to exist for this contract to be respected.
```{=include=} options
id-prefix: contracts-databasebackup-options-
list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```
## Usage {#contract-databasebackup-usage}
A database that can be backed up will provide a `databasebackup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define how to create a database dump,
the user to backup with
and how to restore from a database dump.
Here is an example module defining such a `databasebackup` option:
```nix
{
options = {
myservice.databasebackup = mkOption {
type = contracts.databasebackup.request;
default = {
user = "myservice";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
'';
restoreCmd = ''
${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres
'';
};
};
};
};
```
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `databasebackupservice` option
and that one can create multiple backup instances under `databasebackupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
databasebackupservice.instances.myservice = {
request = myservice.databasebackup;
settings = {
enable = true;
repository = {
path = "/srv/backup/myservice";
};
# ... Other options specific to backupservice like scheduling.
};
};
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
```nix
databasebackupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
repository = {
path = "<remote path>";
};
};
};
```
Or with another module `databasebackupservice_2`!
## Providers of the Database Backup Contract {#contract-databasebackup-providers}
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
## Requester Blocks and Services {#contract-databasebackup-requesters}
- [PostgreSQL](blocks-postgresql.html#blocks-postgresql-contract-databasebackup).

43
modules/contracts/databasebackup/dummyModule.nix Normal file
View file

@ -0,0 +1,43 @@
{ pkgs, lib, ... }:
let
contracts = pkgs.callPackage ../. {};
inherit (lib) mkOption;
inherit (lib.types) anything submodule;
in
{
options.shb.contracts.databasebackup = mkOption {
description = ''
Contract for database backup between a requester module
and a provider module.
The requester communicates to the provider
how to backup the database
through the `request` options.
The provider reads from the `request` options
and backs up the database as requested.
It communicates to the requester what script is used
to backup and restore the database
through the `result` options.
'';
type = submodule {
options = {
request = mkOption {
description = ''
Options set by a requester module of the database contract.
'';
type = contracts.databasebackup.requestType;
};
result = mkOption {
description = ''
Options set by a provider module of the database contract.
'';
type = contracts.databasebackup.resultType;
};
};
};
};
}

84
modules/contracts/databasebackup/test.nix Normal file
View file

@ -0,0 +1,84 @@
{ pkgs, lib, ... }:
let
pkgs' = pkgs;
testLib = pkgs.callPackage ../../../test/common.nix {};
inherit (lib) getAttrFromPath mkIf optionalAttrs setAttrByPath;
in
{ name,
requesterRoot,
providerRoot,
providerExtraConfig ? null, # { username, database } -> attrset
modules ? [],
username ? "me",
database ? "me",
settings, # repository -> attrset
}: pkgs.testers.runNixOSTest {
inherit name;
nodes.machine = { config, ... }: {
imports = ( testLib.baseImports pkgs' ) ++ modules;
config = lib.mkMerge [
(setAttrByPath providerRoot {
request = (getAttrFromPath requesterRoot config).databasebackup;
settings = settings "/opt/repos/database";
})
(mkIf (username != "root") {
users.users.${username} = {
isSystemUser = true;
extraGroups = [ "sudoers" ];
group = "root";
};
})
(optionalAttrs (providerExtraConfig != null) (providerExtraConfig { inherit username database; }))
];
};
testScript = { nodes, ... }: let
provider = (getAttrFromPath providerRoot nodes.machine).result;
in ''
import csv
start_all()
machine.wait_for_unit("postgresql.service")
machine.wait_for_open_port(5432)
def peer_cmd(cmd, db="me"):
return "sudo -u me psql -U me {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
def query(query):
res = machine.succeed(peer_cmd(query))
return list(dict(l) for l in csv.DictReader(res.splitlines()))
def cmp_tables(a, b):
for i in range(max(len(a), len(b))):
diff = set(a[i]) ^ set(b[i])
if len(diff) > 0:
raise Exception(i, diff)
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
with subtest("create fixture"):
machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
res = query("SELECT * FROM test")
cmp_tables(res, table)
with subtest("backup"):
print(machine.succeed("systemctl cat ${provider.backupService}"))
machine.succeed("systemctl start ${provider.backupService}")
with subtest("drop database"):
machine.succeed(peer_cmd("DROP DATABASE me", db="postgres"))
with subtest("restore"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed("${provider.restoreScript} restore latest ")
with subtest("check restoration"):
res = query("SELECT * FROM test")
cmp_tables(res, table)
'';
}

3
modules/contracts/default.nix
View file

@ -1,10 +1,13 @@
{ pkgs, lib }:
{
databasebackup = import ./databasebackup.nix { inherit lib; };
backup = import ./backup.nix { inherit lib; };
mount = import ./mount.nix { inherit lib; };
secret = import ./secret.nix { inherit lib; };
ssl = import ./ssl.nix { inherit lib; };
test = {
secret = import ./secret/test.nix { inherit pkgs lib; };
databasebackup = import ./databasebackup/test.nix { inherit pkgs lib; };
backup = import ./backup/test.nix { inherit pkgs lib; };
};
}

11
modules/services/arr.nix
View file

@ -316,7 +316,7 @@ let
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -325,10 +325,11 @@ let
```
shb.restic.instances."${name}" = {
enable = true;
# Options specific to Restic.
} // config.shb.${name}.backup;
request = config.shb.${name}.backup;
settings = {
enable = true;
};
}
```
'';
readOnly = true;

11
modules/services/audiobookshelf.nix
View file

@ -83,7 +83,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -92,10 +92,11 @@ in
```
shb.restic.instances."audiobookshelf" = {
enable = true;
# Options specific to Restic.
} // config.shb.audiobookshelf.backup;
request = config.shb.audiobookshelf.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

11
modules/services/deluge.nix
View file

@ -234,7 +234,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -243,10 +243,11 @@ in
```
shb.restic.instances."vaultwarden" = {
enable = true;
# Options specific to Restic.
} // config.shb.vaultwarden.backup;
request = config.shb.vaultwarden.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

11
modules/services/forgejo.nix
View file

@ -218,7 +218,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -227,10 +227,11 @@ in
```
shb.restic.instances."forgejo" = {
enable = true;
# Options specific to Restic.
} // config.shb.forgejo.backup;
request = config.shb.forgejo.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

7
modules/services/forgejo/docs/default.md
View file

@ -213,8 +213,11 @@ twice with a future secrets SHB block.
Backing up Forgejo using the [Restic block](blocks-restic.html) is done like so:
```nix
shb.restic.instances."forgejo" = config.shb.forgejo.backup // {
enable = true;
shb.restic.instances."forgejo" = {
request = config.shb.forgejo.backup;
settings = {
enable = true;
};
};
```

11
modules/services/grocy.nix
View file

@ -63,7 +63,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -72,10 +72,11 @@ in
```
shb.restic.instances."grocy" = {
enable = true;
# Options specific to Restic.
} // config.shb.grocy.backup;
request = config.shb.grocy.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

11
modules/services/hledger.nix
View file

@ -55,7 +55,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -64,10 +64,11 @@ in
```
shb.restic.instances."hledger" = {
enable = true;
# Options specific to Restic.
} // config.shb.hledger.backup;
request = config.shb.hledger.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

11
modules/services/home-assistant.nix
View file

@ -137,7 +137,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -146,10 +146,11 @@ in
```
shb.restic.instances."home-assistant" = {
enable = true;
# Options specific to Restic.
} // config.shb.home-assistant.backup;
request = config.shb.home-assistant.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

11
modules/services/jellyfin.nix
View file

@ -139,7 +139,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -148,10 +148,11 @@ in
```
shb.restic.instances."jellyfin" = {
enable = true;
# Options specific to Restic.
} // config.shb.jellyfin.backup;
request = config.shb.jellyfin.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

11
modules/services/nextcloud-server.nix
View file

@ -498,7 +498,7 @@ in
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -507,10 +507,11 @@ in
```
shb.restic.instances."nextcloud" = {
enable = true;
# Options specific to Restic.
} // config.shb.nextcloud.backup;
request = config.shb.nextcloud.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

7
modules/services/nextcloud-server/docs/default.md
View file

@ -281,8 +281,11 @@ shb.nextcloud.postgresSettings = {
Backing up Nextcloud using the [Restic block](blocks-restic.html) is done like so:
```nix
shb.restic.instances."nextcloud" = config.shb.nextcloud.backup // {
enable = true;
shb.restic.instances."nextcloud" = {
request = config.shb.nextcloud.backup;
settings = {
enable = true;
};
};
```

11
modules/services/vaultwarden.nix
View file

@ -115,7 +115,7 @@ in
};
backup = lib.mkOption {
type = contracts.backup;
type = contracts.backup.request;
description = ''
Backup configuration. This is an output option.
@ -124,10 +124,11 @@ in
```
shb.restic.instances."vaultwarden" = {
enable = true;
# Options specific to Restic.
} // config.shb.vaultwarden.backup;
request = config.shb.vaultwarden.backup;
settings = {
enable = true;
};
};
```
'';
readOnly = true;

7
modules/services/vaultwarden/docs/default.md
View file

@ -97,8 +97,11 @@ shb.zfs.datasets."postgresql".path = "/var/lib/postgresql";
Backing up Vaultwarden using the [Restic block](blocks-restic.html) is done like so:
```nix
shb.restic.instances."vaultwarden" = config.shb.vaultwarden.backup // {
enable = true;
shb.restic.instances."vaultwarden" = {
request = config.shb.vaultwarden.backup;
settings = {
enable = true;
};
};
```

90
test/blocks/restic.nix
View file

@ -32,19 +32,12 @@ let
};
shb.restic.instances."testinstance" = {
enable = true;
settings = {
enable = true;
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
sourceDirectories = [
"/opt/files/A"
"/opt/files/B"
];
user = user;
repositories = [
{
repository = {
path = "/opt/repos/A";
timerConfig = {
OnCalendar = "00:00:00";
@ -56,39 +49,45 @@ let
A.source = config.shb.hardcodedsecret.A.path;
B.source = config.shb.hardcodedsecret.B.path;
};
}
{
path = "/opt/repos/B";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
}
];
};
};
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets_restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
request = {
inherit user;
sourceDirectories = [
"/opt/files/A"
"/opt/files/B"
];
hooks.before_backup = [''
echo $RUNTIME_DIRECTORY
if [ "$RUNTIME_DIRECTORY" = /run/restic-backups-testinstance_opt_repos_A ]; then
if ! [ -f /run/secrets_restic/restic-backups-testinstance_opt_repos_A ]; then
exit 10
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "B:$B"
exit 12
fi
fi
if [ -z "$A" ] || ! [ "$A" = "secretA" ]; then
echo "A:$A"
exit 11
fi
if [ -z "$B" ] || ! [ "$B" = "secretB" ]; then
echo "B:$B"
exit 12
fi
fi
''];
''];
};
};
};
extraPythonPackages = p: [ p.dictdiffer ];
skipTypeCheck = true;
testScript = ''
testScript = { nodes, ... }: let
provider = nodes.machine.shb.restic.instances."testinstance";
backupService = provider.result.backupService;
restoreScript = provider.result.restoreScript;
in ''
from dictdiffer import diff
def list_files(dir):
@ -133,7 +132,7 @@ let
})
with subtest("First backup in repo A"):
machine.succeed("systemctl start restic-backups-testinstance_opt_repos_A")
machine.succeed("systemctl start ${backupService}")
with subtest("New content"):
machine.succeed("""
@ -150,9 +149,6 @@ let
'/opt/files/A/fileB': 'repoA_fileB_2',
})
with subtest("Second backup in repo B"):
machine.succeed("systemctl start restic-backups-testinstance_opt_repos_B")
with subtest("Delete content"):
machine.succeed("""
rm -r /opt/files/A /opt/files/B
@ -162,7 +158,7 @@ let
with subtest("Restore initial content from repo A"):
machine.succeed("""
restic-testinstance_opt_repos_A restore latest -t /
${restoreScript} restore latest
""")
assert_files("/opt/files", {
@ -171,18 +167,6 @@ let
'/opt/files/A/fileA': 'repoA_fileA_1',
'/opt/files/A/fileB': 'repoA_fileB_1',
})
with subtest("Restore initial content from repo B"):
machine.succeed("""
restic-testinstance_opt_repos_B restore latest -t /
""")
assert_files("/opt/files", {
'/opt/files/B/fileA': 'repoB_fileA_2',
'/opt/files/B/fileB': 'repoB_fileB_2',
'/opt/files/A/fileA': 'repoA_fileA_2',
'/opt/files/A/fileB': 'repoA_fileB_2',
})
'';
};

43
test/contracts/backup.nix Normal file
View file

@ -0,0 +1,43 @@
{ pkgs, ... }:
let
contracts = pkgs.callPackage ../../modules/contracts {};
in
{
restic_root = contracts.test.backup {
name = "restic_root";
username = "root";
providerRoot = [ "shb" "restic" "instances" "mytest" ];
modules = [
../../modules/blocks/restic.nix
];
settings = repository: {
enable = true;
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
repository = {
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};
};
};
};
restic_me = contracts.test.backup {
name = "restic_me";
username = "me";
providerRoot = [ "shb" "restic" "instances" "mytest" ];
modules = [
../../modules/blocks/restic.nix
];
settings = repository: {
enable = true;
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
repository = {
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};
};
};
};
}

32
test/contracts/databasebackup.nix Normal file
View file

@ -0,0 +1,32 @@
{ pkgs, ... }:
let
contracts = pkgs.callPackage ../../modules/contracts {};
in
{
restic_postgres = contracts.test.databasebackup {
name = "restic_postgres";
requesterRoot = [ "shb" "postgresql" ];
providerRoot = [ "shb" "restic" "databases" "postgresql" ];
modules = [
../../modules/blocks/postgresql.nix
../../modules/blocks/restic.nix
];
settings = repository: {
enable = true;
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
repository = {
path = repository;
timerConfig = {
OnCalendar = "00:00:00";
};
};
};
providerExtraConfig = { username, database, ... }: {
shb.postgresql.ensures = [
{
inherit username database;
}
];
};
};
}

15
test/services/vaultwarden.nix
View file

@ -93,18 +93,19 @@ let
imports = [
../../modules/blocks/restic.nix
];
shb.restic.instances."testinstance" = config.shb.vaultwarden.backup // {
enable = true;
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
repositories = [
{
shb.restic.instances."testinstance" = {
request = config.shb.vaultwarden.backup;
settings = {
enable = true;
passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
repository = {
path = "/opt/repos/A";
timerConfig = {
OnCalendar = "00:00:00";
RandomizedDelaySec = "5h";
};
}
];
};
};
};
};
in