datasetBackup: add contract with sanoid implementation
This commit is contained in:
ibizaman
Pierre Penninckx
parent
a0d29e9d97
commit
7ca1f93f3f
13 changed files with 811 additions and 9 deletions
|
|
@ -39,7 +39,9 @@ Provided contracts are:
|
|||
- [Backup contract][] to backup directories.
|
||||
Two providers are implemented: [BorgBackup][] and [Restic][].
|
||||
- [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
|
||||
One provider is implemented: [BorgBackup][] and [Restic][].
|
||||
Two providers are implemented: [BorgBackup][] and [Restic][].
|
||||
- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets.
|
||||
One provider is implemented: [Sanoid][]
|
||||
- [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
|
||||
One provider is implemented: [SOPS][].
|
||||
- [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard.
|
||||
|
|
@ -49,6 +51,7 @@ Provided contracts are:
|
|||
[borgbackup]: blocks-borgbackup.html
|
||||
[homepage]: services-homepage.html
|
||||
[restic]: blocks-restic.html
|
||||
[sanoid]: blocks-sanoid.html
|
||||
[sops]: blocks-sops.html
|
||||
|
||||
```{=include=} chapters html:into-file=//contracts-ssl.html
|
||||
|
|
@ -63,6 +66,10 @@ modules/contracts/backup/docs/default.md
|
|||
modules/contracts/databasebackup/docs/default.md
|
||||
```
|
||||
|
||||
```{=include=} chapters html:into-file=//contracts-datasetbackup.html
|
||||
modules/contracts/datasetbackup/docs/default.md
|
||||
```
|
||||
|
||||
```{=include=} chapters html:into-file=//contracts-secret.html
|
||||
modules/contracts/secret/docs/default.md
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1826,6 +1826,24 @@
|
|||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript"
|
||||
],
|
||||
"blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs": [
|
||||
"blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs"
|
||||
],
|
||||
|
|
@ -1907,6 +1925,21 @@
|
|||
"contract-databasebackup-usage": [
|
||||
"contracts-databasebackup.html#contract-databasebackup-usage"
|
||||
],
|
||||
"contract-dataset-backup": [
|
||||
"contracts-datasetbackup.html#contract-dataset-backup"
|
||||
],
|
||||
"contract-datasetbackup-providers": [
|
||||
"contracts-datasetbackup.html#contract-datasetbackup-providers"
|
||||
],
|
||||
"contract-datasetbackup-requesters": [
|
||||
"contracts-datasetbackup.html#contract-datasetbackup-requesters"
|
||||
],
|
||||
"contract-datasetbackup-usage": [
|
||||
"contracts-datasetbackup.html#contract-datasetbackup-usage"
|
||||
],
|
||||
"contract-datatsetbackup-options": [
|
||||
"contracts-datasetbackup.html#contract-datatsetbackup-options"
|
||||
],
|
||||
"contract-secret": [
|
||||
"contracts-secret.html#contract-secret"
|
||||
],
|
||||
|
|
@ -2036,6 +2069,27 @@
|
|||
"contracts-databasebackup-options-shb.contracts.databasebackup.settings": [
|
||||
"contracts-databasebackup.html#contracts-databasebackup-options-shb.contracts.databasebackup.settings"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup.request": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup.result": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript"
|
||||
],
|
||||
"contracts-datasetbackup-options-shb.contracts.datasetbackup.settings": [
|
||||
"contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.settings"
|
||||
],
|
||||
"contracts-nixpkgs": [
|
||||
"contracts.html#contracts-nixpkgs"
|
||||
],
|
||||
|
|
|
|||
31
flake.nix
31
flake.nix
|
|
@ -78,6 +78,7 @@
|
|||
modules/contracts/backup/dummyModule.nix
|
||||
modules/contracts/dashboard/dummyModule.nix
|
||||
modules/contracts/databasebackup/dummyModule.nix
|
||||
modules/contracts/datasetbackup/dummyModule.nix
|
||||
modules/contracts/secret/dummyModule.nix
|
||||
modules/contracts/ssl/dummyModule.nix
|
||||
];
|
||||
|
|
@ -103,6 +104,13 @@
|
|||
"blocks/authelia" = ./modules/blocks/authelia.nix;
|
||||
"blocks/borgbackup" = ./modules/blocks/borgbackup.nix;
|
||||
"blocks/lldap" = ./modules/blocks/lldap.nix;
|
||||
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
|
||||
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
|
||||
"blocks/nginx" = ./modules/blocks/nginx.nix;
|
||||
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
|
||||
"blocks/restic" = ./modules/blocks/restic.nix;
|
||||
"blocks/sanoid" = ./modules/blocks/sanoid.nix;
|
||||
"blocks/sops" = ./modules/blocks/sops.nix;
|
||||
"blocks/ssl" = {
|
||||
module = ./modules/blocks/ssl.nix;
|
||||
optionRoot = [
|
||||
|
|
@ -110,13 +118,8 @@
|
|||
"certs"
|
||||
];
|
||||
};
|
||||
"blocks/mitmdump" = ./modules/blocks/mitmdump.nix;
|
||||
"blocks/monitoring" = ./modules/blocks/monitoring.nix;
|
||||
"blocks/nginx" = ./modules/blocks/nginx.nix;
|
||||
"blocks/postgresql" = ./modules/blocks/postgresql.nix;
|
||||
"blocks/restic" = ./modules/blocks/restic.nix;
|
||||
"blocks/sops" = ./modules/blocks/sops.nix;
|
||||
"blocks/zfs" = ./modules/blocks/zfs.nix;
|
||||
|
||||
"services/arr" = ./modules/services/arr.nix;
|
||||
"services/firefly-iii" = ./modules/services/firefly-iii.nix;
|
||||
"services/forgejo" = [
|
||||
|
|
@ -138,6 +141,7 @@
|
|||
"services/open-webui" = ./modules/services/open-webui.nix;
|
||||
"services/pinchflat" = ./modules/services/pinchflat.nix;
|
||||
"services/vaultwarden" = ./modules/services/vaultwarden.nix;
|
||||
|
||||
"contracts/backup" = {
|
||||
module = ./modules/contracts/backup/dummyModule.nix;
|
||||
optionRoot = [
|
||||
|
|
@ -162,6 +166,14 @@
|
|||
"databasebackup"
|
||||
];
|
||||
};
|
||||
"contracts/datasetbackup" = {
|
||||
module = ./modules/contracts/datasetbackup/dummyModule.nix;
|
||||
optionRoot = [
|
||||
"shb"
|
||||
"contracts"
|
||||
"datasetbackup"
|
||||
];
|
||||
};
|
||||
"contracts/secret" = {
|
||||
module = ./modules/contracts/secret/dummyModule.nix;
|
||||
optionRoot = [
|
||||
|
|
@ -397,8 +409,8 @@
|
|||
|
||||
// (vm_test "authelia" ./test/blocks/authelia.nix)
|
||||
// (vm_test "borgbackup" ./test/blocks/borgbackup.nix)
|
||||
// (vm_test "lldap" ./test/blocks/lldap.nix)
|
||||
// (vm_test "lib" ./test/blocks/lib.nix)
|
||||
// (vm_test "lldap" ./test/blocks/lldap.nix)
|
||||
// (vm_test "mitmdump" ./test/blocks/mitmdump.nix)
|
||||
// (vm_test "monitoring" ./test/blocks/monitoring.nix)
|
||||
// (vm_test "postgresql" ./test/blocks/postgresql.nix)
|
||||
|
|
@ -408,6 +420,7 @@
|
|||
|
||||
// (vm_test "contracts-backup" ./test/contracts/backup.nix)
|
||||
// (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
|
||||
// (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix)
|
||||
// (vm_test "contracts-secret" ./test/contracts/secret.nix)
|
||||
);
|
||||
|
||||
|
|
@ -429,6 +442,7 @@
|
|||
self.nixosModules.nginx
|
||||
self.nixosModules.postgresql
|
||||
self.nixosModules.restic
|
||||
self.nixosModules.sanoid
|
||||
self.nixosModules.ssl
|
||||
self.nixosModules.tinyproxy
|
||||
self.nixosModules.vpn
|
||||
|
|
@ -468,8 +482,9 @@
|
|||
nixosModules.nginx = modules/blocks/nginx.nix;
|
||||
nixosModules.postgresql = modules/blocks/postgresql.nix;
|
||||
nixosModules.restic = modules/blocks/restic.nix;
|
||||
nixosModules.ssl = modules/blocks/ssl.nix;
|
||||
nixosModules.sanoid = modules/blocks/sanoid.nix;
|
||||
nixosModules.sops = modules/blocks/sops.nix;
|
||||
nixosModules.ssl = modules/blocks/ssl.nix;
|
||||
nixosModules.tinyproxy = modules/blocks/tinyproxy.nix;
|
||||
nixosModules.vpn = modules/blocks/vpn.nix;
|
||||
nixosModules.zfs = modules/blocks/zfs.nix;
|
||||
|
|
|
|||
164
modules/blocks/sanoid.nix
Normal file
164
modules/blocks/sanoid.nix
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
shb,
|
||||
utils,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.shb.sanoid;
|
||||
|
||||
restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore";
|
||||
backupScriptBase = "sanoid";
|
||||
|
||||
restoreScript =
|
||||
name:
|
||||
pkgs.writers.writePython3Bin (restoreScriptName name)
|
||||
{
|
||||
flakeIgnore = [ "E501" ];
|
||||
}
|
||||
''
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
dataset = "${name}"
|
||||
|
||||
|
||||
class ZFSError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def run_command(cmd: list[str]) -> str:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ZFSError(
|
||||
f"Command failed: {' '.join(cmd)}\n"
|
||||
f"Exit code: {e.returncode}\n"
|
||||
f"stderr: {e.stderr.strip()}"
|
||||
) from e
|
||||
|
||||
|
||||
def list_snapshots() -> None:
|
||||
"""List all ZFS snapshots."""
|
||||
output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset])
|
||||
if not output:
|
||||
return []
|
||||
return output.splitlines()
|
||||
|
||||
|
||||
def restore_snapshot(snapshot: str) -> None:
|
||||
"""Rollback to a given snapshot."""
|
||||
if not snapshot:
|
||||
raise ValueError("Snapshot name must not be empty")
|
||||
|
||||
print(f"Rolling back to snapshot: {snapshot}")
|
||||
run_command(["zfs", "rollback", "-r", snapshot])
|
||||
print("Rollback completed successfully.")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=f"Restore script for {dataset}")
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
subparsers.add_parser(
|
||||
"snapshots",
|
||||
help="List all ZFS snapshots",
|
||||
)
|
||||
|
||||
restore_parser = subparsers.add_parser(
|
||||
"restore",
|
||||
help="Rollback to a specific snapshot",
|
||||
)
|
||||
restore_parser.add_argument(
|
||||
"snapshot",
|
||||
help="Snapshot name (e.g. pool/dataset@snapname)",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
if args.command == "snapshots":
|
||||
snapshots = list_snapshots()
|
||||
for s in snapshots:
|
||||
print(s)
|
||||
elif args.command == "restore":
|
||||
restore_snapshot(args.snapshot)
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
except ZFSError as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Unexpected error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
../../lib/module.nix
|
||||
];
|
||||
|
||||
options.shb.sanoid.backup = lib.mkOption {
|
||||
description = "Sanoid prodiver for file backup contract";
|
||||
default = { };
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule (
|
||||
{ name, ... }:
|
||||
{
|
||||
options = shb.contracts.backup.mkProvider {
|
||||
resultCfg = {
|
||||
restoreScript = restoreScriptName name;
|
||||
restoreScriptText = restoreScriptName "<name>";
|
||||
|
||||
backupService = "${backupScriptBase}.service";
|
||||
backupServiceText = "${backupScriptBase}.service";
|
||||
};
|
||||
|
||||
settings = lib.mkOption {
|
||||
description = "Options passed to the `services.sanoid.datasets.<name>` option.";
|
||||
default = { };
|
||||
type = lib.types.attrsOf lib.types.anything;
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
config = lib.mkIf (cfg.backup != { }) {
|
||||
services.sanoid.enable = true;
|
||||
|
||||
services.sanoid.datasets =
|
||||
let
|
||||
mkDataset = name: cfg': {
|
||||
inherit name;
|
||||
value = cfg'.settings;
|
||||
};
|
||||
in
|
||||
lib.mapAttrs' mkDataset cfg.backup;
|
||||
|
||||
environment.systemPackages = map restoreScript (lib.attrNames cfg.backup);
|
||||
};
|
||||
}
|
||||
86
modules/blocks/sanoid/docs/default.md
Normal file
86
modules/blocks/sanoid/docs/default.md
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
# Sanoid Block {#blocks-sanoid}
|
||||
|
||||
Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix):
|
||||
|
||||
```nix
|
||||
{
|
||||
imports = [
|
||||
inputs.selfhostblocks.nixosModules.sanoid
|
||||
];
|
||||
}
|
||||
```
|
||||
|
||||
## Provider Contracts {#blocks-sanoid-contract-provider}
|
||||
|
||||
This block provides the following contracts:
|
||||
|
||||
- [backup contract](contracts-backup.html) under the [`shb.sanoid.backup`][backup] option.
|
||||
It is tested with the [generic contract tests][backup contract tests].
|
||||
|
||||
[backup]: #blocks-sanoid-options-shb.restic.backup
|
||||
[backup contract tests]: @REPO@/test/contracts/backup.nix
|
||||
|
||||
## Usage {#blocks-sanoid-usage}
|
||||
|
||||
Sanoid uses templates to know when snapshots should be kept or pruned.
|
||||
|
||||
### Default Template {#blocks-sanoid-usage-default-template}
|
||||
|
||||
Backup a dataset using the default Sanoid template:
|
||||
|
||||
```nix
|
||||
{
|
||||
shb.zfs.pools.root.datasets.home = {
|
||||
path = "/home";
|
||||
};
|
||||
|
||||
shb.sanoid.backup."root/home" = { };
|
||||
}
|
||||
```
|
||||
|
||||
### Custom Template {#blocks-sanoid-usage-custom-template}
|
||||
|
||||
Create a custom template and use it:
|
||||
|
||||
```nix
|
||||
{
|
||||
shb.zfs.pools.root.datasets.home = {
|
||||
path = "/home";
|
||||
};
|
||||
|
||||
services.sanoid.templates."yearly" = {
|
||||
hourly = 10;
|
||||
daily = 3;
|
||||
monthly = 3;
|
||||
yearly = 2;
|
||||
};
|
||||
|
||||
shb.sanoid.backup."root/home" = {
|
||||
template = "yearly";
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
Note we use the upstream `services.sanoid.templates` option to define the templates.
|
||||
|
||||
### Custom Template {#blocks-sanoid-usage-custom-template}
|
||||
|
||||
Create a custom template and use it:
|
||||
|
||||
```nix
|
||||
{
|
||||
shb.zfs.pools.root.datasets.home = {
|
||||
path = "/home";
|
||||
};
|
||||
|
||||
shb.sanoid.backup."root/home";
|
||||
}
|
||||
```
|
||||
|
||||
## Options Reference {#blocks-sanoid-options}
|
||||
|
||||
```{=include=} options
|
||||
id-prefix: blocks-sanoid-options-
|
||||
list-id: selfhostblocks-blocks-sanoid-options
|
||||
source: @OPTIONS_JSON@
|
||||
```
|
||||
|
|
@ -122,6 +122,19 @@ in
|
|||
};
|
||||
};
|
||||
};
|
||||
|
||||
datasetbackup = lib.mkOption {
|
||||
description = ''
|
||||
ZFS dataset backup contract configuration.
|
||||
|
||||
This contract will take snaphots of the dataset.
|
||||
'';
|
||||
type = lib.types.submodule {
|
||||
options = shb.contracts.datasetbackup.mkRequester {
|
||||
dataset = name;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@ This block creates ZFS datasets, optionally mounts them and sets permissions on
|
|||
|
||||
- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
|
||||
- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
|
||||
- Backup of the files in the dataset [`shb.zfs.<name>.backup`][backup] through the [backup contract](./contracts-backup.html).
|
||||
- Backup of the dataset itself [`shb.zfs.<name>.datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html).
|
||||
|
||||
[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup
|
||||
[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup
|
||||
|
||||
## Usage {#blocks-zfs-usage}
|
||||
|
||||
|
|
|
|||
113
modules/contracts/datasetbackup.nix
Normal file
113
modules/contracts/datasetbackup.nix
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
{ lib, ... }:
|
||||
let
|
||||
inherit (lib)
|
||||
literalMD
|
||||
mkOption
|
||||
optionalAttrs
|
||||
;
|
||||
inherit (lib.types)
|
||||
submodule
|
||||
str
|
||||
;
|
||||
in
|
||||
{
|
||||
mkRequest =
|
||||
{
|
||||
dataset ? "",
|
||||
datasetText ? null,
|
||||
}:
|
||||
mkOption {
|
||||
description = ''
|
||||
Request part of the backup contract.
|
||||
|
||||
Options set by the requester module
|
||||
enforcing how to backup files.
|
||||
'';
|
||||
|
||||
default = { };
|
||||
|
||||
type = submodule {
|
||||
options = {
|
||||
dataset =
|
||||
mkOption {
|
||||
description = ''
|
||||
Dataset to backup, including the pool name.
|
||||
'';
|
||||
type = str;
|
||||
example = "root/home";
|
||||
default = dataset;
|
||||
}
|
||||
// optionalAttrs (datasetText != null) {
|
||||
defaultText = literalMD datasetText;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
mkResult =
|
||||
{
|
||||
restoreScript ? "restore",
|
||||
restoreScriptText ? null,
|
||||
backupService ? "backup.service",
|
||||
backupServiceText ? null,
|
||||
}:
|
||||
mkOption {
|
||||
description = ''
|
||||
Result part of the backup contract.
|
||||
|
||||
Options set by the provider module that indicates the name of the backup and restore scripts.
|
||||
'';
|
||||
default = { };
|
||||
|
||||
type = submodule {
|
||||
options = {
|
||||
restoreScript =
|
||||
mkOption {
|
||||
description = ''
|
||||
Name of script that can restore the database.
|
||||
One can then list snapshots with:
|
||||
|
||||
```bash
|
||||
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
|
||||
<snapshot 1> <metadata>
|
||||
<snapshot 2> <metadata>
|
||||
```
|
||||
|
||||
And restore the database with:
|
||||
|
||||
```bash
|
||||
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
|
||||
```
|
||||
|
||||
It is not garanteed to be able to restore back to a snapshot in the future.
|
||||
With the above example, it may not be possible to restore `<snapshot 2>`
|
||||
after having restored `<snapshot 1>`.
|
||||
'';
|
||||
type = str;
|
||||
default = restoreScript;
|
||||
}
|
||||
// optionalAttrs (restoreScriptText != null) {
|
||||
defaultText = literalMD restoreScriptText;
|
||||
};
|
||||
|
||||
backupService =
|
||||
mkOption {
|
||||
description = ''
|
||||
Name of service backing up the database.
|
||||
|
||||
This script can be ran manually to backup the database:
|
||||
|
||||
```bash
|
||||
$ systemctl start ${if backupServiceText != null then backupServiceText else backupService}
|
||||
```
|
||||
'';
|
||||
type = str;
|
||||
default = backupService;
|
||||
}
|
||||
// optionalAttrs (backupServiceText != null) {
|
||||
defaultText = literalMD backupServiceText;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
85
modules/contracts/datasetbackup/docs/default.md
Normal file
85
modules/contracts/datasetbackup/docs/default.md
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# Dataset Backup Contract {#contract-dataset-backup}
|
||||
|
||||
This NixOS contract represents a backup job
|
||||
that will backup one ZFS dataset.
|
||||
|
||||
It is a contract between a service that manages ZFS datasets
|
||||
and a service that backs up ZFS datasets.
|
||||
|
||||
## Contract Reference {#contract-datatsetbackup-options}
|
||||
|
||||
These are all the options that are expected to exist for this contract to be respected.
|
||||
|
||||
```{=include=} options
|
||||
id-prefix: contracts-datasetbackup-options-
|
||||
list-id: selfhostblocks-options
|
||||
source: @OPTIONS_JSON@
|
||||
```
|
||||
|
||||
## Usage {#contract-datasetbackup-usage}
|
||||
|
||||
A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option.
|
||||
|
||||
Here is an example module defining such a `backup` option,
|
||||
which defines what directories to backup (`sourceDirectories`)
|
||||
and the user to backup with (`user`).
|
||||
|
||||
```nix
|
||||
{
|
||||
options = {
|
||||
myservice.datasetbackup = mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = shb.contracts.datasetbackup.mkRequester {
|
||||
dataset = "root/test";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset.
|
||||
This service is a `provider` of this contract and will provide a `result` option.
|
||||
|
||||
Let's assume such a module is available under the `backupService` option
|
||||
and that one can create multiple backup instances under `backupService.instances`.
|
||||
Then, to actually backup the `myservice` service, one would write:
|
||||
|
||||
```nix
|
||||
backupService.instances.myservice = {
|
||||
request = myservice.datasetbackup.request;
|
||||
|
||||
settings = {
|
||||
enable = true;
|
||||
|
||||
targetDataset = "backup/myservice";
|
||||
|
||||
# ... Other options specific to backupService like scheduling.
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
It is advised to backup files to different location, to improve redundancy.
|
||||
Thanks to using contracts, this can be made easily either with the same `backupService`:
|
||||
|
||||
```nix
|
||||
backupService.instances.myservice_2 = {
|
||||
request = myservice.datasetbackup.request;
|
||||
|
||||
settings = {
|
||||
enable = true;
|
||||
|
||||
targetDataset = "backup2/myservice";
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
Or with another module `backupService_2`!
|
||||
|
||||
## Providers of the Backup Contract {#contract-datasetbackup-providers}
|
||||
|
||||
- [Sanoid block](blocks-sanoid.html).
|
||||
|
||||
## Requester Blocks and Services {#contract-datasetbackup-requesters}
|
||||
|
||||
- [ZFS](blocks-zfs.html#blocks-zfs-backup).
|
||||
30
modules/contracts/datasetbackup/dummyModule.nix
Normal file
30
modules/contracts/datasetbackup/dummyModule.nix
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
{ lib, shb, ... }:
|
||||
let
|
||||
inherit (lib) mkOption;
|
||||
inherit (lib.types) submodule;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
../../../lib/module.nix
|
||||
];
|
||||
|
||||
options.shb.contracts.datasetbackup = mkOption {
|
||||
description = ''
|
||||
Contract for backing up ZFS datasets.
|
||||
|
||||
The requester communicates to the provider
|
||||
the dataset to backup
|
||||
through the `request` options.
|
||||
|
||||
The provider reads from the `request` options
|
||||
and backs up the requested dataset.
|
||||
It communicates to the requester what script is used
|
||||
to backup and restore the files
|
||||
through the `result` options.
|
||||
'';
|
||||
|
||||
type = submodule {
|
||||
options = shb.contracts.datasetbackup.contract;
|
||||
};
|
||||
};
|
||||
}
|
||||
164
modules/contracts/datasetbackup/test.nix
Normal file
164
modules/contracts/datasetbackup/test.nix
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
{
|
||||
pkgs,
|
||||
lib,
|
||||
shb,
|
||||
}:
|
||||
let
|
||||
inherit (lib)
|
||||
concatMapStringsSep
|
||||
getAttrFromPath
|
||||
setAttrByPath
|
||||
;
|
||||
in
|
||||
{
|
||||
name,
|
||||
providerRoot,
|
||||
modules ? [ ],
|
||||
dataset ? "root/test",
|
||||
sourceDirectories ? [
|
||||
"/opt/files/A"
|
||||
"/opt/files/B"
|
||||
],
|
||||
settings ? { ... }: { }, # { filesRoot, config } -> attrset
|
||||
extraConfig ? null, # { filesRoot, config } -> attrset
|
||||
}:
|
||||
shb.test.runNixOSTest {
|
||||
inherit name;
|
||||
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
{
|
||||
imports = [ shb.test.baseImports ] ++ modules;
|
||||
|
||||
config = lib.mkMerge [
|
||||
(setAttrByPath providerRoot {
|
||||
request = {
|
||||
inherit dataset;
|
||||
};
|
||||
settings = settings {
|
||||
inherit config;
|
||||
filesRoot = "/opt/files";
|
||||
};
|
||||
})
|
||||
(lib.optionalAttrs (extraConfig != null) (extraConfig {
|
||||
inherit config;
|
||||
filesRoot = "/opt/files";
|
||||
}))
|
||||
];
|
||||
};
|
||||
|
||||
extraPythonPackages = p: [ p.dictdiffer ];
|
||||
skipTypeCheck = true;
|
||||
|
||||
testScript =
|
||||
{ nodes, ... }:
|
||||
let
|
||||
provider = (getAttrFromPath providerRoot nodes.machine).result;
|
||||
in
|
||||
''
|
||||
from datetime import datetime, timedelta
|
||||
from dictdiffer import diff
|
||||
import re
|
||||
|
||||
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
|
||||
|
||||
def list_files(dir):
|
||||
files_and_content = {}
|
||||
|
||||
files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1]
|
||||
|
||||
for f in files:
|
||||
content = machine.succeed(f"""cat {f}""").strip()
|
||||
files_and_content[f] = content
|
||||
|
||||
return files_and_content
|
||||
|
||||
def assert_files(dir, files):
|
||||
result = list(diff(list_files(dir), files))
|
||||
if len(result) > 0:
|
||||
raise Exception("Unexpected files:", result)
|
||||
|
||||
with subtest("Create initial content"):
|
||||
for path in sourceDirectories:
|
||||
machine.succeed(f"""
|
||||
mkdir -p {path}
|
||||
echo repo_fileA_1 > {path}/fileA
|
||||
echo repo_fileB_1 > {path}/fileB
|
||||
""")
|
||||
|
||||
for path in sourceDirectories:
|
||||
assert_files(path, {
|
||||
f'{path}/fileA': 'repo_fileA_1',
|
||||
f'{path}/fileB': 'repo_fileB_1',
|
||||
})
|
||||
|
||||
with subtest("Initial snapshot"):
|
||||
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
|
||||
if len(out) != 0:
|
||||
raise Exception(f"Unexpected snapshots:\n{out}")
|
||||
|
||||
with subtest("First backup in repo"):
|
||||
machine.succeed("systemctl start --wait ${provider.backupService}")
|
||||
|
||||
with subtest("One snapshot"):
|
||||
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
|
||||
print(f"Found snapshots:\n{out}")
|
||||
if len(out) != 1:
|
||||
raise Exception(f"Unexpected snapshots:\n{out}")
|
||||
|
||||
# To accomodate for snapshot orchestrators which keep only a given amount
|
||||
# of snapshots per unit of time, we set the time to now + 2h.
|
||||
new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
machine.succeed(f"timedatectl set-time '{new_date}'")
|
||||
|
||||
with subtest("New content"):
|
||||
for path in sourceDirectories:
|
||||
machine.succeed(f"""
|
||||
echo repo_fileA_2 > {path}/fileA
|
||||
echo repo_fileB_2 > {path}/fileB
|
||||
""")
|
||||
|
||||
assert_files(path, {
|
||||
f'{path}/fileA': 'repo_fileA_2',
|
||||
f'{path}/fileB': 'repo_fileB_2',
|
||||
})
|
||||
|
||||
with subtest("Second backup in repo"):
|
||||
machine.succeed("systemctl start --wait ${provider.backupService}")
|
||||
|
||||
with subtest("two snapshots"):
|
||||
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
|
||||
print(f"Found snapshots:\n{out}")
|
||||
if len(out) != 2:
|
||||
raise Exception(f"Unexpected snapshots:\n{out}")
|
||||
|
||||
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
|
||||
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
|
||||
print(f"First snapshot {firstSnapshot}")
|
||||
print(f"Second snapshot {secondSnapshot}")
|
||||
|
||||
with subtest("Delete content"):
|
||||
for path in sourceDirectories:
|
||||
machine.succeed(f"""rm -r {path}/*""")
|
||||
|
||||
assert_files(path, {})
|
||||
|
||||
with subtest("Restore second backup"):
|
||||
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
|
||||
|
||||
for path in sourceDirectories:
|
||||
assert_files(path, {
|
||||
f'{path}/fileA': 'repo_fileA_2',
|
||||
f'{path}/fileB': 'repo_fileB_2',
|
||||
})
|
||||
|
||||
with subtest("Restore first backup"):
|
||||
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
|
||||
|
||||
for path in sourceDirectories:
|
||||
assert_files(path, {
|
||||
f'{path}/fileA': 'repo_fileA_1',
|
||||
f'{path}/fileB': 'repo_fileB_1',
|
||||
})
|
||||
'';
|
||||
}
|
||||
|
|
@ -61,6 +61,7 @@ let
|
|||
|
||||
contracts = {
|
||||
databasebackup = importContract ./databasebackup.nix;
|
||||
datasetbackup = importContract ./datasetbackup.nix;
|
||||
dashboard = importContract ./dashboard.nix;
|
||||
backup = importContract ./backup.nix;
|
||||
mount = pkgs.callPackage ./mount.nix { };
|
||||
|
|
@ -69,6 +70,7 @@ let
|
|||
test = {
|
||||
secret = pkgs.callPackage ./secret/test.nix { inherit shb; };
|
||||
databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; };
|
||||
datasetbackup = pkgs.callPackage ./datasetbackup/test.nix { inherit shb; };
|
||||
backup = pkgs.callPackage ./backup/test.nix { inherit shb; };
|
||||
};
|
||||
};
|
||||
|
|
|
|||
64
test/contracts/datasetbackup.nix
Normal file
64
test/contracts/datasetbackup.nix
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
{ shb, pkgs, ... }:
|
||||
{
|
||||
sanoid = shb.contracts.test.datasetbackup {
|
||||
name = "sanoid";
|
||||
providerRoot = [
|
||||
"shb"
|
||||
"sanoid"
|
||||
"backup"
|
||||
# This is the name of the dataset
|
||||
"root/mytest"
|
||||
];
|
||||
modules = [
|
||||
../../modules/blocks/sanoid.nix
|
||||
# We use the zfs module to test the sanoid one
|
||||
../../modules/blocks/zfs.nix
|
||||
];
|
||||
settings =
|
||||
{ ... }:
|
||||
{
|
||||
useTemplate = [ "test" ];
|
||||
};
|
||||
extraConfig =
|
||||
{ filesRoot, ... }:
|
||||
{
|
||||
# Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix
|
||||
networking.hostId = "deadbeef";
|
||||
boot.supportedFilesystems = [ "zfs" ];
|
||||
|
||||
virtualisation = {
|
||||
emptyDiskImages = [
|
||||
512
|
||||
];
|
||||
};
|
||||
|
||||
# The test expects to keep one snapshot per hour.
|
||||
services.sanoid.templates."test" = {
|
||||
hourly = 1;
|
||||
daily = 0;
|
||||
monthly = 0;
|
||||
yearly = 0;
|
||||
};
|
||||
|
||||
systemd.services."zfs-zpool-create" = {
|
||||
unitConfig.DefaultDependencies = false;
|
||||
after = [ "systemd-modules-load.service" ];
|
||||
requiredBy = [
|
||||
"zfs-import-root.service"
|
||||
"zfs-mount.service"
|
||||
];
|
||||
before = [
|
||||
"zfs-import-root.service"
|
||||
"zfs-mount.service"
|
||||
];
|
||||
script = ''
|
||||
${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl -O mountpoint=none root /dev/vdb
|
||||
'';
|
||||
};
|
||||
|
||||
shb.zfs.pools.root.datasets."mytest" = {
|
||||
path = filesRoot;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue