From 89f7aa711820b0a7ec36ef0fa3b7755a66554293 Mon Sep 17 00:00:00 2001 From: ibizaman Date: Sat, 25 Apr 2026 23:36:37 +0200 Subject: [PATCH] datasetBackup: add contract with sanoid implementation --- docs/contracts.md | 9 +- docs/redirects.json | 54 ++++++ flake.nix | 31 +++- modules/blocks/sanoid.nix | 164 ++++++++++++++++++ modules/blocks/sanoid/docs/default.md | 86 +++++++++ modules/blocks/zfs.nix | 13 ++ modules/blocks/zfs/docs/default.md | 5 + modules/contracts/datasetbackup.nix | 113 ++++++++++++ .../contracts/datasetbackup/docs/default.md | 85 +++++++++ .../contracts/datasetbackup/dummyModule.nix | 30 ++++ modules/contracts/datasetbackup/test.nix | 164 ++++++++++++++++++ modules/contracts/default.nix | 2 + test/contracts/datasetbackup.nix | 64 +++++++ 13 files changed, 811 insertions(+), 9 deletions(-) create mode 100644 modules/blocks/sanoid.nix create mode 100644 modules/blocks/sanoid/docs/default.md create mode 100644 modules/contracts/datasetbackup.nix create mode 100644 modules/contracts/datasetbackup/docs/default.md create mode 100644 modules/contracts/datasetbackup/dummyModule.nix create mode 100644 modules/contracts/datasetbackup/test.nix create mode 100644 test/contracts/datasetbackup.nix diff --git a/docs/contracts.md b/docs/contracts.md index dbb4b0f..bf0fdb6 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -39,7 +39,9 @@ Provided contracts are: - [Backup contract][] to backup directories. Two providers are implemented: [BorgBackup][] and [Restic][]. - [Database Backup contract](contracts-databasebackup.html) to backup database dumps. - One provider is implemented: [BorgBackup][] and [Restic][]. + Two providers are implemented: [BorgBackup][] and [Restic][]. +- [Dataset Backup contract](contracts-datasetbackup.html) to backup ZFS datasets. + One provider is implemented: [Sanoid][] - [Contract for Secrets](contracts-secret.html) to provide secrets that are deployed outside of the Nix store. One provider is implemented: [SOPS][]. - [Dashboard contract](contracts-dashboard.html) to show services in a nice user-facing dashboard. @@ -49,6 +51,7 @@ Provided contracts are: [borgbackup]: blocks-borgbackup.html [homepage]: services-homepage.html [restic]: blocks-restic.html +[sanoid]: blocks-sanoid.html [sops]: blocks-sops.html ```{=include=} chapters html:into-file=//contracts-ssl.html @@ -63,6 +66,10 @@ modules/contracts/backup/docs/default.md modules/contracts/databasebackup/docs/default.md ``` +```{=include=} chapters html:into-file=//contracts-datasetbackup.html +modules/contracts/datasetbackup/docs/default.md +``` + ```{=include=} chapters html:into-file=//contracts-secret.html modules/contracts/secret/docs/default.md ``` diff --git a/docs/redirects.json b/docs/redirects.json index 0aeba3e..133d3f8 100644 --- a/docs/redirects.json +++ b/docs/redirects.json @@ -1826,6 +1826,24 @@ "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript": [ "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript" ], + "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup": [ + "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup" + ], + "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request": [ + "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request" + ], + "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset": [ + "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.request.dataset" + ], + "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result": [ + "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result" + ], + "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService": [ + "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.backupService" + ], + "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript": [ + "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup.result.restoreScript" + ], "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs": [ "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs" ], @@ -1907,6 +1925,21 @@ "contract-databasebackup-usage": [ "contracts-databasebackup.html#contract-databasebackup-usage" ], + "contract-dataset-backup": [ + "contracts-datasetbackup.html#contract-dataset-backup" + ], + "contract-datasetbackup-providers": [ + "contracts-datasetbackup.html#contract-datasetbackup-providers" + ], + "contract-datasetbackup-requesters": [ + "contracts-datasetbackup.html#contract-datasetbackup-requesters" + ], + "contract-datasetbackup-usage": [ + "contracts-datasetbackup.html#contract-datasetbackup-usage" + ], + "contract-datatsetbackup-options": [ + "contracts-datasetbackup.html#contract-datatsetbackup-options" + ], "contract-secret": [ "contracts-secret.html#contract-secret" ], @@ -2036,6 +2069,27 @@ "contracts-databasebackup-options-shb.contracts.databasebackup.settings": [ "contracts-databasebackup.html#contracts-databasebackup-options-shb.contracts.databasebackup.settings" ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup" + ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup.request": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request" + ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.request.dataset" + ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup.result": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result" + ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.backupService" + ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.result.restoreScript" + ], + "contracts-datasetbackup-options-shb.contracts.datasetbackup.settings": [ + "contracts-datasetbackup.html#contracts-datasetbackup-options-shb.contracts.datasetbackup.settings" + ], "contracts-nixpkgs": [ "contracts.html#contracts-nixpkgs" ], diff --git a/flake.nix b/flake.nix index be7a479..48010f7 100644 --- a/flake.nix +++ b/flake.nix @@ -78,6 +78,7 @@ modules/contracts/backup/dummyModule.nix modules/contracts/dashboard/dummyModule.nix modules/contracts/databasebackup/dummyModule.nix + modules/contracts/datasetbackup/dummyModule.nix modules/contracts/secret/dummyModule.nix modules/contracts/ssl/dummyModule.nix ]; @@ -103,6 +104,13 @@ "blocks/authelia" = ./modules/blocks/authelia.nix; "blocks/borgbackup" = ./modules/blocks/borgbackup.nix; "blocks/lldap" = ./modules/blocks/lldap.nix; + "blocks/mitmdump" = ./modules/blocks/mitmdump.nix; + "blocks/monitoring" = ./modules/blocks/monitoring.nix; + "blocks/nginx" = ./modules/blocks/nginx.nix; + "blocks/postgresql" = ./modules/blocks/postgresql.nix; + "blocks/restic" = ./modules/blocks/restic.nix; + "blocks/sanoid" = ./modules/blocks/sanoid.nix; + "blocks/sops" = ./modules/blocks/sops.nix; "blocks/ssl" = { module = ./modules/blocks/ssl.nix; optionRoot = [ @@ -110,13 +118,8 @@ "certs" ]; }; - "blocks/mitmdump" = ./modules/blocks/mitmdump.nix; - "blocks/monitoring" = ./modules/blocks/monitoring.nix; - "blocks/nginx" = ./modules/blocks/nginx.nix; - "blocks/postgresql" = ./modules/blocks/postgresql.nix; - "blocks/restic" = ./modules/blocks/restic.nix; - "blocks/sops" = ./modules/blocks/sops.nix; "blocks/zfs" = ./modules/blocks/zfs.nix; + "services/arr" = ./modules/services/arr.nix; "services/firefly-iii" = ./modules/services/firefly-iii.nix; "services/forgejo" = [ @@ -138,6 +141,7 @@ "services/open-webui" = ./modules/services/open-webui.nix; "services/pinchflat" = ./modules/services/pinchflat.nix; "services/vaultwarden" = ./modules/services/vaultwarden.nix; + "contracts/backup" = { module = ./modules/contracts/backup/dummyModule.nix; optionRoot = [ @@ -162,6 +166,14 @@ "databasebackup" ]; }; + "contracts/datasetbackup" = { + module = ./modules/contracts/datasetbackup/dummyModule.nix; + optionRoot = [ + "shb" + "contracts" + "datasetbackup" + ]; + }; "contracts/secret" = { module = ./modules/contracts/secret/dummyModule.nix; optionRoot = [ @@ -397,8 +409,8 @@ // (vm_test "authelia" ./test/blocks/authelia.nix) // (vm_test "borgbackup" ./test/blocks/borgbackup.nix) - // (vm_test "lldap" ./test/blocks/lldap.nix) // (vm_test "lib" ./test/blocks/lib.nix) + // (vm_test "lldap" ./test/blocks/lldap.nix) // (vm_test "mitmdump" ./test/blocks/mitmdump.nix) // (vm_test "monitoring" ./test/blocks/monitoring.nix) // (vm_test "postgresql" ./test/blocks/postgresql.nix) @@ -408,6 +420,7 @@ // (vm_test "contracts-backup" ./test/contracts/backup.nix) // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix) + // (vm_test "contracts-datasetbackup" ./test/contracts/datasetbackup.nix) // (vm_test "contracts-secret" ./test/contracts/secret.nix) ); @@ -429,6 +442,7 @@ self.nixosModules.nginx self.nixosModules.postgresql self.nixosModules.restic + self.nixosModules.sanoid self.nixosModules.ssl self.nixosModules.tinyproxy self.nixosModules.vpn @@ -468,8 +482,9 @@ nixosModules.nginx = modules/blocks/nginx.nix; nixosModules.postgresql = modules/blocks/postgresql.nix; nixosModules.restic = modules/blocks/restic.nix; - nixosModules.ssl = modules/blocks/ssl.nix; + nixosModules.sanoid = modules/blocks/sanoid.nix; nixosModules.sops = modules/blocks/sops.nix; + nixosModules.ssl = modules/blocks/ssl.nix; nixosModules.tinyproxy = modules/blocks/tinyproxy.nix; nixosModules.vpn = modules/blocks/vpn.nix; nixosModules.zfs = modules/blocks/zfs.nix; diff --git a/modules/blocks/sanoid.nix b/modules/blocks/sanoid.nix new file mode 100644 index 0000000..cc5f644 --- /dev/null +++ b/modules/blocks/sanoid.nix @@ -0,0 +1,164 @@ +{ + config, + lib, + pkgs, + shb, + utils, + ... +}: +let + cfg = config.shb.sanoid; + + restoreScriptName = name: "sanoid-${utils.escapeSystemdPath name}-restore"; + backupScriptBase = "sanoid"; + + restoreScript = + name: + pkgs.writers.writePython3Bin (restoreScriptName name) + { + flakeIgnore = [ "E501" ]; + } + '' + import argparse + import subprocess + import sys + + + dataset = "${name}" + + + class ZFSError(Exception): + pass + + + def run_command(cmd: list[str]) -> str: + try: + result = subprocess.run( + cmd, + check=True, + text=True, + capture_output=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + raise ZFSError( + f"Command failed: {' '.join(cmd)}\n" + f"Exit code: {e.returncode}\n" + f"stderr: {e.stderr.strip()}" + ) from e + + + def list_snapshots() -> None: + """List all ZFS snapshots.""" + output = run_command(["zfs", "list", "-H", "-t", "snapshot", dataset]) + if not output: + return [] + return output.splitlines() + + + def restore_snapshot(snapshot: str) -> None: + """Rollback to a given snapshot.""" + if not snapshot: + raise ValueError("Snapshot name must not be empty") + + print(f"Rolling back to snapshot: {snapshot}") + run_command(["zfs", "rollback", "-r", snapshot]) + print("Rollback completed successfully.") + + + def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=f"Restore script for {dataset}") + + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser( + "snapshots", + help="List all ZFS snapshots", + ) + + restore_parser = subparsers.add_parser( + "restore", + help="Rollback to a specific snapshot", + ) + restore_parser.add_argument( + "snapshot", + help="Snapshot name (e.g. pool/dataset@snapname)", + ) + + return parser + + + def main(): + parser = build_parser() + args = parser.parse_args() + + try: + if args.command == "snapshots": + snapshots = list_snapshots() + for s in snapshots: + print(s) + elif args.command == "restore": + restore_snapshot(args.snapshot) + else: + parser.print_help() + sys.exit(1) + + except ZFSError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Unexpected error: {e}", file=sys.stderr) + sys.exit(1) + + + if __name__ == "__main__": + main() + ''; +in +{ + imports = [ + ../../lib/module.nix + ]; + + options.shb.sanoid.backup = lib.mkOption { + description = "Sanoid prodiver for file backup contract"; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule ( + { name, ... }: + { + options = shb.contracts.backup.mkProvider { + resultCfg = { + restoreScript = restoreScriptName name; + restoreScriptText = restoreScriptName ""; + + backupService = "${backupScriptBase}.service"; + backupServiceText = "${backupScriptBase}.service"; + }; + + settings = lib.mkOption { + description = "Options passed to the `services.sanoid.datasets.` option."; + default = { }; + type = lib.types.attrsOf lib.types.anything; + }; + }; + } + ) + ); + }; + + config = lib.mkIf (cfg.backup != { }) { + services.sanoid.enable = true; + + services.sanoid.datasets = + let + mkDataset = name: cfg': { + inherit name; + value = cfg'.settings; + }; + in + lib.mapAttrs' mkDataset cfg.backup; + + environment.systemPackages = map restoreScript (lib.attrNames cfg.backup); + }; +} diff --git a/modules/blocks/sanoid/docs/default.md b/modules/blocks/sanoid/docs/default.md new file mode 100644 index 0000000..a71e0f8 --- /dev/null +++ b/modules/blocks/sanoid/docs/default.md @@ -0,0 +1,86 @@ +# Sanoid Block {#blocks-sanoid} + +Defined in [`/modules/blocks/sanoid.nix`](@REPO@/modules/blocks/sanoid.nix): + +```nix +{ + imports = [ + inputs.selfhostblocks.nixosModules.sanoid + ]; +} +``` + +## Provider Contracts {#blocks-sanoid-contract-provider} + +This block provides the following contracts: + +- [backup contract](contracts-backup.html) under the [`shb.sanoid.backup`][backup] option. + It is tested with the [generic contract tests][backup contract tests]. + +[backup]: #blocks-sanoid-options-shb.restic.backup +[backup contract tests]: @REPO@/test/contracts/backup.nix + +## Usage {#blocks-sanoid-usage} + +Sanoid uses templates to know when snapshots should be kept or pruned. + +### Default Template {#blocks-sanoid-usage-default-template} + +Backup a dataset using the default Sanoid template: + +```nix +{ + shb.zfs.pools.root.datasets.home = { + path = "/home"; + }; + + shb.sanoid.backup."root/home" = { }; +} +``` + +### Custom Template {#blocks-sanoid-usage-custom-template} + +Create a custom template and use it: + +```nix +{ + shb.zfs.pools.root.datasets.home = { + path = "/home"; + }; + + services.sanoid.templates."yearly" = { + hourly = 10; + daily = 3; + monthly = 3; + yearly = 2; + }; + + shb.sanoid.backup."root/home" = { + template = "yearly"; + }; +} +``` + +Note we use the upstream `services.sanoid.templates` option to define the templates. + +### Custom Template {#blocks-sanoid-usage-custom-template} + +Create a custom template and use it: + +```nix +{ + shb.zfs.pools.root.datasets.home = { + path = "/home"; + }; + + shb.sanoid.backup."root/home"; +} +``` + +## Options Reference {#blocks-sanoid-options} + +```{=include=} options +id-prefix: blocks-sanoid-options- +list-id: selfhostblocks-blocks-sanoid-options +source: @OPTIONS_JSON@ +``` diff --git a/modules/blocks/zfs.nix b/modules/blocks/zfs.nix index 8465b66..898ec6f 100644 --- a/modules/blocks/zfs.nix +++ b/modules/blocks/zfs.nix @@ -122,6 +122,19 @@ in }; }; }; + + datasetbackup = lib.mkOption { + description = '' + ZFS dataset backup contract configuration. + + This contract will take snaphots of the dataset. + ''; + type = lib.types.submodule { + options = shb.contracts.datasetbackup.mkRequester { + dataset = name; + }; + }; + }; }; } ) diff --git a/modules/blocks/zfs/docs/default.md b/modules/blocks/zfs/docs/default.md index 34a0b0a..a636fb5 100644 --- a/modules/blocks/zfs/docs/default.md +++ b/modules/blocks/zfs/docs/default.md @@ -20,6 +20,11 @@ This block creates ZFS datasets, optionally mounts them and sets permissions on - Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path). - Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point. +- Backup of the files in the dataset [`shb.zfs..backup`][backup] through the [backup contract](./contracts-backup.html). +- Backup of the dataset itself [`shb.zfs..datasetbackup`][datasetbackup] through the [dataset backup contract](./contracts-datasetbackup.html). + +[backup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup +[datasetbackup]: #blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.datasetbackup ## Usage {#blocks-zfs-usage} diff --git a/modules/contracts/datasetbackup.nix b/modules/contracts/datasetbackup.nix new file mode 100644 index 0000000..6bc88ff --- /dev/null +++ b/modules/contracts/datasetbackup.nix @@ -0,0 +1,113 @@ +{ lib, ... }: +let + inherit (lib) + literalMD + mkOption + optionalAttrs + ; + inherit (lib.types) + submodule + str + ; +in +{ + mkRequest = + { + dataset ? "", + datasetText ? null, + }: + mkOption { + description = '' + Request part of the backup contract. + + Options set by the requester module + enforcing how to backup files. + ''; + + default = { }; + + type = submodule { + options = { + dataset = + mkOption { + description = '' + Dataset to backup, including the pool name. + ''; + type = str; + example = "root/home"; + default = dataset; + } + // optionalAttrs (datasetText != null) { + defaultText = literalMD datasetText; + }; + }; + }; + }; + + mkResult = + { + restoreScript ? "restore", + restoreScriptText ? null, + backupService ? "backup.service", + backupServiceText ? null, + }: + mkOption { + description = '' + Result part of the backup contract. + + Options set by the provider module that indicates the name of the backup and restore scripts. + ''; + default = { }; + + type = submodule { + options = { + restoreScript = + mkOption { + description = '' + Name of script that can restore the database. + One can then list snapshots with: + + ```bash + $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots + + + ``` + + And restore the database with: + + ```bash + $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore + ``` + + It is not garanteed to be able to restore back to a snapshot in the future. + With the above example, it may not be possible to restore `` + after having restored ``. + ''; + type = str; + default = restoreScript; + } + // optionalAttrs (restoreScriptText != null) { + defaultText = literalMD restoreScriptText; + }; + + backupService = + mkOption { + description = '' + Name of service backing up the database. + + This script can be ran manually to backup the database: + + ```bash + $ systemctl start ${if backupServiceText != null then backupServiceText else backupService} + ``` + ''; + type = str; + default = backupService; + } + // optionalAttrs (backupServiceText != null) { + defaultText = literalMD backupServiceText; + }; + }; + }; + }; +} diff --git a/modules/contracts/datasetbackup/docs/default.md b/modules/contracts/datasetbackup/docs/default.md new file mode 100644 index 0000000..f2bb4d4 --- /dev/null +++ b/modules/contracts/datasetbackup/docs/default.md @@ -0,0 +1,85 @@ +# Dataset Backup Contract {#contract-dataset-backup} + +This NixOS contract represents a backup job +that will backup one ZFS dataset. + +It is a contract between a service that manages ZFS datasets +and a service that backs up ZFS datasets. + +## Contract Reference {#contract-datatsetbackup-options} + +These are all the options that are expected to exist for this contract to be respected. + +```{=include=} options +id-prefix: contracts-datasetbackup-options- +list-id: selfhostblocks-options +source: @OPTIONS_JSON@ +``` + +## Usage {#contract-datasetbackup-usage} + +A service that manages ZFS datasets that can be backed up will provide a `datasetbackup` option. + +Here is an example module defining such a `backup` option, +which defines what directories to backup (`sourceDirectories`) +and the user to backup with (`user`). + +```nix +{ + options = { + myservice.datasetbackup = mkOption { + type = lib.types.submodule { + options = shb.contracts.datasetbackup.mkRequester { + dataset = "root/test"; + }; + }; + }; + }; +}; +``` + +Now, on the other side we have a service that uses this `datasetbackup` option and actually backs up the dataset. +This service is a `provider` of this contract and will provide a `result` option. + +Let's assume such a module is available under the `backupService` option +and that one can create multiple backup instances under `backupService.instances`. +Then, to actually backup the `myservice` service, one would write: + +```nix +backupService.instances.myservice = { + request = myservice.datasetbackup.request; + + settings = { + enable = true; + + targetDataset = "backup/myservice"; + + # ... Other options specific to backupService like scheduling. + }; +}; +``` + +It is advised to backup files to different location, to improve redundancy. +Thanks to using contracts, this can be made easily either with the same `backupService`: + +```nix +backupService.instances.myservice_2 = { + request = myservice.datasetbackup.request; + + settings = { + enable = true; + + targetDataset = "backup2/myservice"; + }; +}; +``` + +Or with another module `backupService_2`! + +## Providers of the Backup Contract {#contract-datasetbackup-providers} + +- [Sanoid block](blocks-sanoid.html). + +## Requester Blocks and Services {#contract-datasetbackup-requesters} + +- [ZFS](blocks-zfs.html#blocks-zfs-backup). diff --git a/modules/contracts/datasetbackup/dummyModule.nix b/modules/contracts/datasetbackup/dummyModule.nix new file mode 100644 index 0000000..d7f4973 --- /dev/null +++ b/modules/contracts/datasetbackup/dummyModule.nix @@ -0,0 +1,30 @@ +{ lib, shb, ... }: +let + inherit (lib) mkOption; + inherit (lib.types) submodule; +in +{ + imports = [ + ../../../lib/module.nix + ]; + + options.shb.contracts.datasetbackup = mkOption { + description = '' + Contract for backing up ZFS datasets. + + The requester communicates to the provider + the dataset to backup + through the `request` options. + + The provider reads from the `request` options + and backs up the requested dataset. + It communicates to the requester what script is used + to backup and restore the files + through the `result` options. + ''; + + type = submodule { + options = shb.contracts.datasetbackup.contract; + }; + }; +} diff --git a/modules/contracts/datasetbackup/test.nix b/modules/contracts/datasetbackup/test.nix new file mode 100644 index 0000000..6d0c363 --- /dev/null +++ b/modules/contracts/datasetbackup/test.nix @@ -0,0 +1,164 @@ +{ + pkgs, + lib, + shb, +}: +let + inherit (lib) + concatMapStringsSep + getAttrFromPath + setAttrByPath + ; +in +{ + name, + providerRoot, + modules ? [ ], + dataset ? "root/test", + sourceDirectories ? [ + "/opt/files/A" + "/opt/files/B" + ], + settings ? { ... }: { }, # { filesRoot, config } -> attrset + extraConfig ? null, # { filesRoot, config } -> attrset +}: +shb.test.runNixOSTest { + inherit name; + + nodes.machine = + { config, ... }: + { + imports = [ shb.test.baseImports ] ++ modules; + + config = lib.mkMerge [ + (setAttrByPath providerRoot { + request = { + inherit dataset; + }; + settings = settings { + inherit config; + filesRoot = "/opt/files"; + }; + }) + (lib.optionalAttrs (extraConfig != null) (extraConfig { + inherit config; + filesRoot = "/opt/files"; + })) + ]; + }; + + extraPythonPackages = p: [ p.dictdiffer ]; + skipTypeCheck = true; + + testScript = + { nodes, ... }: + let + provider = (getAttrFromPath providerRoot nodes.machine).result; + in + '' + from datetime import datetime, timedelta + from dictdiffer import diff + import re + + sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ] + + def list_files(dir): + files_and_content = {} + + files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1] + + for f in files: + content = machine.succeed(f"""cat {f}""").strip() + files_and_content[f] = content + + return files_and_content + + def assert_files(dir, files): + result = list(diff(list_files(dir), files)) + if len(result) > 0: + raise Exception("Unexpected files:", result) + + with subtest("Create initial content"): + for path in sourceDirectories: + machine.succeed(f""" + mkdir -p {path} + echo repo_fileA_1 > {path}/fileA + echo repo_fileB_1 > {path}/fileB + """) + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_1', + f'{path}/fileB': 'repo_fileB_1', + }) + + with subtest("Initial snapshot"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + if len(out) != 0: + raise Exception(f"Unexpected snapshots:\n{out}") + + with subtest("First backup in repo"): + machine.succeed("systemctl start --wait ${provider.backupService}") + + with subtest("One snapshot"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 1: + raise Exception(f"Unexpected snapshots:\n{out}") + + # To accomodate for snapshot orchestrators which keep only a given amount + # of snapshots per unit of time, we set the time to now + 2h. + new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S") + machine.succeed(f"timedatectl set-time '{new_date}'") + + with subtest("New content"): + for path in sourceDirectories: + machine.succeed(f""" + echo repo_fileA_2 > {path}/fileA + echo repo_fileB_2 > {path}/fileB + """) + + assert_files(path, { + f'{path}/fileA': 'repo_fileA_2', + f'{path}/fileB': 'repo_fileB_2', + }) + + with subtest("Second backup in repo"): + machine.succeed("systemctl start --wait ${provider.backupService}") + + with subtest("two snapshots"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 2: + raise Exception(f"Unexpected snapshots:\n{out}") + + firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0] + secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0] + print(f"First snapshot {firstSnapshot}") + print(f"Second snapshot {secondSnapshot}") + + with subtest("Delete content"): + for path in sourceDirectories: + machine.succeed(f"""rm -r {path}/*""") + + assert_files(path, {}) + + with subtest("Restore second backup"): + machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}") + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_2', + f'{path}/fileB': 'repo_fileB_2', + }) + + with subtest("Restore first backup"): + machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}") + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_1', + f'{path}/fileB': 'repo_fileB_1', + }) + ''; +} diff --git a/modules/contracts/default.nix b/modules/contracts/default.nix index 3a68511..2a57547 100644 --- a/modules/contracts/default.nix +++ b/modules/contracts/default.nix @@ -61,6 +61,7 @@ let contracts = { databasebackup = importContract ./databasebackup.nix; + datasetbackup = importContract ./datasetbackup.nix; dashboard = importContract ./dashboard.nix; backup = importContract ./backup.nix; mount = pkgs.callPackage ./mount.nix { }; @@ -69,6 +70,7 @@ let test = { secret = pkgs.callPackage ./secret/test.nix { inherit shb; }; databasebackup = pkgs.callPackage ./databasebackup/test.nix { inherit shb; }; + datasetbackup = pkgs.callPackage ./datasetbackup/test.nix { inherit shb; }; backup = pkgs.callPackage ./backup/test.nix { inherit shb; }; }; }; diff --git a/test/contracts/datasetbackup.nix b/test/contracts/datasetbackup.nix new file mode 100644 index 0000000..c064d18 --- /dev/null +++ b/test/contracts/datasetbackup.nix @@ -0,0 +1,64 @@ +{ shb, pkgs, ... }: +{ + sanoid = shb.contracts.test.datasetbackup { + name = "sanoid"; + providerRoot = [ + "shb" + "sanoid" + "backup" + # This is the name of the dataset + "root/mytest" + ]; + modules = [ + ../../modules/blocks/sanoid.nix + # We use the zfs module to test the sanoid one + ../../modules/blocks/zfs.nix + ]; + settings = + { ... }: + { + useTemplate = [ "test" ]; + }; + extraConfig = + { filesRoot, ... }: + { + # Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix + networking.hostId = "deadbeef"; + boot.supportedFilesystems = [ "zfs" ]; + + virtualisation = { + emptyDiskImages = [ + 512 + ]; + }; + + # The test expects to keep one snapshot per hour. + services.sanoid.templates."test" = { + hourly = 1; + daily = 0; + monthly = 0; + yearly = 0; + }; + + systemd.services."zfs-zpool-create" = { + unitConfig.DefaultDependencies = false; + after = [ "systemd-modules-load.service" ]; + requiredBy = [ + "zfs-import-root.service" + "zfs-mount.service" + ]; + before = [ + "zfs-import-root.service" + "zfs-mount.service" + ]; + script = '' + ${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl -O mountpoint=none root /dev/vdb + ''; + }; + + shb.zfs.pools.root.datasets."mytest" = { + path = filesRoot; + }; + }; + }; +}