diff --git a/modules/blocks/borgbackup.nix b/modules/blocks/borgbackup.nix index fb83b0f..929aea4 100644 --- a/modules/blocks/borgbackup.nix +++ b/modules/blocks/borgbackup.nix @@ -457,39 +457,16 @@ in let mkBorgBackupBinary = name: instance: - pkgs.writeShellApplication { + shb.contracts.backup.mkRestoreScript { name = fullName name instance.settings.repository; - text = '' - usage() { - echo "$0 restore latest" - } - - if ! [ "$1" = "restore" ]; then - usage - exit 1 - fi - shift - - if ! [ "$1" = "latest" ]; then - usage - exit 1 - fi - shift - - sudocmd() { - sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@" - } - - set -a - # shellcheck disable=SC1090 - source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}") - set +a - - archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)" - echo "Will restore archive $archive" - - (cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive") - ''; + user = instance.request.user; + sudoPreserveEnvs = [ + "BORG_REPO" + "BORG_PASSCOMMAND" + ]; + secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}"; + restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")''; + listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi''; }; in flatten (mapAttrsToList mkBorgBackupBinary cfg.instances); @@ -499,39 +476,16 @@ in let mkBorgBackupBinary = name: instance: - pkgs.writeShellApplication { + shb.contracts.backup.mkRestoreScript { name = fullName name instance.settings.repository; - text = '' - usage() { - echo "$0 restore latest" - } - - if ! [ "$1" = "restore" ]; then - usage - exit 1 - fi - shift - - if ! [ "$1" = "latest" ]; then - usage - exit 1 - fi - shift - - sudocmd() { - sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@" - } - - set -a - # shellcheck disable=SC1090 - source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}") - set +a - - archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)" - echo "Will restore archive $archive" - - sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}" - ''; + user = instance.request.user; + sudoPreserveEnvs = [ + "BORG_REPO" + "BORG_PASSCOMMAND" + ]; + secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}"; + restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}''; + listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi''; }; in flatten (mapAttrsToList mkBorgBackupBinary cfg.databases); diff --git a/modules/blocks/postgresql.nix b/modules/blocks/postgresql.nix index 95e6584..8ed49ca 100644 --- a/modules/blocks/postgresql.nix +++ b/modules/blocks/postgresql.nix @@ -73,7 +73,7 @@ in backupName = "postgres.sql"; backupCmd = '' - ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable + ${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable ''; restoreCmd = '' diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix index 6a0f711..a5a4075 100644 --- a/modules/blocks/restic.nix +++ b/modules/blocks/restic.nix @@ -440,38 +440,16 @@ in let mkResticBinary = name: instance: - pkgs.writeShellApplication { + shb.contracts.backup.mkRestoreScript { name = fullName name instance.settings.repository; - text = '' - usage() { - echo "$0 restore latest" - } - - if ! [ "$1" = "restore" ]; then - usage - exit 1 - fi - shift - - if ! [ "$1" = "latest" ]; then - usage - exit 1 - fi - shift - - sudocmd() { - sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@" - } - - set -a - # shellcheck disable=SC1090 - source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}") - set +a - - echo "Will restore archive 'latest'" - - sudocmd ${pkgs.restic}/bin/restic restore latest --target / - ''; + user = instance.request.user; + sudoPreserveEnvs = [ + "RESTIC_REPOSITORY" + "RESTIC_PASSWORD_FILE" + ]; + secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}"; + restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /''; + listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi''; }; in flatten (mapAttrsToList mkResticBinary cfg.instances); @@ -481,38 +459,16 @@ in let mkResticBinary = name: instance: - pkgs.writeShellApplication { + shb.contracts.backup.mkRestoreScript { name = fullName name instance.settings.repository; - text = '' - usage() { - echo "$0 restore latest" - } - - if ! [ "$1" = "restore" ]; then - usage - exit 1 - fi - shift - - if ! [ "$1" = "latest" ]; then - usage - exit 1 - fi - shift - - sudocmd() { - sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@" - } - - set -a - # shellcheck disable=SC1090 - source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}") - set +a - - echo "Will restore archive 'latest'" - - sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}" - ''; + user = instance.request.user; + sudoPreserveEnvs = [ + "RESTIC_REPOSITORY" + "RESTIC_PASSWORD_FILE" + ]; + secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}"; + restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}''; + listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi''; }; in flatten (mapAttrsToList mkResticBinary cfg.databases); diff --git a/modules/contracts/backup.nix b/modules/contracts/backup.nix index 27c790c..ff1b77b 100644 --- a/modules/contracts/backup.nix +++ b/modules/contracts/backup.nix @@ -1,4 +1,9 @@ -{ lib, shb, ... }: +{ + lib, + shb, + pkgs, + ... +}: let inherit (lib) concatStringsSep @@ -187,12 +192,14 @@ in ```bash $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots + + ``` And restore the database with: ```bash - $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest + $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore ``` ''; type = str; @@ -222,4 +229,56 @@ in }; }; }; + + passthru.mkRestoreScript = + { + name, + user, + sudoPreserveEnvs ? [ ], + secretsFile, + restoreCmd, + listCmd, + }: + pkgs.writeShellApplication { + inherit name; + + text = '' + usage() { + echo "$0 snapshots" + echo "$0 restore " + } + + sudocmd() { + sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@" + } + + loadenv() { + set -a + # shellcheck disable=SC1090 + source <(sudocmd cat "${secretsFile}") + set +a + } + + if [ "$1" = "restore" ]; then + shift + snapshot="$1" + + loadenv + + echo "Will restore snapshot '$snapshot'" + + sudocmd sh -c "${restoreCmd}" + elif [ "$1" = "snapshots" ]; then + shift + + loadenv + + sudocmd sh -c "${listCmd}" + else + usage + exit 1 + fi + ''; + }; + } diff --git a/modules/contracts/backup/docs/default.md b/modules/contracts/backup/docs/default.md index 7ead513..a77fc6a 100644 --- a/modules/contracts/backup/docs/default.md +++ b/modules/contracts/backup/docs/default.md @@ -88,6 +88,7 @@ Or with another module `backupService_2`! - [Restic block](blocks-restic.html). - [Borgbackup block](blocks-borgbackup.html). +- [ZFS block](blocks-zfs.html). ## Requester Blocks and Services {#contract-backup-requesters} diff --git a/modules/contracts/backup/test.nix b/modules/contracts/backup/test.nix index 86a6c90..402c063 100644 --- a/modules/contracts/backup/test.nix +++ b/modules/contracts/backup/test.nix @@ -21,8 +21,8 @@ in "/opt/files/A" "/opt/files/B" ], - settings, # { repository, config } -> attrset - extraConfig ? null, # { username, config } -> attrset + settings ? { ... }: { }, # { filesRoot, config } -> attrset + extraConfig ? null, # { filesRoot, username, config } -> attrset }: shb.test.runNixOSTest { inherit name; @@ -40,7 +40,7 @@ shb.test.runNixOSTest { }; settings = settings { inherit config; - repository = "/opt/repos/${name}"; + filesRoot = "/opt/files"; }; }) (mkIf (username != "root") { @@ -52,6 +52,7 @@ shb.test.runNixOSTest { }) (optionalAttrs (extraConfig != null) (extraConfig { inherit username config; + filesRoot = "/opt/files"; })) ]; }; @@ -65,7 +66,9 @@ shb.test.runNixOSTest { provider = (getAttrFromPath providerRoot nodes.machine).result; in '' + from datetime import datetime, timedelta from dictdiffer import diff + import re username = "${username}" sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ] @@ -103,10 +106,26 @@ shb.test.runNixOSTest { f'{path}/fileB': 'repo_fileB_1', }) + with subtest("Initial snapshot"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + if len(out) != 0: + raise Exception(f"Unexpected snapshots:\n{out}") + with subtest("First backup in repo"): print(machine.succeed("systemctl cat ${provider.backupService}")) machine.succeed("systemctl start --wait ${provider.backupService}") + with subtest("One snapshot"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 1: + raise Exception(f"Unexpected snapshots:\n{out}") + + # To accomodate for snapshot orchestrators which keep only a given amount + # of snapshots per unit of time, we set the time to now + 2h. + new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S") + machine.succeed(f"timedatectl set-time '{new_date}'") + with subtest("New content"): for path in sourceDirectories: machine.succeed(f""" @@ -119,14 +138,37 @@ shb.test.runNixOSTest { f'{path}/fileB': 'repo_fileB_2', }) + with subtest("Second backup in repo"): + machine.succeed("systemctl start --wait ${provider.backupService}") + + with subtest("two snapshots"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 2: + raise Exception(f"Unexpected snapshots:\n{out}") + + firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0] + secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0] + print(f"First snapshot {firstSnapshot}") + print(f"Second snapshot {secondSnapshot}") + with subtest("Delete content"): for path in sourceDirectories: machine.succeed(f"""rm -r {path}/*""") assert_files(path, {}) - with subtest("Restore initial content from repo"): - machine.succeed("""${provider.restoreScript} restore latest""") + with subtest("Restore second backup"): + machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}") + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_2', + f'{path}/fileB': 'repo_fileB_2', + }) + + with subtest("Restore first backup"): + machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}") for path in sourceDirectories: assert_files(path, { diff --git a/modules/contracts/databasebackup/test.nix b/modules/contracts/databasebackup/test.nix index 09a638f..f61a2f4 100644 --- a/modules/contracts/databasebackup/test.nix +++ b/modules/contracts/databasebackup/test.nix @@ -51,16 +51,18 @@ shb.test.runNixOSTest { testScript = { nodes, ... }: let - provider = getAttrFromPath providerRoot nodes.machine; + provider = (getAttrFromPath providerRoot nodes.machine).result; in '' import csv + import re start_all() machine.wait_for_unit("postgresql.service") + machine.wait_for_unit("postgresql-setup.service") machine.wait_for_open_port(5432) - def peer_cmd(cmd, db="me"): + def peer_cmd(cmd, db="${database}"): return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db) def query(query): @@ -73,30 +75,68 @@ shb.test.runNixOSTest { if len(diff) > 0: raise Exception(i, diff) - table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}] - with subtest("create fixture"): machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)")) - machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)")) + machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)")) res = query("SELECT * FROM test") + table = [{'name': 'car', 'count': '1'}] + cmp_tables(res, table) + + with subtest("Initial snapshots"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 0: + raise Exception(f"Unexpected snapshots:\n{out}") + + with subtest("backup"): + machine.succeed("systemctl start --wait ${provider.backupService}") + + with subtest("One snapshot"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 1: + raise Exception(f"Unexpected snapshots:\n{out}") + + with subtest("New content"): + machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)")) + + res = query("SELECT * FROM test") + table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}] cmp_tables(res, table) with subtest("backup"): - print(machine.succeed("systemctl cat ${provider.result.backupService}")) - print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase")) - machine.succeed("systemctl start --wait ${provider.result.backupService}") + machine.succeed("systemctl start --wait ${provider.backupService}") + + with subtest("Two snapshots"): + out = machine.succeed("${provider.restoreScript} snapshots").splitlines() + print(f"Found snapshots:\n{out}") + if len(out) != 2: + raise Exception(f"Unexpected snapshots:\n{out}") + + firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0] + secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0] + print(f"First snapshot {firstSnapshot}") + print(f"Second snapshot {secondSnapshot}") with subtest("drop database"): machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres")) machine.fail(peer_cmd("SELECT * FROM test")) - with subtest("restore"): - print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})")) - machine.succeed("${provider.result.restoreScript} restore latest ") + with subtest("restore second snapshot"): + print(machine.succeed("readlink -f $(type ${provider.restoreScript})")) + machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}") - with subtest("check restoration"): res = query("SELECT * FROM test") + table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}] + cmp_tables(res, table) + + with subtest("restore first snapshot"): + print(machine.succeed("readlink -f $(type ${provider.restoreScript})")) + machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}") + + res = query("SELECT * FROM test") + table = [{'name': 'car', 'count': '1'}] cmp_tables(res, table) ''; } diff --git a/modules/contracts/default.nix b/modules/contracts/default.nix index 0f90360..3a68511 100644 --- a/modules/contracts/default.nix +++ b/modules/contracts/default.nix @@ -54,9 +54,10 @@ let }; }; in - mkContractFunctions { + (mkContractFunctions { inherit (importedModule) mkRequest mkResult; - }; + }) + // (importedModule.passthru or { }); contracts = { databasebackup = importContract ./databasebackup.nix; diff --git a/test/blocks/borgbackup.nix b/test/blocks/borgbackup.nix index f7e8e07..eaeb625 100644 --- a/test/blocks/borgbackup.nix +++ b/test/blocks/borgbackup.nix @@ -168,8 +168,12 @@ let assert_files("/opt/files", {}) with subtest("Restore initial content from repo A"): - machine.succeed(""" - ${restoreScript} restore latest + snapshot = machine.succeed(""" + ${restoreScript} snapshots + """) + print(snapshot) + machine.succeed(f""" + ${restoreScript} restore {snapshot} """) assert_files("/opt/files", { diff --git a/test/blocks/restic.nix b/test/blocks/restic.nix index d7826b1..35bf2bf 100644 --- a/test/blocks/restic.nix +++ b/test/blocks/restic.nix @@ -168,8 +168,12 @@ let assert_files("/opt/files", {}) with subtest("Restore initial content from repo A"): - machine.succeed(""" - ${restoreScript} restore latest + snapshot = machine.succeed(""" + ${restoreScript} snapshots + """) + print(snapshot) + machine.succeed(f""" + ${restoreScript} restore {snapshot} """) assert_files("/opt/files", { diff --git a/test/contracts/backup.nix b/test/contracts/backup.nix index 2cd2404..e999264 100644 --- a/test/contracts/backup.nix +++ b/test/contracts/backup.nix @@ -14,12 +14,12 @@ ../../modules/blocks/hardcodedsecret.nix ]; settings = - { repository, config, ... }: + { config, ... }: { enable = true; passphrase.result = config.shb.hardcodedsecret.passphrase.result; repository = { - path = repository; + path = "/opt/repos/mytest"; timerConfig = { OnCalendar = "00:00:00"; }; @@ -49,12 +49,12 @@ ../../modules/blocks/hardcodedsecret.nix ]; settings = - { repository, config, ... }: + { config, ... }: { enable = true; passphrase.result = config.shb.hardcodedsecret.passphrase.result; repository = { - path = repository; + path = "/opt/repos/mytest"; timerConfig = { OnCalendar = "00:00:00"; }; @@ -84,12 +84,12 @@ ../../modules/blocks/hardcodedsecret.nix ]; settings = - { repository, config, ... }: + { config, ... }: { enable = true; passphrase.result = config.shb.hardcodedsecret.passphrase.result; repository = { - path = repository; + path = "/opt/repos/mytest"; timerConfig = { OnCalendar = "00:00:00"; }; @@ -119,12 +119,12 @@ ../../modules/blocks/hardcodedsecret.nix ]; settings = - { repository, config, ... }: + { config, ... }: { enable = true; passphrase.result = config.shb.hardcodedsecret.passphrase.result; repository = { - path = repository; + path = "/opt/repos/mytest"; timerConfig = { OnCalendar = "00:00:00"; };