backup: enhance restore script and more tests

modules/blocks/borgbackup.nix

@@ -457,39 +457,16 @@ in
           let
             mkBorgBackupBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "BORG_REPO"
-                  }
+                  "BORG_PASSCOMMAND"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
-                    exit 1
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
-                  echo "Will restore archive $archive"
-
-                  (cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive")
-                '';
               };
           in
           flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
@@ -499,39 +476,16 @@ in
           let
             mkBorgBackupBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "BORG_REPO"
-                  }
+                  "BORG_PASSCOMMAND"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
-                    exit 1
+                listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
-                  echo "Will restore archive $archive"
-
-                  sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}"
-                '';
               };
           in
           flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);

modules/blocks/postgresql.nix

@@ -73,7 +73,7 @@ in
           backupName = "postgres.sql";
 
           backupCmd = ''
-            ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
+            ${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
           '';
 
           restoreCmd = ''

modules/blocks/restic.nix

@@ -440,38 +440,16 @@ in
           let
             mkResticBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "RESTIC_REPOSITORY"
-                  }
+                  "RESTIC_PASSWORD_FILE"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
-                    exit 1
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  echo "Will restore archive 'latest'"
-
-                  sudocmd ${pkgs.restic}/bin/restic restore latest --target /
-                '';
               };
           in
           flatten (mapAttrsToList mkResticBinary cfg.instances);
@@ -481,38 +459,16 @@ in
           let
             mkResticBinary =
               name: instance:
-              pkgs.writeShellApplication {
+              shb.contracts.backup.mkRestoreScript {
                 name = fullName name instance.settings.repository;
-                text = ''
+                user = instance.request.user;
-                  usage() {
+                sudoPreserveEnvs = [
-                    echo "$0 restore latest"
+                  "RESTIC_REPOSITORY"
-                  }
+                  "RESTIC_PASSWORD_FILE"
-
+                ];
-                  if ! [ "$1" = "restore" ]; then
+                secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                    usage
+                restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
-                    exit 1
+                listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
-                  fi
-                  shift
-
-                  if ! [ "$1" = "latest" ]; then
-                    usage
-                    exit 1
-                  fi
-                  shift
-
-                  sudocmd() {
-                    sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
-                  }
-
-                  set -a
-                  # shellcheck disable=SC1090
-                  source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
-                  set +a
-
-                  echo "Will restore archive 'latest'"
-
-                  sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}"
-                '';
               };
           in
           flatten (mapAttrsToList mkResticBinary cfg.databases);

modules/contracts/backup.nix

@@ -1,4 +1,9 @@
-{ lib, shb, ... }:
+{
+  lib,
+  shb,
+  pkgs,
+  ...
+}:
 let
   inherit (lib)
     concatStringsSep
@@ -187,12 +192,14 @@ in
 
                 ```bash
                 $ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
+                <snapshot 1> <metadata>
+                <snapshot 2> <metadata>
                 ```
 
                 And restore the database with:
 
                 ```bash
-                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
+                $ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
                 ```
               '';
               type = str;
@@ -222,4 +229,56 @@ in
         };
       };
     };
+
+  passthru.mkRestoreScript =
+    {
+      name,
+      user,
+      sudoPreserveEnvs ? [ ],
+      secretsFile,
+      restoreCmd,
+      listCmd,
+    }:
+    pkgs.writeShellApplication {
+      inherit name;
+
+      text = ''
+        usage() {
+          echo "$0 snapshots"
+          echo "$0 restore <snapshot>"
+        }
+
+        sudocmd() {
+          sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
+        }
+
+        loadenv() {
+          set -a
+          # shellcheck disable=SC1090
+          source <(sudocmd cat "${secretsFile}")
+          set +a
+        }
+
+        if [ "$1" = "restore" ]; then
+          shift
+          snapshot="$1"
+
+          loadenv
+
+          echo "Will restore snapshot '$snapshot'"
+
+          sudocmd sh -c "${restoreCmd}"
+        elif [ "$1" = "snapshots" ]; then
+          shift
+
+          loadenv
+
+          sudocmd sh -c "${listCmd}"
+        else
+          usage
+          exit 1
+        fi
+      '';
+    };
+
 }

modules/contracts/backup/docs/default.md

@@ -88,6 +88,7 @@ Or with another module `backupService_2`!
 
 - [Restic block](blocks-restic.html).
 - [Borgbackup block](blocks-borgbackup.html).
+- [ZFS block](blocks-zfs.html).
 
 ## Requester Blocks and Services {#contract-backup-requesters}
 

modules/contracts/backup/test.nix

@@ -21,8 +21,8 @@ in
     "/opt/files/A"
     "/opt/files/B"
   ],
-  settings, # { repository, config } -> attrset
+  settings ? { ... }: { }, # { filesRoot, config } -> attrset
-  extraConfig ? null, # { username, config } -> attrset
+  extraConfig ? null, # { filesRoot, username, config } -> attrset
 }:
 shb.test.runNixOSTest {
   inherit name;
@@ -40,7 +40,7 @@ shb.test.runNixOSTest {
           };
           settings = settings {
             inherit config;
-            repository = "/opt/repos/${name}";
+            filesRoot = "/opt/files";
           };
         })
         (mkIf (username != "root") {
@@ -52,6 +52,7 @@ shb.test.runNixOSTest {
         })
         (optionalAttrs (extraConfig != null) (extraConfig {
           inherit username config;
+          filesRoot = "/opt/files";
         }))
       ];
     };
@@ -65,7 +66,9 @@ shb.test.runNixOSTest {
       provider = (getAttrFromPath providerRoot nodes.machine).result;
     in
     ''
+      from datetime import datetime, timedelta
       from dictdiffer import diff
+      import re
 
       username = "${username}"
       sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
@@ -103,10 +106,26 @@ shb.test.runNixOSTest {
                   f'{path}/fileB': 'repo_fileB_1',
               })
 
+      with subtest("Initial snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          if len(out) != 0:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
       with subtest("First backup in repo"):
           print(machine.succeed("systemctl cat ${provider.backupService}"))
           machine.succeed("systemctl start --wait ${provider.backupService}")
 
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+            raise Exception(f"Unexpected snapshots:\n{out}")
+
+      # To accomodate for snapshot orchestrators which keep only a given amount
+      # of snapshots per unit of time, we set the time to now + 2h.
+      new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
+      machine.succeed(f"timedatectl set-time '{new_date}'")
+
       with subtest("New content"):
           for path in sourceDirectories:
               machine.succeed(f"""
@@ -119,14 +138,37 @@ shb.test.runNixOSTest {
                   f'{path}/fileB': 'repo_fileB_2',
               })
 
+      with subtest("Second backup in repo"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
+
       with subtest("Delete content"):
           for path in sourceDirectories:
               machine.succeed(f"""rm -r {path}/*""")
 
               assert_files(path, {})
 
-      with subtest("Restore initial content from repo"):
+      with subtest("Restore second backup"):
-          machine.succeed("""${provider.restoreScript} restore latest""")
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
+
+          for path in sourceDirectories:
+              assert_files(path, {
+                  f'{path}/fileA': 'repo_fileA_2',
+                  f'{path}/fileB': 'repo_fileB_2',
+              })
+
+      with subtest("Restore first backup"):
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
 
           for path in sourceDirectories:
               assert_files(path, {

modules/contracts/databasebackup/test.nix

@@ -51,16 +51,18 @@ shb.test.runNixOSTest {
   testScript =
     { nodes, ... }:
     let
-      provider = getAttrFromPath providerRoot nodes.machine;
+      provider = (getAttrFromPath providerRoot nodes.machine).result;
     in
     ''
       import csv
+      import re
 
       start_all()
       machine.wait_for_unit("postgresql.service")
+      machine.wait_for_unit("postgresql-setup.service")
       machine.wait_for_open_port(5432)
 
-      def peer_cmd(cmd, db="me"):
+      def peer_cmd(cmd, db="${database}"):
           return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
 
       def query(query):
@@ -73,30 +75,68 @@ shb.test.runNixOSTest {
               if len(diff) > 0:
                   raise Exception(i, diff)
 
-      table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
-
       with subtest("create fixture"):
           machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
-          machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
 
           res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
+          cmp_tables(res, table)
+
+      with subtest("Initial snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 0:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("backup"):
+          machine.succeed("systemctl start --wait ${provider.backupService}")
+
+      with subtest("One snapshot"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 1:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      with subtest("New content"):
+          machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
           cmp_tables(res, table)
 
       with subtest("backup"):
-          print(machine.succeed("systemctl cat ${provider.result.backupService}"))
+          machine.succeed("systemctl start --wait ${provider.backupService}")
-          print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
+
-          machine.succeed("systemctl start --wait ${provider.result.backupService}")
+      with subtest("Two snapshots"):
+          out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
+          print(f"Found snapshots:\n{out}")
+          if len(out) != 2:
+              raise Exception(f"Unexpected snapshots:\n{out}")
+
+      firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
+      secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
+      print(f"First snapshot {firstSnapshot}")
+      print(f"Second snapshot {secondSnapshot}")
 
       with subtest("drop database"):
           machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
           machine.fail(peer_cmd("SELECT * FROM test"))
 
-      with subtest("restore"):
+      with subtest("restore second snapshot"):
-          print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
-          machine.succeed("${provider.result.restoreScript} restore latest ")
+          machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
 
-      with subtest("check restoration"):
           res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+          cmp_tables(res, table)
+
+      with subtest("restore first snapshot"):
+          print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+          machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
+
+          res = query("SELECT * FROM test")
+          table = [{'name': 'car', 'count': '1'}]
           cmp_tables(res, table)
     '';
 }

modules/contracts/default.nix

@@ -54,9 +54,10 @@ let
         };
       };
     in
-    mkContractFunctions {
+    (mkContractFunctions {
       inherit (importedModule) mkRequest mkResult;
-    };
+    })
+    // (importedModule.passthru or { });
 
   contracts = {
     databasebackup = importContract ./databasebackup.nix;

test/blocks/borgbackup.nix

@@ -168,8 +168,12 @@ let
               assert_files("/opt/files", {})
 
           with subtest("Restore initial content from repo A"):
-              machine.succeed("""
+              snapshot = machine.succeed("""
-              ${restoreScript} restore latest
+              ${restoreScript} snapshots
+              """)
+              print(snapshot)
+              machine.succeed(f"""
+              ${restoreScript} restore {snapshot}
               """)
 
               assert_files("/opt/files", {

test/blocks/restic.nix

@@ -168,8 +168,12 @@ let
               assert_files("/opt/files", {})
 
           with subtest("Restore initial content from repo A"):
-              machine.succeed("""
+              snapshot = machine.succeed("""
-              ${restoreScript} restore latest
+              ${restoreScript} snapshots
+              """)
+              print(snapshot)
+              machine.succeed(f"""
+              ${restoreScript} restore {snapshot}
               """)
 
               assert_files("/opt/files", {

test/contracts/backup.nix

@@ -14,12 +14,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };
@@ -49,12 +49,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };
@@ -84,12 +84,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };
@@ -119,12 +119,12 @@
       ../../modules/blocks/hardcodedsecret.nix
     ];
     settings =
-      { repository, config, ... }:
+      { config, ... }:
       {
         enable = true;
         passphrase.result = config.shb.hardcodedsecret.passphrase.result;
         repository = {
-          path = repository;
+          path = "/opt/repos/mytest";
           timerConfig = {
             OnCalendar = "00:00:00";
           };