backup: enhance restore script and more tests

This commit is contained in:
ibizaman 2026-04-25 23:34:55 +02:00 committed by Pierre Penninckx
parent bdd8a0cc2a
commit a0d29e9d97
11 changed files with 221 additions and 160 deletions

View file

@ -457,39 +457,16 @@ in
let
mkBorgBackupBinary =
name: instance:
pkgs.writeShellApplication {
shb.contracts.backup.mkRestoreScript {
name = fullName name instance.settings.repository;
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
set +a
archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
echo "Will restore archive $archive"
(cd / && sudocmd ${pkgs.borgbackup}/bin/borg extract "$BORG_REPO"::"$archive")
'';
user = instance.request.user;
sudoPreserveEnvs = [
"BORG_REPO"
"BORG_PASSCOMMAND"
];
secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
restoreCmd = ''(cd / && ${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\")'';
listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
};
in
flatten (mapAttrsToList mkBorgBackupBinary cfg.instances);
@ -499,39 +476,16 @@ in
let
mkBorgBackupBinary =
name: instance:
pkgs.writeShellApplication {
shb.contracts.backup.mkRestoreScript {
name = fullName name instance.settings.repository;
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=BORG_REPO,BORG_PASSCOMMAND -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}")
set +a
archive="$(sudocmd borg list --short "$BORG_REPO" | tail -n 1)"
echo "Will restore archive $archive"
sudocmd sh -c "${pkgs.borgbackup}/bin/borg extract $BORG_REPO::$archive --stdout | ${instance.request.restoreCmd}"
'';
user = instance.request.user;
sudoPreserveEnvs = [
"BORG_REPO"
"BORG_PASSCOMMAND"
];
secretsFile = "/run/secrets_borgbackup_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.borgbackup}/bin/borg extract \"$BORG_REPO::$snapshot\" --stdout | ${instance.request.restoreCmd}'';
listCmd = ''if [ -e \"$BORG_REPO/data\" ]; then borg list --short \"$BORG_REPO\"; fi'';
};
in
flatten (mapAttrsToList mkBorgBackupBinary cfg.databases);

View file

@ -73,7 +73,7 @@ in
backupName = "postgres.sql";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
${pkgs.postgresql}/bin/pg_dumpall --clean --if-exists | ${pkgs.gzip}/bin/gzip --rsyncable
'';
restoreCmd = ''

View file

@ -440,38 +440,16 @@ in
let
mkResticBinary =
name: instance:
pkgs.writeShellApplication {
shb.contracts.backup.mkRestoreScript {
name = fullName name instance.settings.repository;
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
set +a
echo "Will restore archive 'latest'"
sudocmd ${pkgs.restic}/bin/restic restore latest --target /
'';
user = instance.request.user;
sudoPreserveEnvs = [
"RESTIC_REPOSITORY"
"RESTIC_PASSWORD_FILE"
];
secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.restic}/bin/restic restore \"$snapshot\" --target /'';
listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
};
in
flatten (mapAttrsToList mkResticBinary cfg.instances);
@ -481,38 +459,16 @@ in
let
mkResticBinary =
name: instance:
pkgs.writeShellApplication {
shb.contracts.backup.mkRestoreScript {
name = fullName name instance.settings.repository;
text = ''
usage() {
echo "$0 restore latest"
}
if ! [ "$1" = "restore" ]; then
usage
exit 1
fi
shift
if ! [ "$1" = "latest" ]; then
usage
exit 1
fi
shift
sudocmd() {
sudo --preserve-env=RESTIC_REPOSITORY,RESTIC_PASSWORD_FILE -u ${instance.request.user} "$@"
}
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "/run/secrets_restic_env/${fullName name instance.settings.repository}")
set +a
echo "Will restore archive 'latest'"
sudocmd sh -c "${pkgs.restic}/bin/restic dump latest ${instance.request.backupName} | ${instance.request.restoreCmd}"
'';
user = instance.request.user;
sudoPreserveEnvs = [
"RESTIC_REPOSITORY"
"RESTIC_PASSWORD_FILE"
];
secretsFile = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
restoreCmd = ''${pkgs.restic}/bin/restic dump \"$snapshot\" ${instance.request.backupName} | ${instance.request.restoreCmd}'';
listCmd = ''if [ -e \"$RESTIC_REPOSITORY/index\" ]; then ${pkgs.restic}/bin/restic snapshots --json | ${pkgs.jq}/bin/jq '.[].id'; fi'';
};
in
flatten (mapAttrsToList mkResticBinary cfg.databases);

View file

@ -1,4 +1,9 @@
{ lib, shb, ... }:
{
lib,
shb,
pkgs,
...
}:
let
inherit (lib)
concatStringsSep
@ -187,12 +192,14 @@ in
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} snapshots
<snapshot 1> <metadata>
<snapshot 2> <metadata>
```
And restore the database with:
```bash
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore latest
$ ${if restoreScriptText != null then restoreScriptText else restoreScript} restore <snapshot 1>
```
'';
type = str;
@ -222,4 +229,56 @@ in
};
};
};
passthru.mkRestoreScript =
{
name,
user,
sudoPreserveEnvs ? [ ],
secretsFile,
restoreCmd,
listCmd,
}:
pkgs.writeShellApplication {
inherit name;
text = ''
usage() {
echo "$0 snapshots"
echo "$0 restore <snapshot>"
}
sudocmd() {
sudo --preserve-env=${lib.concatStringsSep "," sudoPreserveEnvs} -u ${user} "$@"
}
loadenv() {
set -a
# shellcheck disable=SC1090
source <(sudocmd cat "${secretsFile}")
set +a
}
if [ "$1" = "restore" ]; then
shift
snapshot="$1"
loadenv
echo "Will restore snapshot '$snapshot'"
sudocmd sh -c "${restoreCmd}"
elif [ "$1" = "snapshots" ]; then
shift
loadenv
sudocmd sh -c "${listCmd}"
else
usage
exit 1
fi
'';
};
}

View file

@ -88,6 +88,7 @@ Or with another module `backupService_2`!
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html).
- [ZFS block](blocks-zfs.html).
## Requester Blocks and Services {#contract-backup-requesters}

View file

@ -21,8 +21,8 @@ in
"/opt/files/A"
"/opt/files/B"
],
settings, # { repository, config } -> attrset
extraConfig ? null, # { username, config } -> attrset
settings ? { ... }: { }, # { filesRoot, config } -> attrset
extraConfig ? null, # { filesRoot, username, config } -> attrset
}:
shb.test.runNixOSTest {
inherit name;
@ -40,7 +40,7 @@ shb.test.runNixOSTest {
};
settings = settings {
inherit config;
repository = "/opt/repos/${name}";
filesRoot = "/opt/files";
};
})
(mkIf (username != "root") {
@ -52,6 +52,7 @@ shb.test.runNixOSTest {
})
(optionalAttrs (extraConfig != null) (extraConfig {
inherit username config;
filesRoot = "/opt/files";
}))
];
};
@ -65,7 +66,9 @@ shb.test.runNixOSTest {
provider = (getAttrFromPath providerRoot nodes.machine).result;
in
''
from datetime import datetime, timedelta
from dictdiffer import diff
import re
username = "${username}"
sourceDirectories = [ ${concatMapStringsSep ", " (x: ''"${x}"'') sourceDirectories} ]
@ -103,10 +106,26 @@ shb.test.runNixOSTest {
f'{path}/fileB': 'repo_fileB_1',
})
with subtest("Initial snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("First backup in repo"):
print(machine.succeed("systemctl cat ${provider.backupService}"))
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
# To accomodate for snapshot orchestrators which keep only a given amount
# of snapshots per unit of time, we set the time to now + 2h.
new_date = (datetime.now() + timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S")
machine.succeed(f"timedatectl set-time '{new_date}'")
with subtest("New content"):
for path in sourceDirectories:
machine.succeed(f"""
@ -119,14 +138,37 @@ shb.test.runNixOSTest {
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Second backup in repo"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
with subtest("Delete content"):
for path in sourceDirectories:
machine.succeed(f"""rm -r {path}/*""")
assert_files(path, {})
with subtest("Restore initial content from repo"):
machine.succeed("""${provider.restoreScript} restore latest""")
with subtest("Restore second backup"):
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
for path in sourceDirectories:
assert_files(path, {
f'{path}/fileA': 'repo_fileA_2',
f'{path}/fileB': 'repo_fileB_2',
})
with subtest("Restore first backup"):
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
for path in sourceDirectories:
assert_files(path, {

View file

@ -51,16 +51,18 @@ shb.test.runNixOSTest {
testScript =
{ nodes, ... }:
let
provider = getAttrFromPath providerRoot nodes.machine;
provider = (getAttrFromPath providerRoot nodes.machine).result;
in
''
import csv
import re
start_all()
machine.wait_for_unit("postgresql.service")
machine.wait_for_unit("postgresql-setup.service")
machine.wait_for_open_port(5432)
def peer_cmd(cmd, db="me"):
def peer_cmd(cmd, db="${database}"):
return "sudo -u ${database} psql -U ${database} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
def query(query):
@ -73,30 +75,68 @@ shb.test.runNixOSTest {
if len(diff) > 0:
raise Exception(i, diff)
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
with subtest("create fixture"):
machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1)"))
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}]
cmp_tables(res, table)
with subtest("Initial snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 0:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("backup"):
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("One snapshot"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 1:
raise Exception(f"Unexpected snapshots:\n{out}")
with subtest("New content"):
machine.succeed(peer_cmd("INSERT INTO test VALUES ('lollipop', 2)"))
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
cmp_tables(res, table)
with subtest("backup"):
print(machine.succeed("systemctl cat ${provider.result.backupService}"))
print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
machine.succeed("systemctl start --wait ${provider.result.backupService}")
machine.succeed("systemctl start --wait ${provider.backupService}")
with subtest("Two snapshots"):
out = machine.succeed("${provider.restoreScript} snapshots").splitlines()
print(f"Found snapshots:\n{out}")
if len(out) != 2:
raise Exception(f"Unexpected snapshots:\n{out}")
firstSnapshot = re.split("[ \t+]", out[0], maxsplit=1)[0]
secondSnapshot = re.split("[ \t+]", out[1], maxsplit=1)[0]
print(f"First snapshot {firstSnapshot}")
print(f"Second snapshot {secondSnapshot}")
with subtest("drop database"):
machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))
machine.fail(peer_cmd("SELECT * FROM test"))
with subtest("restore"):
print(machine.succeed("readlink -f $(type ${provider.result.restoreScript})"))
machine.succeed("${provider.result.restoreScript} restore latest ")
with subtest("restore second snapshot"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed(f"${provider.restoreScript} restore {secondSnapshot}")
with subtest("check restoration"):
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
cmp_tables(res, table)
with subtest("restore first snapshot"):
print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
machine.succeed(f"${provider.restoreScript} restore {firstSnapshot}")
res = query("SELECT * FROM test")
table = [{'name': 'car', 'count': '1'}]
cmp_tables(res, table)
'';
}

View file

@ -54,9 +54,10 @@ let
};
};
in
mkContractFunctions {
(mkContractFunctions {
inherit (importedModule) mkRequest mkResult;
};
})
// (importedModule.passthru or { });
contracts = {
databasebackup = importContract ./databasebackup.nix;

View file

@ -168,8 +168,12 @@ let
assert_files("/opt/files", {})
with subtest("Restore initial content from repo A"):
machine.succeed("""
${restoreScript} restore latest
snapshot = machine.succeed("""
${restoreScript} snapshots
""")
print(snapshot)
machine.succeed(f"""
${restoreScript} restore {snapshot}
""")
assert_files("/opt/files", {

View file

@ -168,8 +168,12 @@ let
assert_files("/opt/files", {})
with subtest("Restore initial content from repo A"):
machine.succeed("""
${restoreScript} restore latest
snapshot = machine.succeed("""
${restoreScript} snapshots
""")
print(snapshot)
machine.succeed(f"""
${restoreScript} restore {snapshot}
""")
assert_files("/opt/files", {

View file

@ -14,12 +14,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ repository, config, ... }:
{ config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = repository;
path = "/opt/repos/mytest";
timerConfig = {
OnCalendar = "00:00:00";
};
@ -49,12 +49,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ repository, config, ... }:
{ config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = repository;
path = "/opt/repos/mytest";
timerConfig = {
OnCalendar = "00:00:00";
};
@ -84,12 +84,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ repository, config, ... }:
{ config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = repository;
path = "/opt/repos/mytest";
timerConfig = {
OnCalendar = "00:00:00";
};
@ -119,12 +119,12 @@
../../modules/blocks/hardcodedsecret.nix
];
settings =
{ repository, config, ... }:
{ config, ... }:
{
enable = true;
passphrase.result = config.shb.hardcodedsecret.passphrase.result;
repository = {
path = repository;
path = "/opt/repos/mytest";
timerConfig = {
OnCalendar = "00:00:00";
};