diff --git a/modules/blocks/postgresql.nix b/modules/blocks/postgresql.nix index bc3b755..600eeb1 100644 --- a/modules/blocks/postgresql.nix +++ b/modules/blocks/postgresql.nix @@ -51,7 +51,7 @@ in }; backup = lib.mkOption { - type = contracts.backup; + type = contracts.databasebackup.request; description = '' Backup configuration. This is an output option. @@ -63,32 +63,22 @@ in enable = true; # Options specific to Restic. - } // config.shb.nextcloud.backup; + } // config.shb.postgresl.backup; ``` ''; readOnly = true; default = { - user = "postgresql"; - sourceDirectories = [ - /tmp/postgresql_backup - ]; - excludePatterns = [ ]; + user = "postgres"; - hooks.before_backup = ['' - set -e -o pipefail + backupFile = "postgres.sql"; - umask 077 # Ensure backup is only readable by postgres user + backupCmd = '' + ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable + ''; - rm -rf /tmp/postgresql_backup # Clean up in case after_backup hook wasn't run. - mkdir /tmp/postgresql_backup - - ${pkgs.psql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable > /tmp/postgresql_backup/pg_dumpall.sql.gz - '']; - - hooks.after_backup = ['' - set -e -o pipefail - rm -rf /tmp/postgresql_backup - '']; + restoreCmd = '' + ${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres + ''; }; }; diff --git a/modules/blocks/restic.nix b/modules/blocks/restic.nix index a5b209f..a57f635 100644 --- a/modules/blocks/restic.nix +++ b/modules/blocks/restic.nix @@ -4,125 +4,90 @@ let cfg = config.shb.restic; shblib = pkgs.callPackage ../../lib {}; + contracts = pkgs.callPackage ../contracts {}; inherit (lib) concatStringsSep filterAttrs flatten literalExpression optionals optionalString listToAttrs mapAttrsToList mkEnableOption mkOption mkMerge; inherit (lib) generators hasPrefix mkIf nameValuePair optionalAttrs removePrefix; inherit (lib.types) attrsOf enum int ints listOf oneOf nonEmptyListOf nonEmptyStr nullOr path str submodule; - instanceOptions = { - enable = mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots"; + commonOptions = submodule { + options = { + enable = mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots"; - passphraseFile = mkOption { - description = "Encryption key for the backups."; - type = path; - }; + passphraseFile = mkOption { + description = "Encryption key for the backups."; + type = path; + }; - user = mkOption { - description = '' - Unix user doing the backups. Must be the user owning the files to be backed up. - ''; - type = str; - }; + repositories = mkOption { + description = "Repositories to back this instance to."; + type = nonEmptyListOf (submodule { + options = { + path = mkOption { + type = str; + description = "Repository location"; + }; - sourceDirectories = mkOption { - description = "Source directories."; - type = nonEmptyListOf str; - }; + secrets = mkOption { + type = attrsOf shblib.secretFileType; + default = {}; + description = '' + Secrets needed to access the repository where the backups will be stored. - excludePatterns = mkOption { - description = "Exclude patterns."; - type = listOf str; - default = []; - }; - - repositories = mkOption { - description = "Repositories to back this instance to."; - type = nonEmptyListOf (submodule { - options = { - path = mkOption { - type = str; - description = "Repository location"; - }; - - secrets = mkOption { - type = attrsOf shblib.secretFileType; - default = {}; - description = '' - Secrets needed to access the repository where the backups will be stored. - - See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example - and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets. + See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example + and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets. ''; - example = literalExpression '' - { - AWS_ACCESS_KEY_ID = ; - AWS_SECRET_ACCESS_KEY = ; - } + example = literalExpression '' + { + AWS_ACCESS_KEY_ID = ; + AWS_SECRET_ACCESS_KEY = ; + } ''; - }; - - timerConfig = mkOption { - type = attrsOf utils.systemdUtils.unitOptions.unitOption; - default = { - OnCalendar = "daily"; - Persistent = true; }; - description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.''; - example = { - OnCalendar = "00:05"; - RandomizedDelaySec = "5h"; - Persistent = true; + + timerConfig = mkOption { + type = attrsOf utils.systemdUtils.unitOptions.unitOption; + default = { + OnCalendar = "daily"; + Persistent = true; + }; + description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.''; + example = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + Persistent = true; + }; }; }; - }; - }); - }; - - retention = mkOption { - description = "For how long to keep backup files."; - type = attrsOf (oneOf [ int nonEmptyStr ]); - default = { - keep_within = "1d"; - keep_hourly = 24; - keep_daily = 7; - keep_weekly = 4; - keep_monthly = 6; + }); }; - }; - hooks = mkOption { - description = "Hooks to run before or after the backup."; - default = {}; - type = submodule { - options = { - before_backup = mkOption { - description = "Hooks to run before backup"; - type = listOf str; - default = []; - }; - - after_backup = mkOption { - description = "Hooks to run after backup"; - type = listOf str; - default = []; - }; + retention = mkOption { + description = "For how long to keep backup files."; + type = attrsOf (oneOf [ int nonEmptyStr ]); + default = { + keep_within = "1d"; + keep_hourly = 24; + keep_daily = 7; + keep_weekly = 4; + keep_monthly = 6; }; }; - }; - limitUploadKiBs = mkOption { - type = nullOr int; - description = "Limit upload bandwidth to the given KiB/s amount."; - default = null; - example = 8000; - }; + limitUploadKiBs = mkOption { + type = nullOr int; + description = "Limit upload bandwidth to the given KiB/s amount."; + default = null; + example = 8000; + }; - limitDownloadKiBs = mkOption { - type = nullOr int; - description = "Limit download bandwidth to the given KiB/s amount."; - default = null; - example = 8000; + limitDownloadKiBs = mkOption { + type = nullOr int; + description = "Limit download bandwidth to the given KiB/s amount."; + default = null; + example = 8000; + }; }; }; @@ -133,13 +98,45 @@ in { options.shb.restic = { instances = mkOption { - description = "Each instance is a backup setting"; + description = "Each instance is backing up some directories to one or more repositories."; default = {}; type = attrsOf (submodule { - options = instanceOptions; + options = { + request = mkOption { + type = contracts.backup.request; + }; + + settings = mkOption { + type = commonOptions; + }; + }; }); }; + databases = mkOption { + description = "Each item is backing up some database to one or more repositories."; + default = {}; + type = attrsOf (submodule ({ options, ... }: { + options = { + request = mkOption { + type = contracts.databasebackup.request; + }; + + settings = mkOption { + type = commonOptions; + }; + + result = mkOption { + type = contracts.databasebackup.result; + default = { + restoreScript = fullName options.name options.repository; + backupService = "${fullName options.name options.repository}.service"; + }; + }; + }; + })); + }; + # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23 performance = mkOption { description = "Reduce performance impact of backup jobs."; @@ -166,34 +163,38 @@ in }; }; - config = mkIf (cfg.instances != {}) ( + config = mkIf (cfg.instances != {} || cfg.databases != {}) ( let - enabledInstances = filterAttrs (k: i: i.enable) cfg.instances; + enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances; + enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases; in mkMerge [ { - environment.systemPackages = optionals (enabledInstances != {}) [ pkgs.restic ]; - + environment.systemPackages = optionals (enabledInstances != {} || enabledDatabases != {}) [ pkgs.restic ]; + } + { + # Create repository if it is a local path. systemd.tmpfiles.rules = let mkRepositorySettings = name: instance: repository: optionals (hasPrefix "/" repository.path) [ - "d '${repository.path}' 0750 ${instance.user} root - -" + "d '${repository.path}' 0750 ${instance.request.user} root - -" ]; - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; + mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories; in - flatten (mapAttrsToList mkSettings cfg.instances); - + flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases)); + } + { services.restic.backups = let mkRepositorySettings = name: instance: repository: { "${name}_${repoSlugName repository.path}" = { - inherit (instance) user; + inherit (instance.request) user; repository = repository.path; - paths = instance.sourceDirectories; + paths = instance.request.sourceDirectories; - passwordFile = toString instance.passphraseFile; + passwordFile = toString instance.settings.passphraseFile; initialize = true; @@ -201,28 +202,71 @@ in pruneOpts = mapAttrsToList (name: value: "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}" - ) instance.retention; + ) instance.settings.retention; - backupPrepareCommand = concatStringsSep "\n" instance.hooks.before_backup; + backupPrepareCommand = concatStringsSep "\n" instance.request.hooks.before_backup; - backupCleanupCommand = concatStringsSep "\n" instance.hooks.after_backup; - } // optionalAttrs (builtins.length instance.excludePatterns > 0) { - exclude = instance.excludePatterns; + backupCleanupCommand = concatStringsSep "\n" instance.request.hooks.after_backup; extraBackupArgs = - (optionals (instance.limitUploadKiBs != null) [ - "--limit-upload=${toString instance.limitUploadKiBs}" + (optionals (instance.settings.limitUploadKiBs != null) [ + "--limit-upload=${toString instance.settings.limitUploadKiBs}" ]) - ++ (optionals (instance.limitDownloadKiBs != null) [ - "--limit-download=${toString instance.limitDownloadKiBs}" + ++ (optionals (instance.settings.limitDownloadKiBs != null) [ + "--limit-download=${toString instance.settings.limitDownloadKiBs}" ]); + } // optionalAttrs (builtins.length instance.request.excludePatterns > 0) { + exclude = instance.request.excludePatterns; }; }; - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; + mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories; in mkMerge (flatten (mapAttrsToList mkSettings enabledInstances)); + } + { + services.restic.backups = + let + mkRepositorySettings = name: instance: repository: { + "${name}_${repoSlugName repository.path}" = { + inherit (instance.request) user; + repository = repository.path; + + dynamicFilesFrom = "echo"; + + passwordFile = toString instance.settings.passphraseFile; + + initialize = true; + + inherit (repository) timerConfig; + + pruneOpts = mapAttrsToList (name: value: + "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}" + ) instance.settings.retention; + + extraBackupArgs = + (optionals (instance.settings.limitUploadKiBs != null) [ + "--limit-upload=${toString instance.settings.limitUploadKiBs}" + ]) + ++ (optionals (instance.settings.limitDownloadKiBs != null) [ + "--limit-download=${toString instance.settings.limitDownloadKiBs}" + ]) + ++ + (let + cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd; + in + [ + "--stdin-filename ${instance.request.backupFile} --stdin-from-command -- ${cmd}/bin/dump.sh" + ]); + }; + }; + + mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories; + in + mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases)); + } + { systemd.services = let mkRepositorySettings = name: instance: repository: @@ -236,7 +280,7 @@ in Nice = cfg.performance.niceness; IOSchedulingClass = cfg.performance.ioSchedulingClass; IOSchedulingPriority = cfg.performance.ioPriority; - BindReadOnlyPaths = instance.sourceDirectories; + # BindReadOnlyPaths = instance.sourceDirectories; }; } (optionalAttrs (repository.secrets != {}) @@ -255,7 +299,7 @@ in config = repository.secrets; configLocation = "/run/secrets_restic/${serviceName}"; generator = name: v: pkgs.writeText "template" (generators.toINIWithGlobalSection {} { globalSection = v; }); - user = instance.user; + user = instance.request.user; }; in { @@ -264,9 +308,9 @@ in serviceConfig.LoadCredential = script.loadCredentials; }); }; - mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories; + mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories; in - mkMerge (flatten (mapAttrsToList mkSettings enabledInstances)); + mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases))); } { system.activationScripts = let @@ -274,17 +318,18 @@ in nameValuePair "${fullName name repository}_gen" (shblib.replaceSecrets { userConfig = repository.secrets // { - RESTIC_PASSWORD_FILE = instance.passphraseFile; + RESTIC_PASSWORD_FILE = instance.settings.passphraseFile; RESTIC_REPOSITORY = repository.path; }; resultPath = "/run/secrets_restic_env/${fullName name repository}"; generator = name: v: pkgs.writeText (fullName name repository) (generators.toINIWithGlobalSection {} { globalSection = v; }); - user = instance.user; + user = instance.request.user; }); - mkSettings = name: instance: builtins.map (mkEnv name instance) instance.repositories; + mkSettings = name: instance: builtins.map (mkEnv name instance) instance.settings.repositories; in - listToAttrs (flatten (mapAttrsToList mkSettings cfg.instances)); - + listToAttrs (flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases))); + } + { environment.systemPackages = let mkResticBinary = name: instance: repository: pkgs.writeShellScriptBin (fullName name repository) '' @@ -292,9 +337,33 @@ in | xargs -d '\n') ${pkgs.restic}/bin/restic $@ ''; - mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories; + mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.settings.repositories; in flatten (mapAttrsToList mkSettings cfg.instances); } + { + environment.systemPackages = let + mkResticBinary = name: instance: repository: + pkgs.writeShellScriptBin (fullName name repository) '' + set -euo pipefail + + ls /run/secrets_restic_env/${fullName name repository} + + export $(grep -v '^#' "/run/secrets_restic_env/${fullName name repository}" \ + | xargs -d '\n') + + set -x + + if ! [ "$1" = "restore" ]; then + sudo -u ${instance.request.user} ${pkgs.restic}/bin/restic $@ + else + shift + sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupFile} | ${instance.request.restoreCmd}" + fi + ''; + mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.settings.repositories; + in + flatten (mapAttrsToList mkSettings cfg.databases); + } ]); } diff --git a/modules/contracts/databasebackup.nix b/modules/contracts/databasebackup.nix new file mode 100644 index 0000000..b7ed733 --- /dev/null +++ b/modules/contracts/databasebackup.nix @@ -0,0 +1,45 @@ +{ lib, ... }: +let + inherit (lib) mkOption; + inherit (lib.types) anything submodule str; +in +{ + request = submodule { + freeformType = anything; + options = { + user = mkOption { + description = "Unix user doing the backups."; + type = str; + }; + + backupFile = mkOption { + description = "Filename of the backup."; + type = str; + }; + + backupCmd = mkOption { + description = "Command that produces the database dump on stdout."; + type = str; + }; + + restoreCmd = mkOption { + description = "Command that reads the database dump on stdin and restores the database."; + type = str; + }; + }; + }; + + result = submodule { + options = { + restoreScript = mkOption { + description = "Name of script that can restore the database."; + type = str; + }; + + backupService = mkOption { + description = "Name of service backing up the database."; + type = str; + }; + }; + }; +} diff --git a/modules/contracts/default.nix b/modules/contracts/default.nix index c19f509..44079f7 100644 --- a/modules/contracts/default.nix +++ b/modules/contracts/default.nix @@ -1,5 +1,6 @@ { pkgs, lib }: { + databasebackup = import ./databasebackup.nix { inherit lib; }; backup = import ./backup.nix { inherit lib; }; mount = import ./mount.nix { inherit lib; }; secret = import ./secret.nix { inherit lib; }; diff --git a/test/blocks/postgresql.nix b/test/blocks/postgresql.nix index 9bd0163..ab05a79 100644 --- a/test/blocks/postgresql.nix +++ b/test/blocks/postgresql.nix @@ -179,4 +179,91 @@ in machine.fail(tcpip_cmd("me", "me", "5432", "oops"), timeout=10) ''; }; + + backup = pkgs.testers.runNixOSTest { + name = "postgresql-backup"; + + nodes.machine = { config, pkgs, ... }: { + imports = [ + (pkgs'.path + "/nixos/modules/profiles/headless.nix") + (pkgs'.path + "/nixos/modules/profiles/qemu-guest.nix") + ../../modules/blocks/postgresql.nix + ../../modules/blocks/restic.nix + ]; + + users.users.me = { + isSystemUser = true; + group = "me"; + extraGroups = [ "sudoers" ]; + }; + users.groups.me = {}; + + services.postgresql.enable = true; + shb.postgresql.ensures = [ + { + username = "me"; + database = "me"; + } + ]; + shb.restic.databases."postgres".request = config.shb.postgresql.backup; + shb.restic.databases."postgres".settings = { + enable = true; + + passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase"); + repositories = [ + { + path = "/opt/repos/postgres"; + timerConfig = { + OnCalendar = "00:00:00"; + }; + } + ]; + }; + }; + + testScript = { nodes, ... }: '' + import csv + + start_all() + machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) + + def peer_cmd(cmd, db="me"): + return "sudo -u me psql -U me {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db) + + def query(query): + res = machine.succeed(peer_cmd(query)) + return list(dict(l) for l in csv.DictReader(res.splitlines())) + + def cmp_tables(a, b): + for i in range(max(len(a), len(b))): + diff = set(a[i]) ^ set(b[i]) + if len(diff) > 0: + raise Exception(i, diff) + + table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}] + + with subtest("create fixture"): + machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)")) + machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)")) + + res = query("SELECT * FROM test") + cmp_tables(res, table) + + with subtest("backup"): + print(machine.succeed("systemctl cat restic-backups-postgres_opt_repos_postgres.service")) + machine.succeed("systemctl start restic-backups-postgres_opt_repos_postgres.service") + + with subtest("drop database"): + machine.succeed(peer_cmd("DROP DATABASE me", db="postgres")) + + with subtest("restore"): + print(machine.succeed("readlink -f $(type restic-backups-postgres_opt_repos_postgres)")) + machine.succeed("restic-backups-postgres_opt_repos_postgres restore latest ") + + with subtest("check restoration"): + res = query("SELECT * FROM test") + cmp_tables(res, table) + ''; + }; }