add test for postgres backup

modules/blocks/postgresql.nix

@@ -51,7 +51,7 @@ in
     };
 
     backup = lib.mkOption {
-      type = contracts.backup;
+      type = contracts.databasebackup.request;
       description = ''
         Backup configuration. This is an output option.
 
@@ -63,32 +63,22 @@ in
           enable = true;
 
           # Options specific to Restic.
-        } // config.shb.nextcloud.backup;
+        } // config.shb.postgresl.backup;
         ```
       '';
       readOnly = true;
       default = {
-        user = "postgresql";
+        user = "postgres";
-        sourceDirectories = [
-          /tmp/postgresql_backup
-        ];
-        excludePatterns = [ ];
 
-        hooks.before_backup = [''
+        backupFile = "postgres.sql";
-          set -e -o pipefail
 
-          umask 077 # Ensure backup is only readable by postgres user
+        backupCmd = ''
+            ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
+          '';
 
-          rm -rf /tmp/postgresql_backup # Clean up in case after_backup hook wasn't run.
+        restoreCmd = ''
-          mkdir /tmp/postgresql_backup
+            ${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres
-
+          '';
-          ${pkgs.psql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable > /tmp/postgresql_backup/pg_dumpall.sql.gz
-        ''];
-
-        hooks.after_backup = [''
-          set -e -o pipefail
-          rm -rf /tmp/postgresql_backup
-        ''];
       };
     };
 

modules/blocks/restic.nix

@@ -4,125 +4,90 @@ let
   cfg = config.shb.restic;
 
   shblib = pkgs.callPackage ../../lib {};
+  contracts = pkgs.callPackage ../contracts {};
 
   inherit (lib) concatStringsSep filterAttrs flatten literalExpression optionals optionalString listToAttrs mapAttrsToList mkEnableOption mkOption mkMerge;
   inherit (lib) generators hasPrefix mkIf nameValuePair optionalAttrs removePrefix;
   inherit (lib.types) attrsOf enum int ints listOf oneOf nonEmptyListOf nonEmptyStr nullOr path str submodule;
 
-  instanceOptions = {
+  commonOptions = submodule {
-    enable = mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots";
+    options = {
+      enable = mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots";
 
-    passphraseFile = mkOption {
+      passphraseFile = mkOption {
-      description = "Encryption key for the backups.";
+        description = "Encryption key for the backups.";
-      type = path;
+        type = path;
-    };
+      };
 
-    user = mkOption {
+      repositories = mkOption {
-      description = ''
+        description = "Repositories to back this instance to.";
-        Unix user doing the backups. Must be the user owning the files to be backed up.
+        type = nonEmptyListOf (submodule {
-      '';
+          options = {
-      type = str;
+            path = mkOption {
-    };
+              type = str;
+              description = "Repository location";
+            };
 
-    sourceDirectories = mkOption {
+            secrets = mkOption {
-      description = "Source directories.";
+              type = attrsOf shblib.secretFileType;
-      type = nonEmptyListOf str;
+              default = {};
-    };
+              description = ''
+                Secrets needed to access the repository where the backups will be stored.
 
-    excludePatterns = mkOption {
+                See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
-      description = "Exclude patterns.";
+                and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
-      type = listOf str;
-      default = [];
-    };
-
-    repositories = mkOption {
-      description = "Repositories to back this instance to.";
-      type = nonEmptyListOf (submodule {
-        options = {
-          path = mkOption {
-            type = str;
-            description = "Repository location";
-          };
-
-          secrets = mkOption {
-            type = attrsOf shblib.secretFileType;
-            default = {};
-            description = ''
-              Secrets needed to access the repository where the backups will be stored.
-
-              See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
-              and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
 
               '';
-            example = literalExpression ''
+              example = literalExpression ''
-              {
+                {
-                AWS_ACCESS_KEY_ID = <path/to/secret>;
+                  AWS_ACCESS_KEY_ID = <path/to/secret>;
-                AWS_SECRET_ACCESS_KEY = <path/to/secret>;
+                  AWS_SECRET_ACCESS_KEY = <path/to/secret>;
-              }
+                }
               '';
-          };
-
-          timerConfig = mkOption {
-            type = attrsOf utils.systemdUtils.unitOptions.unitOption;
-            default = {
-              OnCalendar = "daily";
-              Persistent = true;
             };
-            description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
+
-            example = {
+            timerConfig = mkOption {
-              OnCalendar = "00:05";
+              type = attrsOf utils.systemdUtils.unitOptions.unitOption;
-              RandomizedDelaySec = "5h";
+              default = {
-              Persistent = true;
+                OnCalendar = "daily";
+                Persistent = true;
+              };
+              description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
+              example = {
+                OnCalendar = "00:05";
+                RandomizedDelaySec = "5h";
+                Persistent = true;
+              };
             };
           };
-        };
+        });
-      });
-    };
-
-    retention = mkOption {
-      description = "For how long to keep backup files.";
-      type = attrsOf (oneOf [ int nonEmptyStr ]);
-      default = {
-        keep_within = "1d";
-        keep_hourly = 24;
-        keep_daily = 7;
-        keep_weekly = 4;
-        keep_monthly = 6;
       };
-    };
 
-    hooks = mkOption {
+      retention = mkOption {
-      description = "Hooks to run before or after the backup.";
+        description = "For how long to keep backup files.";
-      default = {};
+        type = attrsOf (oneOf [ int nonEmptyStr ]);
-      type = submodule {
+        default = {
-        options = {
+          keep_within = "1d";
-          before_backup = mkOption {
+          keep_hourly = 24;
-            description = "Hooks to run before backup";
+          keep_daily = 7;
-            type = listOf str;
+          keep_weekly = 4;
-            default = [];
+          keep_monthly = 6;
-          };
-
-          after_backup = mkOption {
-            description = "Hooks to run after backup";
-            type = listOf str;
-            default = [];
-          };
         };
       };
-    };
 
-    limitUploadKiBs = mkOption {
+      limitUploadKiBs = mkOption {
-      type = nullOr int;
+        type = nullOr int;
-      description = "Limit upload bandwidth to the given KiB/s amount.";
+        description = "Limit upload bandwidth to the given KiB/s amount.";
-      default = null;
+        default = null;
-      example = 8000;
+        example = 8000;
-    };
+      };
 
-    limitDownloadKiBs = mkOption {
+      limitDownloadKiBs = mkOption {
-      type = nullOr int;
+        type = nullOr int;
-      description = "Limit download bandwidth to the given KiB/s amount.";
+        description = "Limit download bandwidth to the given KiB/s amount.";
-      default = null;
+        default = null;
-      example = 8000;
+        example = 8000;
+      };
     };
   };
 
@@ -133,13 +98,45 @@ in
 {
   options.shb.restic = {
     instances = mkOption {
-      description = "Each instance is a backup setting";
+      description = "Each instance is backing up some directories to one or more repositories.";
       default = {};
       type = attrsOf (submodule {
-        options = instanceOptions;
+        options = {
+          request = mkOption {
+            type = contracts.backup.request;
+          };
+
+          settings = mkOption {
+            type = commonOptions;
+          };
+        };
       });
     };
 
+    databases = mkOption {
+      description = "Each item is backing up some database to one or more repositories.";
+      default = {};
+      type = attrsOf (submodule ({ options, ... }: {
+        options = {
+          request = mkOption {
+            type = contracts.databasebackup.request;
+          };
+
+          settings = mkOption {
+            type = commonOptions;
+          };
+
+          result = mkOption {
+            type = contracts.databasebackup.result;
+            default = {
+              restoreScript = fullName options.name options.repository;
+              backupService = "${fullName options.name options.repository}.service";
+            };
+          };
+        };
+      }));
+    };
+
     # Taken from https://github.com/HubbeKing/restic-kubernetes/blob/73bfbdb0ba76939a4c52173fa2dbd52070710008/README.md?plain=1#L23
     performance = mkOption {
       description = "Reduce performance impact of backup jobs.";
@@ -166,34 +163,38 @@ in
     };
   };
 
-  config = mkIf (cfg.instances != {}) (
+  config = mkIf (cfg.instances != {} || cfg.databases != {}) (
     let
-      enabledInstances = filterAttrs (k: i: i.enable) cfg.instances;
+      enabledInstances = filterAttrs (k: i: i.settings.enable) cfg.instances;
+      enabledDatabases = filterAttrs (k: i: i.settings.enable) cfg.databases;
     in mkMerge [
       {
-        environment.systemPackages = optionals (enabledInstances != {}) [ pkgs.restic ];
+        environment.systemPackages = optionals (enabledInstances != {} || enabledDatabases != {}) [ pkgs.restic ];
-
+      }
+      {
+        # Create repository if it is a local path.
         systemd.tmpfiles.rules =
           let
             mkRepositorySettings = name: instance: repository: optionals (hasPrefix "/" repository.path) [
-              "d '${repository.path}' 0750 ${instance.user} root - -"
+              "d '${repository.path}' 0750 ${instance.request.user} root - -"
             ];
 
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
-            flatten (mapAttrsToList mkSettings cfg.instances);
+            flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
-
+      }
+      {
         services.restic.backups =
           let
             mkRepositorySettings = name: instance: repository: {
               "${name}_${repoSlugName repository.path}" = {
-                inherit (instance) user;
+                inherit (instance.request) user;
 
                 repository = repository.path;
 
-                paths = instance.sourceDirectories;
+                paths = instance.request.sourceDirectories;
 
-                passwordFile = toString instance.passphraseFile;
+                passwordFile = toString instance.settings.passphraseFile;
 
                 initialize = true;
 
@@ -201,28 +202,71 @@ in
 
                 pruneOpts = mapAttrsToList (name: value:
                   "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
-                ) instance.retention;
+                ) instance.settings.retention;
 
-                backupPrepareCommand = concatStringsSep "\n" instance.hooks.before_backup;
+                backupPrepareCommand = concatStringsSep "\n" instance.request.hooks.before_backup;
 
-                backupCleanupCommand = concatStringsSep "\n" instance.hooks.after_backup;
+                backupCleanupCommand = concatStringsSep "\n" instance.request.hooks.after_backup;
-              } // optionalAttrs (builtins.length instance.excludePatterns > 0) {
-                exclude = instance.excludePatterns;
 
                 extraBackupArgs =
-                  (optionals (instance.limitUploadKiBs != null) [
+                  (optionals (instance.settings.limitUploadKiBs != null) [
-                    "--limit-upload=${toString instance.limitUploadKiBs}"
+                    "--limit-upload=${toString instance.settings.limitUploadKiBs}"
                   ])
-                  ++ (optionals (instance.limitDownloadKiBs != null) [
+                  ++ (optionals (instance.settings.limitDownloadKiBs != null) [
-                    "--limit-download=${toString instance.limitDownloadKiBs}"
+                    "--limit-download=${toString instance.settings.limitDownloadKiBs}"
                   ]);
+              } // optionalAttrs (builtins.length instance.request.excludePatterns > 0) {
+                exclude = instance.request.excludePatterns;
               };
             };
 
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
             mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
+      }
+      {
+        services.restic.backups =
+          let
+            mkRepositorySettings = name: instance: repository: {
+              "${name}_${repoSlugName repository.path}" = {
+                inherit (instance.request) user;
 
+                repository = repository.path;
+
+                dynamicFilesFrom = "echo";
+
+                passwordFile = toString instance.settings.passphraseFile;
+
+                initialize = true;
+
+                inherit (repository) timerConfig;
+
+                pruneOpts = mapAttrsToList (name: value:
+                  "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
+                ) instance.settings.retention;
+
+                extraBackupArgs =
+                  (optionals (instance.settings.limitUploadKiBs != null) [
+                    "--limit-upload=${toString instance.settings.limitUploadKiBs}"
+                  ])
+                  ++ (optionals (instance.settings.limitDownloadKiBs != null) [
+                    "--limit-download=${toString instance.settings.limitDownloadKiBs}"
+                  ])
+                  ++ 
+                  (let
+                    cmd = pkgs.writeShellScriptBin "dump.sh" instance.request.backupCmd;
+                  in
+                    [
+                      "--stdin-filename ${instance.request.backupFile} --stdin-from-command -- ${cmd}/bin/dump.sh"
+                    ]);
+              };
+            };
+
+            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
+          in
+            mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
+      }
+      {
         systemd.services =
           let
             mkRepositorySettings = name: instance: repository:
@@ -236,7 +280,7 @@ in
                         Nice = cfg.performance.niceness;
                         IOSchedulingClass = cfg.performance.ioSchedulingClass;
                         IOSchedulingPriority = cfg.performance.ioPriority;
-                        BindReadOnlyPaths = instance.sourceDirectories;
+                        # BindReadOnlyPaths = instance.sourceDirectories;
                       };
                     }
                     (optionalAttrs (repository.secrets != {})
@@ -255,7 +299,7 @@ in
                         config = repository.secrets;
                         configLocation = "/run/secrets_restic/${serviceName}";
                         generator = name: v: pkgs.writeText "template" (generators.toINIWithGlobalSection {} { globalSection = v; });
-                        user = instance.user;
+                        user = instance.request.user;
                       };
                     in
                       {
@@ -264,9 +308,9 @@ in
                         serviceConfig.LoadCredential = script.loadCredentials;
                       });
             };
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.repositories;
+            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
-            mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
+            mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
       }
       {
         system.activationScripts = let
@@ -274,17 +318,18 @@ in
             nameValuePair "${fullName name repository}_gen"
               (shblib.replaceSecrets {
                 userConfig = repository.secrets // {
-                  RESTIC_PASSWORD_FILE = instance.passphraseFile;
+                  RESTIC_PASSWORD_FILE = instance.settings.passphraseFile;
                   RESTIC_REPOSITORY = repository.path;
                 };
                 resultPath = "/run/secrets_restic_env/${fullName name repository}";
                 generator = name: v: pkgs.writeText (fullName name repository) (generators.toINIWithGlobalSection {} { globalSection = v; });
-                user = instance.user;
+                user = instance.request.user;
               });
-          mkSettings = name: instance: builtins.map (mkEnv name instance) instance.repositories;
+          mkSettings = name: instance: builtins.map (mkEnv name instance) instance.settings.repositories;
         in
-          listToAttrs (flatten (mapAttrsToList mkSettings cfg.instances));
+          listToAttrs (flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases)));
-
+      }
+      {
         environment.systemPackages = let
           mkResticBinary = name: instance: repository:
             pkgs.writeShellScriptBin (fullName name repository) ''
@@ -292,9 +337,33 @@ in
                        | xargs -d '\n')
               ${pkgs.restic}/bin/restic $@
               '';
-          mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.repositories;
+          mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.settings.repositories;
         in
           flatten (mapAttrsToList mkSettings cfg.instances);
       }
+      {
+        environment.systemPackages = let
+          mkResticBinary = name: instance: repository:
+            pkgs.writeShellScriptBin (fullName name repository) ''
+              set -euo pipefail
+
+              ls /run/secrets_restic_env/${fullName name repository}
+
+              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name repository}" \
+                       | xargs -d '\n')
+
+              set -x
+
+              if ! [ "$1" = "restore" ]; then
+                sudo -u ${instance.request.user} ${pkgs.restic}/bin/restic $@
+              else
+                shift
+                sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupFile} | ${instance.request.restoreCmd}"
+              fi
+              '';
+          mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.settings.repositories;
+        in
+          flatten (mapAttrsToList mkSettings cfg.databases);
+      }
     ]);
 }

modules/contracts/databasebackup.nix

@@ -0,0 +1,45 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption;
+  inherit (lib.types) anything submodule str;
+in
+{
+  request = submodule {
+    freeformType = anything;
+    options = {
+      user = mkOption {
+        description = "Unix user doing the backups.";
+        type = str;
+      };
+
+      backupFile = mkOption {
+        description = "Filename of the backup.";
+        type = str;
+      };
+
+      backupCmd = mkOption {
+        description = "Command that produces the database dump on stdout.";
+        type = str;
+      };
+
+      restoreCmd = mkOption {
+        description = "Command that reads the database dump on stdin and restores the database.";
+        type = str;
+      };
+    };
+  };
+
+  result = submodule {
+    options = {
+      restoreScript = mkOption {
+        description = "Name of script that can restore the database.";
+        type = str;
+      };
+
+      backupService = mkOption {
+        description = "Name of service backing up the database.";
+        type = str;
+      };
+    };
+  };
+}

modules/contracts/default.nix

@@ -1,5 +1,6 @@
 { pkgs, lib }:
 {
+  databasebackup = import ./databasebackup.nix { inherit lib; };
   backup = import ./backup.nix { inherit lib; };
   mount = import ./mount.nix { inherit lib; };
   secret = import ./secret.nix { inherit lib; };

test/blocks/postgresql.nix

@@ -179,4 +179,91 @@ in
         machine.fail(tcpip_cmd("me", "me", "5432", "oops"), timeout=10)
     '';
   };
+
+  backup = pkgs.testers.runNixOSTest {
+    name = "postgresql-backup";
+
+    nodes.machine = { config, pkgs, ... }: {
+      imports = [
+        (pkgs'.path + "/nixos/modules/profiles/headless.nix")
+        (pkgs'.path + "/nixos/modules/profiles/qemu-guest.nix")
+        ../../modules/blocks/postgresql.nix
+        ../../modules/blocks/restic.nix
+      ];
+
+      users.users.me = {
+        isSystemUser = true;
+        group = "me";
+        extraGroups = [ "sudoers" ];
+      };
+      users.groups.me = {};
+
+      services.postgresql.enable = true;
+      shb.postgresql.ensures = [
+        {
+          username = "me";
+          database = "me";
+        }
+      ];
+      shb.restic.databases."postgres".request = config.shb.postgresql.backup;
+      shb.restic.databases."postgres".settings = {
+        enable = true;
+
+        passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
+        repositories = [
+          {
+            path = "/opt/repos/postgres";
+            timerConfig = {
+              OnCalendar = "00:00:00";
+            };
+          }
+        ];
+      };
+    };
+
+    testScript = { nodes, ... }: ''
+    import csv
+
+    start_all()
+    machine.wait_for_unit("postgresql.service")
+    machine.wait_for_open_port(5432)
+
+    def peer_cmd(cmd, db="me"):
+        return "sudo -u me psql -U me {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
+
+    def query(query):
+        res = machine.succeed(peer_cmd(query))
+        return list(dict(l) for l in csv.DictReader(res.splitlines()))
+
+    def cmp_tables(a, b):
+        for i in range(max(len(a), len(b))):
+            diff = set(a[i]) ^ set(b[i])
+            if len(diff) > 0:
+                raise Exception(i, diff)
+
+    table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+
+    with subtest("create fixture"):
+        machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
+        machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
+
+        res = query("SELECT * FROM test")
+        cmp_tables(res, table)
+
+    with subtest("backup"):
+        print(machine.succeed("systemctl cat restic-backups-postgres_opt_repos_postgres.service"))
+        machine.succeed("systemctl start restic-backups-postgres_opt_repos_postgres.service")
+
+    with subtest("drop database"):
+        machine.succeed(peer_cmd("DROP DATABASE me", db="postgres"))
+
+    with subtest("restore"):
+        print(machine.succeed("readlink -f $(type restic-backups-postgres_opt_repos_postgres)"))
+        machine.succeed("restic-backups-postgres_opt_repos_postgres restore latest ")
+
+    with subtest("check restoration"):
+        res = query("SELECT * FROM test")
+        cmp_tables(res, table)
+    '';
+  };
 }