use contract test for databasebackup

flake.nix

@@ -136,6 +136,7 @@
           // (vm_test "restic" ./test/blocks/restic.nix)
           // (vm_test "ssl" ./test/blocks/ssl.nix)
 
+          // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)
           // (vm_test "contracts-secret" ./test/contracts/secret.nix)
           ));
       }

modules/blocks/postgresql.nix

@@ -73,12 +73,12 @@ in
         backupFile = "postgres.sql";
 
         backupCmd = ''
-            ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
+          ${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
-          '';
+        '';
 
         restoreCmd = ''
-            ${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres
+          ${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres
-          '';
+        '';
       };
     };
 

modules/blocks/restic.nix

@@ -19,9 +19,9 @@ let
         type = path;
       };
 
-      repositories = mkOption {
+      repository = mkOption {
         description = "Repositories to back this instance to.";
-        type = nonEmptyListOf (submodule {
+        type = submodule {
           options = {
             path = mkOption {
               type = str;
@@ -60,7 +60,7 @@ let
               };
             };
           };
-        });
+        };
       };
 
       retention = mkOption {
@@ -98,9 +98,9 @@ in
 {
   options.shb.restic = {
     instances = mkOption {
-      description = "Each instance is backing up some directories to one or more repositories.";
+      description = "Each instance is backing up some directories to one repository.";
       default = {};
-      type = attrsOf (submodule {
+      type = attrsOf (submodule ({ name, config, ... }: {
         options = {
           request = mkOption {
             type = contracts.backup.request;
@@ -109,14 +109,22 @@ in
           settings = mkOption {
             type = commonOptions;
           };
+
+          result = mkOption {
+            type = contracts.databasebackup.result;
+            default = {
+              restoreScript = fullName name config.settings.repository;
+              backupService = "${fullName name config.settings.repository}.service";
+            };
+          };
         };
-      });
+      }));
     };
 
     databases = mkOption {
-      description = "Each item is backing up some database to one or more repositories.";
+      description = "Each item is backing up some database to one repository.";
       default = {};
-      type = attrsOf (submodule ({ options, ... }: {
+      type = attrsOf (submodule ({ name, config, ... }: {
         options = {
           request = mkOption {
             type = contracts.databasebackup.request;
@@ -129,8 +137,8 @@ in
           result = mkOption {
             type = contracts.databasebackup.result;
             default = {
-              restoreScript = fullName options.name options.repository;
+              restoreScript = fullName name config.settings.repository;
-              backupService = "${fullName options.name options.repository}.service";
+              backupService = "${fullName name config.settings.repository}.service";
             };
           };
         };
@@ -175,22 +183,20 @@ in
         # Create repository if it is a local path.
         systemd.tmpfiles.rules =
           let
-            mkRepositorySettings = name: instance: repository: optionals (hasPrefix "/" repository.path) [
+            mkSettings = name: instance: optionals (hasPrefix "/" instance.settings.repository.path) [
-              "d '${repository.path}' 0750 ${instance.request.user} root - -"
+              "d '${instance.settings.repository.path}' 0750 ${instance.request.user} root - -"
             ];
-
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
             flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases));
       }
       {
         services.restic.backups =
           let
-            mkRepositorySettings = name: instance: repository: {
+            mkSettings = name: instance: {
-              "${name}_${repoSlugName repository.path}" = {
+              "${name}_${repoSlugName instance.settings.repository.path}" = {
                 inherit (instance.request) user;
 
-                repository = repository.path;
+                repository = instance.settings.repository.path;
 
                 paths = instance.request.sourceDirectories;
 
@@ -198,7 +204,7 @@ in
 
                 initialize = true;
 
-                inherit (repository) timerConfig;
+                inherit (instance.settings.repository) timerConfig;
 
                 pruneOpts = mapAttrsToList (name: value:
                   "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
@@ -219,19 +225,17 @@ in
                 exclude = instance.request.excludePatterns;
               };
             };
-
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
             mkMerge (flatten (mapAttrsToList mkSettings enabledInstances));
       }
       {
         services.restic.backups =
           let
-            mkRepositorySettings = name: instance: repository: {
+            mkSettings = name: instance: {
-              "${name}_${repoSlugName repository.path}" = {
+              "${name}_${repoSlugName instance.settings.repository.path}" = {
                 inherit (instance.request) user;
 
-                repository = repository.path;
+                repository = instance.settings.repository.path;
 
                 dynamicFilesFrom = "echo";
 
@@ -239,7 +243,7 @@ in
 
                 initialize = true;
 
-                inherit (repository) timerConfig;
+                inherit (instance.settings.repository) timerConfig;
 
                 pruneOpts = mapAttrsToList (name: value:
                   "--${builtins.replaceStrings ["_"] ["-"] name} ${builtins.toString value}"
@@ -261,17 +265,15 @@ in
                     ]);
               };
             };
-
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
             mkMerge (flatten (mapAttrsToList mkSettings enabledDatabases));
       }
       {
         systemd.services =
           let
-            mkRepositorySettings = name: instance: repository:
+            mkSettings = name: instance:
               let
-                serviceName = fullName name repository;
+                serviceName = fullName name instance.settings.repository;
               in
                 {
                   ${serviceName} = mkMerge [
@@ -283,7 +285,7 @@ in
                         # BindReadOnlyPaths = instance.sourceDirectories;
                       };
                     }
-                    (optionalAttrs (repository.secrets != {})
+                    (optionalAttrs (instance.settings.repository.secrets != {})
                       {
                         serviceConfig.EnvironmentFile = [
                           "/run/secrets_restic/${serviceName}"
@@ -293,10 +295,10 @@ in
                       })
                   ];
 
-                  "${serviceName}-pre" = mkIf (repository.secrets != {})
+                  "${serviceName}-pre" = mkIf (instance.settings.repository.secrets != {})
                     (let
                       script = shblib.genConfigOutOfBandSystemd {
-                        config = repository.secrets;
+                        config = instance.settings.repository.secrets;
                         configLocation = "/run/secrets_restic/${serviceName}";
                         generator = name: v: pkgs.writeText "template" (generators.toINIWithGlobalSection {} { globalSection = v; });
                         user = instance.request.user;
@@ -308,48 +310,45 @@ in
                         serviceConfig.LoadCredential = script.loadCredentials;
                       });
             };
-            mkSettings = name: instance: builtins.map (mkRepositorySettings name instance) instance.settings.repositories;
           in
             mkMerge (flatten (mapAttrsToList mkSettings (enabledInstances // enabledDatabases)));
       }
       {
         system.activationScripts = let
-          mkEnv = name: instance: repository:
+          mkEnv = name: instance:
-            nameValuePair "${fullName name repository}_gen"
+            nameValuePair "${fullName name instance.settings.repository}_gen"
               (shblib.replaceSecrets {
-                userConfig = repository.secrets // {
+                userConfig = instance.settings.repository.secrets // {
                   RESTIC_PASSWORD_FILE = instance.settings.passphraseFile;
-                  RESTIC_REPOSITORY = repository.path;
+                  RESTIC_REPOSITORY = instance.settings.repository.path;
                 };
-                resultPath = "/run/secrets_restic_env/${fullName name repository}";
+                resultPath = "/run/secrets_restic_env/${fullName name instance.settings.repository}";
-                generator = name: v: pkgs.writeText (fullName name repository) (generators.toINIWithGlobalSection {} { globalSection = v; });
+                generator = name: v: pkgs.writeText (fullName name instance.settings.repository) (generators.toINIWithGlobalSection {} { globalSection = v; });
                 user = instance.request.user;
               });
-          mkSettings = name: instance: builtins.map (mkEnv name instance) instance.settings.repositories;
         in
-          listToAttrs (flatten (mapAttrsToList mkSettings (cfg.instances // cfg.databases)));
+          listToAttrs (flatten (mapAttrsToList mkEnv (cfg.instances // cfg.databases)));
       }
       {
         environment.systemPackages = let
-          mkResticBinary = name: instance: repository:
+          mkResticBinary = name: instance:
-            pkgs.writeShellScriptBin (fullName name repository) ''
+            pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
-              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name repository}" \
+              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
                        | xargs -d '\n')
               ${pkgs.restic}/bin/restic $@
               '';
-          mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.settings.repositories;
         in
-          flatten (mapAttrsToList mkSettings cfg.instances);
+          flatten (mapAttrsToList mkResticBinary cfg.instances);
       }
       {
         environment.systemPackages = let
-          mkResticBinary = name: instance: repository:
+          mkResticBinary = name: instance:
-            pkgs.writeShellScriptBin (fullName name repository) ''
+            pkgs.writeShellScriptBin (fullName name instance.settings.repository) ''
               set -euo pipefail
 
-              ls /run/secrets_restic_env/${fullName name repository}
+              ls /run/secrets_restic_env/${fullName name instance.settings.repository}
 
-              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name repository}" \
+              export $(grep -v '^#' "/run/secrets_restic_env/${fullName name instance.settings.repository}" \
                        | xargs -d '\n')
 
               set -x
@@ -361,9 +360,8 @@ in
                 sudo --preserve-env -u ${instance.request.user} sh -c "${pkgs.restic}/bin/restic dump $@ ${instance.request.backupFile} | ${instance.request.restoreCmd}"
               fi
               '';
-          mkSettings = name: instance: builtins.map (mkResticBinary name instance) instance.settings.repositories;
         in
-          flatten (mapAttrsToList mkSettings cfg.databases);
+          flatten (mapAttrsToList mkResticBinary cfg.databases);
       }
     ]);
 }

modules/contracts/databasebackup/test.nix

@@ -0,0 +1,84 @@
+{ pkgs, lib, ... }:
+let
+  pkgs' = pkgs;
+
+  testLib = pkgs.callPackage ../../../test/common.nix {};
+
+  inherit (lib) getAttrFromPath mkIf optionalAttrs setAttrByPath;
+in
+{ name,
+  requesterRoot,
+  providerRoot,
+  providerExtraConfig ? null, # { username, database } -> attrset
+  modules ? [],
+  username ? "me",
+  database ? "me",
+  settings, # repository -> attrset
+}: pkgs.testers.runNixOSTest {
+  inherit name;
+
+  nodes.machine = { config, ... }: {
+    imports = ( testLib.baseImports pkgs' ) ++ modules;
+    config = lib.mkMerge [
+      (setAttrByPath providerRoot {
+        request = (getAttrFromPath requesterRoot config).backup;
+        settings = settings "/opt/repos/database";
+      })
+      (mkIf (username != "root") {
+        users.users.${username} = {
+          isSystemUser = true;
+          extraGroups = [ "sudoers" ];
+          group = "root";
+        };
+      })
+      (optionalAttrs (providerExtraConfig != null) (providerExtraConfig { inherit username database; }))
+    ];
+  };
+
+  testScript = { nodes, ... }: let
+    provider = (getAttrFromPath providerRoot nodes.machine).result;
+  in ''
+    import csv
+
+    start_all()
+    machine.wait_for_unit("postgresql.service")
+    machine.wait_for_open_port(5432)
+
+    def peer_cmd(cmd, db="me"):
+        return "sudo -u me psql -U me {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
+
+    def query(query):
+        res = machine.succeed(peer_cmd(query))
+        return list(dict(l) for l in csv.DictReader(res.splitlines()))
+
+    def cmp_tables(a, b):
+        for i in range(max(len(a), len(b))):
+            diff = set(a[i]) ^ set(b[i])
+            if len(diff) > 0:
+                raise Exception(i, diff)
+
+    table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
+
+    with subtest("create fixture"):
+        machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
+        machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
+
+        res = query("SELECT * FROM test")
+        cmp_tables(res, table)
+
+    with subtest("backup"):
+        print(machine.succeed("systemctl cat ${provider.backupService}"))
+        machine.succeed("systemctl start ${provider.backupService}")
+
+    with subtest("drop database"):
+        machine.succeed(peer_cmd("DROP DATABASE me", db="postgres"))
+
+    with subtest("restore"):
+        print(machine.succeed("readlink -f $(type ${provider.restoreScript})"))
+        machine.succeed("${provider.restoreScript} restore latest ")
+
+    with subtest("check restoration"):
+        res = query("SELECT * FROM test")
+        cmp_tables(res, table)
+  '';
+}

modules/contracts/default.nix

@@ -7,5 +7,6 @@
   ssl = import ./ssl.nix { inherit lib; };
   test = {
     secret = import ./secret/test.nix { inherit pkgs lib; };
+    databasebackup = import ./databasebackup/test.nix { inherit pkgs lib; };
   };
 }

test/blocks/postgresql.nix

@@ -179,91 +179,4 @@ in
         machine.fail(tcpip_cmd("me", "me", "5432", "oops"), timeout=10)
     '';
   };
-
-  backup = pkgs.testers.runNixOSTest {
-    name = "postgresql-backup";
-
-    nodes.machine = { config, pkgs, ... }: {
-      imports = [
-        (pkgs'.path + "/nixos/modules/profiles/headless.nix")
-        (pkgs'.path + "/nixos/modules/profiles/qemu-guest.nix")
-        ../../modules/blocks/postgresql.nix
-        ../../modules/blocks/restic.nix
-      ];
-
-      users.users.me = {
-        isSystemUser = true;
-        group = "me";
-        extraGroups = [ "sudoers" ];
-      };
-      users.groups.me = {};
-
-      services.postgresql.enable = true;
-      shb.postgresql.ensures = [
-        {
-          username = "me";
-          database = "me";
-        }
-      ];
-      shb.restic.databases."postgres".request = config.shb.postgresql.backup;
-      shb.restic.databases."postgres".settings = {
-        enable = true;
-
-        passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
-        repositories = [
-          {
-            path = "/opt/repos/postgres";
-            timerConfig = {
-              OnCalendar = "00:00:00";
-            };
-          }
-        ];
-      };
-    };
-
-    testScript = { nodes, ... }: ''
-    import csv
-
-    start_all()
-    machine.wait_for_unit("postgresql.service")
-    machine.wait_for_open_port(5432)
-
-    def peer_cmd(cmd, db="me"):
-        return "sudo -u me psql -U me {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db)
-
-    def query(query):
-        res = machine.succeed(peer_cmd(query))
-        return list(dict(l) for l in csv.DictReader(res.splitlines()))
-
-    def cmp_tables(a, b):
-        for i in range(max(len(a), len(b))):
-            diff = set(a[i]) ^ set(b[i])
-            if len(diff) > 0:
-                raise Exception(i, diff)
-
-    table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}]
-
-    with subtest("create fixture"):
-        machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)"))
-        machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)"))
-
-        res = query("SELECT * FROM test")
-        cmp_tables(res, table)
-
-    with subtest("backup"):
-        print(machine.succeed("systemctl cat restic-backups-postgres_opt_repos_postgres.service"))
-        machine.succeed("systemctl start restic-backups-postgres_opt_repos_postgres.service")
-
-    with subtest("drop database"):
-        machine.succeed(peer_cmd("DROP DATABASE me", db="postgres"))
-
-    with subtest("restore"):
-        print(machine.succeed("readlink -f $(type restic-backups-postgres_opt_repos_postgres)"))
-        machine.succeed("restic-backups-postgres_opt_repos_postgres restore latest ")
-
-    with subtest("check restoration"):
-        res = query("SELECT * FROM test")
-        cmp_tables(res, table)
-    '';
-  };
 }

test/contracts/databasebackup.nix

@@ -0,0 +1,32 @@
+{ pkgs, ... }:
+let
+  contracts = pkgs.callPackage ../../modules/contracts {};
+in
+{
+  restic_postgres = contracts.test.databasebackup {
+    name = "restic_postgres";
+    requesterRoot = [ "shb" "postgresql" ];
+    providerRoot = [ "shb" "restic" "databases" "postgresql" ];
+    modules = [
+      ../../modules/blocks/postgresql.nix
+      ../../modules/blocks/restic.nix
+    ];
+    settings = repository: {
+      enable = true;
+      passphraseFile = toString (pkgs.writeText "passphrase" "PassPhrase");
+      repository = {
+        path = repository;
+        timerConfig = {
+          OnCalendar = "00:00:00";
+        };
+      };
+    };
+    providerExtraConfig = { username, database, ... }: {
+      shb.postgresql.ensures = [
+        {
+          inherit username database;
+        }
+      ];
+    };
+  };
+}