zfs: add doc and tests and convert to systemd services

docs/blocks.md

@@ -43,6 +43,12 @@ modules/blocks/postgresql/docs/default.md
 modules/blocks/sops/docs/default.md
 ```
 
+## Filesystem {#blocks-category-filesystem}
+
+```{=include=} chapters html:into-file=//blocks-zfs.html
+modules/blocks/zfs/docs/default.md
+```
+
 ## Network {#blocks-category-network}
 
 ```{=include=} chapters html:into-file=//blocks-ssl.html

docs/redirects.json

@@ -587,6 +587,9 @@
   "blocks-category-database": [
     "blocks.html#blocks-category-database"
   ],
+  "blocks-category-filesystem": [
+    "blocks.html#blocks-category-filesystem"
+  ],
   "blocks-category-introspection": [
     "blocks.html#blocks-category-introspection"
   ],
@@ -1772,6 +1775,78 @@
   "blocks-ssl-options-shb.certs.systemdService": [
     "blocks-ssl.html#blocks-ssl-options-shb.certs.systemdService"
   ],
+  "blocks-zfs": [
+    "blocks-zfs.html#blocks-zfs"
+  ],
+  "blocks-zfs-features": [
+    "blocks-zfs.html#blocks-zfs-features"
+  ],
+  "blocks-zfs-options": [
+    "blocks-zfs.html#blocks-zfs-options"
+  ],
+  "blocks-zfs-options-shb.zfs.pools": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.after": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.after"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.excludePatterns": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.excludePatterns"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.afterBackup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.afterBackup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.beforeBackup": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.hooks.beforeBackup"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.sourceDirectories": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.sourceDirectories"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.user": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.request.user"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.backupService": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.backupService"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.backup.result.restoreScript"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.enable": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.enable"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.mode": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.mode"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner"
+  ],
+  "blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path": [
+    "blocks-zfs.html#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path"
+  ],
+  "blocks-zfs-usage": [
+    "blocks-zfs.html#blocks-zfs-usage"
+  ],
   "build-time-validation": [
     "service-implementation-guide.html#build-time-validation"
   ],

flake.nix

@@ -116,6 +116,7 @@
             "blocks/postgresql" = ./modules/blocks/postgresql.nix;
             "blocks/restic" = ./modules/blocks/restic.nix;
             "blocks/sops" = ./modules/blocks/sops.nix;
+            "blocks/zfs" = ./modules/blocks/zfs.nix;
             "services/arr" = ./modules/services/arr.nix;
             "services/firefly-iii" = ./modules/services/firefly-iii.nix;
             "services/forgejo" = [
@@ -403,6 +404,7 @@
             // (vm_test "postgresql" ./test/blocks/postgresql.nix)
             // (vm_test "restic" ./test/blocks/restic.nix)
             // (vm_test "ssl" ./test/blocks/ssl.nix)
+            // (vm_test "zfs" ./test/blocks/zfs.nix)
 
             // (vm_test "contracts-backup" ./test/contracts/backup.nix)
             // (vm_test "contracts-databasebackup" ./test/contracts/databasebackup.nix)

modules/blocks/zfs.nix

@@ -2,6 +2,8 @@
   config,
   pkgs,
   lib,
+  shb,
+  utils,
   ...
 }:
 
@@ -9,121 +11,184 @@ let
   cfg = config.shb.zfs;
 in
 {
+  imports = [
+    ../../lib/module.nix
+  ];
+
   options.shb.zfs = {
-    defaultPoolName = lib.mkOption {
+    pools = lib.mkOption {
-      type = lib.types.nullOr lib.types.str;
-      default = null;
-      description = "ZFS pool name datasets should be created on if no pool name is given in the dataset.";
-    };
-
-    datasets = lib.mkOption {
       description = ''
-        ZFS Datasets.
+        Attrset of ZFS pools under which datasets will be created.
 
-        Each entry in the attrset will be created and mounted in the given path.
+        The ZFS pools are not managed by this module, they should already exist.
-        The attrset name is the dataset name.
 
-        This block implements the following contracts:
+        Each pool named here will be added to the [`boot.zfs.extraPools`](https://search.nixos.org/options?channel=unstable&include_modular_service_options=0&include_nixos_options=1&query=boot.zfs.extrapools&show=option:boot.zfs.extraPools) option.
-          - mount
       '';
       default = { };
-      example = lib.literalExpression ''
-        shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
-      '';
       type = lib.types.attrsOf (
         lib.types.submodule {
           options = {
-            enable = lib.mkEnableOption "shb.zfs.datasets";
+            datasets = lib.mkOption {
-
-            poolName = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              default = null;
-              description = "ZFS pool name this dataset should be created on. Overrides the defaultPoolName.";
-            };
-
-            path = lib.mkOption {
-              type = lib.types.str;
-              description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
-            };
-
-            mode = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              description = "If non null, unix mode to apply to the dataset root folder.";
-              default = null;
-              example = "ug=rwx,g+s";
-            };
-
-            owner = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              description = "If non null, unix user to apply to the dataset root folder.";
-              default = null;
-              example = "syncthing";
-            };
-
-            group = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
-              description = "If non null, unix group to apply to the dataset root folder.";
-              default = null;
-              example = "syncthing";
-            };
-
-            defaultACLs = lib.mkOption {
-              type = lib.types.nullOr lib.types.str;
               description = ''
-                If non null, default ACL to set on the dataset root folder.
+                ZFS Datasets.
 
-                Executes "setfacl -d -m $acl $path"
+                Each entry in the attrset will be created and mounted in the given path.
+                The attrset name is the dataset name.
+
+                This block implements the following contracts:
+                  - mount
               '';
-              default = null;
+              default = { };
-              example = "g:syncthing:rwX";
+              example = lib.literalExpression ''
+                shb.zfs."safe/postgresql".path = "/var/lib/postgresql";
+              '';
+              type = lib.types.attrsOf (
+                lib.types.submodule (
+                  { config, name, ... }:
+                  {
+                    options = {
+                      enable = lib.mkEnableOption "shb.zfs.datasets";
+
+                      path = lib.mkOption {
+                        type = lib.types.str;
+                        description = "Path this dataset should be mounted on. If the string 'none' is given, the dataset will not be mounted.";
+                      };
+
+                      mode = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix mode to apply to the dataset root folder.";
+                        default = null;
+                        example = "ug=rwx,g+s";
+                      };
+
+                      owner = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix user to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      group = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = "If non null, unix group to apply to the dataset root folder.";
+                        default = null;
+                        example = "syncthing";
+                      };
+
+                      defaultACLs = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = ''
+                          If non null, default ACL to set on the dataset root folder.
+
+                          Executes "setfacl -d -m $acl $path"
+                        '';
+                        default = null;
+                        example = "g:syncthing:rwX";
+                      };
+
+                      after = lib.mkOption {
+                        type = lib.types.listOf lib.types.str;
+                        description = ''
+                          Order creating this dataset after the mentioned ones.
+                          This only works with datasets managed by this module.
+
+                          Use the name of the dataset without the pool name.
+                        '';
+                        default = [ ];
+                        example = lib.literalExpression ''
+                          [
+                            "backup"
+                          ]
+                        '';
+                      };
+
+                      backup = lib.mkOption {
+                        description = ''
+                          Backup contract consumer configuration.
+
+                          This contract will backup the files inside the dataset.
+                        '';
+                        type = lib.types.submodule {
+                          options = shb.contracts.backup.mkRequester {
+                            user = if config.owner == null then "root" else config.owner;
+                            sourceDirectories = [
+                              config.path
+                            ];
+                            sourceDirectoriesText = ''
+                              [
+                                shb.zfs.pools.<name>.datasets.<name>.path
+                              ]
+                            '';
+                          };
+                        };
+                      };
+                    };
+                  }
+                )
+              );
             };
           };
         }
+
       );
     };
   };
 
+  # The implementation is greatly inspired by https://discourse.nixos.org/t/configure-zfs-filesystems-after-install/48633/3
   config = {
-    assertions = [
+    boot.zfs.extraPools = lib.uniqueStrings (builtins.attrNames cfg.pools);
-      {
-        assertion =
-          lib.any (x: x.poolName == null) (lib.mapAttrsToList (n: v: v) cfg.datasets)
-          -> cfg.defaultPoolName != null;
-        message = "Cannot have both datasets.poolName and defaultPoolName set to null";
-      }
-    ];
 
-    system.activationScripts = lib.mapAttrs' (
+    systemd.services =
-      name: cfg':
       let
-        dataset = (if cfg'.poolName != null then cfg'.poolName else cfg.defaultPoolName) + "/" + name;
+        mkPool =
+          poolName: poolCfg: lib.listToAttrs (lib.mapAttrsToList (mkDataset poolName) poolCfg.datasets);
+
+        mkDataset =
+          poolName: name: cfg':
+          let
+            dataset = poolName + "/" + name;
+          in
+          lib.attrsets.nameValuePair "zfs-create-${utils.escapeSystemdPath dataset}" {
+            # oneshot is used to make the systemd service wait on completion of the script.
+            serviceConfig.Type = "oneshot";
+            unitConfig.DefaultDependencies = false;
+            requiredBy = [ "local-fs.target" ];
+            before = [ "local-fs.target" ];
+            after = [
+              "zfs-import-${poolName}.service"
+              "zfs-mount.service"
+            ]
+            ++ map (n: "zfs-create-${poolName}-${n}.service") cfg'.after;
+
+            unitConfig.ConditionPathIsMountPoint = lib.mkIf (cfg'.path != "none") "!${cfg'.path}";
+
+            script = ''
+              ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
+                || ${pkgs.zfs}/bin/zfs create \
+                   -o mountpoint=none \
+                   ${dataset} || :
+
+              [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
+                || ${pkgs.zfs}/bin/zfs set \
+                   mountpoint="${cfg'.path}" \
+                   ${dataset}
+
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
+              chmod "${cfg'.mode}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
+              chown "${cfg'.owner}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
+              chown :"${cfg'.group}" "${cfg'.path}"
+            ''
+            + lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
+              ${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
+            '';
+          };
+        mergeAttrs = lib.foldl lib.mergeAttrs { };
       in
-      lib.attrsets.nameValuePair "zfsCreate-${name}" {
+      mergeAttrs (lib.mapAttrsToList mkPool cfg.pools);
-        text = ''
-          ${pkgs.zfs}/bin/zfs list ${dataset} > /dev/null 2>&1 \
-            || ${pkgs.zfs}/bin/zfs create \
-               -o mountpoint=none \
-               ${dataset} || :
-
-          [ "$(${pkgs.zfs}/bin/zfs get -H mountpoint -o value ${dataset})" = ${cfg'.path} ] \
-            || ${pkgs.zfs}/bin/zfs set \
-               mountpoint="${cfg'.path}" \
-               ${dataset}
-
-        ''
-        + lib.optionalString (cfg'.path != "none" && cfg'.mode != null) ''
-          chmod "${cfg'.mode}" "${cfg'.path}"
-        ''
-        + lib.optionalString (cfg'.path != "none" && cfg'.owner != null) ''
-          chown "${cfg'.owner}" "${cfg'.path}"
-        ''
-        + lib.optionalString (cfg'.path != "none" && cfg'.group != null) ''
-          chown :"${cfg'.group}" "${cfg'.path}"
-        ''
-        + lib.optionalString (cfg'.path != "none" && cfg'.defaultACLs != null) ''
-          ${pkgs.acl}/bin/setfacl -d -m "${cfg'.defaultACLs}" "${cfg'.path}"
-        '';
-      }
-    ) cfg.datasets;
   };
 }

modules/blocks/zfs/docs/default.md

@@ -0,0 +1,58 @@
+# ZFS Block {#blocks-zfs}
+
+Defined in [`/modules/blocks/zfs.nix`](@REPO@/modules/blocks/zfs.nix):
+
+```nix
+{
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.selfhostblocks.nixosModules.zfs
+  ];
+}
+```
+
+This block creates ZFS datasets, optionally mounts them and sets permissions on the mount point.
+
+## Features {#blocks-zfs-features}
+
+- Creates ZFS dataset which is [optionally mounted](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.path).
+- Sets permissions, [owner](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.owner), [group](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.group) and [ACL](#blocks-zfs-options-shb.zfs.pools._name_.datasets._name_.defaultACLs) on the mount point.
+
+## Usage {#blocks-zfs-usage}
+
+Create a dataset at `root/safe/users` mounted on `/var/lib/nixos`:
+
+```nix
+shb.zfs.pools.root.datasets."safe/users".path = "/var/lib/nixos";
+```
+
+Create a dataset at `backup/syncoid` but do not mount it:
+
+```nix
+shb.zfs.pools.backup.datasets."syncoid".path = "none";
+```
+
+Create a dataset at `root/syncthing` and set custom permissions and ACL.
+Permission and ACL are only enforced for the mount point.
+
+```nix
+shb.zfs.pools.root.datasets."syncthing" = {
+  path = "/srv/syncthing";
+
+  mode = "ug=rwx,g+s,o=";
+  owner = "syncthing";
+  group = "syncthing";
+  defaultACLs = "g:syncthing:rwX";
+};
+```
+
+## Options Reference {#blocks-zfs-options}
+
+```{=include=} options
+id-prefix: blocks-zfs-options-
+list-id: selfhostblocks-block-zfs-options
+source: @OPTIONS_JSON@
+```

test/blocks/zfs.nix

@@ -0,0 +1,115 @@
+{
+  shb,
+  ...
+}:
+{
+  default = shb.test.runNixOSTest {
+    name = "zfs-default";
+
+    nodes.machine =
+      { config, pkgs, ... }:
+      {
+        imports = [
+          ../../modules/blocks/zfs.nix
+        ];
+
+        # Inspiration from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/zfs.nix
+        networking.hostId = "deadbeef";
+        boot.supportedFilesystems = [ "zfs" ];
+
+        users.users.syncthing = {
+          isSystemUser = true;
+          group = "syncthing";
+        };
+        users.groups.syncthing = { };
+        virtualisation = {
+          emptyDiskImages = [
+            512
+            512
+          ];
+        };
+
+        systemd.services."zfs-zpool-create" = {
+          unitConfig.DefaultDependencies = false;
+          after = [ "systemd-modules-load.service" ];
+          requiredBy = [
+            "zfs-import-root.service"
+            "zfs-import-data.service"
+            "zfs-mount.service"
+          ];
+          before = [
+            "zfs-import-root.service"
+            "zfs-import-data.service"
+            "zfs-mount.service"
+          ];
+          script = ''
+            if [ ! -f /var/done ]; then
+              ${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl root /dev/vdb
+              ${pkgs.zfs}/bin/zpool create -m none -O acltype=posixacl data /dev/vdc
+              sync
+            fi
+            touch /var/done
+          '';
+        };
+
+        shb.zfs.pools.root.datasets.one.path = "/var/root/one";
+        shb.zfs.pools.root.datasets.two.path = "/var/root/two";
+        shb.zfs.pools.root.datasets.none.path = "none";
+        shb.zfs.pools.data.datasets.two = {
+          path = "/var/data/two";
+
+          mode = "ug=rwx,g+s,o=";
+          owner = "syncthing";
+          group = "syncthing";
+          defaultACLs = "g:syncthing:rwX";
+        };
+      };
+
+    testScript =
+      { nodes, ... }:
+      ''
+        import difflib
+
+        def assert_facl():
+            out = machine.succeed("getfacl /var/data/two").splitlines()
+            expect = """\
+        # file: var/data/two
+        # owner: syncthing
+        # group: syncthing
+        # flags: -s-
+        user::rwx
+        group::rwx
+        other::---
+        default:user::rwx
+        default:group::rwx
+        default:group:syncthing:rwx
+        default:mask::rwx
+        default:other::---
+
+        """.splitlines()
+            if out != expect:
+                diff = difflib.context_diff(expect, out)
+                raise Exception(f"Unexpected getfacl:\n{"\n".join(diff)}")
+
+        def assert_mounts():
+            out = sorted([l.split()[0] for l in machine.succeed("mount | grep /var/").splitlines()])
+            expect = ["data/two", "root/one", "root/two"]
+            if out != expect:
+                diff = difflib.context_diff(expect, out)
+                raise Exception(f"Unexpected mounts:\n{"\n".join(diff)}")
+
+        machine.start(allow_reboot=True)
+        machine.wait_for_unit("multi-user.target")
+
+        assert_facl()
+        assert_mounts()
+
+        # TODO: make this work. zpool import does not work after reboot.
+        # machine.crash()
+        # machine.wait_for_unit("multi-user.target")
+
+        # assert_facl()
+        # assert_mounts()
+      '';
+  };
+}