backup: do not make systemd service wait for backup to complete

fixes #421

docs/redirects.json

@@ -347,6 +347,12 @@
   "blocks-borgbackup-maintenance": [
     "blocks-borgbackup.html#blocks-borgbackup-maintenance"
   ],
+  "blocks-borgbackup-maintenance-manuql": [
+    "blocks-borgbackup.html#blocks-borgbackup-maintenance-manuql"
+  ],
+  "blocks-borgbackup-maintenance-restore": [
+    "blocks-borgbackup.html#blocks-borgbackup-maintenance-restore"
+  ],
   "blocks-borgbackup-maintenance-troubleshooting": [
     "blocks-borgbackup.html#blocks-borgbackup-maintenance-troubleshooting"
   ],
@@ -1373,6 +1379,12 @@
   "blocks-restic-maintenance": [
     "blocks-restic.html#blocks-restic-maintenance"
   ],
+  "blocks-restic-maintenance-manuql": [
+    "blocks-restic.html#blocks-restic-maintenance-manuql"
+  ],
+  "blocks-restic-maintenance-restore": [
+    "blocks-restic.html#blocks-restic-maintenance-restore"
+  ],
   "blocks-restic-maintenance-troubleshooting": [
     "blocks-restic.html#blocks-restic-maintenance-troubleshooting"
   ],

modules/blocks/borgbackup.nix

@@ -386,8 +386,13 @@ in
                 ${serviceName} = mkMerge [
                   {
                     serviceConfig = {
-                      # Makes the systemd service wait for the backup to be done before changing state to inactive.
+                      # Purposely not a oneshot systemd service otherwise
-                      Type = "oneshot";
+                      # the service waits on the completion the backup before deactivating.
+                      # This seems like a nice property at first but there is one annoying
+                      # edge case when deploying. If a backup job somehow is started when
+                      # the deploy happens, the deploy will wait on the service to finish
+                      # before considering the deploy done. And worse, it will consider the
+                      # deploy as failed if the backup fails, which is not what you want.
                       Nice = lib.mkForce cfg.performance.niceness;
                       IOSchedulingClass = lib.mkForce cfg.performance.ioSchedulingClass;
                       IOSchedulingPriority = lib.mkForce cfg.performance.ioPriority;
@@ -414,6 +419,7 @@ in
                   in
                   {
                     script = script.preStart;
+                    # Makes the systemd service wait for the backup to be done before changing state to inactive.
                     serviceConfig.Type = "oneshot";
                     serviceConfig.LoadCredential = script.loadCredentials;
                   }

modules/blocks/borgbackup/docs/default.md

@@ -3,6 +3,7 @@
 Defined in [`/modules/blocks/borgbackup.nix`](@REPO@/modules/blocks/borgbackup.nix).
 
 This block sets up a backup job using [BorgBackup][].
+It is heavily based on the nixpkgs BorgBackup module.
 
 [borgbackup]: https://www.borgbackup.org/
 
@@ -229,6 +230,36 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
 
 ## Maintenance {#blocks-borgbackup-maintenance}
 
+### Manual Backup {#blocks-borgbackup-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'borgbackup*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start borgbackup<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-borgbackup-maintenance-restore}
+
 One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
 
 The restore script has all the secrets needed to access the repo,

modules/blocks/restic.nix

@@ -413,7 +413,13 @@ in
               nameValuePair "${fullName name instance.settings.repository}_restore_gen" {
                 enable = true;
                 wantedBy = [ "multi-user.target" ];
-                serviceConfig.Type = "oneshot";
+                # Purposely not a oneshot systemd service otherwise
+                # the service waits on the completion the backup before deactivating.
+                # This seems like a nice property at first but there is one annoying
+                # edge case when deploying. If a backup job somehow is started when
+                # the deploy happens, the deploy will wait on the service to finish
+                # before considering the deploy done. And worse, it will consider the
+                # deploy as failed if the backup fails, which is not what you want.
                 script = (
                   shb.replaceSecrets {
                     userConfig = instance.settings.repository.secrets // {

modules/blocks/restic/docs/default.md

@@ -3,12 +3,12 @@
 Defined in [`/modules/blocks/restic.nix`](@REPO@/modules/blocks/restic.nix).
 
 This block sets up a backup job using [Restic][].
+It is heavily based on the nixpkgs Restic module.
 
 [restic]: https://restic.net/
 
 ## Provider Contracts {#blocks-restic-contract-provider}
 
-
 This block provides the following contracts:
 
 - [backup contract](contracts-backup.html) under the [`shb.restic.instances`][instances] option.
@@ -26,7 +26,6 @@ a backup Systemd service and a [restore script](#blocks-restic-maintenance) are
 
 ## Usage {#blocks-restic-usage}
 
-
 The following examples assume usage of the [sops block][] to provide secrets
 although any blocks providing the [secrets contract][] works too.
 
@@ -35,7 +34,6 @@ although any blocks providing the [secrets contract][] works too.
 
 ### One folder backed up manually {#blocks-restic-usage-provider-manual}
 
-
 The following snippet shows how to configure
 the backup of 1 folder to 1 repository.
 We assume that the folder `/var/lib/myfolder` of the service `myservice` must be backed up.
@@ -232,11 +230,44 @@ See [Backups Dashboard and Alert](blocks-monitoring.html#blocks-monitoring-backu
 
 ## Maintenance {#blocks-restic-maintenance}
 
-One command-line helper is provided per backup instance and repository pair to automatically supply the needed secrets.
+### Manual Backup {#blocks-restic-maintenance-manuql}
+
+To launch a backup manually, just run:
+
+```bash
+systemctl start <systemd-service-name>
+```
+
+You can easily discover the systemd service name you need by either listing the units:
+
+```bash
+systemctl list-units 'restic*'
+```
+
+Or by autocompleting the unit name with `<TAB>`:
+
+```bash
+systemctl start restic<TAB><TAB>
+```
+
+Note that the systemd services are of `Type=simple` which means the systemd service
+will not wait for the backup completion to terminate.
+If you want instead to wait for the backup to complete, use the `--wait` flag:
+
+```bash
+systemctl start --wait <systemd-service-name>
+```
+
+### Restore {#blocks-restic-maintenance-restore}
+
+One command-line helper is provided per backup instance and repository pair which allows to:
+
+- list snapshots: `<script> snapshots`
+- to restore a snapshot: `<script> restore <snapshot>`
 
 The restore script has all the secrets needed to access the repo,
 it will run `sudo` automatically
-and the user running it needs to have correct permissions for privilege escalation
+and the user running it needs to have correct permissions for privilege escalation.
 
 In the [multiple directories example](#blocks-restic-usage-multiple) above, the following 6 helpers are provided in the `$PATH`:
 

modules/contracts/backup/test.nix

@@ -105,7 +105,7 @@ shb.test.runNixOSTest {
 
       with subtest("First backup in repo"):
           print(machine.succeed("systemctl cat ${provider.backupService}"))
-          machine.succeed("systemctl start ${provider.backupService}")
+          machine.succeed("systemctl start --wait ${provider.backupService}")
 
       with subtest("New content"):
           for path in sourceDirectories:

modules/contracts/databasebackup/test.nix

@@ -85,7 +85,7 @@ shb.test.runNixOSTest {
       with subtest("backup"):
           print(machine.succeed("systemctl cat ${provider.result.backupService}"))
           print(machine.succeed("ls -l /run/hardcodedsecrets/hardcodedsecret_passphrase"))
-          machine.succeed("systemctl start ${provider.result.backupService}")
+          machine.succeed("systemctl start --wait ${provider.result.backupService}")
 
       with subtest("drop database"):
           machine.succeed(peer_cmd("DROP DATABASE ${database}", db="postgres"))

test/blocks/borgbackup.nix

@@ -143,7 +143,7 @@ let
               })
 
           with subtest("First backup in repo A"):
-              machine.succeed("systemctl start ${backupService}")
+              machine.succeed("systemctl start --wait ${backupService}")
 
           with subtest("New content"):
               machine.succeed("""