danvics/selfhostblocks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

update documentation

Browse source
This commit is contained in:
ibizaman 2024-11-11 22:38:16 +01:00 committed by Pierre Penninckx
parent 64d337e1a6
commit 07f6de7409
15 changed files with 419 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/blocks.md
View file

@ -34,6 +34,10 @@ Not all blocks are yet documented. You can find all available blocks [in the rep
modules/blocks/ssl/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-postgresql.html
modules/blocks/postgresql/docs/default.md
```
```{=include=} chapters html:into-file=//blocks-restic.html
modules/blocks/restic/docs/default.md
```

11
docs/contracts.md
View file

@ -20,10 +20,13 @@ as possible, reducing the quite thick layer that it is now.
Provided contracts are:
- [SSL generator contract](contracts-ssl.html) to generate SSL certificates.
Two implementations are provided: self-signed and Let's Encrypt.
Two providers are implemented: self-signed and Let's Encrypt.
- [Backup contract](contracts-backup.html) to backup directories.
This contract allows to backup multiple times the same directories for extra protection.
One provider is implemented: Restic.
- [Database Backup contract](contracts-databasebackup.html) to backup database dumps.
One provider is implemented: Restic.
- [Secret contract](contracts-secret.html) to provide secrets that are deployed outside of the Nix store.
One provider is implemented: SOPS.
```{=include=} chapters html:into-file=//contracts-ssl.html
modules/contracts/ssl/docs/default.md
@ -33,6 +36,10 @@ modules/contracts/ssl/docs/default.md
modules/contracts/backup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-databasebackup.html
modules/contracts/databasebackup/docs/default.md
```
```{=include=} chapters html:into-file=//contracts-secret.html
modules/contracts/secret/docs/default.md
```

15
docs/default.nix
View file

@ -132,13 +132,15 @@ in stdenv.mkDerivation {
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/blocks/ssl.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/blocks/postgresql/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/blocks/postgresql.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/blocks/restic/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [
../modules/blocks/restic.nix
../modules/blocks/restic/dummyModule.nix
]}/share/doc/nixos/options.json
${individualModuleOptionsDocs [ ../modules/blocks/restic.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/services/nextcloud-server/docs/default.md \
--replace \
@ -163,6 +165,11 @@ in stdenv.mkDerivation {
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/contracts/backup/dummyModule.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/contracts/databasebackup/docs/default.md \
--replace \
'@OPTIONS_JSON@' \
${individualModuleOptionsDocs [ ../modules/contracts/databasebackup/dummyModule.nix ]}/share/doc/nixos/options.json
substituteInPlace ./modules/contracts/secret/docs/default.md \
--replace \
'@OPTIONS_JSON@' \

7
modules/blocks/postgresql.nix
View file

@ -50,8 +50,7 @@ in
default = false;
};
backup = lib.mkOption {
type = contracts.databasebackup.request;
databasebackup = lib.mkOption {
description = ''
Backup configuration. This is an output option.
@ -67,7 +66,9 @@ in
};
```
'';
readOnly = true;
type = contracts.databasebackup.requestType;
default = {
user = "postgres";

39
modules/blocks/postgresql/docs/default.md Normal file
View file

@ -0,0 +1,39 @@
# PostgreSQL Block {#blocks-postgresql}
Defined in [`/modules/blocks/postgresql.nix`](@REPO@/modules/blocks/postgresql.nix).
This block sets up a [PostgreSQL][] database.
[postgresql]: https://www.postgresql.org/
## Tests {#blocks-postgresql-tests}
Specific integration tests are defined in [`/test/blocks/postgresql.nix`](@REPO@/test/blocks/postgresql.nix).
## Database Backup Requester Contracts {#blocks-postgresql-contract-databasebackup}
This block can be backed up using the [database backup](contracts-databasebackup.html) contract.
Contract integration tests are defined in [`/test/contracts/databasebackup.nix`](@REPO@/test/contracts/databasebackup.nix).
### Backing up All Databases {#blocks-postgresql-contract-databasebackup-all}
```nix
{
my.backup.provider."postgresql" = {
request = config.shb.postgresql.databasebackup;
settings = {
// Specific options for the backup provider.
};
};
}
```
## Options Reference {#blocks-postgresql-options}
```{=include=} options
id-prefix: blocks-postgresql-options-
list-id: selfhostblocks-block-postgresql-options
source: @OPTIONS_JSON@
```

197
modules/blocks/restic.nix
View file

@ -10,84 +10,87 @@ let
inherit (lib) generators hasPrefix mkIf nameValuePair optionalAttrs removePrefix;
inherit (lib.types) attrsOf enum int ints listOf oneOf nonEmptyListOf nonEmptyStr nullOr path str submodule;
commonOptions = submodule {
options = {
enable = mkEnableOption "shb restic. A disabled instance will not backup data anymore but still provides the helper tool to introspect and rollback snapshots";
commonOptions = {
enable = mkEnableOption ''
this backup intance.
passphraseFile = mkOption {
description = "Encryption key for the backups.";
type = path;
};
A disabled instance will not backup data anymore
but still provides the helper tool to restore snapshots
'';
repository = mkOption {
description = "Repositories to back this instance to.";
type = submodule {
options = {
path = mkOption {
type = str;
description = "Repository location";
passphraseFile = mkOption {
description = "Encryption key for the backups.";
type = path;
};
repository = mkOption {
description = "Repositories to back this instance to.";
type = submodule {
options = {
path = mkOption {
type = str;
description = "Repository location";
};
secrets = mkOption {
type = attrsOf shblib.secretFileType;
default = {};
description = ''
Secrets needed to access the repository where the backups will be stored.
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
'';
example = literalExpression ''
{
AWS_ACCESS_KEY_ID.source = <path/to/secret>;
AWS_SECRET_ACCESS_KEY.source = <path/to/secret>;
}
'';
};
timerConfig = mkOption {
type = attrsOf utils.systemdUtils.unitOptions.unitOption;
default = {
OnCalendar = "daily";
Persistent = true;
};
secrets = mkOption {
type = attrsOf shblib.secretFileType;
default = {};
description = ''
Secrets needed to access the repository where the backups will be stored.
See [s3 config](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for an example
and [list](https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables) for the list of all secrets.
'';
example = literalExpression ''
{
AWS_ACCESS_KEY_ID = <path/to/secret>;
AWS_SECRET_ACCESS_KEY = <path/to/secret>;
}
'';
};
timerConfig = mkOption {
type = attrsOf utils.systemdUtils.unitOptions.unitOption;
default = {
OnCalendar = "daily";
Persistent = true;
};
description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
example = {
OnCalendar = "00:05";
RandomizedDelaySec = "5h";
Persistent = true;
};
description = ''When to run the backup. See {manpage}`systemd.timer(5)` for details.'';
example = {
OnCalendar = "00:05";
RandomizedDelaySec = "5h";
Persistent = true;
};
};
};
};
};
retention = mkOption {
description = "For how long to keep backup files.";
type = attrsOf (oneOf [ int nonEmptyStr ]);
default = {
keep_within = "1d";
keep_hourly = 24;
keep_daily = 7;
keep_weekly = 4;
keep_monthly = 6;
};
retention = mkOption {
description = "For how long to keep backup files.";
type = attrsOf (oneOf [ int nonEmptyStr ]);
default = {
keep_within = "1d";
keep_hourly = 24;
keep_daily = 7;
keep_weekly = 4;
keep_monthly = 6;
};
};
limitUploadKiBs = mkOption {
type = nullOr int;
description = "Limit upload bandwidth to the given KiB/s amount.";
default = null;
example = 8000;
};
limitUploadKiBs = mkOption {
type = nullOr int;
description = "Limit upload bandwidth to the given KiB/s amount.";
default = null;
example = 8000;
};
limitDownloadKiBs = mkOption {
type = nullOr int;
description = "Limit download bandwidth to the given KiB/s amount.";
default = null;
example = 8000;
};
limitDownloadKiBs = mkOption {
type = nullOr int;
description = "Limit download bandwidth to the given KiB/s amount.";
default = null;
example = 8000;
};
};
@ -103,43 +106,89 @@ in
type = attrsOf (submodule ({ name, options, ... }: {
options = {
request = mkOption {
description = ''
Request part of the backup contract.
Accepts values from a requester.
'';
type = contracts.backup.request;
};
settings = mkOption {
type = commonOptions;
description = ''
Settings specific to the Restic provider.
'';
type = submodule {
options = commonOptions;
};
};
result = mkOption {
type = contracts.databasebackup.result;
description = ''
Result part of the backup contract.
Contains the output of the Restic provider.
'';
type = lib.types.anything; # contracts.databasebackup.result;
default = {
restoreScript = fullName name options.settings.value.repository;
backupService = "${fullName name options.settings.value.repository}.service";
};
defaultText = literalExpression ''
{
restoreScript = "${fullName "<name>" { path = "path/to/repository"; }}";
backupService = "${fullName "<name>" { path = "path/to/repository"; }}.service";
}
'';
};
};
}));
};
databases = mkOption {
description = "Each item is backing up some database to one repository.";
description = "Databases to backup following the database backup contract.";
default = {};
type = attrsOf (submodule ({ name, options, ... }: {
options = {
request = mkOption {
type = contracts.databasebackup.request;
description = ''
Request part of the backup contract.
Accepts values from a requester.
'';
type = contracts.databasebackup.requestType;
};
settings = mkOption {
type = commonOptions;
description = ''
Settings specific to the Restic provider.
'';
type = submodule {
options = commonOptions;
};
};
result = mkOption {
type = contracts.databasebackup.result;
description = ''
Result part of the backup contract.
Contains the output of the Restic provider.
'';
type = contracts.databasebackup.resultType;
default = {
restoreScript = fullName name options.settings.value.repository;
backupService = "${fullName name (lib.debug.traceValSeqN 2 (lib.debug.traceValSeqN 2 (lib.debug.traceValSeqN 2 options).settings).value).repository}.service";
backupService = "${fullName name options.settings.value.repository}.service";
};
defaultText = literalExpression ''
{
restoreScript = "${fullName "<name>" { path = "path/to/repository"; }}";
backupService = "${fullName "<name>" { path = "path/to/repository"; }}.service";
}
'';
};
};
}));

24
modules/blocks/restic/docs/default.md
View file

@ -6,15 +6,17 @@ This block sets up a backup job using [Restic][restic].
[restic]: https://restic.net/
## Contract {#blocks-restic-features}
## Tests {#blocks-restic-tests}
This block implements the [backup](contracts-backup.html) contract.
Specific integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
Integration tests are defined in [`/test/blocks/restic.nix`](@REPO@/test/blocks/restic.nix).
## Provider Contracts {#blocks-restic-contract-provider}
## Usage {#blocks-backup-usage}
This block implements the [backup](contracts-backup.html) and [database backup](contracts-databasebackup.html) contracts.
### One folder backed up to mounted hard drives {#blocks-backup-usage-one}
Contract integration tests are defined in [`/test/contracts/backup.nix`](@REPO@/test/contracts/backup.nix).
### One folder backed up to mounted hard drives {#blocks-restic-contract-provider-one}
The following snippet shows how to configure
the backup of 1 folder to 1 repository.
@ -54,7 +56,7 @@ shb.restic.instances.myservice = {
};
```
### One folder backed up to S3 {#blocks-restic-usage-remote}
### One folder backed up to S3 {#blocks-restic-contract-provider-remote}
Here we will only highlight the differences with the previous configuration.
@ -79,7 +81,7 @@ This assumes you have access to such a remote S3 store, for example by using [Ba
}
```
### Secrets {#blocks-restic-secrets}
## Secrets {#blocks-restic-secrets}
To be secure, the secrets should deployed out of band, otherwise they will be world-readable in the nix store.
@ -119,7 +121,7 @@ sops.secrets."backup/b2/secret_access_key" = {
Pay attention that the owner must be the `myservice` user, the one owning the files to be backed up.
A `secrets` contract is in progress that will allow one to not care about such details.
### Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
## Multiple directories to multiple destinations {#blocks-restic-usage-multiple}
The following snippet shows how to configure backup of any number of folders to 3 repositories,
each happening at different times to avoid I/O contention.
@ -230,7 +232,7 @@ restic-myfolder2_s3_s3.us-west-000.backblazeb2.com_backups
Discovering those is easy thanks to tab-completion.
One can then restore a backup with:
One can then restore a backup from a given repository with:
```bash
restic-myfolder1_srv_pool1_backups restore latest
@ -243,7 +245,7 @@ In case something bad happens with a backup, the [official documentation](https:
## Options Reference {#blocks-restic-options}
```{=include=} options
id-prefix: blocks-backup-options-
list-id: selfhostblocks-block-backup-options
id-prefix: blocks-restic-options-
list-id: selfhostblocks-block-restic-options
source: @OPTIONS_JSON@
```

5
modules/blocks/restic/dummyModule.nix
View file

@ -1,8 +1,3 @@
{ lib, ... }:
{
config = {
shb.restic.databases."<name>".settings = {
repository = "";
};
};
}

2
modules/contracts/backup.nix
View file

@ -5,8 +5,6 @@ let
in
{
request = submodule {
freeformType = anything;
options = {
user = mkOption {
description = "Unix user doing the backups.";

59
modules/contracts/backup/docs/default.md
View file

@ -1,13 +1,13 @@
# Backup Contract {#backup-contract}
# Backup Contract {#contract-backup}
This NixOS contract represents a backup job
that will backup one or more files or directories
at a regular schedule.
on a regular schedule.
It is a contract between a service that has files to be backed up
and a service that backs up files.
## Contract Reference {#backup-contract-options}
## Contract Reference {#contract-backup-options}
These are all the options that are expected to exist for this contract to be respected.
@ -17,9 +17,11 @@ list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```
## Usage {#backup-contract-usage}
## Usage {#contract-backup-usage}
A service that can be backed up will provide a `backup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define what directories to backup,
the user to backup with
@ -30,9 +32,8 @@ Here is an example module defining such a `backup` option:
```nix
{
options = {
myservice.backup = lib.mkOption {
myservice.backup = mkOption {
type = contracts.backup.request;
readOnly = true;
default = {
user = "myservice";
sourceDirectories = [
@ -44,23 +45,26 @@ Here is an example module defining such a `backup` option:
};
```
As you can see, NixOS modules are a bit abused to make contracts work.
Default values are set as well as the `readOnly` attribute to ensure those values stay as defined.
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `backupservice` option
and that one can create multiple backup instances under `backupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
backupservice.instances.myservice = myservice.backup // {
enable = true;
backupservice.instances.myservice = {
request = myservice.backup;
settings = {
enable = true;
repository = {
path = "/srv/backup/myservice";
repository = {
path = "/srv/backup/myservice";
};
# ... Other options specific to backupservice like scheduling.
};
# ... Other options specific to backupservice like scheduling.
};
```
@ -68,28 +72,27 @@ It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `backupservice`:
```nix
backupservice.instances.myservice_2 = myservice.backup // {
enable = true;
repository = {
path = "<remote path>";
backupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
repository = {
path = "<remote path>";
};
};
};
```
Or with another module `backupservice_2`!
## Provided Implementations {#backup-contract-impl}
## Providers of the Backup Contract {#contract-backup-providers}
An implementation here is a service that understands the `backup` contract
and will backup the files accordingly.
One implementation is provided out of the box:
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
A second one based on `borgbackup` is in progress.
## Services Providing `backup` Option {#backup-contract-services}
## Requester Blocks and Services {#contract-backup-requesters}
- <!-- [ -->Audiobookshelf<!-- ](services-audiobookshelf.html). --> (no manual yet)
- <!-- [ -->Deluge<!--](services-deluge.html). --> (no manual yet)

4
modules/contracts/backup/dummyModule.nix
View file

@ -1,9 +1,11 @@
{ pkgs, lib, ... }:
let
contracts = pkgs.callPackage ../. {};
inherit (lib) mkOption;
in
{
options.shb.contracts.backup = lib.mkOption {
options.shb.contracts.backup = mkOption {
description = "Contract for backups.";
type = contracts.backup.request;
};

45
modules/contracts/databasebackup.nix
View file

@ -1,45 +1,76 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib) mkIf mkOption literalExpression;
inherit (lib.types) anything submodule str;
in
{
request = submodule {
freeformType = anything;
requestType = submodule {
options = {
user = mkOption {
description = "Unix user doing the backups.";
type = str;
example = "postgres";
};
backupFile = mkOption {
description = "Filename of the backup.";
type = str;
default = "dump";
example = "postgresql.sql";
};
backupCmd = mkOption {
description = "Command that produces the database dump on stdout.";
type = str;
example = literalExpression ''
''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable
'';
};
restoreCmd = mkOption {
description = "Command that reads the database dump on stdin and restores the database.";
type = str;
example = literalExpression ''
''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres
'';
};
};
};
result = submodule {
resultType = submodule {
options = {
restoreScript = mkOption {
description = "Name of script that can restore the database.";
description = ''
Name of script that can restore the database.
One can then list snapshots with:
```bash
$ my_restore_script snapshots
```
And restore the database with:
```bash
$ my_restore_script restore latest
```
'';
type = str;
example = "my_restore_script";
};
backupService = mkOption {
description = "Name of service backing up the database.";
description = ''
Name of service backing up the database.
This script can be ran manually to backup the database:
```bash
$ systemctl start my_backup_service
```
'';
type = str;
example = "my_backup_service.service";
};
};
};

100
modules/contracts/databasebackup/docs/default.md Normal file
View file

@ -0,0 +1,100 @@
# Database Backup Contract {#contract-databasebackup}
This NixOS contract represents a backup job
that will backup everything in one database
on a regular schedule.
It is a contract between a service that has database dumps to be backed up
and a service that backs up databases dumps.
## Contract Reference {#contract-databasebackup-options}
These are all the options that are expected to exist for this contract to be respected.
```{=include=} options
id-prefix: contracts-databasebackup-options-
list-id: selfhostblocks-options
source: @OPTIONS_JSON@
```
## Usage {#contract-databasebackup-usage}
A database that can be backed up will provide a `databasebackup` option.
Such a service is a `requester` providing a `request` for a module `provider` of this contract.
What this option defines is, from the user perspective - that is _you_ - an implementation detail
but it will at least define how to create a database dump,
the user to backup with
and how to restore from a database dump.
Here is an example module defining such a `databasebackup` option:
```nix
{
options = {
myservice.databasebackup = mkOption {
type = contracts.databasebackup.request;
default = {
user = "myservice";
backupCmd = ''
${pkgs.postgresql}/bin/pg_dumpall | ${pkgs.gzip}/bin/gzip --rsyncable
'';
restoreCmd = ''
${pkgs.gzip}/bin/gunzip | ${pkgs.postgresql}/bin/psql postgres
'';
};
};
};
};
```
Now, on the other side we have a service that uses this `backup` option and actually backs up files.
This service is a `provider` of this contract and will provide a `result` option.
Let's assume such a module is available under the `databasebackupservice` option
and that one can create multiple backup instances under `databasebackupservice.instances`.
Then, to actually backup the `myservice` service, one would write:
```nix
databasebackupservice.instances.myservice = {
request = myservice.databasebackup;
settings = {
enable = true;
repository = {
path = "/srv/backup/myservice";
};
# ... Other options specific to backupservice like scheduling.
};
};
```
It is advised to backup files to different location, to improve redundancy.
Thanks to using contracts, this can be made easily either with the same `databasebackupservice`:
```nix
databasebackupservice.instances.myservice_2 = {
request = myservice.backup;
settings = {
enable = true;
repository = {
path = "<remote path>";
};
};
};
```
Or with another module `databasebackupservice_2`!
## Providers of the Database Backup Contract {#contract-databasebackup-providers}
- [Restic block](blocks-restic.html).
- [Borgbackup block](blocks-borgbackup.html) [WIP].
## Requester Blocks and Services {#contract-databasebackup-requesters}
- [PostgreSQL](blocks-postgresql.html#blocks-postgresql-contract-databasebackup).

43
modules/contracts/databasebackup/dummyModule.nix Normal file
View file

@ -0,0 +1,43 @@
{ pkgs, lib, ... }:
let
contracts = pkgs.callPackage ../. {};
inherit (lib) mkOption;
inherit (lib.types) anything submodule;
in
{
options.shb.contracts.databasebackup = mkOption {
description = ''
Contract for database backup between a requester module
and a provider module.
The requester communicates to the provider
how to backup the database
through the `request` options.
The provider reads from the `request` options
and backs up the database as requested.
It communicates to the requester what script is used
to backup and restore the database
through the `result` options.
'';
type = submodule {
options = {
request = mkOption {
description = ''
Options set by a requester module of the database contract.
'';
type = contracts.databasebackup.requestType;
};
result = mkOption {
description = ''
Options set by a provider module of the database contract.
'';
type = contracts.databasebackup.resultType;
};
};
};
};
}

2
modules/contracts/databasebackup/test.nix
View file

@ -21,7 +21,7 @@ in
imports = ( testLib.baseImports pkgs' ) ++ modules;
config = lib.mkMerge [
(setAttrByPath providerRoot {
request = (getAttrFromPath requesterRoot config).backup;
request = (getAttrFromPath requesterRoot config).databasebackup;
settings = settings "/opt/repos/database";
})
(mkIf (username != "root") {