From 442c948a1aabd9938596f9540a31556f9e020af4 Mon Sep 17 00:00:00 2001 From: Pierre Penninckx Date: Wed, 13 Nov 2024 22:43:12 +0100 Subject: [PATCH] more contracts usage in Nextcloud and docs enhancement (#345) --- README.md | 137 +++--- docs/contracts.md | 9 + docs/usage.md | 22 +- modules/services/nextcloud-server.nix | 82 ++-- .../services/nextcloud-server/docs/default.md | 421 +++++++++++------- test/services/nextcloud.nix | 41 +- 6 files changed, 408 insertions(+), 304 deletions(-) diff --git a/README.md b/README.md index 0325476..dbc72b1 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,58 @@ that is not like the other server management tools. ## TOC -- [Unified Interfaces](#unified-interfaces) -- [Incremental Adoption](#incremental-adoption) - [Usage](#usage) -- [More Benefits of SHB](#more-benefits-of-shb) -- [Manual](#manual) +- [Server Management](#server-management) + - [Unified Interfaces](#unified-interfaces) + - [Incremental Adoption](#incremental-adoption) + - [More Benefits of SHB](#more-benefits-of-shb) - [Roadmap](#roadmap) -- [Available Blocks](#available-blocks) -- [Provided Services](#provided-services) - [Demos](#demos) - [Community](#community) - [License](#license) -## Unified Interfaces +## Usage + +> **Caution:** You should know that although I am using everything in this repo for my personal +> production server, this is really just a one person effort for now and there are most certainly +> bugs that I didn't discover yet. + +Self Host Blocks is available as a flake. +To use it in your project, add the following flake input: + +```nix +inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks"; +``` + +To get started using Self Host Blocks, +follow [the usage section](https://shb.skarabox.com/usage.html) of the manual. +It goes over how to deploy with [Colmena][], [nixos-rebuild][] +and also goes over secrets management with [SOPS][]. + +[Colmena]: https://colmena.cli.rs/ +[nixos-rebuild]: https://nixos.org/manual/nixos/stable/#sec-changing-config +[SOPS]: https://github.com/Mic92/sops-nix + +Then, to actually configure services, you can choose which one interests you in +[the services section](https://shb.skarabox.com/services.html) of the manual. + +Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org) +for any remaining question, or just to say hi :) + +## Server Management + +Self Host Blocks provides a standardized configuration for [some services](https://shb.skarabox.com/services.html) provided by nixpkgs. +The goal is to help spread adoption of self-hosting by providing an opinionated configuration with best practices by default. + +Self Host Blocks takes care of common self-hosting needs: +- Backup for all services. +- LDAP and SSO integration for most services. +- Monitoring with Grafana and Prometheus stack with provided dashboards. +- Automatic reverse proxy and certificate management for HTTPS. +- VPN and proxy tunneling services. + +### Unified Interfaces SHB's first goal is to provide unified [building blocks](#available-blocks) and by extension configuration interface, for self-hosting. @@ -92,7 +130,7 @@ As you can see, they are pretty similar! SHB provides an ever growing list of [services](#provided-services) that are configured in the same way. -## Incremental Adoption +### Incremental Adoption SHB's second goal is to facilitate testing NixOS and slowly switching an existing installation to NixOS. @@ -104,35 +142,15 @@ any reverse proxy you want or any database you want, without requiring work from maintainers of the services you want to self host. (See [manual][contracts] for a complete explanation) +Two videos exist of me presenting the topic, +the first at [NixCon North America in spring of 2024][NixConNA2024] +and the second at [NixCon in Berlin in fall of 2024][NixConBerlin2024]. + [contracts]: https://shb.skarabox.com/contracts.html +[NixConNA2024]: https://www.youtube.com/watch?v=lw7PgphB9qM +[NixConBerlin2024]: https://www.youtube.com/watch?v=CP0hR6w1csc -## Usage - -> **Caution:** You should know that although I am using everything in this repo for my personal -> production server, this is really just a one person effort for now and there are most certainly -> bugs that I didn't discover yet. - -Self Host Blocks is available as a flake. -To use it in your project, add the following flake input: - -```nix -inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks"; -``` - -More information is provided in [the manual](https://shb.skarabox.com/usage.html), -like secrets management. - -- You are new to self hosting and want pre-configured services to deploy easily. - Look at the [services section](https://shb.skarabox.com/services.html). -- You are a seasoned self-hoster but want to enhance some services you deploy already. - Go to the [blocks section](https://shb.skarabox.com/blocks.html). -- You are a user of Self Host Blocks but would like to use your own implementation for a block. - Go to the [contracts section](https://shb.skarabox.com/contracts.html). - -Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org) -for any remaining question, or just to say hi :) - -## More Benefits of SHB +### More Benefits of SHB By using Self Host Blocks, you get all the benefits of NixOS which are, for self hosted applications specifically: @@ -163,51 +181,23 @@ from other server management projects: the `nixpkgs` input in the root `flakes.nix`, runs the tests and merges a PR with the new input if the tests pass. -## Manual - -The manual can be found at [shb.skarabox.com](https://shb.skarabox.com/). - -Work is in progress to document everything in the manual but I'm not there yet. For what's not yet -documented, unfortunately the source code is the best place to read about them. -[Here](./modules/services) for services and [here](./modules/blocks) for blocks. - ## Roadmap -Currently, the Nextcloud service and SSL block are the most advanced and most documented. +Currently, the Nextcloud, Vaultwarden services and the SSL and backup blocks are the most advanced and most documented. Documenting all services and blocks will be done as I make all blocks and services use the contracts. Upstreaming changes is also on the roadmap. -Check [the issues](https://github.com/ibizaman/selfhostblocks/issues) to see planned works. Feel -free to add more! +Check the [issues][] and the [milestones]() to see planned work. +Feel free to add more or to contribute! -That being said, I am personally using all the blocks and services in this project, so they do work -to some extent. +[issues]: (https://github.com/ibizaman/selfhostblocks/issues) +[milestones]: https://github.com/ibizaman/selfhostblocks/milestones -## Available Blocks - -- [`authelia.nix`](./modules/blocks/authelia.nix) for Single Sign On. -- [`backup.nix`](./modules/blocks/backup.nix). -- [`ldap.nix`](./modules/blocks/ldap.nix) for user management. -- [`monitoring.nix`](./modules/blocks/monitoring.nix) for dashboards, logs and alerts. -- [`nginx.nix`](./modules/blocks/nginx.nix) for reverse proxy with SSL termination. -- [`postgresql.nix`](./modules/blocks/postgresql.nix) for database setup. -- [`ssl.nix`](./modules/blocks/ssl.nix) for maintaining self-signed SSL certificates or certificates provided by Let's Encrypt. -- [`tinyproxy.nix`](./modules/blocks/tinyproxy.nix) to forward traffic to a VPN tunnel. -- [`vpn.nix`](./modules/blocks/vpn.nix) to setup a VPN tunnel. - -## Provided Services - -- [`arr.nix`](./modules/services/arr.nix) for finding media https://wiki.servarr.com/. -- [`deluge.nix`](./modules/services/deluge.nix) for downloading linux isos https://deluge-torrent.org/. -- [`hledger.nix`](./modules/services/hledger.nix) for managing finances https://hledger.org/. -- [`home-assistant.nix`](./modules/services/home-assistant.nix) for private IoT https://www.home-assistant.io/. -- [`jellyfin.nix`](./modules/services/jellyfin.nix) for watching media https://jellyfin.org/. -- [Nextcloud Server](https://shb.skarabox.com/services-nextcloud.html) for private documents, contacts, calendar, etc https://nextcloud.com. -- [`vaultwarden.nix`](./modules/services/vaultwarden.nix) for passwords https://github.com/dani-garcia/vaultwarden. -- [`audiobookshelf.nix`](./modules/services/audiobookshelf.nix) for hosting podcasts and audio books https://www.audiobookshelf.org/. +All blocks and services have NixOS tests. +Also, I am personally using all the blocks and services in this project, so they do work to some extent. ## Demos @@ -228,6 +218,7 @@ when I'm satisfied with how things look, I'll upstream changes. ## License -I'm following the [Nextcloud](https://github.com/nextcloud/server) license which is AGPLv3. See -[this article](https://www.fsf.org/bulletin/2021/fall/the-fundamentals-of-the-agplv3) from the FSF that explains what this license adds to the GPL -one. +I'm following the [Nextcloud](https://github.com/nextcloud/server) license which is AGPLv3. +See [this article][why agplv3] from the FSF that explains what this license adds to the GPL one. + +[why agplv3]: (https://www.fsf.org/bulletin/2021/fall/the-fundamentals-of-the-agplv3) diff --git a/docs/contracts.md b/docs/contracts.md index 31ed94e..7c7c786 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -11,6 +11,15 @@ In practice, a contract is a set of options that any user of a contract expects values of these options dictate the behavior of the implementation. This is enforced with NixOS VM tests. +## Videos {#contracts-videos} + +Two videos exist of me presenting the topic, +the first at [NixCon North America in spring of 2024][NixConNA2024] +and the second at [NixCon in Berlin in fall of 2024][NixConBerlin2024]. + +[NixConNA2024]: https://www.youtube.com/watch?v=lw7PgphB9qM +[NixConBerlin2024]: https://www.youtube.com/watch?v=CP0hR6w1csc + ## Provided contracts {#contracts-provided} Self Host Blocks is a proving ground of contracts. This repository adds a layer on top of services diff --git a/docs/usage.md b/docs/usage.md index 28daa25..03e513c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -279,15 +279,15 @@ One way to setup secrets management using `sops-nix`: selfhostblocks.inputs.sops-nix.nixosModules.default ]; ``` -6. Reference the secrets in nix: - ```nix - shb.nextcloud.adminPassFile = config.sops.secrets."nextcloud/adminpass".path; - - sops.secrets."nextcloud/adminpass" = { - sopsFile = ./secrets.yaml; - mode = "0440"; - owner = "nextcloud"; - group = "nextcloud"; - restartUnits = [ "phpfpm-nextcloud.service" ]; - }; +6. Set default sops file: + ```bash + sops.defaultSopsFile = ./secrets.yaml; ``` + Setting the default this way makes all sops instances use that same file. +7. Reference the secrets in nix: + ```nix + shb.nextcloud.adminPass.result.path = config.sops.secrets."nextcloud/adminpass".path; + + sops.secrets."nextcloud/adminpass" = config.shb.nextcloud.adminPass.request; + ``` + The above snippet uses the [secrets contract](./contracts-secret.html) to ease configuration. diff --git a/modules/services/nextcloud-server.nix b/modules/services/nextcloud-server.nix index e662513..f675326 100644 --- a/modules/services/nextcloud-server.nix +++ b/modules/services/nextcloud-server.nix @@ -95,10 +95,11 @@ in default = "root"; }; - adminPassFile = lib.mkOption { - type = lib.types.nullOr lib.types.path; - description = "File containing the Nextcloud admin password. Required."; - default = null; + adminPass = contracts.secret.mkOption { + description = "Nextcloud admin password."; + mode = "0400"; + owner = "nextcloud"; + restartUnits = [ "phpfpm-nextcloud.service" ]; }; maxUploadSize = lib.mkOption { @@ -121,7 +122,11 @@ in postgresSettings = lib.mkOption { type = lib.types.nullOr (lib.types.attrsOf lib.types.str); default = null; - description = "Settings for the PostgreSQL database. Go to https://pgtune.leopard.in.ua/ and copy the generated configuration here."; + description = '' + Settings for the PostgreSQL database. + + Go to https://pgtune.leopard.in.ua/ and copy the generated configuration here. + ''; example = lib.literalExpression '' { # From https://pgtune.leopard.in.ua/ with: @@ -369,14 +374,11 @@ in default = "admin"; }; - adminPasswordFile = lib.mkOption { - type = lib.types.path; - description = '' - File containing the admin password of the LDAP server. - - Must be readable by the nextcloud system user. - ''; - default = ""; + adminPassword = contracts.secret.mkOption { + description = "LDAP server admin password."; + mode = "0400"; + owner = "nextcloud"; + restartUnits = [ "phpfpm-nextcloud.service" ]; }; userGroup = lib.mkOption { @@ -439,24 +441,17 @@ in default = "one_factor"; }; - secretFile = lib.mkOption { - type = lib.types.path; - description = '' - File containing the secret for the OIDC endpoint. - - Must be readable by the nextcloud system user. - ''; - default = ""; + secret = contracts.secret.mkOption { + description = "OIDC shared secret."; + mode = "0400"; + owner = "nextcloud"; + restartUnits = [ "phpfpm-nextcloud.service" ]; }; - secretFileForAuthelia = lib.mkOption { - type = lib.types.path; - description = '' - File containing the secret for the OIDC endpoint, must be readable by the Authelia user. - - Must be readable by the authelia system user. - ''; - default = ""; + secretForAuthelia = contracts.secret.mkOption { + description = "OIDC shared secret. Content must be the same as `secretFile` option."; + mode = "0400"; + owner = "authelia"; }; fallbackDefaultAuth = lib.mkOption { @@ -478,9 +473,15 @@ in extraApps = lib.mkOption { type = lib.types.raw; description = '' - Extra apps to install. Should be a function returning an attrSet of appid to packages - generated by fetchNextcloudApp. The appid must be identical to the “id” value in the apps - appinfo/info.xml. You can still install apps through the appstore. + Extra apps to install. + + Should be a function returning an `attrSet` of `appid` as keys to `packages` as values, + like generated by `fetchNextcloudApp`. + The appid must be identical to the `id` value in the apps' + `appinfo/info.xml`. + Search in [nixpkgs](https://github.com/NixOS/nixpkgs/tree/master/pkgs/servers/nextcloud/packages) for the `NN.json` files for existing apps. + + You can still install apps through the appstore. ''; default = null; example = lib.literalExpression '' @@ -576,13 +577,6 @@ in config = lib.mkMerge [ (lib.mkIf cfg.enable { - assertions = [ - { - assertion = !(isNull cfg.adminPassFile); - message = "Must set shb.nextcloud.adminPassFile."; - } - ]; - users.users = { nextcloud = { name = "nextcloud"; @@ -617,7 +611,7 @@ in config = { dbtype = "pgsql"; adminuser = cfg.adminUser; - adminpassFile = toString cfg.adminPassFile; + adminpassFile = cfg.adminPass.result.path; }; database.createLocally = true; @@ -819,7 +813,7 @@ in systemd.services.nextcloud-setup.script = '' ${occ} app:install files_external || : ${occ} app:enable files_external - '' + lib.optionalString (cfg.apps.externalStorage.userLocalMount != "") ( + '' + lib.optionalString (cfg.apps.externalStorage.userLocalMount != null) ( let cfg' = cfg.apps.externalStorage.userLocalMount; @@ -859,7 +853,7 @@ in ${occ} ldap:set-config "${cID}" 'ldapAgentName' \ 'uid=${cfg'.adminName},ou=people,${cfg'.dcdomain}' ${occ} ldap:set-config "${cID}" 'ldapAgentPassword' \ - "$(cat ${cfg'.adminPasswordFile})" + "$(cat ${cfg'.adminPassword.result.path})" ${occ} ldap:set-config "${cID}" 'ldapBase' \ '${cfg'.dcdomain}' ${occ} ldap:set-config "${cID}" 'ldapBaseGroups' \ @@ -936,7 +930,7 @@ in mkdir -p ${cfg.dataDir}/config cat < "${cfg.dataDir}/config/secretFile" { - "oidc_login_client_secret": "$(cat ${cfg.apps.sso.secretFile})" + "oidc_login_client_secret": "$(cat ${cfg.apps.sso.secret.result.path})" } EOF ''; @@ -1001,7 +995,7 @@ in { client_id = cfg.apps.sso.clientID; client_name = "Nextcloud"; - client_secret.source = cfg.apps.sso.secretFileForAuthelia; + client_secret.source = cfg.apps.sso.secretForAuthelia.result.path; public = false; authorization_policy = cfg.apps.sso.authorization_policy; redirect_uris = [ "${protocol}://${fqdnWithPort}/apps/oidc_login/oidc" ]; diff --git a/modules/services/nextcloud-server/docs/default.md b/modules/services/nextcloud-server/docs/default.md index 8422684..7eac7bc 100644 --- a/modules/services/nextcloud-server/docs/default.md +++ b/modules/services/nextcloud-server/docs/default.md @@ -3,92 +3,129 @@ Defined in [`/modules/services/nextcloud-server.nix`](@REPO@/modules/services/nextcloud-server.nix). This NixOS module is a service that sets up a [Nextcloud Server](https://nextcloud.com/). +It is based on the nixpkgs Nextcloud server and provides opinionated defaults. ## Features {#services-nextcloud-server-features} - Declarative [Apps](#services-nextcloud-server-options-shb.nextcloud.apps) Configuration - no need to configure those with the UI. - - [LDAP](#services-nextcloud-server-usage-ldap) app: enables app and sets up integration with an existing LDAP server. - - [OIDC](#services-nextcloud-server-usage-oidc) app: enables app and sets up integration with an existing OIDC server. - - [Preview Generator](#services-nextcloud-server-usage-previewgenerator) app: enables app and sets - up required cron job. - - [External Storage](#services-nextcloud-server-usage-externalstorage) app: enables app and - optionally configures one local mount. - - [Only Office](#services-nextcloud-server-usage-onlyoffice) app: enables app and sets up Only - Office service. + - [LDAP](#services-nextcloud-server-usage-ldap) app: + enables app and sets up integration with an existing LDAP server, in this case LLDAP. + - [OIDC](#services-nextcloud-server-usage-oidc) app: + enables app and sets up integration with an existing OIDC server, in this case Authelia. + - [Preview Generator](#services-nextcloud-server-usage-previewgenerator) app: + enables app and sets up required cron job. + - [External Storage](#services-nextcloud-server-usage-externalstorage) app: + enables app and optionally configures one local mount. + This enables having data living on separate hard drives. + - [Only Office](#services-nextcloud-server-usage-onlyoffice) app: + enables app and sets up Only Office service. - Any other app through the [shb.nextcloud.extraApps](#services-nextcloud-server-options-shb.nextcloud.extraApps) option. -- [Demo](./demo-nextcloud-server.html) - - Demo deploying a Nextcloud server with [Colmena](https://colmena.cli.rs/) and with proper - secrets management with [sops-nix](https://github.com/Mic92/sops-nix). - Access through subdomain using reverse proxy. +- Forces Nginx as the reverse proxy. (This is hardcoded in the upstream nixpkgs module). +- Sets good defaults for trusted proxies settings, chunk size, opcache php options. - Access through HTTPS using reverse proxy. -- Automatic setup of PostgreSQL database. -- Automatic setup of Redis database for caching. -- Backup of the [`shb.nextcloud.dataDir`][1] through the [backup block](./blocks-backup.html). +- Forces PostgreSQL as the database. +- Forces Redis as the cache and sets good defaults. +- Backup of the [`shb.nextcloud.dataDir`][dataDir] through the [backup block](./blocks-backup.html). - Monitoring of reverse proxy, PHP-FPM, and database backups through the [monitoring block](./blocks-monitoring.html). - [Integration Tests](@REPO@/test/services/nextcloud.nix) - Tests system cron job is setup correctly. - Tests initial admin user and password are setup correctly. - Tests admin user can create and retrieve a file through WebDAV. +- Enables easy setup of xdebug for PHP debugging if needed. +- Easily add other apps declaratively through [extraApps][] +- By default automatically disables maintenance mode on start. +- By default automatically launches repair mode with expensive migrations on start. - Access to advanced options not exposed here thanks to how NixOS modules work. +- Has a [demo](#services-nextcloud-server-demo). -[1]: ./services-nextcloud.html#services-nextcloud-server-options-shb.nextcloud.dataDir +[dataDir]: ./services-nextcloud.html#services-nextcloud-server-options-shb.nextcloud.dataDir ## Usage {#services-nextcloud-server-usage} -### Secrets {#services-nextcloud-server-secrets} - -All the secrets should be readable by the nextcloud user. - -Secrets should not be stored in the nix store. If you're using -[sops-nix](https://github.com/Mic92/sops-nix) and assuming your secrets file is located at -`./secrets.yaml`, you can define a secret with: - -```nix -sops.secrets."nextcloud/adminpass" = { - sopsFile = ./secrets.yaml; - mode = "0400"; - owner = "nextcloud"; - group = "nextcloud"; - restartUnits = [ "phpfpm-nextcloud.service" ]; -}; -``` - -Then you can use that secret: - -```nix -shb.nextcloud.adminPassFile = config.sops.secrets."nextcloud/adminpass".path; -``` - ### Nextcloud through HTTP {#services-nextcloud-server-usage-basic} +[HTTP]: #services-nextcloud-server-usage-basic + :::: {.note} This section corresponds to the `basic` section of the [Nextcloud demo](demo-nextcloud-server.html#demo-nextcloud-deploy-basic). :::: -This will set up a Nextcloud service that runs on the NixOS target machine, reachable at -`http://nextcloud.example.com`. If the `shb.ssl` block is [enabled](blocks-ssl.html#usage), the -instance will be reachable at `https://nextcloud.example.com`. +Configuring Nextcloud to be accessible through Nginx reverse proxy +at the address `http://n.example.com`, +with PostgreSQL and Redis configured, +is done like so: ```nix shb.nextcloud = { enable = true; domain = "example.com"; - subdomain = "nextcloud"; - dataDir = "/var/lib/nextcloud"; - adminPassFile = ; + subdomain = "n"; + defaultPhoneRegion = "US"; + adminPass.result.path = config.sops.secrets."nextcloud/adminpass".path; +}; + +sops.secrets."nextcloud/adminpass" = config.shb.nextcloud.adminPass.request; +``` + +This assumes secrets are setup with SOPS as mentioned in [the secrets setup section](usage.html#usage-secrets) of the manual. +Secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`. + +Note though that Nextcloud will not be very happy to be accessed through HTTP, +it much prefers - rightfully - to be accessed through HTTPS. +We will set that up in the next section. + +You can now login as the admin user using the username `admin` +and the password defined in `sops.secrets."nextcloud/adminpass"`. + +### Nextcloud through HTTPS {#services-nextcloud-server-usage-https} + +[HTTPS]: #services-nextcloud-server-usage-https + +To setup HTTPS, we will get our certificates from Let's Encrypt using the HTTP method. +This is the easiest way to get started and does not require you to programmatically +configure a DNS provider. + +Under the hood, we use the Self Host Block [SSL contract](./contracts-ssl.html). +It allows the end user to choose how to generate the certificates. +If you want other options to generate the certificate, follow the SSL contract link. + +Building upon the [Basic Configuration](#services-nextcloud-server-usage-basic) above, we add: + +```nix +shb.certs.certs.letsencrypt."example.com" = { + domain = "example.com"; + group = "nginx"; + reloadServices = [ "nginx.service" ]; + adminEmail = "myemail@mydomain.com"; +}; + +shb.certs.certs.letsencrypt."example.com".extraDomains = [ "n.example.com" ]; + +shb.nextcloud = { + ssl = config.shb.certs.certs.letsencrypt."example.com"; }; ``` -After deploying, the Nextcloud server will be reachable at `http://nextcloud.example.com`. +### Choose Nextcloud Version {#services-nextcloud-server-usage-version} -### Mount Point {#services-nextcloud-server-mount-point} +Self Host Blocks is conservative in the version of Nextcloud it's using. +To choose the version and upgrade at the time of your liking, +just use the [version](#services-nextcloud-server-options-shb.nextcloud.version) option: -If the `dataDir` exists in a mount point, it is highly recommended to make the various Nextcloud -services wait on the mount point before starting. Doing that is just a matter of setting the `mountPointServices` option. +```nix +shb.nextcloud.version = 29; +``` + +### Mount Point {#services-nextcloud-server-usage-mount-point} + +If the `dataDir` exists in a mount point, +it is highly recommended to make the various Nextcloud services wait on the mount point before starting. +Doing that is just a matter of setting the `mountPointServices` option. Assuming a mount point on `/var`, the configuration would look like so: @@ -99,16 +136,18 @@ shb.nextcloud.mountPointServices = [ "var.mount" ]; ### With LDAP Support {#services-nextcloud-server-usage-ldap} +[LDAP]: #services-nextcloud-server-usage-ldap + :::: {.note} This section corresponds to the `ldap` section of the [Nextcloud demo](demo-nextcloud-server.html#demo-nextcloud-deploy-ldap). :::: -We will build upon the [Basic Configuration](#services-nextcloud-server-usage-basic) section, so -please read that first. - +We will build upon the [HTTP][] and [HTTPS][] sections, +so please read those first. We will use the LDAP block provided by Self Host Blocks to setup a [LLDAP](https://github.com/lldap/lldap) service. +If did already configure this for another service, you can skip this snippet. ```nix shb.ldap = { @@ -118,37 +157,50 @@ shb.ldap = { ldapPort = 3890; webUIListenPort = 17170; dcdomain = "dc=example,dc=com"; - ldapUserPasswordFile = ; - jwtSecretFile = ; + ldapUserPassword.result.path = config.sops.secrets."ldap/userPassword".path; + jwtSecret.result.path = config.sops.secrets."ldap/jwtSecret".path; }; + +sops.secrets."ldap/userPassword" = config.shb.ldap.userPassword.request; +sops.secrets."ldap/jwtSecret" = config.shb.ldap.jwtSecret.request; ``` -We also need to configure the `nextcloud` Self Host Blocks service to talk to the LDAP server we +On the `nextcloud` module side, we need to configure it to talk to the LDAP server we just defined: ```nix -shb.nextcloud.apps.ldap +shb.nextcloud.apps.ldap = { enable = true; host = "127.0.0.1"; port = config.shb.ldap.ldapPort; dcdomain = config.shb.ldap.dcdomain; adminName = "admin"; - adminPasswordFile = ; + adminPassword.result.path = config.sops.secrets."nextcloud/ldapUserPassword".path userGroup = "nextcloud_user"; }; + +sops.secrets."nextcloud/ldapUserPassword" = config.shb.nextcloud.adminPasswordFile.request // { + key = "ldap/userPassword"; +}; ``` -The `shb.nextcloud.apps.ldap.adminPasswordFile` must be the same as the -`shb.ldap.ldapUserPasswordFile`. The other secret can be randomly generated with `nix run -nixpkgs#openssl -- rand -hex 64`. +The LDAP admin password must be shared between `shb.ldap` and `shb.nextcloud`, +to do that with SOPS we use the `key` option so that both +`sops.secrets."ldap/userPassword"` +and `sops.secrets."nextcloud/ldapUserPassword"` +secrets have the same content. -And that's it. Now, go to the LDAP server at `http://ldap.example.com`, create the `nextcloud_user` -group, create a user and add it to the group. When that's done, go back to the Nextcloud server at -`http://nextcloud.example.com` and login with that user. +Creating LDAP users and groups is not declarative yet, +so go to the LDAP server at `http://ldap.example.com`, +create the `nextcloud_user` group, +create a user and add it to the group. +When that's done, go back to the Nextcloud server at +`https://nextcloud.example.com` and login with that user. -Note that we cannot create an admin user from the LDAP server, so you need to create a normal user -like above, login with it once so it is known to Nextcloud, then logout, login with the admin -Nextcloud user and promote that new user to admin level. +Note that we cannot create an admin user from the LDAP server, +so you need to create a normal user like above, +login with it once so it is known to Nextcloud, then logout, +login with the admin Nextcloud user and promote that new user to admin level. ### With OIDC Support {#services-nextcloud-server-usage-oidc} @@ -157,128 +209,146 @@ This section corresponds to the `sso` section of the [Nextcloud demo](demo-nextcloud-server.html#demo-nextcloud-deploy-sso). :::: -We will build upon the [Basic Configuration](#services-nextcloud-server-usage-basic) and [With LDAP -Support](#services-nextcloud-server-usage-ldap) sections, so please read those first and setup the -LDAP app as described above. - -Here though, we must setup SSL certificates because the SSO provider only works with the https -protocol. This is actually quite easy thanks to the [SSL block](blocks-ssl.html). For example, with -self-signed certificates: - -```nix -shb.certs = { - cas.selfsigned.myca = { - name = "My CA"; - }; - certs.selfsigned = { - nextcloud = { - ca = config.shb.certs.cas.selfsigned.myca; - domain = "nextcloud.example.com"; - }; - auth = { - ca = config.shb.certs.cas.selfsigned.myca; - domain = "auth.example.com"; - }; - ldap = { - ca = config.shb.certs.cas.selfsigned.myca; - domain = "ldap.example.com"; - }; - }; -}; -``` - -We need to setup the SSO provider, here Authelia thanks to the corresponding SHB block: +We will build upon the [HTTP][], [HTTPS][] and [LDAP][] sections, +so please read those first. +We need to setup the SSO provider, here Authelia, thanks to the corresponding SHB block +and we link it to the LDAP server: ```nix shb.authelia = { enable = true; domain = "example.com"; subdomain = "auth"; - ssl = config.shb.certs.certs.selfsigned.auth; + ssl = config.shb.certs.certs.letsencrypt."example.com"; ldapHostname = "127.0.0.1"; ldapPort = config.shb.ldap.ldapPort; dcdomain = config.shb.ldap.dcdomain; + smtp = { + host = "smtp.eu.mailgun.org"; + port = 587; + username = "postmaster@mg.example.com"; + from_address = "authelia@example.com"; + password.result.path = config.sops.secrets."authelia/smtp_password".path; + }; + secrets = { - jwtSecretFile = ; - ldapAdminPasswordFile = ; - sessionSecretFile = ; - storageEncryptionKeyFile = ; - identityProvidersOIDCHMACSecretFile = ; - identityProvidersOIDCIssuerPrivateKeyFile = ; + jwtSecret.result.path = config.sops.secrets."authelia/jwt_secret".path; + ldapAdminPassword.result.path = config.sops.secrets."authelia/ldap_admin_password".path; + sessionSecret.result.path = config.sops.secrets."authelia/session_secret".path; + storageEncryptionKey.result.path = config.sops.secrets."authelia/storage_encryption_key".path; + identityProvidersOIDCHMACSecret.result.path = config.sops.secrets."authelia/hmac_secret".path; + identityProvidersOIDCIssuerPrivateKey.result.path = config.sops.secrets."authelia/private_key".path; }; }; + +sops.secrets."authelia/jwt_secret" = config.shb.authelia.secrets.jwtSecret.request; +sops.secrets."authelia/ldap_admin_password" = config.shb.authelia.secrets.ldapAdminPassword.request; +sops.secrets."authelia/session_secret" = config.shb.authelia.secrets.sessionSecret.request; +sops.secrets."authelia/storage_encryption_key" = config.shb.authelia.secrets.storageEncryptionKey.request; +sops.secrets."authelia/hmac_secret" = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request; +sops.secrets."authelia/private_key" = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request; +sops.secrets."authelia/smtp_password" = config.shb.authelia.smtp.password.request; ``` -The `shb.authelia.secrets.ldapAdminPasswordFile` must be the same as the -`shb.ldap.ldapUserPasswordFile` defined in the previous section. The secrets can be randomly -generated with `nix run nixpkgs#openssl -- rand -hex 64`. +The secrets can be randomly generated with `nix run nixpkgs#openssl -- rand -hex 64`. Now, on the Nextcloud side, you need to add the following options: ```nix -shb.nextcloud.ssl = config.shb.certs.certs.selfsigned.nextcloud; - shb.nextcloud.apps.sso = { enable = true; endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; clientID = "nextcloud"; fallbackDefaultAuth = false; - secretFile = ; - secretFileForAuthelia = ; + secret.result.path = config.sops.secrets."nextcloud/sso/secret".path; + secretForAuthelia.result.path = config.sops.secrets."nextcloud/sso/secretForAuthelia".path; +}; + +sops.secret."nextcloud/sso/secret" = config.shb.nextcloud.apps.sso.secret.request; +sops.secret."nextcloud/sso/secretForAuthelia" = config.shb.nextcloud.apps.sso.secretForAuthelia.request // { + key = "nextcloud/sso/secret"; }; ``` -Passing the `ssl` option will auto-configure nginx to force SSL connections with the given -certificate. +The SSO secret must be shared between `shb.authelia` and `shb.nextcloud`, +to do that with SOPS we use the `key` option so that both +`sops.secrets."nextcloud/sso/secret"` +and `sops.secrets."nextcloud/sso/secretForAuthelia"` +secrets have the same content. -The `shb.nextcloud.apps.sso.secretFile` and `shb.nextcloud.apps.sso.secretFileForAuthelia` options -must have the same content. The former is a file that must be owned by the `nextcloud` user while -the latter must be owned by the `authelia` user. I want to avoid needing to define the same secret -twice with a future secrets SHB block. +Setting the `fallbackDefaultAuth` to `false` means the only way to login is through Authelia. +If this does not work for any reason, you can let users login through Nextcloud directly by setting this option to `true`. ### Tweak PHPFpm Config {#services-nextcloud-server-usage-phpfpm} +For instances with more users, or if you feel the pages are loading slowly, +you can tweak the `php-fpm` pool settings. + ```nix shb.nextcloud.phpFpmPoolSettings = { - "pm" = "dynamic"; - "pm.max_children" = 800; - "pm.start_servers" = 300; - "pm.min_spare_servers" = 300; - "pm.max_spare_servers" = 500; - "pm.max_spawn_rate" = 50; - "pm.max_requests" = 50; - "pm.process_idle_timeout" = "20s"; + "pm" = "static"; # Can be dynamic + "pm.max_children" = 150; + # "pm.start_servers" = 300; + # "pm.min_spare_servers" = 300; + # "pm.max_spare_servers" = 500; + # "pm.max_spawn_rate" = 50; + # "pm.max_requests" = 50; + # "pm.process_idle_timeout" = "20s"; }; ``` +I don't have a good heuristic for what are good values here but what I found +is that you don't want too high of a `max_children` value +to avoid I/O strain on the hard drives, especially if you use spinning drives. + ### Tweak PostgreSQL Settings {#services-nextcloud-server-usage-postgres} -These settings will impact all databases. +These settings will impact all databases since the NixOS Postgres module +configures only one Postgres instance. + +To know what values to put here, use [https://pgtune.leopard.in.ua/](https://pgtune.leopard.in.ua/). +Remember the server hosting PostgreSQL is shared at least with the Nextcloud service and probably others. +So to avoid PostgreSQL hogging all the resources, reduce the values you give on that website +for CPU, available memory, etc. +For example, I put 12 GB of memory and 4 CPUs while I had more: + +- `DB Version`: 14 +- `OS Type`: linux +- `DB Type`: dw +- `Total Memory (RAM)`: 12 GB +- `CPUs num`: 4 +- `Data Storage`: ssd + +And got the following values: ```nix shb.nextcloud.postgresSettings = { - max_connections = "100"; - shared_buffers = "512MB"; - effective_cache_size = "1536MB"; - maintenance_work_mem = "128MB"; + max_connections = "400"; + shared_buffers = "3GB"; + effective_cache_size = "9GB"; + maintenance_work_mem = "768MB"; checkpoint_completion_target = "0.9"; wal_buffers = "16MB"; default_statistics_target = "100"; random_page_cost = "1.1"; effective_io_concurrency = "200"; - work_mem = "2621kB"; + work_mem = "7864kB"; huge_pages = "off"; min_wal_size = "1GB"; max_wal_size = "4GB"; + max_worker_processes = "4"; + max_parallel_workers_per_gather = "2"; + max_parallel_workers = "4"; + max_parallel_maintenance_workers = "2"; }; ``` ### Backup {#services-nextcloud-server-usage-backup} -Backing up Nextcloud using the [Restic block](blocks-restic.html) is done like so: +Backing up Nextcloud data files using the [Restic block](blocks-restic.html) is done like so: ```nix shb.restic.instances."nextcloud" = { @@ -293,6 +363,21 @@ The name `"nextcloud"` in the `instances` can be anything. The `config.shb.nextcloud.backup` option provides what directories to backup. You can define any number of Restic instances to backup Nextcloud multiple times. +For backing up the Nextcloud database using the same Restic block, do like so: + +```nix +shb.restic.instances."postgres" = { + request = config.shb.postgresql.databasebackup; + settings = { + enable = true; + }; +}; +``` + +Note that this will backup the whole PostgreSQL instance, +not just the Nextcloud database. +This limitation will be lifted in the future. + ### Enable Preview Generator App {#services-nextcloud-server-usage-previewgenerator} The following snippet installs and enables the [Preview @@ -327,25 +412,33 @@ Storage](https://docs.nextcloud.com/server/28/go.php?to=admin-external-storage) shb.nextcloud.apps.externalStorage.enable = true; ``` -Optionally creates a local mount point with: +Adding external storage can then be done through the UI. +For the special case of mounting a local folder as an external storage, +Self Host Blocks provides options. +The following snippet will mount the `/srv/nextcloud/$user` local file +in each user's `/home` Nextcloud directory. ```nix -externalStorage = { - userLocalMount.rootDirectory = "/srv/nextcloud/$user"; - userLocalMount.mountName = "home"; +shb.nextcloud.apps.externalStorage.userLocalMount = { + rootDirectory = "/srv/nextcloud/$user"; + mountName = "home"; }; ``` -You can even make the external storage be at the root with: +You can even make the external storage mount in the root `/` Nextcloud directory with: ```nix -externalStorage.userLocalMount.mountName = "/"; +shb.nextcloud.apps.externalStorage.userLocalMount = { + mountName = "/"; +}; ``` -Recommended use of this app is to have the Nextcloud's `dataDir` on a SSD and the -`userLocalRooDirectory` on a HDD. Indeed, a SSD is much quicker than a spinning hard drive, which is -well suited for randomly accessing small files like thumbnails. On the other side, a spinning hard -drive can store more data which is well suited for storing user data. +Recommended use of this app is to have the Nextcloud's `dataDir` on a SSD +and the `userLocalMount` on a HDD. +Indeed, a SSD is much quicker than a spinning hard drive, +which is well suited for randomly accessing small files like thumbnails. +On the other side, a spinning hard drive can store more data +which is well suited for storing user data. ### Enable OnlyOffice App {#services-nextcloud-server-usage-onlyoffice} @@ -371,8 +464,11 @@ nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) ### Enable Monitoring {#services-nextcloud-server-server-usage-monitoring} -Enable the [monitoring block](./blocks-monitoring.html). The metrics will automatically appear in -the corresponding dashboards. +Enable the [monitoring block](./blocks-monitoring.html). +A [Grafana dashboard][] for overall server performance will be created +and the Nextcloud metrics will automatically appear there. + +[Grafana dashboard]: ./blocks-monitoring.html#blocks-monitoring-performance-dashboard ### Enable Tracing {#services-nextcloud-server-server-usage-tracing} @@ -383,26 +479,31 @@ shb.nextcloud.debug = true; ``` Traces will be located at `/var/log/xdebug`. +See [my blog post][] for how to look at the traces. +I want to make the traces available in Grafana directly +but that's not the case yet. -See [my blog -post](http://blog.tiserbox.com/posts/2023-08-12-what%27s-up-with-nextcloud-webdav-slowness.html) for -how to look at the traces. +[my blog post]: http://blog.tiserbox.com/posts/2023-08-12-what%27s-up-with-nextcloud-webdav-slowness.html ### Appdata Location {#services-nextcloud-server-server-usage-appdata} -The appdata folder is a special folder located under the `shb.nextcloud.dataDir` directory. It is -named `appdata_` with the Nextcloud's instance ID as a suffix. You can find your current -instance ID with `nextcloud-occ config:system:get instanceid`. In there, you will find one subfolder -for every installed app that needs to store files. +The appdata folder is a special folder located under the `shb.nextcloud.dataDir` directory. +It is named `appdata_` with the Nextcloud's instance ID as a suffix. +You can find your current instance ID with `nextcloud-occ config:system:get instanceid`. +In there, you will find one subfolder for every installed app that needs to store files. -For performance reasons, it is recommended to store this folder on a fast drive that is optimized -for randomized read and write access. The best would be either an SSD or an NVMe drive. +For performance reasons, it is recommended to store this folder on a fast drive +that is optimized for randomized read and write access. +The best would be either an SSD or an NVMe drive. -If you intentionally put Nextcloud's `shb.nextcloud.dataDir` folder on a HDD with spinning disks, -for example because they offer more disk space, then the appdata folder is also located on spinning -drives. You are thus faced with a conundrum. The only way to solve this is to bind mount a folder -from an SSD over the appdata folder. SHB does not provide (yet?) a declarative way to setup this but -this command should be enough: +The best way to solve this is to use the [External Storage app](#services-nextcloud-server-usage-externalstorage). + +If you have an existing installation and put Nextcloud's `shb.nextcloud.dataDir` folder on a HDD with spinning disks, +then the appdata folder is also located on spinning drives. +One way to solve this is to bind mount a folder from an SSD over the appdata folder. +SHB does not provide a declarative way to setup this +as the external storage app is the preferred way +but this command should be enough: ```bash mount /dev/sdd /srv/sdd @@ -410,8 +511,8 @@ mkdir -p /srv/sdd/appdata_nextcloud mount --bind /srv/sdd/appdata_nextcloud /var/lib/nextcloud/data/appdata_ocxvky2f5ix7 ``` -Note that you can re-generate a new appdata folder by issuing the command `occ config:system:delete -instanceid`. +Note that you can re-generate a new appdata folder +by issuing the command `nextcloud-occ config:system:delete instanceid`. ## Demo {#services-nextcloud-server-demo} diff --git a/test/services/nextcloud.nix b/test/services/nextcloud.nix index e4dfaa2..f3eaf29 100644 --- a/test/services/nextcloud.nix +++ b/test/services/nextcloud.nix @@ -3,6 +3,7 @@ let pkgs' = pkgs; adminUser = "root"; adminPass = "rootpw"; + oidcSecret = "oidcSecret"; subdomain = "n"; domain = "example.com"; @@ -131,9 +132,13 @@ let externalFqdn = "${fqdn}:8080"; adminUser = adminUser; - adminPassFile = pkgs.writeText "adminPassFile" adminPass; + adminPass.result.path = config.shb.hardcodedsecret.adminPass.path; debug = true; }; + + shb.hardcodedsecret.adminPass = config.shb.nextcloud.adminPass.request // { + content = adminPass; + }; }; https = { config, ...}: { @@ -152,30 +157,34 @@ let port = config.shb.ldap.ldapPort; dcdomain = config.shb.ldap.dcdomain; adminName = "admin"; - adminPasswordFile = config.shb.ldap.ldapUserPassword.result.path; + adminPassword.result.path = config.shb.ldap.ldapUserPassword.result.path; userGroup = "nextcloud_user"; }; }; }; sso = { config, ... }: - let - authSecret = pkgs.writeText "authSecret" "authSecret"; - in - { - shb.nextcloud = { - apps.sso = { - enable = true; - endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; - clientID = "nextcloud"; - # adminUserGroup = "nextcloud_admin"; + { + shb.nextcloud = { + apps.sso = { + enable = true; + endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + clientID = "nextcloud"; + # adminUserGroup = "nextcloud_admin"; - secretFile = authSecret; - secretFileForAuthelia = authSecret; + secret.result.path = config.shb.hardcodedsecret.oidcSecret.path; + secretForAuthelia.result.path = config.shb.hardcodedsecret.oidcAutheliaSecret.path; - fallbackDefaultAuth = false; - }; + fallbackDefaultAuth = false; }; + }; + + shb.hardcodedsecret.oidcSecret = config.shb.nextcloud.apps.sso.secret.request // { + content = oidcSecret; + }; + shb.hardcodedsecret.oidcAutheliaSecret = config.shb.nextcloud.apps.sso.secretForAuthelia.request // { + content = oidcSecret; + }; }; previewgenerator = { config, ...}: {