selfhostblocks/modules/blocks/monitoring.nix

823 lines
25 KiB
Nix

{
config,
pkgs,
lib,
shb,
...
}:
let
cfg = config.shb.monitoring;
fqdn = "${cfg.subdomain}.${cfg.domain}";
commonLabels = {
hostname = config.networking.hostName;
domain = cfg.domain;
};
roleClaim = "grafana_groups";
oauthScopes = [
"openid"
"email"
"profile"
"groups"
"${roleClaim}"
];
in
{
imports = [
../blocks/authelia.nix
../blocks/lldap.nix
];
options.shb.monitoring = {
enable = lib.mkEnableOption "selfhostblocks.monitoring";
subdomain = lib.mkOption {
type = lib.types.str;
description = "Subdomain under which Grafana will be served.";
example = "grafana";
};
domain = lib.mkOption {
type = lib.types.str;
description = "domain under which Grafana will be served.";
example = "mydomain.com";
};
ssl = lib.mkOption {
description = "Path to SSL files";
type = lib.types.nullOr shb.contracts.ssl.certs;
default = null;
};
grafanaPort = lib.mkOption {
type = lib.types.port;
description = "Port where Grafana listens to HTTP requests.";
default = 3000;
};
prometheusPort = lib.mkOption {
type = lib.types.port;
description = "Port where Prometheus listens to HTTP requests.";
default = 3001;
};
lokiPort = lib.mkOption {
type = lib.types.port;
description = "Port where Loki listens to HTTP requests.";
default = 3002;
};
lokiMajorVersion = lib.mkOption {
type = lib.types.enum [
2
3
];
description = ''
Switching from version 2 to 3 requires manual intervention
https://grafana.com/docs/loki/latest/setup/upgrade/#main--unreleased. So this let's the user
upgrade at their own pace.
'';
default = 2;
};
debugLog = lib.mkOption {
type = lib.types.bool;
description = "Set to true to enable debug logging of the infrastructure serving Grafana.";
default = false;
example = true;
};
orgId = lib.mkOption {
type = lib.types.int;
description = "Org ID where all self host blocks related config will be stored.";
default = 1;
};
provisionDashboards = lib.mkOption {
type = lib.types.bool;
description = "Provision Self Host Blocks dashboards under 'Self Host Blocks' folder.";
default = true;
};
contactPoints = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "List of email addresses to send alerts to";
default = [ ];
};
adminPassword = lib.mkOption {
description = "Initial admin password.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = "grafana";
group = "grafana";
restartUnits = [ "grafana.service" ];
};
};
};
secretKey = lib.mkOption {
description = "Secret key used for signing.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
owner = "grafana";
group = "grafana";
restartUnits = [ "grafana.service" ];
};
};
};
smtp = lib.mkOption {
description = "SMTP options.";
default = null;
type = lib.types.nullOr (
lib.types.submodule {
options = {
from_address = lib.mkOption {
type = lib.types.str;
description = "SMTP address from which the emails originate.";
example = "vaultwarden@mydomain.com";
};
from_name = lib.mkOption {
type = lib.types.str;
description = "SMTP name from which the emails originate.";
default = "Grafana";
};
host = lib.mkOption {
type = lib.types.str;
description = "SMTP host to send the emails to.";
};
port = lib.mkOption {
type = lib.types.port;
description = "SMTP port to send the emails to.";
default = 25;
};
username = lib.mkOption {
type = lib.types.str;
description = "Username to connect to the SMTP host.";
};
passwordFile = lib.mkOption {
type = lib.types.str;
description = "File containing the password to connect to the SMTP host.";
};
};
}
);
};
ldap = lib.mkOption {
description = ''
Setup LDAP integration.
'';
default = { };
type = lib.types.submodule {
options = {
userGroup = lib.mkOption {
type = lib.types.str;
description = "Group users must belong to to be able to login to Grafana.";
default = "monitoring_user";
};
adminGroup = lib.mkOption {
type = lib.types.str;
description = "Group users must belong to to be admins in Grafana.";
default = "monitoring_admin";
};
};
};
};
sso = lib.mkOption {
description = ''
Setup SSO integration.
'';
default = { };
type = lib.types.submodule {
options = {
enable = lib.mkEnableOption "SSO integration.";
authEndpoint = lib.mkOption {
type = lib.types.str;
default = null;
description = "Endpoint to the SSO provider.";
example = "https://authelia.example.com";
};
clientID = lib.mkOption {
type = lib.types.str;
description = "Client ID for the OIDC endpoint.";
default = "grafana";
};
authorization_policy = lib.mkOption {
type = lib.types.enum [
"one_factor"
"two_factor"
];
description = "Require one factor (password) or two factor (device) authentication.";
default = "one_factor";
};
sharedSecret = lib.mkOption {
description = "OIDC shared secret for Grafana.";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
owner = "grafana";
restartUnits = [
"grafana.service"
];
};
};
};
sharedSecretForAuthelia = lib.mkOption {
description = "OIDC shared secret for Authelia. Must be the same as `sharedSecret`";
type = lib.types.submodule {
options = shb.contracts.secret.mkRequester {
mode = "0400";
ownerText = "config.shb.authelia.autheliaUser";
owner = config.shb.authelia.autheliaUser;
};
};
};
};
};
};
};
config = lib.mkMerge [
(lib.mkIf cfg.enable {
assertions = [
{
assertion = (!(isNull cfg.smtp)) -> builtins.length cfg.contactPoints > 0;
message = "Must have at least one contact point for alerting";
}
];
shb.postgresql.ensures = [
{
username = "grafana";
database = "grafana";
}
];
services.grafana = {
enable = true;
settings = {
database = {
host = "/run/postgresql";
user = "grafana";
name = "grafana";
type = "postgres";
# Uses peer auth for local users, so we don't need a password.
# Here's the syntax anyway for future refence:
# password = "$__file{/run/secrets/homeassistant/dbpass}";
};
security = {
secret_key = "$__file{${cfg.secretKey.result.path}}";
disable_initial_admin_creation = false; # Enable when LDAP support is configured.
admin_password = "$__file{${cfg.adminPassword.result.path}}"; # Remove when LDAP support is configured.
};
server = {
http_addr = "127.0.0.1";
http_port = cfg.grafanaPort;
domain = fqdn;
root_url = "https://${fqdn}";
router_logging = cfg.debugLog;
};
smtp = lib.mkIf (!(isNull cfg.smtp)) {
enabled = true;
inherit (cfg.smtp) from_address from_name;
host = "${cfg.smtp.host}:${toString cfg.smtp.port}";
user = cfg.smtp.username;
password = "$__file{${cfg.smtp.passwordFile}}";
};
};
};
services.grafana.provision = {
dashboards.settings = lib.mkIf cfg.provisionDashboards {
apiVersion = 1;
providers = [
{
folder = "Self Host Blocks";
options.path = ./monitoring/dashboards;
allowUiUpdates = true;
disableDeletion = true;
}
];
};
datasources.settings = {
apiVersion = 1;
datasources = [
{
inherit (cfg) orgId;
name = "Prometheus";
type = "prometheus";
url = "http://127.0.0.1:${toString config.services.prometheus.port}";
uid = "df80f9f5-97d7-4112-91d8-72f523a02b09";
isDefault = true;
version = 1;
}
{
inherit (cfg) orgId;
name = "Loki";
type = "loki";
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
uid = "cd6cc53e-840c-484d-85f7-96fede324006";
version = 1;
}
];
deleteDatasources = [
{
inherit (cfg) orgId;
name = "Prometheus";
}
{
inherit (cfg) orgId;
name = "Loki";
}
];
};
alerting.contactPoints.settings = {
apiVersion = 1;
contactPoints = [
{
inherit (cfg) orgId;
name = "grafana-default-email";
receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [
{
uid = "sysadmin";
type = "email";
settings.addresses = lib.concatStringsSep ";" cfg.contactPoints;
}
];
}
];
};
alerting.policies.settings = {
apiVersion = 1;
policies = [
{
inherit (cfg) orgId;
receiver = "grafana-default-email";
group_by = [
"grafana_folder"
"alertname"
];
group_wait = "30s";
group_interval = "5m";
repeat_interval = "4h";
}
];
# resetPolicies seems to happen after setting the above policies, effectively rolling back
# any updates.
};
alerting.rules.settings =
let
rules = builtins.fromJSON (builtins.readFile ./monitoring/rules.json);
in
{
apiVersion = 1;
groups = [
{
inherit (cfg) orgId;
name = "SysAdmin";
folder = "Self Host Blocks";
interval = "10m";
inherit rules;
}
];
# deleteRules seems to happen after creating the above rules, effectively rolling back
# any updates.
};
};
services.prometheus = {
enable = true;
port = cfg.prometheusPort;
};
services.loki = {
enable = true;
dataDir = "/var/lib/loki";
package =
if cfg.lokiMajorVersion == 3 then
pkgs.grafana-loki
else
# Comes from https://github.com/NixOS/nixpkgs/commit/8f95320f39d7e4e4a29ee70b8718974295a619f4
(pkgs.grafana-loki.overrideAttrs (
finalAttrs: previousAttrs: rec {
version = "2.9.6";
src = pkgs.fetchFromGitHub {
owner = "grafana";
repo = "loki";
rev = "v${version}";
hash = "sha256-79hK7axHf6soku5DvdXkE/0K4WKc4pnS9VMbVc1FS2I=";
};
subPackages = [
"cmd/loki"
"cmd/loki-canary"
"clients/cmd/promtail"
"cmd/logcli"
# Removes "cmd/lokitool"
];
ldflags =
let
t = "github.com/grafana/loki/pkg/util/build";
in
[
"-s"
"-w"
"-X ${t}.Version=${version}"
"-X ${t}.BuildUser=nix@nixpkgs"
"-X ${t}.BuildDate=unknown"
"-X ${t}.Branch=unknown"
"-X ${t}.Revision=unknown"
];
}
));
configuration = {
auth_enabled = false;
server.http_listen_port = cfg.lokiPort;
ingester = {
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore.store = "inmemory";
replication_factor = 1;
};
final_sleep = "0s";
};
chunk_idle_period = "5m";
chunk_retain_period = "30s";
};
schema_config = {
configs = [
{
from = "2018-04-15";
store = "boltdb";
object_store = "filesystem";
schema = "v9";
index.prefix = "index_";
index.period = "168h";
}
];
};
storage_config = {
boltdb.directory = "/tmp/loki/index";
filesystem.directory = "/tmp/loki/chunks";
};
limits_config = {
enforce_metric_name = false;
reject_old_samples = true;
reject_old_samples_max_age = "168h";
};
chunk_store_config = {
max_look_back_period = 0;
};
table_manager = {
chunk_tables_provisioning = {
inactive_read_throughput = 0;
inactive_write_throughput = 0;
provisioned_read_throughput = 0;
provisioned_write_throughput = 0;
};
index_tables_provisioning = {
inactive_read_throughput = 0;
inactive_write_throughput = 0;
provisioned_read_throughput = 0;
provisioned_write_throughput = 0;
};
retention_deletes_enabled = false;
retention_period = 0;
};
};
};
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 9080;
grpc_listen_port = 0;
};
positions.filename = "/tmp/positions.yaml";
client.url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/api/prom/push";
scrape_configs = [
{
job_name = "systemd";
journal = {
json = false;
max_age = "12h";
path = "/var/log/journal";
# matches = "_TRANSPORT=kernel";
labels = {
domain = cfg.domain;
hostname = config.networking.hostName;
job = "systemd-journal";
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "unit";
}
];
}
];
};
};
services.nginx = {
enable = true;
virtualHosts.${fqdn} = {
forceSSL = !(isNull cfg.ssl);
sslCertificate = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.cert;
sslCertificateKey = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.key;
locations."/" = {
proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header Host $host;
'';
};
};
};
services.prometheus.scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
labels = commonLabels;
}
];
}
{
job_name = "netdata";
metrics_path = "/api/v1/allmetrics";
params.format = [ "prometheus" ];
honor_labels = true;
static_configs = [
{
targets = [ "127.0.0.1:19999" ];
labels = commonLabels;
}
];
}
{
job_name = "smartctl";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" ];
labels = commonLabels;
}
];
}
{
job_name = "prometheus_internal";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.prometheus.port}" ];
labels = commonLabels;
}
];
}
{
job_name = "systemd";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}" ];
labels = commonLabels;
}
];
}
]
++ (lib.lists.optional config.services.nginx.enable {
job_name = "nginx";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}" ];
labels = commonLabels;
}
];
# }) ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
# job_name = "redis";
# static_configs = [
# {
# targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"];
# }
# ];
# }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
# job_name = "openvpn";
# static_configs = [
# {
# targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"];
# }
# ];
})
++ (lib.optional config.services.dnsmasq.enable {
job_name = "dnsmasq";
static_configs = [
{
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}" ];
labels = commonLabels;
}
];
});
services.prometheus.exporters.nginx = lib.mkIf config.services.nginx.enable {
enable = true;
port = 9111;
listenAddress = "127.0.0.1";
scrapeUri = "http://localhost:80/nginx_status";
};
services.prometheus.exporters.node = {
enable = true;
# https://github.com/prometheus/node_exporter#collectors
enabledCollectors = [
"arp"
"cpu"
"cpufreq"
"diskstats"
"dmi"
"edac"
"entropy"
"filefd"
"filesystem"
"hwmon"
"loadavg"
"meminfo"
"netclass"
"netdev"
"netstat"
"nvme"
"os"
"pressure"
"rapl"
"schedstat"
"stat"
"thermal_zone"
"time"
"uname"
"vmstat"
"zfs"
# Disabled by default
"cgroups"
"drm"
"ethtool"
"logind"
"wifi"
];
port = 9112;
listenAddress = "127.0.0.1";
};
# https://github.com/nixos/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80
# TODO: remove when https://github.com/NixOS/nixpkgs/pull/205165 is merged
services.udev.extraRules = ''
SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
'';
services.prometheus.exporters.smartctl = {
enable = true;
port = 9115;
listenAddress = "127.0.0.1";
};
# services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
# enable = true;
# port = 9119;
# listenAddress = "127.0.0.1";
# };
# services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
# enable = true;
# port = 9121;
# listenAddress = "127.0.0.1";
# statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers;
# };
services.prometheus.exporters.dnsmasq = lib.mkIf config.services.dnsmasq.enable {
enable = true;
port = 9211;
listenAddress = "127.0.0.1";
};
services.prometheus.exporters.systemd = {
enable = true;
port = 9116;
listenAddress = "127.0.0.1";
};
services.nginx.statusPage = lib.mkDefault config.services.nginx.enable;
services.netdata = {
enable = true;
config = {
# web.mode = "none";
# web."bind to" = "127.0.0.1:19999";
global = {
"debug log" = "syslog";
"access log" = "syslog";
"error log" = "syslog";
};
};
};
nixpkgs.overlays = [
(final: prev: {
prometheus-systemd-exporter = prev.prometheus-systemd-exporter.overrideAttrs {
src = final.fetchFromGitHub {
owner = "ibizaman";
repo = prev.prometheus-systemd-exporter.pname;
# rev = "v${prev.prometheus-systemd-exporter.version}";
rev = "next_timer";
sha256 = "sha256-jzkh/616tsJbNxFtZ0xbdBQc16TMIYr9QOkPaeQw8xA=";
};
vendorHash = "sha256-4hsQ1417jLNOAqGkfCkzrmEtYR4YLLW2j0CiJtPg6GI=";
};
})
];
})
(lib.mkIf (cfg.enable && cfg.sso.enable) {
shb.lldap.ensureGroups = {
${cfg.ldap.userGroup} = { };
${cfg.ldap.adminGroup} = { };
};
shb.authelia.extraDefinitions = {
user_attributes.${roleClaim}.expression =
# Roles are: None, Viewer, Editor, Admin, GrafanaAdmin
''"${cfg.ldap.adminGroup}" in groups ? "Admin" : ("${cfg.ldap.userGroup}" in groups ? "Editor" : "Invalid")'';
};
shb.authelia.extraOidcClaimsPolicies.${roleClaim} = {
custom_claims = {
"${roleClaim}" = { };
};
};
shb.authelia.extraOidcScopes."${roleClaim}" = {
claims = [ "${roleClaim}" ];
};
services.grafana.settings."auth.generic_oauth" = {
enabled = true;
name = "Authelia";
icon = "signin";
client_id = cfg.sso.clientID;
client_secret = "$__file{${cfg.sso.sharedSecret.result.path}}";
scopes = oauthScopes;
empty_scopes = false;
allow_sign_up = true;
auto_login = true;
auth_url = "${cfg.sso.authEndpoint}/api/oidc/authorization";
token_url = "${cfg.sso.authEndpoint}/api/oidc/token";
# use_refresh_token = true; ? # https://grafana.com/docs/grafana/latest/setup-grafana/configure-access/configure-authentication/generic-oauth/#configure-generic-oauth-authentication-client-using-the-grafana-configuration-file
api_url = "${cfg.sso.authEndpoint}/api/oidc/userinfo";
login_attribute_path = "preferred_username";
groups_attribute_path = "groups";
name_attribute_path = "name";
use_pkce = true;
allow_assign_grafana_admin = true;
skip_org_role_sync = false;
role_attribute_path = roleClaim;
role_attribute_strict = true;
};
shb.authelia.oidcClients = [
{
client_id = cfg.sso.clientID;
client_secret.source = cfg.sso.sharedSecretForAuthelia.result.path;
claims_policy = "${roleClaim}";
scopes = oauthScopes;
authorization_policy = cfg.sso.authorization_policy;
redirect_uris = [
"https://${cfg.subdomain}.${cfg.domain}/login/generic_oauth"
];
require_pkce = true;
pkce_challenge_method = "S256";
response_types = [ "code" ];
token_endpoint_auth_method = "client_secret_basic";
}
];
})
];
}