990 lines
30 KiB
Nix
990 lines
30 KiB
Nix
{
|
|
config,
|
|
pkgs,
|
|
lib,
|
|
shb,
|
|
...
|
|
}:
|
|
|
|
let
|
|
cfg = config.shb.monitoring;
|
|
|
|
fqdn = "${cfg.subdomain}.${cfg.domain}";
|
|
|
|
commonLabels = {
|
|
hostname = config.networking.hostName;
|
|
domain = cfg.domain;
|
|
};
|
|
|
|
roleClaim = "grafana_groups";
|
|
oauthScopes = [
|
|
"openid"
|
|
"email"
|
|
"profile"
|
|
"groups"
|
|
"${roleClaim}"
|
|
];
|
|
in
|
|
{
|
|
imports = [
|
|
../../lib/module.nix
|
|
../blocks/authelia.nix
|
|
../blocks/lldap.nix
|
|
../blocks/nginx.nix
|
|
];
|
|
|
|
options.shb.monitoring = {
|
|
enable = lib.mkEnableOption "selfhostblocks.monitoring";
|
|
|
|
subdomain = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Subdomain under which Grafana will be served.";
|
|
example = "grafana";
|
|
};
|
|
|
|
domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "domain under which Grafana will be served.";
|
|
example = "mydomain.com";
|
|
};
|
|
|
|
ssl = lib.mkOption {
|
|
description = "Path to SSL files";
|
|
type = lib.types.nullOr shb.contracts.ssl.certs;
|
|
default = null;
|
|
};
|
|
|
|
grafanaPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
description = "Port where Grafana listens to HTTP requests.";
|
|
default = 3000;
|
|
};
|
|
|
|
prometheusPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
description = "Port where Prometheus listens to HTTP requests.";
|
|
default = 3001;
|
|
};
|
|
|
|
lokiPort = lib.mkOption {
|
|
type = lib.types.port;
|
|
description = "Port where Loki listens to HTTP requests.";
|
|
default = 3002;
|
|
};
|
|
|
|
lokiMajorVersion = lib.mkOption {
|
|
type = lib.types.enum [
|
|
2
|
|
3
|
|
];
|
|
description = ''
|
|
Switching from version 2 to 3 requires manual intervention
|
|
https://grafana.com/docs/loki/latest/setup/upgrade/#main--unreleased. So this let's the user
|
|
upgrade at their own pace.
|
|
'';
|
|
default = 2;
|
|
};
|
|
|
|
debugLog = lib.mkOption {
|
|
type = lib.types.bool;
|
|
description = "Set to true to enable debug logging of the infrastructure serving Grafana.";
|
|
default = false;
|
|
example = true;
|
|
};
|
|
|
|
orgId = lib.mkOption {
|
|
type = lib.types.int;
|
|
description = "Org ID where all self host blocks related config will be stored.";
|
|
default = 1;
|
|
};
|
|
|
|
dashboards = lib.mkOption {
|
|
type = lib.types.listOf lib.types.path;
|
|
description = "Dashboards to provision under 'Self Host Blocks' folder.";
|
|
default = [ ];
|
|
};
|
|
|
|
contactPoints = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
description = "List of email addresses to send alerts to";
|
|
default = [ ];
|
|
};
|
|
|
|
scrutiny = {
|
|
enable = lib.mkEnableOption "scrutiny service" // {
|
|
default = true;
|
|
};
|
|
subdomain = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
description = ''
|
|
If a string, this will be the subdomain under which the scrutiny web interface will be servced.
|
|
|
|
If null, the web interface will not be served and only the prometheus metrics will be accessible.
|
|
'';
|
|
default = "scrutiny";
|
|
};
|
|
dashboard = lib.mkOption {
|
|
description = ''
|
|
Dashboard contract consumer
|
|
'';
|
|
default = { };
|
|
type = lib.types.submodule {
|
|
options = shb.contracts.dashboard.mkRequester {
|
|
externalUrl = "https://${cfg.scrutiny.subdomain}.${cfg.domain}";
|
|
externalUrlText = "https://\${config.shb.monitoring.scrutiny.subdomain}.\${config.shb.monitoring.domain}";
|
|
internalUrl = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
|
|
internalUrlText = "https://127.0.0.1.\${config.services.scrutiny.settings.web.listen.port}";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
adminPassword = lib.mkOption {
|
|
description = "Initial admin password.";
|
|
type = lib.types.submodule {
|
|
options = shb.contracts.secret.mkRequester {
|
|
mode = "0400";
|
|
owner = "grafana";
|
|
group = "grafana";
|
|
restartUnits = [ "grafana.service" ];
|
|
};
|
|
};
|
|
};
|
|
|
|
secretKey = lib.mkOption {
|
|
description = "Secret key used for signing.";
|
|
type = lib.types.submodule {
|
|
options = shb.contracts.secret.mkRequester {
|
|
mode = "0400";
|
|
owner = "grafana";
|
|
group = "grafana";
|
|
restartUnits = [ "grafana.service" ];
|
|
};
|
|
};
|
|
};
|
|
|
|
smtp = lib.mkOption {
|
|
description = "SMTP options.";
|
|
default = null;
|
|
type = lib.types.nullOr (
|
|
lib.types.submodule {
|
|
options = {
|
|
from_address = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "SMTP address from which the emails originate.";
|
|
example = "vaultwarden@mydomain.com";
|
|
};
|
|
from_name = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "SMTP name from which the emails originate.";
|
|
default = "Grafana";
|
|
};
|
|
host = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "SMTP host to send the emails to.";
|
|
};
|
|
port = lib.mkOption {
|
|
type = lib.types.port;
|
|
description = "SMTP port to send the emails to.";
|
|
default = 25;
|
|
};
|
|
username = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Username to connect to the SMTP host.";
|
|
};
|
|
passwordFile = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "File containing the password to connect to the SMTP host.";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
};
|
|
|
|
ldap = lib.mkOption {
|
|
description = ''
|
|
Setup LDAP integration.
|
|
'';
|
|
default = { };
|
|
type = lib.types.submodule {
|
|
options = {
|
|
userGroup = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Group users must belong to to be able to login to Grafana.";
|
|
default = "monitoring_user";
|
|
};
|
|
adminGroup = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Group users must belong to to be admins in Grafana.";
|
|
default = "monitoring_admin";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
sso = lib.mkOption {
|
|
description = ''
|
|
Setup SSO integration.
|
|
'';
|
|
default = { };
|
|
type = lib.types.submodule {
|
|
options = {
|
|
enable = lib.mkEnableOption "SSO integration.";
|
|
|
|
authEndpoint = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = null;
|
|
description = "Endpoint to the SSO provider.";
|
|
example = "https://authelia.example.com";
|
|
};
|
|
|
|
clientID = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Client ID for the OIDC endpoint.";
|
|
default = "grafana";
|
|
};
|
|
|
|
authorization_policy = lib.mkOption {
|
|
type = lib.types.enum [
|
|
"one_factor"
|
|
"two_factor"
|
|
];
|
|
description = "Require one factor (password) or two factor (device) authentication.";
|
|
default = "one_factor";
|
|
};
|
|
|
|
sharedSecret = lib.mkOption {
|
|
description = "OIDC shared secret for Grafana.";
|
|
type = lib.types.submodule {
|
|
options = shb.contracts.secret.mkRequester {
|
|
owner = "grafana";
|
|
restartUnits = [
|
|
"grafana.service"
|
|
];
|
|
};
|
|
};
|
|
};
|
|
|
|
sharedSecretForAuthelia = lib.mkOption {
|
|
description = "OIDC shared secret for Authelia. Must be the same as `sharedSecret`";
|
|
type = lib.types.submodule {
|
|
options = shb.contracts.secret.mkRequester {
|
|
mode = "0400";
|
|
ownerText = "config.shb.authelia.autheliaUser";
|
|
owner = config.shb.authelia.autheliaUser;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
dashboard = lib.mkOption {
|
|
description = ''
|
|
Dashboard contract consumer
|
|
'';
|
|
default = { };
|
|
type = lib.types.submodule {
|
|
options = shb.contracts.dashboard.mkRequester {
|
|
externalUrl = "https://${cfg.subdomain}.${cfg.domain}";
|
|
externalUrlText = "https://\${config.shb.monitoring.subdomain}.\${config.shb.monitoring.domain}";
|
|
internalUrl = "https://${cfg.subdomain}.${cfg.domain}";
|
|
internalUrlText = "https://\${config.shb.monitoring.subdomain}.\${config.shb.monitoring.domain}";
|
|
};
|
|
};
|
|
};
|
|
|
|
impermanence = lib.mkOption {
|
|
description = ''
|
|
Paths to save when using impermanence setup.
|
|
'';
|
|
type = lib.types.attrsOf lib.types.str;
|
|
default = {
|
|
fluent-bit = "/var/fluent-bit";
|
|
};
|
|
};
|
|
};
|
|
|
|
config = lib.mkMerge [
|
|
(lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = builtins.length cfg.contactPoints > 0;
|
|
message = "Must have at least one contact point for alerting";
|
|
}
|
|
];
|
|
|
|
shb.postgresql.ensures = [
|
|
{
|
|
username = "grafana";
|
|
database = "grafana";
|
|
}
|
|
];
|
|
|
|
services.grafana = {
|
|
enable = true;
|
|
|
|
settings = {
|
|
database = {
|
|
host = "/run/postgresql";
|
|
user = "grafana";
|
|
name = "grafana";
|
|
type = "postgres";
|
|
# Uses peer auth for local users, so we don't need a password.
|
|
# Here's the syntax anyway for future refence:
|
|
# password = "$__file{/run/secrets/homeassistant/dbpass}";
|
|
};
|
|
|
|
security = {
|
|
secret_key = "$__file{${cfg.secretKey.result.path}}";
|
|
disable_initial_admin_creation = false; # Enable when LDAP support is configured.
|
|
admin_password = "$__file{${cfg.adminPassword.result.path}}"; # Remove when LDAP support is configured.
|
|
};
|
|
|
|
server = {
|
|
http_addr = "127.0.0.1";
|
|
http_port = cfg.grafanaPort;
|
|
domain = fqdn;
|
|
root_url = "https://${fqdn}";
|
|
router_logging = cfg.debugLog;
|
|
};
|
|
|
|
smtp = lib.mkIf (!(isNull cfg.smtp)) {
|
|
enabled = true;
|
|
inherit (cfg.smtp) from_address from_name;
|
|
host = "${cfg.smtp.host}:${toString cfg.smtp.port}";
|
|
user = cfg.smtp.username;
|
|
password = "$__file{${cfg.smtp.passwordFile}}";
|
|
};
|
|
};
|
|
};
|
|
})
|
|
|
|
(lib.mkIf cfg.enable {
|
|
shb.monitoring.dashboards = [
|
|
./monitoring/dashboards/Errors.json
|
|
./monitoring/dashboards/Performance.json
|
|
./monitoring/dashboards/Scraping_Jobs.json
|
|
];
|
|
|
|
services.grafana.provision = {
|
|
dashboards.settings = lib.mkIf (cfg.dashboards != [ ]) {
|
|
apiVersion = 1;
|
|
providers = [
|
|
{
|
|
folder = "Self Host Blocks";
|
|
options.path = pkgs.symlinkJoin {
|
|
name = "dashboards";
|
|
paths = map (p: pkgs.runCommand "dashboard" { } "mkdir $out; cp ${p} $out") cfg.dashboards;
|
|
};
|
|
allowUiUpdates = true;
|
|
disableDeletion = true;
|
|
}
|
|
];
|
|
};
|
|
datasources.settings = {
|
|
apiVersion = 1;
|
|
datasources = [
|
|
{
|
|
inherit (cfg) orgId;
|
|
name = "Prometheus";
|
|
type = "prometheus";
|
|
url = "http://127.0.0.1:${toString config.services.prometheus.port}";
|
|
uid = "df80f9f5-97d7-4112-91d8-72f523a02b09";
|
|
isDefault = true;
|
|
version = 1;
|
|
}
|
|
{
|
|
inherit (cfg) orgId;
|
|
name = "Loki";
|
|
type = "loki";
|
|
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
|
|
uid = "cd6cc53e-840c-484d-85f7-96fede324006";
|
|
version = 1;
|
|
}
|
|
];
|
|
deleteDatasources = [
|
|
{
|
|
inherit (cfg) orgId;
|
|
name = "Prometheus";
|
|
}
|
|
{
|
|
inherit (cfg) orgId;
|
|
name = "Loki";
|
|
}
|
|
];
|
|
};
|
|
alerting.contactPoints.settings = {
|
|
apiVersion = 1;
|
|
contactPoints = [
|
|
{
|
|
inherit (cfg) orgId;
|
|
name = "grafana-default-email";
|
|
receivers = lib.optionals ((builtins.length cfg.contactPoints) > 0) [
|
|
{
|
|
uid = "sysadmin";
|
|
type = "email";
|
|
settings.addresses = lib.concatStringsSep ";" cfg.contactPoints;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
};
|
|
alerting.policies.settings = {
|
|
apiVersion = 1;
|
|
policies = [
|
|
{
|
|
inherit (cfg) orgId;
|
|
receiver = "grafana-default-email";
|
|
group_by = [
|
|
"grafana_folder"
|
|
"alertname"
|
|
];
|
|
group_wait = "30s";
|
|
group_interval = "5m";
|
|
repeat_interval = "4h";
|
|
}
|
|
];
|
|
# resetPolicies seems to happen after setting the above policies, effectively rolling back
|
|
# any updates.
|
|
};
|
|
alerting.rules.settings =
|
|
let
|
|
rules = builtins.fromJSON (builtins.readFile ./monitoring/rules.json);
|
|
in
|
|
{
|
|
apiVersion = 1;
|
|
groups = [
|
|
{
|
|
inherit (cfg) orgId;
|
|
name = "SysAdmin";
|
|
folder = "Self Host Blocks";
|
|
interval = "10m";
|
|
inherit rules;
|
|
}
|
|
];
|
|
# deleteRules seems to happen after creating the above rules, effectively rolling back
|
|
# any updates.
|
|
};
|
|
};
|
|
})
|
|
|
|
(lib.mkIf cfg.enable {
|
|
services.prometheus = {
|
|
enable = true;
|
|
port = cfg.prometheusPort;
|
|
globalConfig = {
|
|
scrape_interval = "15s";
|
|
};
|
|
};
|
|
|
|
services.loki = {
|
|
enable = true;
|
|
dataDir = "/var/lib/loki";
|
|
package =
|
|
if cfg.lokiMajorVersion == 3 then
|
|
pkgs.grafana-loki
|
|
else
|
|
# Comes from https://github.com/NixOS/nixpkgs/commit/8f95320f39d7e4e4a29ee70b8718974295a619f4
|
|
(pkgs.grafana-loki.overrideAttrs (
|
|
finalAttrs: previousAttrs: rec {
|
|
version = "2.9.6";
|
|
|
|
src = pkgs.fetchFromGitHub {
|
|
owner = "grafana";
|
|
repo = "loki";
|
|
rev = "v${version}";
|
|
hash = "sha256-79hK7axHf6soku5DvdXkE/0K4WKc4pnS9VMbVc1FS2I=";
|
|
};
|
|
|
|
subPackages = [
|
|
"cmd/loki"
|
|
"cmd/loki-canary"
|
|
"clients/cmd/promtail"
|
|
"cmd/logcli"
|
|
# Removes "cmd/lokitool"
|
|
];
|
|
|
|
ldflags =
|
|
let
|
|
t = "github.com/grafana/loki/pkg/util/build";
|
|
in
|
|
[
|
|
"-s"
|
|
"-w"
|
|
"-X ${t}.Version=${version}"
|
|
"-X ${t}.BuildUser=nix@nixpkgs"
|
|
"-X ${t}.BuildDate=unknown"
|
|
"-X ${t}.Branch=unknown"
|
|
"-X ${t}.Revision=unknown"
|
|
];
|
|
}
|
|
));
|
|
configuration = {
|
|
auth_enabled = false;
|
|
|
|
server.http_listen_port = cfg.lokiPort;
|
|
|
|
ingester = {
|
|
lifecycler = {
|
|
address = "127.0.0.1";
|
|
ring = {
|
|
kvstore.store = "inmemory";
|
|
replication_factor = 1;
|
|
};
|
|
final_sleep = "0s";
|
|
};
|
|
chunk_idle_period = "5m";
|
|
chunk_retain_period = "30s";
|
|
};
|
|
|
|
schema_config = {
|
|
configs = [
|
|
{
|
|
from = "2018-04-15";
|
|
store = "boltdb";
|
|
object_store = "filesystem";
|
|
schema = "v9";
|
|
index.prefix = "index_";
|
|
index.period = "168h";
|
|
}
|
|
];
|
|
};
|
|
|
|
storage_config = {
|
|
boltdb.directory = "/tmp/loki/index";
|
|
filesystem.directory = "/tmp/loki/chunks";
|
|
};
|
|
|
|
limits_config = {
|
|
enforce_metric_name = false;
|
|
reject_old_samples = true;
|
|
reject_old_samples_max_age = "168h";
|
|
};
|
|
|
|
chunk_store_config = {
|
|
max_look_back_period = 0;
|
|
};
|
|
|
|
table_manager = {
|
|
chunk_tables_provisioning = {
|
|
inactive_read_throughput = 0;
|
|
inactive_write_throughput = 0;
|
|
provisioned_read_throughput = 0;
|
|
provisioned_write_throughput = 0;
|
|
};
|
|
index_tables_provisioning = {
|
|
inactive_read_throughput = 0;
|
|
inactive_write_throughput = 0;
|
|
provisioned_read_throughput = 0;
|
|
provisioned_write_throughput = 0;
|
|
};
|
|
retention_deletes_enabled = false;
|
|
retention_period = 0;
|
|
};
|
|
};
|
|
};
|
|
|
|
# I decided to switch to fluent-bit because it can be tested locally https://docs.fluentbit.io/manual/local-testing/logging-pipeline
|
|
services.fluent-bit = {
|
|
enable = true;
|
|
settings = {
|
|
service = {
|
|
flush = 1;
|
|
log_level = "info";
|
|
http_server = "true";
|
|
http_listen = "127.0.0.1";
|
|
http_port = 9080;
|
|
grace = 30;
|
|
};
|
|
|
|
pipeline = {
|
|
inputs = [
|
|
{
|
|
name = "systemd";
|
|
|
|
# The asterisk appends the _SYSTEMD_UNIT to the prefix.
|
|
tag = "systemd.*";
|
|
|
|
# Read logs from this systemd journal directory.
|
|
path = "/var/log/journal";
|
|
|
|
# Database file to keep track of the journald cursor.
|
|
db = "/var/fluent-bit/systemd.db";
|
|
|
|
# Start reading new entries. Skip entries already stored in journald.
|
|
read_from_tail = true;
|
|
|
|
# Max entries to lookback on start.
|
|
max_entries = 10000;
|
|
}
|
|
];
|
|
|
|
outputs = [
|
|
{
|
|
name = "loki";
|
|
|
|
match = "systemd.*";
|
|
|
|
host = "localhost";
|
|
port = config.services.loki.configuration.server.http_listen_port;
|
|
|
|
labels = {
|
|
job = "systemd-journal";
|
|
domain = cfg.domain;
|
|
hostname = config.networking.hostName;
|
|
};
|
|
|
|
label_keys = "unit";
|
|
}
|
|
];
|
|
};
|
|
};
|
|
graceLimit = "1m";
|
|
};
|
|
|
|
services.nginx = {
|
|
enable = true;
|
|
|
|
virtualHosts.${fqdn} = {
|
|
forceSSL = !(isNull cfg.ssl);
|
|
sslCertificate = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.cert;
|
|
sslCertificateKey = lib.mkIf (!(isNull cfg.ssl)) cfg.ssl.paths.key;
|
|
|
|
locations."/" = {
|
|
proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
|
|
proxyWebsockets = true;
|
|
extraConfig = ''
|
|
proxy_set_header Host $host;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
})
|
|
|
|
(lib.mkIf cfg.enable {
|
|
services.prometheus.scrapeConfigs = [
|
|
{
|
|
job_name = "node";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "netdata";
|
|
metrics_path = "/api/v1/allmetrics";
|
|
params.format = [ "prometheus" ];
|
|
honor_labels = true;
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:19999" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "smartctl";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "prometheus_internal";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.prometheus.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "systemd";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
}
|
|
]
|
|
++ (lib.lists.optional config.services.nginx.enable {
|
|
job_name = "nginx";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
# }) ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
|
|
# job_name = "redis";
|
|
# static_configs = [
|
|
# {
|
|
# targets = ["127.0.0.1:${toString config.services.prometheus.exporters.redis.port}"];
|
|
# }
|
|
# ];
|
|
# }) ++ (lib.optional (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
|
|
# job_name = "openvpn";
|
|
# static_configs = [
|
|
# {
|
|
# targets = ["127.0.0.1:${toString config.services.prometheus.exporters.openvpn.port}"];
|
|
# }
|
|
# ];
|
|
})
|
|
++ (lib.optional config.services.dnsmasq.enable {
|
|
job_name = "dnsmasq";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
});
|
|
services.prometheus.exporters.nginx = lib.mkIf config.services.nginx.enable {
|
|
enable = true;
|
|
port = 9111;
|
|
listenAddress = "127.0.0.1";
|
|
scrapeUri = "http://localhost:80/nginx_status";
|
|
};
|
|
services.prometheus.exporters.node = {
|
|
enable = true;
|
|
# https://github.com/prometheus/node_exporter#collectors
|
|
enabledCollectors = [
|
|
"arp"
|
|
"cpu"
|
|
"cpufreq"
|
|
"diskstats"
|
|
"dmi"
|
|
"edac"
|
|
"entropy"
|
|
"filefd"
|
|
"filesystem"
|
|
"hwmon"
|
|
"loadavg"
|
|
"meminfo"
|
|
"netclass"
|
|
"netdev"
|
|
"netstat"
|
|
"nvme"
|
|
"os"
|
|
"pressure"
|
|
"rapl"
|
|
"schedstat"
|
|
"stat"
|
|
"thermal_zone"
|
|
"time"
|
|
"uname"
|
|
"vmstat"
|
|
"zfs"
|
|
|
|
# Disabled by default
|
|
"cgroups"
|
|
"drm"
|
|
"ethtool"
|
|
"logind"
|
|
"wifi"
|
|
];
|
|
port = 9112;
|
|
listenAddress = "127.0.0.1";
|
|
};
|
|
# https://github.com/nixos/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80
|
|
# TODO: remove when https://github.com/NixOS/nixpkgs/pull/205165 is merged
|
|
services.udev.extraRules = ''
|
|
SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
|
|
'';
|
|
services.prometheus.exporters.smartctl = {
|
|
enable = true;
|
|
port = 9115;
|
|
listenAddress = "127.0.0.1";
|
|
};
|
|
# services.prometheus.exporters.redis = lib.mkIf (builtins.length (lib.attrNames config.services.redis.servers) > 0) {
|
|
# enable = true;
|
|
# port = 9119;
|
|
# listenAddress = "127.0.0.1";
|
|
# };
|
|
# services.prometheus.exporters.openvpn = lib.mkIf (builtins.length (lib.attrNames config.services.openvpn.servers) > 0) {
|
|
# enable = true;
|
|
# port = 9121;
|
|
# listenAddress = "127.0.0.1";
|
|
# statusPaths = lib.mapAttrsToList (name: _config: "/tmp/openvpn/${name}.status") config.services.openvpn.servers;
|
|
# };
|
|
services.prometheus.exporters.dnsmasq = lib.mkIf config.services.dnsmasq.enable {
|
|
enable = true;
|
|
port = 9211;
|
|
listenAddress = "127.0.0.1";
|
|
};
|
|
services.prometheus.exporters.systemd = {
|
|
enable = true;
|
|
port = 9116;
|
|
listenAddress = "127.0.0.1";
|
|
};
|
|
services.nginx.statusPage = lib.mkDefault config.services.nginx.enable;
|
|
services.netdata = {
|
|
enable = true;
|
|
config = {
|
|
# web.mode = "none";
|
|
# web."bind to" = "127.0.0.1:19999";
|
|
global = {
|
|
"debug log" = "syslog";
|
|
"access log" = "syslog";
|
|
"error log" = "syslog";
|
|
};
|
|
};
|
|
};
|
|
|
|
nixpkgs.overlays = [
|
|
(final: prev: {
|
|
prometheus-systemd-exporter = prev.prometheus-systemd-exporter.overrideAttrs {
|
|
src = final.fetchFromGitHub {
|
|
owner = "ibizaman";
|
|
repo = prev.prometheus-systemd-exporter.pname;
|
|
# rev = "v${prev.prometheus-systemd-exporter.version}";
|
|
rev = "next_timer";
|
|
sha256 = "sha256-jzkh/616tsJbNxFtZ0xbdBQc16TMIYr9QOkPaeQw8xA=";
|
|
};
|
|
|
|
vendorHash = "sha256-4hsQ1417jLNOAqGkfCkzrmEtYR4YLLW2j0CiJtPg6GI=";
|
|
};
|
|
})
|
|
];
|
|
})
|
|
(lib.mkIf (cfg.enable && cfg.sso.enable) {
|
|
shb.lldap.ensureGroups = {
|
|
${cfg.ldap.userGroup} = { };
|
|
${cfg.ldap.adminGroup} = { };
|
|
};
|
|
|
|
shb.authelia.extraDefinitions = {
|
|
user_attributes.${roleClaim}.expression =
|
|
# Roles are: None, Viewer, Editor, Admin, GrafanaAdmin
|
|
''"${cfg.ldap.adminGroup}" in groups ? "Admin" : ("${cfg.ldap.userGroup}" in groups ? "Editor" : "Invalid")'';
|
|
};
|
|
shb.authelia.extraOidcClaimsPolicies.${roleClaim} = {
|
|
custom_claims = {
|
|
"${roleClaim}" = { };
|
|
};
|
|
};
|
|
shb.authelia.extraOidcScopes."${roleClaim}" = {
|
|
claims = [ "${roleClaim}" ];
|
|
};
|
|
|
|
services.grafana.settings."auth.generic_oauth" = {
|
|
enabled = true;
|
|
name = "Authelia";
|
|
icon = "signin";
|
|
client_id = cfg.sso.clientID;
|
|
client_secret = "$__file{${cfg.sso.sharedSecret.result.path}}";
|
|
scopes = oauthScopes;
|
|
empty_scopes = false;
|
|
allow_sign_up = true;
|
|
auto_login = true;
|
|
auth_url = "${cfg.sso.authEndpoint}/api/oidc/authorization";
|
|
token_url = "${cfg.sso.authEndpoint}/api/oidc/token";
|
|
# use_refresh_token = true; ? # https://grafana.com/docs/grafana/latest/setup-grafana/configure-access/configure-authentication/generic-oauth/#configure-generic-oauth-authentication-client-using-the-grafana-configuration-file
|
|
api_url = "${cfg.sso.authEndpoint}/api/oidc/userinfo";
|
|
login_attribute_path = "preferred_username";
|
|
groups_attribute_path = "groups";
|
|
name_attribute_path = "name";
|
|
use_pkce = true;
|
|
allow_assign_grafana_admin = true;
|
|
skip_org_role_sync = false;
|
|
role_attribute_path = roleClaim;
|
|
role_attribute_strict = true;
|
|
};
|
|
|
|
shb.authelia.oidcClients = [
|
|
{
|
|
client_id = cfg.sso.clientID;
|
|
client_secret.source = cfg.sso.sharedSecretForAuthelia.result.path;
|
|
claims_policy = "${roleClaim}";
|
|
scopes = oauthScopes;
|
|
authorization_policy = cfg.sso.authorization_policy;
|
|
redirect_uris = [
|
|
"https://${cfg.subdomain}.${cfg.domain}/login/generic_oauth"
|
|
];
|
|
require_pkce = true;
|
|
pkce_challenge_method = "S256";
|
|
response_types = [ "code" ];
|
|
token_endpoint_auth_method = "client_secret_basic";
|
|
}
|
|
];
|
|
})
|
|
|
|
(lib.mkIf (cfg.enable && cfg.scrutiny.enable) {
|
|
services.scrutiny = {
|
|
enable = true;
|
|
|
|
openFirewall = false;
|
|
|
|
# This src includes Prometheus metrics exporter.
|
|
package = pkgs.scrutiny.overrideAttrs ({
|
|
src = pkgs.fetchFromGitHub {
|
|
owner = "ibizaman";
|
|
repo = "scrutiny";
|
|
rev = "74faf06f77df83f29e7e1806cd88b2fafc0bbb82";
|
|
hash = "sha256-r0AVWL+E046xHxitwMPfRNTOpjuOk+W6tB41YgmLTPg=";
|
|
};
|
|
|
|
vendorHash = "sha256-kAlnlWnBMFCdgdak5L5hRquRtyLi5MTmDa/kxwqPs4E=";
|
|
});
|
|
|
|
settings = {
|
|
web = {
|
|
metrics.enabled = true; # Enables Prometheus exporter
|
|
listenHost = "127.0.0.1";
|
|
};
|
|
};
|
|
|
|
collector = {
|
|
enable = true;
|
|
};
|
|
};
|
|
|
|
services.prometheus.scrapeConfigs = [
|
|
{
|
|
job_name = "scrutiny";
|
|
metrics_path = "/api/metrics";
|
|
static_configs = [
|
|
{
|
|
targets = [ "127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}" ];
|
|
labels = commonLabels;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
|
|
shb.monitoring.dashboards = [
|
|
./monitoring/dashboards/Health.json
|
|
];
|
|
|
|
shb.nginx.vhosts = lib.mkIf (cfg.scrutiny.subdomain != null) [
|
|
(
|
|
{
|
|
inherit (cfg) domain ssl;
|
|
subdomain = cfg.scrutiny.subdomain;
|
|
|
|
upstream = "http://127.0.0.1:${toString config.services.scrutiny.settings.web.listen.port}";
|
|
autheliaRules = lib.optionals (cfg.sso.enable) [
|
|
{
|
|
domain = "${cfg.subdomain}.${cfg.domain}";
|
|
policy = cfg.sso.authorization_policy;
|
|
subject = [
|
|
"group:${cfg.ldap.userGroup}"
|
|
"group:${cfg.ldap.adminGroup}"
|
|
];
|
|
}
|
|
];
|
|
}
|
|
// lib.optionalAttrs cfg.sso.enable {
|
|
inherit (cfg.sso) authEndpoint;
|
|
}
|
|
)
|
|
];
|
|
})
|
|
];
|
|
}
|