danvics/selfhostblocks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

add alert when certs are close to expiring

Browse source
This commit is contained in:
ibizaman 2024-12-26 01:41:08 +01:00 committed by Pierre Penninckx
parent a45f57c960
commit 16f305b9f4
7 changed files with 541 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
CHANGELOG.md
View file

@ -14,6 +14,11 @@ Template:
# Upcoming Release
## New Features
- Add dashboard for SSL certificates validity
and alert they did not renew on time.
# v0.2.7
## New Features

6
flake.nix
View file

@ -14,12 +14,16 @@
outputs = { nixpkgs, nix-flake-tests, flake-utils, nmdsrc, ... }: flake-utils.lib.eachDefaultSystem (system:
let
originPkgs = nixpkgs.legacyPackages.${system};
patches = [
patches = originPkgs.lib.optionals (system == "x86_64-linux") [
# Leaving commented out for an example.
# (originPkgs.fetchpatch {
# url = "https://github.com/NixOS/nixpkgs/pull/317107.patch";
# hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk=";
# })
# Remove when this PR is merged:
# https://github.com/NixOS/nixpkgs/pull/368325
./patches/prometheusnodecertexporter.nix
];
patchedNixpkgs = originPkgs.applyPatches {
name = "nixpkgs-patched";

143
modules/blocks/monitoring/dashboards/SSL.json Normal file
View file

@ -0,0 +1,143 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 16,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "transparent",
"value": 604808
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last *",
"sortDesc": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.4.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"editorMode": "code",
"expr": "ssl_certificate_expiry_seconds",
"legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
"range": true,
"refId": "A"
}
],
"title": "Certificate Remaining Validity",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "SSL Certificates",
"uid": "ae818js0bvw8wb",
"version": 8,
"weekStart": ""
}

129
modules/blocks/monitoring/rules.json
View file

@ -123,7 +123,134 @@
"summary": "The error budget for a service for the last 1 hour is under 99%"
},
"labels": {
"": "",
"role": "sysadmin"
},
"isPaused": false
},
{
"uid": "ee817l3a88s1sd",
"title": "Certificate Did Not Renew",
"condition": "C",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 1800,
"to": 0
},
"datasourceUid": "df80f9f5-97d7-4112-91d8-72f523a02b09",
"model": {
"adhocFilters": [],
"datasource": {
"type": "prometheus",
"uid": "df80f9f5-97d7-4112-91d8-72f523a02b09"
},
"editorMode": "code",
"expr": "ssl_certificate_expiry_seconds",
"interval": "",
"intervalMs": 15000,
"legendFormat": "{{exported_hostname}}: {{subject}} {{path}}",
"maxDataPoints": 43200,
"range": true,
"refId": "A"
}
},
{
"refId": "B",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"B"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "A",
"intervalMs": 1000,
"maxDataPoints": 43200,
"reducer": "last",
"refId": "B",
"type": "reduce"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
604800
],
"type": "lt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"C"
]
},
"reducer": {
"params": [],
"type": "last"
},
"type": "query"
}
],
"datasource": {
"type": "__expr__",
"uid": "__expr__"
},
"expression": "B",
"intervalMs": 1000,
"maxDataPoints": 43200,
"refId": "C",
"type": "threshold"
}
}
],
"dashboardUid": "ae818js0bvw8wb",
"panelId": 3,
"noDataState": "NoData",
"execErrState": "Error",
"for": "20m",
"annotations": {
"__dashboardUid__": "ae818js0bvw8wb",
"__panelId__": "3",
"description": "The expiry date of the certificate is 1 week from now.",
"summary": "Certificate did not renew on time."
},
"labels": {
"role": "sysadmin"
},
"isPaused": false

46
modules/blocks/ssl.nix
View file

@ -4,6 +4,9 @@ let
cfg = config.shb.certs;
contracts = pkgs.callPackage ../contracts {};
inherit (builtins) dirOf;
inherit (lib) flatten mapAttrsToList unique;
in
{
options.shb.certs = {
@ -362,7 +365,7 @@ in
cat /etc/static/ssl/certs/ca-bundle.crt > /etc/ssl/certs/ca-bundle.crt
cat /etc/static/ssl/certs/ca-bundle.crt > /etc/ssl/certs/ca-certificates.crt
for file in ${lib.concatStringsSep " " (lib.mapAttrsToList (_name: caCfg: caCfg.paths.cert) cfg.cas.selfsigned)}; do
for file in ${lib.concatStringsSep " " (mapAttrsToList (_name: caCfg: caCfg.paths.cert) cfg.cas.selfsigned)}; do
cat "$file" >> /etc/ssl/certs/ca-bundle.crt
cat "$file" >> /etc/ssl/certs/ca-certificates.crt
done
@ -431,7 +434,7 @@ in
}
# Config for Let's Encrypt cert.
{
users.users = lib.mkMerge (lib.mapAttrsToList (name: certCfg: {
users.users = lib.mkMerge (mapAttrsToList (name: certCfg: {
${certCfg.makeAvailableToUser}.extraGroups = lib.mkIf (!(isNull certCfg.makeAvailableToUser)) [
config.security.acme.defaults.group
];
@ -447,7 +450,7 @@ in
server = lib.mkIf certCfg.stagingServer "https://acme-staging-v02.api.letsencrypt.org/directory";
};
}) certCfg.extraDomains;
in lib.mkMerge (lib.flatten (lib.mapAttrsToList (name: certCfg:
in lib.mkMerge (flatten (mapAttrsToList (name: certCfg:
[{
"${name}" = {
extraDomainNames = [ certCfg.domain ] ++ certCfg.extraDomains;
@ -470,7 +473,7 @@ in
enableACME = true;
};
}) extraDomains;
in lib.mkMerge (lib.flatten (lib.mapAttrsToList (name: certCfg:
in lib.mkMerge (flatten (mapAttrsToList (name: certCfg:
lib.optionals (certCfg.dnsProvider == null) (
[{
virtualHosts."${name}" = {
@ -482,7 +485,7 @@ in
)) cfg.certs.letsencrypt));
systemd.services = let
extraDomainsCfg = certCfg: lib.flatten (map (name:
extraDomainsCfg = certCfg: flatten (map (name:
lib.optionals (certCfg.additionalEnvironment != {} && certCfg.dnsProvider == null) [{
"acme-${name}".environment = certCfg.additionalEnvironment;
}]
@ -493,7 +496,7 @@ in
};
}]
) certCfg.extraDomains);
in lib.mkMerge (lib.flatten (lib.mapAttrsToList (name: certCfg:
in lib.mkMerge (flatten (mapAttrsToList (name: certCfg:
lib.optionals (certCfg.additionalEnvironment != {} && certCfg.dnsProvider == null) [{
"acme-${certCfg.domain}".environment = certCfg.additionalEnvironment;
}]
@ -505,6 +508,37 @@ in
}]
++ lib.optionals (certCfg.dnsProvider == null) (extraDomainsCfg certCfg)
) cfg.certs.letsencrypt));
services.prometheus.exporters.node-cert = {
enable = true;
listenAddress = "127.0.0.1";
user = "acme";
paths = let
pathCfg = name: certCfg:
let
mainDomainPaths = map dirOf [ certCfg.paths.cert certCfg.paths.key ];
# Not sure this will work for all cases.
mainPath = dirOf (dirOf certCfg.paths.cert);
extraDomainsPath = map (x: "${mainPath}/${x}") certCfg.extraDomains;
in
mainDomainPaths ++ extraDomainsPath;
in
unique (flatten (mapAttrsToList pathCfg cfg.certs.letsencrypt));
};
services.prometheus.scrapeConfigs = let
scrapeCfg = name: certCfg: [{
job_name = "node-cert-${name}";
static_configs = [{
targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node-cert.port}"];
labels = {
"hostname" = config.networking.hostName;
"domain" = certCfg.domain;
};
}];
}];
in
flatten (mapAttrsToList scrapeCfg cfg.certs.letsencrypt);
}
];
}

219
patches/prometheusnodecertexporter.nix Normal file
View file

@ -0,0 +1,219 @@
index f805920c5b87a..b67f41c4fb12c 100644
--- a/nixos/modules/services/monitoring/prometheus/exporters.nix
+++ b/nixos/modules/services/monitoring/prometheus/exporters.nix
@@ -66,6 +66,7 @@ let
"nginx"
"nginxlog"
"node"
+ "node-cert"
"nut"
"nvidia-gpu"
"pgbouncer"
diff --git a/nixos/modules/services/monitoring/prometheus/exporters/node-cert.nix b/nixos/modules/services/monitoring/prometheus/exporters/node-cert.nix
new file mode 100644
index 0000000000000..d8b2004e8e857
--- /dev/null
+++ b/nixos/modules/services/monitoring/prometheus/exporters/node-cert.nix
@@ -0,0 +1,70 @@
+{
+ config,
+ lib,
+ pkgs,
+ ...
+}:
+
+let
+ cfg = config.services.prometheus.exporters.node-cert;
+ inherit (lib) mkOption types concatStringsSep;
+in
+{
+ port = 9141;
+
+ extraOpts = {
+ paths = mkOption {
+ type = types.listOf types.str;
+ description = ''
+ List of paths to search for SSL certificates.
+ '';
+ };
+
+ excludePaths = mkOption {
+ type = types.listOf types.str;
+ description = ''
+ List of paths to exclute from searching for SSL certificates.
+ '';
+ default = [ ];
+ };
+
+ includeGlobs = mkOption {
+ type = types.listOf types.str;
+ description = ''
+ List files matching a pattern to include. Uses Go blob pattern.
+ '';
+ default = [ ];
+ };
+
+ excludeGlobs = mkOption {
+ type = types.listOf types.str;
+ description = ''
+ List files matching a pattern to include. Uses Go blob pattern.
+ '';
+ default = [ ];
+ };
+
+ user = mkOption {
+ type = types.str;
+ description = ''
+ User owning the certs.
+ '';
+ default = "acme";
+ };
+ };
+
+ serviceOpts = {
+ serviceConfig = {
+ User = cfg.user;
+ ExecStart = ''
+ ${lib.getExe pkgs.prometheus-node-cert-exporter} \
+ --listen ${toString cfg.listenAddress}:${toString cfg.port} \
+ --path ${concatStringsSep "," cfg.paths} \
+ --exclude-path "${concatStringsSep "," cfg.excludePaths}" \
+ --include-glob "${concatStringsSep "," cfg.includeGlobs}" \
+ --exclude-glob "${concatStringsSep "," cfg.excludeGlobs}" \
+ ${concatStringsSep " \\\n " cfg.extraFlags}
+ '';
+ };
+ };
+}
diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix
index c15a3fd20b021..f59d61e69b92e 100644
--- a/nixos/tests/prometheus-exporters.nix
+++ b/nixos/tests/prometheus-exporters.nix
@@ -1002,6 +1002,49 @@ let
'';
};
+ node-cert = {
+ nodeName = "node_cert";
+ exporterConfig = {
+ enable = true;
+ paths = ["/run/certs"];
+ };
+ exporterTest = ''
+ wait_for_unit("prometheus-node-cert-exporter.service")
+ wait_for_open_port(9141)
+ wait_until_succeeds(
+ "curl -sSf http://localhost:9141/metrics | grep 'ssl_certificate_expiry_seconds{.\\+path=\"/run/certs/node-cert\\.cert\".\\+}'"
+ )
+ '';
+
+ metricProvider = {
+ system.activationScripts.cert.text = ''
+ mkdir -p /run/certs
+ cd /run/certs
+
+ cat >ca.template <<EOF
+ organization = "prometheus-node-cert-exporter"
+ cn = "prometheus-node-cert-exporter"
+ expiration_days = 365
+ ca
+ cert_signing_key
+ crl_signing_key
+ EOF
+
+ ${pkgs.gnutls}/bin/certtool \
+ --generate-privkey \
+ --key-type rsa \
+ --sec-param High \
+ --outfile node-cert.key
+
+ ${pkgs.gnutls}/bin/certtool \
+ --generate-self-signed \
+ --load-privkey node-cert.key \
+ --template ca.template \
+ --outfile node-cert.cert
+ '';
+ };
+ };
+
pgbouncer = {
exporterConfig = {
enable = true;
diff --git a/pkgs/by-name/pr/prometheus-node-cert-exporter/gomod.patch b/pkgs/by-name/pr/prometheus-node-cert-exporter/gomod.patch
new file mode 100644
index 0000000000000..84626a7477628
--- /dev/null
+++ b/pkgs/by-name/pr/prometheus-node-cert-exporter/gomod.patch
@@ -0,0 +1,33 @@
+diff --git a/go.mod b/go.mod
+index 982eef4..bdb53ee 100644
+--- a/go.mod
++++ b/go.mod
+@@ -7,4 +7,15 @@ require (
+ github.com/spf13/pflag v1.0.3
+ )
+
+-go 1.16
++require (
++ github.com/beorn7/perks v1.0.1 // indirect
++ github.com/cespare/xxhash/v2 v2.1.1 // indirect
++ github.com/golang/protobuf v1.4.3 // indirect
++ github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
++ github.com/prometheus/client_model v0.2.0 // indirect
++ github.com/prometheus/procfs v0.6.0 // indirect
++ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 // indirect
++ google.golang.org/protobuf v1.26.0-rc.1 // indirect
++)
++
++go 1.18
+diff --git a/go.sum b/go.sum
+index 8bebbb3..75f756a 100644
+--- a/go.sum
++++ b/go.sum
+@@ -39,7 +39,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
+ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
diff --git a/pkgs/by-name/pr/prometheus-node-cert-exporter/package.nix b/pkgs/by-name/pr/prometheus-node-cert-exporter/package.nix
new file mode 100644
index 0000000000000..cde041771b296
--- /dev/null
+++ b/pkgs/by-name/pr/prometheus-node-cert-exporter/package.nix
@@ -0,0 +1,33 @@
+{
+ lib,
+ buildGo122Module,
+ fetchFromGitHub,
+ nixosTests,
+}:
+
+buildGo122Module {
+ pname = "node-cert-exporter";
+ version = "1.1.7-unstable-2024-12-26";
+
+ src = fetchFromGitHub {
+ owner = "amimof";
+ repo = "node-cert-exporter";
+ rev = "v1.1.7";
+ sha256 = "sha256-VYJPgNVsfEs/zh/SEdOrFn0FK6S+hNFGDhonj2syutQ=";
+ };
+
+ vendorHash = "sha256-31MHX3YntogvoJmbOytl0rXS6qtdBSBJe8ejKyu6gqM=";
+
+ # Required otherwise we get a few:
+ # vendor/github.com/golang/glog/internal/logsink/logsink.go:129:41:
+ # predeclared any requires go1.18 or later (-lang was set to go1.16; check go.mod)
+ patches = [ ./gomod.patch ];
+
+ meta = with lib; {
+ description = "Prometheus exporter for SSL certificate";
+ mainProgram = "node-cert-exporter";
+ homepage = "https://github.com/amimof/node-cert-exporter";
+ license = licenses.asl20;
+ maintainers = with maintainers; [ ibizaman ];
+ };
+}

2
test/modules/nginx.nix
View file

@ -27,7 +27,7 @@ let
];
}).config;
in lib.attrsets.filterAttrsRecursive (n: v: n != "extraConfig") {
inherit (cfg) services;
services = { inherit (cfg.services) nginx; };
shb = { inherit (cfg.shb) nginx; };
};
in