diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ec2de6c..c39fc60 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -223,6 +223,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout "storageDefault": { "trigger": 85, "clear": 75 }, "dockerDefaults": { "cpu": { "trigger": 75, "clear": 60 }, + "disk": { "trigger": 85, "clear": 75 }, "restartCount": 3, "restartWindow": 300, "memoryWarnPct": 90, @@ -288,6 +289,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout - `dockerIgnoredContainerPrefixes` lets you silence state/metric/restart alerts for ephemeral containers whose names or IDs share a common, case-insensitive prefix. The Docker tab in the UI keeps this list in sync. - Swarm service alerts track missing replicas: `serviceWarnGapPercent` defines when a warning fires, and `serviceCriticalGapPercent` must be greater than or equal to the warning gap (Pulse automatically clamps the critical value upward if an older client submits something smaller). - Docker container state controls live in `dockerDefaults`: flip `stateDisableConnectivity` to silence exit/offline alerts globally, or change `statePoweredOffSeverity` to `critical` when you want exiting containers to page immediately. Per-container overrides still take precedence. +- `dockerDefaults.disk` defines the writable-layer usage threshold (% of the container's upper filesystem compared to its base image). Defaults trigger at 85% and clear at 80%, and can be overridden per container or host when noisy workloads need a different window. - Quiet hours, escalation, deduplication, and restart loop detection are all managed here, and the UI keeps the JSON in sync automatically. > Tip: Back up `alerts.json` alongside `.env` during exports. Restoring it preserves all overrides, quiet-hour schedules, and webhook routing. diff --git a/docs/DOCKER_MONITORING.md b/docs/DOCKER_MONITORING.md index 5bb1327..87698af 100644 --- a/docs/DOCKER_MONITORING.md +++ b/docs/DOCKER_MONITORING.md @@ -165,6 +165,10 @@ docker run -d \ The agent automatically discovers the Docker socket via the usual environment variables. To use SSH tunnels or TCP sockets, export `DOCKER_HOST` as you would for the Docker CLI. +### Disk usage monitoring & alerts + +When `--collect-disk` is enabled (the default), Pulse records each container’s writable layer and root filesystem sizes. The Alerts engine treats the proportion of writable data to total filesystem as the disk usage percentage for that container. A fleet-wide threshold lives under **Alerts → Thresholds → Docker Containers** and defaults to 85% trigger / 80% clear; adjust or disable it per host/container when your workload makes heavy use of copy-on-write layers. Containers that stop reporting disk metrics (for example when size queries are disabled) automatically skip the disk alert evaluation. + ### Suppressing ephemeral containers CI runners and short-lived build containers can generate noisy state alerts when they exit on schedule. In Pulse v4.24.0 and later you can provide a list of prefixes to ignore under **Alerts → Thresholds → Docker → Ignored container prefixes**. Any container whose name *or* ID begins with a configured prefix is skipped for state, health, metric, restart-loop, and OOM alerts. Matching is case-insensitive and the list is saved as `dockerIgnoredContainerPrefixes` inside `alerts.json`. Use one entry per family of ephemeral containers (for example, `runner-` or `gitlab-job-`). diff --git a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx index 386820b..6b44fc8 100644 --- a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx +++ b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx @@ -181,6 +181,7 @@ interface ThresholdsTableProps { dockerDefaults: { cpu: number; memory: number; + disk: number; restartCount: number; restartWindow: number; memoryWarnPct: number; @@ -197,6 +198,7 @@ interface ThresholdsTableProps { | { cpu: number; memory: number; + disk: number; restartCount: number; restartWindow: number; memoryWarnPct: number; @@ -207,6 +209,7 @@ interface ThresholdsTableProps { | ((prev: { cpu: number; memory: number; + disk: number; restartCount: number; restartWindow: number; memoryWarnPct: number; @@ -216,6 +219,7 @@ interface ThresholdsTableProps { }) => { cpu: number; memory: number; + disk: number; restartCount: number; restartWindow: number; memoryWarnPct: number; @@ -2850,6 +2854,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) { columns={[ 'CPU %', 'Memory %', + 'Disk %', 'Restart Count', 'Restart Window (s)', 'Memory Warn %', @@ -2871,6 +2876,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) { globalDefaults={{ cpu: props.dockerDefaults.cpu, memory: props.dockerDefaults.memory, + disk: props.dockerDefaults.disk, restartCount: props.dockerDefaults.restartCount, restartWindow: props.dockerDefaults.restartWindow, memoryWarnPct: props.dockerDefaults.memoryWarnPct, @@ -2880,6 +2886,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) { const current = { cpu: props.dockerDefaults.cpu, memory: props.dockerDefaults.memory, + disk: props.dockerDefaults.disk, restartCount: props.dockerDefaults.restartCount, restartWindow: props.dockerDefaults.restartWindow, memoryWarnPct: props.dockerDefaults.memoryWarnPct, @@ -2892,6 +2899,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) { ...prev, cpu: next.cpu ?? prev.cpu, memory: next.memory ?? prev.memory, + disk: next.disk ?? prev.disk, restartCount: next.restartCount ?? prev.restartCount, restartWindow: next.restartWindow ?? prev.restartWindow, memoryWarnPct: next.memoryWarnPct ?? prev.memoryWarnPct, diff --git a/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx b/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx index 94e9d27..75fea38 100644 --- a/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx +++ b/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx @@ -51,6 +51,7 @@ const DEFAULT_PMG_THRESHOLDS: PMGThresholdDefaults = { const DEFAULT_DOCKER_DEFAULTS = { cpu: 80, memory: 85, + disk: 85, restartCount: 3, restartWindow: 300, memoryWarnPct: 90, diff --git a/frontend-modern/src/pages/Alerts.tsx b/frontend-modern/src/pages/Alerts.tsx index 6a33ddf..184f3df 100644 --- a/frontend-modern/src/pages/Alerts.tsx +++ b/frontend-modern/src/pages/Alerts.tsx @@ -842,6 +842,7 @@ const [appriseConfig, setAppriseConfig] = createSignal( setDockerDefaults({ cpu: getTriggerValue(config.dockerDefaults.cpu) ?? 80, memory: getTriggerValue(config.dockerDefaults.memory) ?? 85, + disk: getTriggerValue(config.dockerDefaults.disk) ?? FACTORY_DOCKER_DEFAULTS.disk, restartCount: config.dockerDefaults.restartCount ?? 3, restartWindow: config.dockerDefaults.restartWindow ?? 300, memoryWarnPct: config.dockerDefaults.memoryWarnPct ?? 90, @@ -1205,6 +1206,7 @@ const [appriseConfig, setAppriseConfig] = createSignal( const FACTORY_DOCKER_DEFAULTS = { cpu: 80, memory: 85, + disk: 85, restartCount: 3, restartWindow: 300, memoryWarnPct: 90, @@ -1526,6 +1528,7 @@ const [appriseConfig, setAppriseConfig] = createSignal( dockerDefaults: { cpu: createHysteresisThreshold(dockerDefaultsValue.cpu), memory: createHysteresisThreshold(dockerDefaultsValue.memory), + disk: createHysteresisThreshold(dockerDefaultsValue.disk), restartCount: dockerDefaultsValue.restartCount, restartWindow: dockerDefaultsValue.restartWindow, memoryWarnPct: dockerDefaultsValue.memoryWarnPct, diff --git a/frontend-modern/src/types/alerts.ts b/frontend-modern/src/types/alerts.ts index 9c02194..6ea69f2 100644 --- a/frontend-modern/src/types/alerts.ts +++ b/frontend-modern/src/types/alerts.ts @@ -60,6 +60,7 @@ export interface CustomAlertRule { export interface DockerThresholdConfig { cpu?: HysteresisThreshold; memory?: HysteresisThreshold; + disk?: HysteresisThreshold; restartCount?: number; restartWindow?: number; memoryWarnPct?: number; diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index e6233a3..fe99602 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -266,6 +266,7 @@ type CustomAlertRule struct { type DockerThresholdConfig struct { CPU HysteresisThreshold `json:"cpu"` // CPU usage % threshold (default: 80%) Memory HysteresisThreshold `json:"memory"` // Memory usage % threshold (default: 85%) + Disk HysteresisThreshold `json:"disk"` // Writable layer usage % threshold (default: 85%) RestartCount int `json:"restartCount"` // Number of restarts to trigger alert (default: 3) RestartWindow int `json:"restartWindow"` // Time window in seconds for restart loop detection (default: 300 = 5min) MemoryWarnPct int `json:"memoryWarnPct"` // Memory limit % to trigger warning (default: 90) @@ -504,6 +505,7 @@ func NewManager() *Manager { DockerDefaults: DockerThresholdConfig{ CPU: HysteresisThreshold{Trigger: 80, Clear: 75}, Memory: HysteresisThreshold{Trigger: 85, Clear: 80}, + Disk: HysteresisThreshold{Trigger: 85, Clear: 80}, RestartCount: 3, RestartWindow: 300, // 5 minutes MemoryWarnPct: 90, @@ -807,6 +809,9 @@ func (m *Manager) UpdateConfig(config AlertConfig) { if config.DockerDefaults.Memory.Trigger <= 0 { config.DockerDefaults.Memory = HysteresisThreshold{Trigger: 85, Clear: 80} } + if config.DockerDefaults.Disk.Trigger <= 0 { + config.DockerDefaults.Disk = HysteresisThreshold{Trigger: 85, Clear: 80} + } if config.DockerDefaults.RestartCount <= 0 { config.DockerDefaults.RestartCount = 3 } @@ -2780,7 +2785,7 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode if state != "running" { m.checkDockerContainerState(host, container, resourceID, containerName, instanceName, nodeName) - m.clearDockerContainerMetricAlerts(resourceID, "cpu", "memory") + m.clearDockerContainerMetricAlerts(resourceID, "cpu", "memory", "disk") } else { m.clearDockerContainerStateAlert(resourceID) @@ -2788,6 +2793,7 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode thresholds := ThresholdConfig{ CPU: &m.config.DockerDefaults.CPU, Memory: &m.config.DockerDefaults.Memory, + Disk: &m.config.DockerDefaults.Disk, } if hasOverride { thresholds = m.applyThresholdOverride(thresholds, overrideConfig) @@ -2832,6 +2838,38 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode } m.checkMetric(resourceID, containerName, nodeName, instanceName, resourceType, "memory", container.MemoryPercent, thresholds.Memory, &metricOptions{Metadata: memMetadata}) } + + if thresholds.Disk != nil { + totalBytes := container.RootFilesystemBytes + usedBytes := container.WritableLayerBytes + if totalBytes > 0 && usedBytes >= 0 { + diskPercent := (float64(usedBytes) / float64(totalBytes)) * 100 + diskMetadata := map[string]interface{}{ + "resourceType": resourceType, + "hostId": host.ID, + "hostName": host.DisplayName, + "hostHostname": host.Hostname, + "containerId": container.ID, + "containerName": containerName, + "image": container.Image, + "state": container.State, + "status": container.Status, + "restartCount": container.RestartCount, + "metric": "disk", + "diskPercent": diskPercent, + "writableLayerBytes": usedBytes, + "rootFilesystemBytes": totalBytes, + "mountCount": len(container.Mounts), + } + if container.BlockIO != nil { + diskMetadata["blockIoReadBytes"] = container.BlockIO.ReadBytes + diskMetadata["blockIoWriteBytes"] = container.BlockIO.WriteBytes + } + m.checkMetric(resourceID, containerName, nodeName, instanceName, resourceType, "disk", diskPercent, thresholds.Disk, &metricOptions{Metadata: diskMetadata}) + } else { + m.clearDockerContainerMetricAlerts(resourceID, "disk") + } + } } m.checkDockerContainerHealth(host, container, resourceID, containerName, instanceName, nodeName) @@ -3552,7 +3590,7 @@ func (m *Manager) checkDockerContainerMemoryLimit(host models.DockerHost, contai func (m *Manager) clearDockerContainerMetricAlerts(resourceID string, metrics ...string) { if len(metrics) == 0 { - metrics = []string{"cpu", "memory"} + metrics = []string{"cpu", "memory", "disk"} } for _, metric := range metrics { alertID := fmt.Sprintf("%s-%s", resourceID, metric) diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 1e7fac0..0d30fce 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -1146,6 +1146,68 @@ func TestDockerContainerMemoryLimitHysteresis(t *testing.T) { } } +func TestDockerContainerDiskUsageAlert(t *testing.T) { + m := NewManager() + + cfg := m.GetConfig() + cfg.Enabled = true + cfg.TimeThreshold = 0 + if cfg.TimeThresholds == nil { + cfg.TimeThresholds = make(map[string]int) + } + cfg.TimeThresholds["docker"] = 0 + cfg.TimeThresholds["guest"] = 0 + cfg.DockerDefaults.Disk = HysteresisThreshold{Trigger: 75, Clear: 65} + m.UpdateConfig(cfg) + + const gib = 1024 * 1024 * 1024 + + host := models.DockerHost{ + ID: "host-disk", + DisplayName: "Docker Host", + Hostname: "docker.disk", + Containers: []models.DockerContainer{ + { + ID: "container-disk", + Name: "disk-hog", + State: "running", + Status: "Up 5 minutes", + WritableLayerBytes: int64(8 * gib), + RootFilesystemBytes: int64(10 * gib), + }, + }, + } + + m.CheckDockerHost(host) + + resourceID := dockerResourceID(host.ID, host.Containers[0].ID) + alertID := fmt.Sprintf("%s-%s", resourceID, "disk") + alert, exists := m.activeAlerts[alertID] + if !exists { + t.Fatalf("expected docker container disk alert %s to be raised", alertID) + } + if alert.Level != AlertLevelWarning { + t.Fatalf("expected warning severity for disk usage alert, got %s", alert.Level) + } + if alert.Metadata == nil { + t.Fatalf("expected disk alert metadata to be populated") + } + if percent, ok := alert.Metadata["diskPercent"].(float64); !ok || percent < 79.5 || percent > 80.5 { + t.Fatalf("expected diskPercent metadata to be ~80%%, got %v", alert.Metadata["diskPercent"]) + } + if used, ok := alert.Metadata["writableLayerBytes"].(int64); !ok || used != int64(8*gib) { + t.Fatalf("expected writableLayerBytes metadata to be %d, got %v", int64(8*gib), alert.Metadata["writableLayerBytes"]) + } + + // Drop usage below the clear threshold and ensure the alert resolves. + host.Containers[0].WritableLayerBytes = int64(4 * gib) + m.CheckDockerHost(host) + + if _, stillActive := m.activeAlerts[alertID]; stillActive { + t.Fatalf("expected docker container disk alert %s to clear after usage dropped", alertID) + } +} + func TestUpdateConfigClampsDockerServiceCriticalGap(t *testing.T) { t.Parallel()