4891f06e76
The original fix in c6c0ac63e only handled per-resource overrides when thresholds were disabled (trigger <= 0 or Disabled=true). It did not handle global DisableAll* flags (DisableAllStorage, DisableAllNodes, DisableAllGuests, etc.). When a user toggled a DisableAll* flag from false to true: - Check* functions returned early without processing - Existing active alerts remained in m.activeAlerts map - Those alerts continued generating webhook notifications - reevaluateActiveAlertsLocked didn't check DisableAll* flags This commit fixes the issue by: 1. Updating reevaluateActiveAlertsLocked to check all DisableAll* flags and resolve alerts for those resource types during config updates 2. Adding alert cleanup to Check* functions before early returns: - CheckStorage: clears usage and offline alerts - CheckNode: clears cpu/memory/disk/temperature and offline alerts - CheckPMG: clears queue/message alerts and offline alerts - CheckPBS: clears cpu/memory and offline alerts - CheckHost: calls existing cleanup helpers 3. Adding comprehensive test coverage for DisableAllStorage scenario Related to #561
1774 lines
48 KiB
Go
1774 lines
48 KiB
Go
package alerts
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
|
)
|
|
|
|
func TestAcknowledgePersistsThroughCheckMetric(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
// Set config fields directly to bypass UpdateConfig's default value enforcement
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.SuppressionWindow = 0
|
|
m.config.MinimumDelta = 0
|
|
m.mu.Unlock()
|
|
|
|
threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}
|
|
m.checkMetric("res1", "Resource", "node1", "inst1", "guest", "usage", 90, threshold, nil)
|
|
if _, exists := m.activeAlerts["res1-usage"]; !exists {
|
|
t.Fatalf("expected alert to be created")
|
|
}
|
|
|
|
if err := m.AcknowledgeAlert("res1-usage", "tester"); err != nil {
|
|
t.Fatalf("ack failed: %v", err)
|
|
}
|
|
|
|
if !m.activeAlerts["res1-usage"].Acknowledged {
|
|
t.Fatalf("acknowledged flag not set")
|
|
}
|
|
|
|
alerts := m.GetActiveAlerts()
|
|
if len(alerts) != 1 || !alerts[0].Acknowledged {
|
|
t.Fatalf("GetActiveAlerts lost acknowledgement")
|
|
}
|
|
|
|
m.checkMetric("res1", "Resource", "node1", "inst1", "guest", "usage", 85, threshold, nil)
|
|
if !m.activeAlerts["res1-usage"].Acknowledged {
|
|
t.Fatalf("acknowledged flag lost after update")
|
|
}
|
|
}
|
|
|
|
func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
|
m := NewManager()
|
|
|
|
vmID := "instance-node-101"
|
|
instanceName := "instance"
|
|
|
|
// Start with default configuration to allow CPU alerts.
|
|
initialConfig := AlertConfig{
|
|
Enabled: true,
|
|
GuestDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
},
|
|
TimeThreshold: 0,
|
|
TimeThresholds: map[string]int{},
|
|
NodeDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
}
|
|
m.UpdateConfig(initialConfig)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.ActivationState = ActivationActive
|
|
m.mu.Unlock()
|
|
|
|
var dispatched []*Alert
|
|
done := make(chan struct{}, 1)
|
|
var resolved []string
|
|
resolvedDone := make(chan struct{}, 1)
|
|
m.SetAlertCallback(func(alert *Alert) {
|
|
dispatched = append(dispatched, alert)
|
|
select {
|
|
case done <- struct{}{}:
|
|
default:
|
|
}
|
|
})
|
|
m.SetResolvedCallback(func(alertID string) {
|
|
resolved = append(resolved, alertID)
|
|
select {
|
|
case resolvedDone <- struct{}{}:
|
|
default:
|
|
}
|
|
})
|
|
|
|
vm := models.VM{
|
|
ID: vmID,
|
|
Name: "test-vm",
|
|
Node: "node",
|
|
Instance: instanceName,
|
|
Status: "running",
|
|
CPU: 1.0, // 100% once multiplied by 100 inside CheckGuest
|
|
Memory: models.Memory{
|
|
Usage: 65,
|
|
},
|
|
Disk: models.Disk{
|
|
Usage: 40,
|
|
},
|
|
}
|
|
|
|
// Initial check should trigger an alert with default thresholds.
|
|
m.CheckGuest(vm, instanceName)
|
|
select {
|
|
case <-done:
|
|
case <-time.After(100 * time.Millisecond):
|
|
t.Fatalf("did not receive initial alert dispatch")
|
|
}
|
|
if len(dispatched) != 1 {
|
|
t.Fatalf("expected 1 alert before disabling metric, got %d", len(dispatched))
|
|
}
|
|
|
|
// Apply override disabling CPU alerts for this VM.
|
|
disabledConfig := initialConfig
|
|
disabledConfig.Overrides = map[string]ThresholdConfig{
|
|
vmID: {
|
|
CPU: &HysteresisThreshold{Trigger: -1, Clear: 0},
|
|
},
|
|
}
|
|
disabledConfig.TimeThreshold = 0
|
|
disabledConfig.TimeThresholds = map[string]int{}
|
|
m.UpdateConfig(disabledConfig)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.ActivationState = ActivationActive
|
|
m.mu.Unlock()
|
|
|
|
// Clear dispatched slice to capture only post-disable notifications.
|
|
dispatched = dispatched[:0]
|
|
done = make(chan struct{}, 1)
|
|
|
|
// Re-run evaluation with high CPU; no alert should be dispatched.
|
|
m.CheckGuest(vm, instanceName)
|
|
select {
|
|
case <-done:
|
|
t.Fatalf("expected no alerts after disabling CPU metric, but callback fired")
|
|
case <-time.After(100 * time.Millisecond):
|
|
// No callback fired as expected.
|
|
}
|
|
|
|
// Active alerts should be cleared by the config update.
|
|
m.mu.RLock()
|
|
activeCount := len(m.activeAlerts)
|
|
m.mu.RUnlock()
|
|
if activeCount != 0 {
|
|
t.Fatalf("expected active alerts to be cleared after disabling metric, got %d", activeCount)
|
|
}
|
|
|
|
select {
|
|
case <-resolvedDone:
|
|
case <-time.After(100 * time.Millisecond):
|
|
t.Fatalf("expected resolved callback to fire after disabling metric")
|
|
}
|
|
if len(resolved) != 1 || resolved[0] != fmt.Sprintf("%s-cpu", vmID) {
|
|
t.Fatalf("expected resolved callback for %s-cpu, got %v", vmID, resolved)
|
|
}
|
|
|
|
m.mu.RLock()
|
|
_, isPending := m.pendingAlerts[fmt.Sprintf("%s-cpu", vmID)]
|
|
m.mu.RUnlock()
|
|
if isPending {
|
|
t.Fatalf("expected pending alert entry to be cleared after disabling metric")
|
|
}
|
|
}
|
|
|
|
func TestPulseNoAlertsSuppressesGuestAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.ActivationState = ActivationActive
|
|
m.mu.Unlock()
|
|
|
|
var dispatched int
|
|
m.SetAlertCallback(func(alert *Alert) {
|
|
dispatched++
|
|
})
|
|
|
|
vm := models.VM{
|
|
ID: "inst/qemu/101",
|
|
Name: "test-vm",
|
|
Node: "node1",
|
|
Instance: "inst",
|
|
Status: "running",
|
|
CPU: 1.0,
|
|
Memory: models.Memory{
|
|
Usage: 95,
|
|
},
|
|
Disk: models.Disk{
|
|
Usage: 95,
|
|
},
|
|
Tags: []string{"pulse-no-alerts"},
|
|
}
|
|
|
|
m.CheckGuest(vm, "inst")
|
|
|
|
if dispatched != 0 {
|
|
t.Fatalf("expected no alert dispatch, got %d", dispatched)
|
|
}
|
|
|
|
if alerts := m.GetActiveAlerts(); len(alerts) != 0 {
|
|
t.Fatalf("expected no active alerts, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
func TestPulseMonitorOnlySkipsDispatchButRetainsAlert(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.ActivationState = ActivationActive
|
|
m.mu.Unlock()
|
|
|
|
var dispatched int
|
|
m.SetAlertCallback(func(alert *Alert) {
|
|
dispatched++
|
|
})
|
|
|
|
vm := models.VM{
|
|
ID: "inst/qemu/102",
|
|
Name: "monitor-vm",
|
|
Node: "node1",
|
|
Instance: "inst",
|
|
Status: "running",
|
|
CPU: 1.0,
|
|
Memory: models.Memory{Usage: 90},
|
|
Disk: models.Disk{Usage: 50},
|
|
Tags: []string{"pulse-monitor-only"},
|
|
}
|
|
|
|
m.CheckGuest(vm, "inst")
|
|
|
|
if dispatched != 0 {
|
|
t.Fatalf("expected monitor-only alert to skip dispatch, got %d callbacks", dispatched)
|
|
}
|
|
|
|
alerts := m.GetActiveAlerts()
|
|
if len(alerts) == 0 {
|
|
t.Fatalf("expected monitor-only alert to remain active")
|
|
}
|
|
|
|
if alerts[0].Metadata == nil || alerts[0].Metadata["monitorOnly"] != true {
|
|
t.Fatalf("expected alert metadata to mark monitorOnly, got %+v", alerts[0].Metadata)
|
|
}
|
|
}
|
|
|
|
func TestPulseRelaxedThresholdsIncreaseCpuTrigger(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
vm := models.VM{
|
|
ID: "inst/qemu/103",
|
|
Name: "relaxed-vm",
|
|
Node: "node1",
|
|
Instance: "inst",
|
|
Status: "running",
|
|
CPU: 0.9, // 90%
|
|
Memory: models.Memory{Usage: 60},
|
|
Disk: models.Disk{Usage: 40},
|
|
Tags: []string{"pulse-relaxed"},
|
|
}
|
|
|
|
m.CheckGuest(vm, "inst")
|
|
|
|
if alerts := m.GetActiveAlerts(); len(alerts) != 0 {
|
|
t.Fatalf("expected no alerts at 90%% CPU with relaxed thresholds, got %d", len(alerts))
|
|
}
|
|
|
|
vm.CPU = 1.0
|
|
m.CheckGuest(vm, "inst")
|
|
|
|
if alerts := m.GetActiveAlerts(); len(alerts) == 0 {
|
|
t.Fatalf("expected alert once CPU exceeds relaxed threshold")
|
|
}
|
|
}
|
|
|
|
func TestClearAlertMarksResolutionAndReturnsStatus(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
vm := models.VM{
|
|
ID: "inst/qemu/104",
|
|
Name: "clear-vm",
|
|
Node: "node1",
|
|
Instance: "inst",
|
|
Status: "running",
|
|
CPU: 1.0,
|
|
Memory: models.Memory{Usage: 80},
|
|
Disk: models.Disk{Usage: 80},
|
|
}
|
|
|
|
m.CheckGuest(vm, "inst")
|
|
alerts := m.GetActiveAlerts()
|
|
if len(alerts) == 0 {
|
|
t.Fatalf("expected alert to be active before clearing")
|
|
}
|
|
|
|
alertID := alerts[0].ID
|
|
if ok := m.ClearAlert(alertID); !ok {
|
|
t.Fatalf("expected manual clear to succeed")
|
|
}
|
|
|
|
if remaining := m.GetActiveAlerts(); len(remaining) != 0 {
|
|
t.Fatalf("expected no active alerts after clear, found %d", len(remaining))
|
|
}
|
|
|
|
resolved := m.GetRecentlyResolved()
|
|
if len(resolved) == 0 || resolved[0].Alert.ID != alertID {
|
|
t.Fatalf("expected alert %s to be tracked as recently resolved", alertID)
|
|
}
|
|
|
|
if ok := m.ClearAlert(alertID); ok {
|
|
t.Fatalf("expected second clear to report missing alert")
|
|
}
|
|
}
|
|
|
|
func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
|
m := NewManager()
|
|
host := models.DockerHost{ID: "host1", DisplayName: "Host One", Hostname: "host-one"}
|
|
containerResourceID := "docker:host1/container1"
|
|
containerAlertID := "docker-container-state-" + containerResourceID
|
|
hostAlertID := "docker-host-offline-host1"
|
|
|
|
m.mu.Lock()
|
|
m.activeAlerts[hostAlertID] = &Alert{ID: hostAlertID, ResourceID: "docker:host1"}
|
|
m.activeAlerts[containerAlertID] = &Alert{ID: containerAlertID, ResourceID: containerResourceID}
|
|
m.dockerOfflineCount[host.ID] = 2
|
|
m.dockerStateConfirm[containerResourceID] = 1
|
|
m.dockerRestartTracking[containerResourceID] = &dockerRestartRecord{}
|
|
m.dockerLastExitCode[containerResourceID] = 137
|
|
m.mu.Unlock()
|
|
|
|
m.HandleDockerHostRemoved(host)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
if _, exists := m.activeAlerts[containerAlertID]; exists {
|
|
t.Fatalf("expected container alerts to be cleared")
|
|
}
|
|
if _, exists := m.activeAlerts[hostAlertID]; exists {
|
|
t.Fatalf("expected host offline alert to be cleared")
|
|
}
|
|
if _, exists := m.dockerOfflineCount[host.ID]; exists {
|
|
t.Fatalf("expected offline tracking to be cleared")
|
|
}
|
|
if _, exists := m.dockerStateConfirm[containerResourceID]; exists {
|
|
t.Fatalf("expected state confirmation to be cleared")
|
|
}
|
|
if _, exists := m.dockerRestartTracking[containerResourceID]; exists {
|
|
t.Fatalf("expected restart tracking to be cleared")
|
|
}
|
|
if _, exists := m.dockerLastExitCode[containerResourceID]; exists {
|
|
t.Fatalf("expected last exit code tracking to be cleared")
|
|
}
|
|
}
|
|
|
|
func TestCheckHostGeneratesMetricAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
host := models.Host{
|
|
ID: "host-1",
|
|
DisplayName: "Test Host",
|
|
Hostname: "host-1.example",
|
|
Platform: "linux",
|
|
OSName: "ubuntu",
|
|
CPUUsage: 95,
|
|
CPUCount: 8,
|
|
Memory: models.Memory{
|
|
Usage: 92,
|
|
Total: 16384,
|
|
Used: 15000,
|
|
Free: 1384,
|
|
},
|
|
Disks: []models.Disk{
|
|
{
|
|
Mountpoint: "/",
|
|
Usage: 93,
|
|
Total: 100,
|
|
Used: 93,
|
|
Free: 7,
|
|
},
|
|
},
|
|
Status: "online",
|
|
IntervalSeconds: 30,
|
|
LastSeen: time.Now(),
|
|
Tags: []string{"prod"},
|
|
}
|
|
|
|
m.CheckHost(host)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
cpuAlertID := fmt.Sprintf("%s-cpu", hostResourceID(host.ID))
|
|
if _, exists := m.activeAlerts[cpuAlertID]; !exists {
|
|
t.Fatalf("expected CPU alert %q to be active", cpuAlertID)
|
|
}
|
|
|
|
memAlertID := fmt.Sprintf("%s-memory", hostResourceID(host.ID))
|
|
if _, exists := m.activeAlerts[memAlertID]; !exists {
|
|
t.Fatalf("expected memory alert %q to be active", memAlertID)
|
|
}
|
|
|
|
diskResourceID, _ := hostDiskResourceID(host, host.Disks[0])
|
|
diskAlertID := fmt.Sprintf("%s-disk", diskResourceID)
|
|
if _, exists := m.activeAlerts[diskAlertID]; !exists {
|
|
t.Fatalf("expected disk alert %q to be active", diskAlertID)
|
|
}
|
|
}
|
|
|
|
func TestHandleHostOfflineRequiresConfirmations(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
host := models.Host{ID: "host-2", DisplayName: "Second Host", Hostname: "host-two"}
|
|
alertID := fmt.Sprintf("host-offline-%s", host.ID)
|
|
resourceKey := hostResourceID(host.ID)
|
|
|
|
m.HandleHostOffline(host)
|
|
m.mu.RLock()
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected no alert after first offline detection")
|
|
}
|
|
if count := m.offlineConfirmations[resourceKey]; count != 1 {
|
|
t.Fatalf("expected confirmation count to be 1, got %d", count)
|
|
}
|
|
m.mu.RUnlock()
|
|
|
|
m.HandleHostOffline(host)
|
|
m.mu.RLock()
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected no alert after second offline detection")
|
|
}
|
|
if count := m.offlineConfirmations[resourceKey]; count != 2 {
|
|
t.Fatalf("expected confirmation count to be 2, got %d", count)
|
|
}
|
|
m.mu.RUnlock()
|
|
|
|
m.HandleHostOffline(host)
|
|
m.mu.RLock()
|
|
if _, exists := m.activeAlerts[alertID]; !exists {
|
|
t.Fatalf("expected alert %q after third offline detection", alertID)
|
|
}
|
|
m.mu.RUnlock()
|
|
|
|
m.HandleHostOnline(host)
|
|
m.mu.RLock()
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected offline alert %q to be cleared after host online", alertID)
|
|
}
|
|
if _, exists := m.offlineConfirmations[resourceKey]; exists {
|
|
t.Fatalf("expected offline confirmations to be cleared when host online")
|
|
}
|
|
m.mu.RUnlock()
|
|
}
|
|
|
|
func TestCheckHostDisabledOverrideClearsAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
host := models.Host{
|
|
ID: "host-3",
|
|
DisplayName: "Override Host",
|
|
Hostname: "override.example",
|
|
CPUUsage: 90,
|
|
Memory: models.Memory{
|
|
Usage: 91,
|
|
Total: 16000,
|
|
Used: 14560,
|
|
Free: 1440,
|
|
},
|
|
Disks: []models.Disk{
|
|
{Mountpoint: "/data", Usage: 92, Total: 200, Used: 184, Free: 16},
|
|
},
|
|
Status: "online",
|
|
IntervalSeconds: 30,
|
|
LastSeen: time.Now(),
|
|
}
|
|
|
|
m.CheckHost(host)
|
|
|
|
m.mu.RLock()
|
|
if len(m.activeAlerts) == 0 {
|
|
m.mu.RUnlock()
|
|
t.Fatalf("expected active alerts prior to disabling host overrides")
|
|
}
|
|
m.mu.RUnlock()
|
|
|
|
cfg := m.GetConfig()
|
|
cfg.Overrides = map[string]ThresholdConfig{
|
|
host.ID: {
|
|
Disabled: true,
|
|
},
|
|
}
|
|
m.UpdateConfig(cfg)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
m.CheckHost(host)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
if len(m.activeAlerts) != 0 {
|
|
t.Fatalf("expected all host alerts to be cleared after disabling override, got %d", len(m.activeAlerts))
|
|
}
|
|
}
|
|
|
|
func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
cfg := AlertConfig{
|
|
Enabled: true,
|
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
SnapshotDefaults: SnapshotAlertConfig{
|
|
Enabled: true,
|
|
WarningDays: 7,
|
|
CriticalDays: 14,
|
|
WarningSizeGiB: 0,
|
|
CriticalSizeGiB: 0,
|
|
},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
}
|
|
m.UpdateConfig(cfg)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
snapshots := []models.GuestSnapshot{
|
|
{
|
|
ID: "inst-node-100-weekly",
|
|
Name: "weekly",
|
|
Node: "node",
|
|
Instance: "inst",
|
|
Type: "qemu",
|
|
VMID: 100,
|
|
Time: now.Add(-15 * 24 * time.Hour),
|
|
SizeBytes: 60 << 30,
|
|
},
|
|
}
|
|
guestNames := map[string]string{
|
|
"inst-node-100": "app-server",
|
|
}
|
|
|
|
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
|
|
|
|
m.mu.RLock()
|
|
alert, exists := m.activeAlerts["snapshot-age-inst-node-100-weekly"]
|
|
m.mu.RUnlock()
|
|
if !exists {
|
|
t.Fatalf("expected snapshot age alert to be created")
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical level for old snapshot, got %s", alert.Level)
|
|
}
|
|
if alert.ResourceName != "app-server snapshot 'weekly'" {
|
|
t.Fatalf("unexpected resource name: %s", alert.ResourceName)
|
|
}
|
|
|
|
m.CheckSnapshotsForInstance("inst", nil, guestNames)
|
|
|
|
m.mu.RLock()
|
|
_, exists = m.activeAlerts["snapshot-age-inst-node-100-weekly"]
|
|
m.mu.RUnlock()
|
|
if exists {
|
|
t.Fatalf("expected snapshot alert to be cleared when snapshot missing")
|
|
}
|
|
}
|
|
|
|
func TestCheckSnapshotsForInstanceTriggersOnSnapshotSize(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
cfg := AlertConfig{
|
|
Enabled: true,
|
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
SnapshotDefaults: SnapshotAlertConfig{
|
|
Enabled: true,
|
|
WarningDays: 0,
|
|
CriticalDays: 0,
|
|
WarningSizeGiB: 50,
|
|
CriticalSizeGiB: 100,
|
|
},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
}
|
|
m.UpdateConfig(cfg)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
snapshots := []models.GuestSnapshot{
|
|
{
|
|
ID: "inst-node-200-sizey",
|
|
Name: "pre-maintenance",
|
|
Node: "node",
|
|
Instance: "inst",
|
|
Type: "qemu",
|
|
VMID: 200,
|
|
Time: now.Add(-2 * time.Hour),
|
|
SizeBytes: int64(120) << 30,
|
|
},
|
|
}
|
|
guestNames := map[string]string{
|
|
"inst-node-200": "db-server",
|
|
}
|
|
|
|
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
|
|
|
|
m.mu.RLock()
|
|
alert, exists := m.activeAlerts["snapshot-age-inst-node-200-sizey"]
|
|
m.mu.RUnlock()
|
|
if !exists {
|
|
t.Fatalf("expected snapshot size alert to be created")
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical level for large snapshot, got %s", alert.Level)
|
|
}
|
|
if alert.Value < 119.5 || alert.Value > 120.5 {
|
|
t.Fatalf("expected alert value near 120 GiB, got %.2f", alert.Value)
|
|
}
|
|
if alert.Threshold != 100 {
|
|
t.Fatalf("expected threshold 100 GiB, got %.2f", alert.Threshold)
|
|
}
|
|
if alert.Metadata == nil {
|
|
t.Fatalf("expected metadata for snapshot alert")
|
|
}
|
|
if metric, ok := alert.Metadata["primaryMetric"].(string); !ok || metric != "size" {
|
|
t.Fatalf("expected primary metric size, got %#v", alert.Metadata["primaryMetric"])
|
|
}
|
|
if sizeBytes, ok := alert.Metadata["snapshotSizeBytes"].(int64); !ok || sizeBytes == 0 {
|
|
t.Fatalf("expected snapshotSizeBytes in metadata")
|
|
}
|
|
metrics, ok := alert.Metadata["triggeredMetrics"].([]string)
|
|
if !ok {
|
|
t.Fatalf("expected triggeredMetrics slice, got %#v", alert.Metadata["triggeredMetrics"])
|
|
}
|
|
foundSize := false
|
|
for _, metric := range metrics {
|
|
if metric == "size" {
|
|
foundSize = true
|
|
break
|
|
}
|
|
}
|
|
if !foundSize {
|
|
t.Fatalf("expected size metric recorded in metadata")
|
|
}
|
|
}
|
|
|
|
func TestCheckSnapshotsForInstanceIncludesAgeAndSizeReasons(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
cfg := AlertConfig{
|
|
Enabled: true,
|
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
SnapshotDefaults: SnapshotAlertConfig{
|
|
Enabled: true,
|
|
WarningDays: 5,
|
|
CriticalDays: 10,
|
|
WarningSizeGiB: 40,
|
|
CriticalSizeGiB: 80,
|
|
},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
}
|
|
m.UpdateConfig(cfg)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
snapshots := []models.GuestSnapshot{
|
|
{
|
|
ID: "inst-node-300-combined",
|
|
Name: "long-running",
|
|
Node: "node",
|
|
Instance: "inst",
|
|
Type: "qemu",
|
|
VMID: 300,
|
|
Time: now.Add(-15 * 24 * time.Hour),
|
|
SizeBytes: int64(90) << 30,
|
|
},
|
|
}
|
|
guestNames := map[string]string{
|
|
"inst-node-300": "app-server",
|
|
}
|
|
|
|
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
|
|
|
|
m.mu.RLock()
|
|
alert, exists := m.activeAlerts["snapshot-age-inst-node-300-combined"]
|
|
m.mu.RUnlock()
|
|
if !exists {
|
|
t.Fatalf("expected combined snapshot alert to be created")
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical level, got %s", alert.Level)
|
|
}
|
|
if !strings.Contains(alert.Message, "days old") || !strings.Contains(strings.ToLower(alert.Message), "gib") {
|
|
t.Fatalf("expected alert message to reference age and size, got %q", alert.Message)
|
|
}
|
|
if alert.Metadata == nil {
|
|
t.Fatalf("expected metadata for combined alert")
|
|
}
|
|
metrics, ok := alert.Metadata["triggeredMetrics"].([]string)
|
|
if !ok {
|
|
t.Fatalf("expected triggeredMetrics slice, got %#v", alert.Metadata["triggeredMetrics"])
|
|
}
|
|
if len(metrics) < 2 {
|
|
t.Fatalf("expected both age and size metrics recorded, got %v", metrics)
|
|
}
|
|
if metric, ok := alert.Metadata["primaryMetric"].(string); !ok || metric != "age" {
|
|
t.Fatalf("expected primary metric age, got %#v", alert.Metadata["primaryMetric"])
|
|
}
|
|
}
|
|
|
|
func TestCheckBackupsCreatesAndClearsAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
m.mu.Lock()
|
|
m.config.Enabled = true
|
|
m.config.BackupDefaults = BackupAlertConfig{
|
|
Enabled: true,
|
|
WarningDays: 7,
|
|
CriticalDays: 14,
|
|
}
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
storageBackups := []models.StorageBackup{
|
|
{
|
|
ID: "inst-node-100-backup",
|
|
Storage: "local",
|
|
Node: "node",
|
|
Instance: "inst",
|
|
Type: "qemu",
|
|
VMID: 100,
|
|
Time: now.Add(-15 * 24 * time.Hour),
|
|
},
|
|
}
|
|
|
|
key := BuildGuestKey("inst", "node", 100)
|
|
guestsByKey := map[string]GuestLookup{
|
|
key: {
|
|
Name: "app-server",
|
|
Instance: "inst",
|
|
Node: "node",
|
|
Type: "qemu",
|
|
VMID: 100,
|
|
},
|
|
}
|
|
guestsByVMID := map[string]GuestLookup{
|
|
"100": guestsByKey[key],
|
|
}
|
|
|
|
m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID)
|
|
|
|
m.mu.RLock()
|
|
alert, exists := m.activeAlerts["backup-age-"+sanitizeAlertKey(key)]
|
|
m.mu.RUnlock()
|
|
if !exists {
|
|
t.Fatalf("expected backup age alert to be created")
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical backup alert, got %s", alert.Level)
|
|
}
|
|
|
|
// Recent backup clears alert
|
|
storageBackups[0].Time = now
|
|
m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID)
|
|
|
|
m.mu.RLock()
|
|
_, exists = m.activeAlerts["backup-age-"+sanitizeAlertKey(key)]
|
|
m.mu.RUnlock()
|
|
if exists {
|
|
t.Fatalf("expected backup-age alert to clear after fresh backup")
|
|
}
|
|
}
|
|
|
|
func TestCheckBackupsHandlesPbsOnlyGuests(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
m.mu.Lock()
|
|
m.config.Enabled = true
|
|
m.config.BackupDefaults = BackupAlertConfig{
|
|
Enabled: true,
|
|
WarningDays: 3,
|
|
CriticalDays: 5,
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
pbsBackups := []models.PBSBackup{
|
|
{
|
|
ID: "pbs-backup-999-0",
|
|
Instance: "pbs-main",
|
|
Datastore: "backup-store",
|
|
BackupType: "qemu",
|
|
VMID: "999",
|
|
BackupTime: now.Add(-6 * 24 * time.Hour),
|
|
},
|
|
}
|
|
|
|
m.CheckBackups(nil, pbsBackups, nil, map[string]GuestLookup{}, map[string]GuestLookup{})
|
|
|
|
m.mu.RLock()
|
|
found := false
|
|
for id, alert := range m.activeAlerts {
|
|
if strings.HasPrefix(id, "backup-age-") {
|
|
found = true
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected PBS backup alert to be critical")
|
|
}
|
|
break
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
if !found {
|
|
t.Fatalf("expected PBS backup alert to be created")
|
|
}
|
|
}
|
|
|
|
func TestCheckBackupsHandlesPmgBackups(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
m.mu.Lock()
|
|
m.config.Enabled = true
|
|
m.config.BackupDefaults = BackupAlertConfig{
|
|
Enabled: true,
|
|
WarningDays: 5,
|
|
CriticalDays: 7,
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
pmgBackups := []models.PMGBackup{
|
|
{
|
|
ID: "pmg-backup-mail-01",
|
|
Instance: "mail",
|
|
Node: "mail-gateway",
|
|
Filename: "pmg-backup_2024-01-01.tgz",
|
|
BackupTime: now.Add(-8 * 24 * time.Hour),
|
|
Size: 123456,
|
|
},
|
|
}
|
|
|
|
m.CheckBackups(nil, nil, pmgBackups, map[string]GuestLookup{}, map[string]GuestLookup{})
|
|
|
|
m.mu.RLock()
|
|
found := false
|
|
for id, alert := range m.activeAlerts {
|
|
if strings.HasPrefix(id, "backup-age-") {
|
|
found = true
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected PMG backup alert to be critical")
|
|
}
|
|
break
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
if !found {
|
|
t.Fatalf("expected PMG backup alert to be created")
|
|
}
|
|
}
|
|
|
|
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
|
|
m := NewManager()
|
|
|
|
m.mu.Lock()
|
|
m.config.DockerIgnoredContainerPrefixes = []string{"runner-"}
|
|
m.mu.Unlock()
|
|
|
|
container := models.DockerContainer{
|
|
ID: "1234567890ab",
|
|
Name: "runner-auto-1",
|
|
State: "exited",
|
|
Status: "Exited (0) 3 seconds ago",
|
|
}
|
|
|
|
host := models.DockerHost{
|
|
ID: "host-ephemeral",
|
|
Hostname: "ci-host",
|
|
DisplayName: "CI Host",
|
|
Containers: []models.DockerContainer{container},
|
|
}
|
|
|
|
resourceID := dockerResourceID(host.ID, container.ID)
|
|
alertID := fmt.Sprintf("docker-container-state-%s", resourceID)
|
|
|
|
// Run twice to satisfy the confirmation threshold when not ignored
|
|
m.CheckDockerHost(host)
|
|
m.CheckDockerHost(host)
|
|
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected no state alert for ignored container")
|
|
}
|
|
if _, exists := m.dockerStateConfirm[resourceID]; exists {
|
|
t.Fatalf("expected no state confirmation tracking for ignored container")
|
|
}
|
|
}
|
|
|
|
func TestDockerServiceReplicaAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
m.mu.RLock()
|
|
cfg := m.config
|
|
m.mu.RUnlock()
|
|
cfg.Enabled = true
|
|
m.UpdateConfig(cfg)
|
|
|
|
host := models.DockerHost{
|
|
ID: "host-1",
|
|
DisplayName: "Prod Swarm",
|
|
Hostname: "swarm-prod",
|
|
Services: []models.DockerService{
|
|
{
|
|
ID: "svc-1",
|
|
Name: "web",
|
|
DesiredTasks: 4,
|
|
RunningTasks: 2,
|
|
Mode: "replicated",
|
|
},
|
|
},
|
|
}
|
|
|
|
m.CheckDockerHost(host)
|
|
|
|
resourceID := dockerServiceResourceID(host.ID, "svc-1", "web")
|
|
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
|
alert, exists := m.activeAlerts[alertID]
|
|
if !exists {
|
|
t.Fatalf("expected service alert %s to be raised", alertID)
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical severity, got %s", alert.Level)
|
|
}
|
|
if missing, ok := alert.Metadata["missingTasks"].(int); !ok || missing != 2 {
|
|
t.Fatalf("expected missingTasks metadata to be 2, got %v", alert.Metadata["missingTasks"])
|
|
}
|
|
|
|
// Resolve by restoring replicas
|
|
host.Services[0].RunningTasks = 4
|
|
m.CheckDockerHost(host)
|
|
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected service alert %s to be cleared when replicas restored", alertID)
|
|
}
|
|
}
|
|
|
|
func TestDockerServiceUpdateStateAlert(t *testing.T) {
|
|
m := NewManager()
|
|
cfg := m.GetConfig()
|
|
cfg.Enabled = true
|
|
m.UpdateConfig(cfg)
|
|
|
|
now := time.Now()
|
|
host := models.DockerHost{
|
|
ID: "host-update",
|
|
DisplayName: "Swarm",
|
|
Hostname: "swarm.local",
|
|
Services: []models.DockerService{
|
|
{
|
|
ID: "svc-update",
|
|
Name: "api",
|
|
DesiredTasks: 1,
|
|
RunningTasks: 1,
|
|
UpdateStatus: &models.DockerServiceUpdate{
|
|
State: "rollback_failed",
|
|
Message: "Rollback failed",
|
|
CompletedAt: &now,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
m.CheckDockerHost(host)
|
|
|
|
resourceID := dockerServiceResourceID(host.ID, "svc-update", "api")
|
|
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
|
alert, exists := m.activeAlerts[alertID]
|
|
if !exists {
|
|
t.Fatalf("expected docker service alert %s to be raised", alertID)
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical severity for rollback failure, got %s", alert.Level)
|
|
}
|
|
if state, ok := alert.Metadata["updateState"].(string); !ok || state != "rollback_failed" {
|
|
t.Fatalf("expected updateState metadata to be rollback_failed, got %v", alert.Metadata["updateState"])
|
|
}
|
|
}
|
|
|
|
func TestDockerContainerStateUsesDockerDefaults(t *testing.T) {
|
|
m := NewManager()
|
|
cfg := m.GetConfig()
|
|
cfg.DockerDefaults.StatePoweredOffSeverity = AlertLevelCritical
|
|
m.UpdateConfig(cfg)
|
|
|
|
container := models.DockerContainer{
|
|
ID: "container-1",
|
|
Name: "web",
|
|
State: "exited",
|
|
Status: "Exited (1) seconds ago",
|
|
}
|
|
host := models.DockerHost{
|
|
ID: "host-1",
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker.local",
|
|
Containers: []models.DockerContainer{container},
|
|
}
|
|
|
|
m.CheckDockerHost(host)
|
|
m.CheckDockerHost(host)
|
|
|
|
resourceID := dockerResourceID(host.ID, container.ID)
|
|
alertID := fmt.Sprintf("docker-container-state-%s", resourceID)
|
|
alert, exists := m.activeAlerts[alertID]
|
|
if !exists {
|
|
t.Fatalf("expected docker container state alert %s to be raised", alertID)
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical severity from docker defaults, got %s", alert.Level)
|
|
}
|
|
}
|
|
|
|
func TestDockerContainerStateRespectsDisableDefault(t *testing.T) {
|
|
m := NewManager()
|
|
cfg := m.GetConfig()
|
|
cfg.DockerDefaults.StateDisableConnectivity = true
|
|
m.UpdateConfig(cfg)
|
|
|
|
container := models.DockerContainer{
|
|
ID: "container-2",
|
|
Name: "batch",
|
|
State: "exited",
|
|
Status: "Exited (0) seconds ago",
|
|
}
|
|
host := models.DockerHost{
|
|
ID: "host-2",
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker.example",
|
|
Containers: []models.DockerContainer{container},
|
|
}
|
|
|
|
m.CheckDockerHost(host)
|
|
m.CheckDockerHost(host)
|
|
|
|
resourceID := dockerResourceID(host.ID, container.ID)
|
|
alertID := fmt.Sprintf("docker-container-state-%s", resourceID)
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("did not expect docker container state alert when defaults disable connectivity")
|
|
}
|
|
}
|
|
|
|
func TestDockerContainerMemoryLimitHysteresis(t *testing.T) {
|
|
m := NewManager()
|
|
|
|
hostID := "host-mem"
|
|
containerID := "container-mem"
|
|
hostHigh := models.DockerHost{
|
|
ID: hostID,
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker.mem",
|
|
Containers: []models.DockerContainer{
|
|
{
|
|
ID: containerID,
|
|
Name: "memory-hog",
|
|
State: "running",
|
|
Status: "Up 10 minutes",
|
|
MemoryUsage: 96 * 1024 * 1024,
|
|
MemoryLimit: 100 * 1024 * 1024,
|
|
},
|
|
},
|
|
}
|
|
|
|
m.CheckDockerHost(hostHigh)
|
|
|
|
resourceID := dockerResourceID(hostID, containerID)
|
|
alertID := fmt.Sprintf("docker-container-memory-limit-%s", resourceID)
|
|
if _, exists := m.activeAlerts[alertID]; !exists {
|
|
t.Fatalf("expected memory limit alert to be raised")
|
|
}
|
|
|
|
hostLow := models.DockerHost{
|
|
ID: hostID,
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker.mem",
|
|
Containers: []models.DockerContainer{
|
|
{
|
|
ID: containerID,
|
|
Name: "memory-hog",
|
|
State: "running",
|
|
Status: "Up 12 minutes",
|
|
MemoryUsage: 80 * 1024 * 1024,
|
|
MemoryLimit: 100 * 1024 * 1024,
|
|
},
|
|
},
|
|
}
|
|
|
|
m.CheckDockerHost(hostLow)
|
|
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected memory limit alert to clear after usage dropped below hysteresis threshold")
|
|
}
|
|
}
|
|
|
|
func TestDockerContainerDiskUsageAlert(t *testing.T) {
|
|
m := NewManager()
|
|
|
|
cfg := m.GetConfig()
|
|
cfg.Enabled = true
|
|
cfg.TimeThreshold = 0
|
|
if cfg.TimeThresholds == nil {
|
|
cfg.TimeThresholds = make(map[string]int)
|
|
}
|
|
cfg.TimeThresholds["docker"] = 0
|
|
cfg.TimeThresholds["guest"] = 0
|
|
cfg.DockerDefaults.Disk = HysteresisThreshold{Trigger: 75, Clear: 65}
|
|
m.UpdateConfig(cfg)
|
|
|
|
const gib = 1024 * 1024 * 1024
|
|
|
|
host := models.DockerHost{
|
|
ID: "host-disk",
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker.disk",
|
|
Containers: []models.DockerContainer{
|
|
{
|
|
ID: "container-disk",
|
|
Name: "disk-hog",
|
|
State: "running",
|
|
Status: "Up 5 minutes",
|
|
WritableLayerBytes: int64(8 * gib),
|
|
RootFilesystemBytes: int64(10 * gib),
|
|
},
|
|
},
|
|
}
|
|
|
|
m.CheckDockerHost(host)
|
|
|
|
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
|
alertID := fmt.Sprintf("%s-%s", resourceID, "disk")
|
|
alert, exists := m.activeAlerts[alertID]
|
|
if !exists {
|
|
t.Fatalf("expected docker container disk alert %s to be raised", alertID)
|
|
}
|
|
if alert.Level != AlertLevelWarning {
|
|
t.Fatalf("expected warning severity for disk usage alert, got %s", alert.Level)
|
|
}
|
|
if alert.Metadata == nil {
|
|
t.Fatalf("expected disk alert metadata to be populated")
|
|
}
|
|
if percent, ok := alert.Metadata["diskPercent"].(float64); !ok || percent < 79.5 || percent > 80.5 {
|
|
t.Fatalf("expected diskPercent metadata to be ~80%%, got %v", alert.Metadata["diskPercent"])
|
|
}
|
|
if used, ok := alert.Metadata["writableLayerBytes"].(int64); !ok || used != int64(8*gib) {
|
|
t.Fatalf("expected writableLayerBytes metadata to be %d, got %v", int64(8*gib), alert.Metadata["writableLayerBytes"])
|
|
}
|
|
|
|
// Drop usage below the clear threshold and ensure the alert resolves.
|
|
host.Containers[0].WritableLayerBytes = int64(4 * gib)
|
|
m.CheckDockerHost(host)
|
|
|
|
if _, stillActive := m.activeAlerts[alertID]; stillActive {
|
|
t.Fatalf("expected docker container disk alert %s to clear after usage dropped", alertID)
|
|
}
|
|
}
|
|
|
|
func TestUpdateConfigClampsDockerServiceCriticalGap(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
m := NewManager()
|
|
|
|
cfg := AlertConfig{
|
|
Enabled: true,
|
|
GuestDefaults: ThresholdConfig{},
|
|
NodeDefaults: ThresholdConfig{},
|
|
HostDefaults: ThresholdConfig{},
|
|
StorageDefault: HysteresisThreshold{},
|
|
DockerDefaults: DockerThresholdConfig{
|
|
ServiceWarnGapPct: 35,
|
|
ServiceCritGapPct: 20,
|
|
},
|
|
PMGDefaults: PMGThresholdConfig{},
|
|
SnapshotDefaults: SnapshotAlertConfig{},
|
|
BackupDefaults: BackupAlertConfig{},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
Schedule: ScheduleConfig{},
|
|
}
|
|
|
|
m.UpdateConfig(cfg)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
if m.config.DockerDefaults.ServiceWarnGapPct != 35 {
|
|
t.Fatalf("expected warning gap to remain 35, got %d", m.config.DockerDefaults.ServiceWarnGapPct)
|
|
}
|
|
if m.config.DockerDefaults.ServiceCritGapPct != 35 {
|
|
t.Fatalf("expected critical gap to be clamped to 35, got %d", m.config.DockerDefaults.ServiceCritGapPct)
|
|
}
|
|
}
|
|
|
|
func TestDockerServiceAlertUsesClampedCriticalGap(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
cfg := AlertConfig{
|
|
Enabled: true,
|
|
GuestDefaults: ThresholdConfig{},
|
|
NodeDefaults: ThresholdConfig{},
|
|
HostDefaults: ThresholdConfig{},
|
|
StorageDefault: HysteresisThreshold{},
|
|
DockerDefaults: DockerThresholdConfig{
|
|
ServiceWarnGapPct: 20,
|
|
ServiceCritGapPct: 5,
|
|
},
|
|
PMGDefaults: PMGThresholdConfig{},
|
|
SnapshotDefaults: SnapshotAlertConfig{},
|
|
BackupDefaults: BackupAlertConfig{},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
Schedule: ScheduleConfig{},
|
|
}
|
|
|
|
m.UpdateConfig(cfg)
|
|
|
|
host := models.DockerHost{
|
|
ID: "docker-host-1",
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker-host.local",
|
|
Services: []models.DockerService{
|
|
{
|
|
ID: "svc-123",
|
|
Name: "api",
|
|
DesiredTasks: 10,
|
|
RunningTasks: 7,
|
|
},
|
|
},
|
|
}
|
|
|
|
m.CheckDockerHost(host)
|
|
|
|
resourceID := dockerServiceResourceID(host.ID, "svc-123", "api")
|
|
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
|
|
|
alert, exists := m.activeAlerts[alertID]
|
|
if !exists {
|
|
t.Fatalf("expected docker service alert %s to be raised", alertID)
|
|
}
|
|
if alert.Level != AlertLevelCritical {
|
|
t.Fatalf("expected critical severity when replicas 7/10, got %s", alert.Level)
|
|
}
|
|
if pct, ok := alert.Metadata["percentMissing"].(float64); !ok || math.Abs(pct-30.0) > 0.01 {
|
|
t.Fatalf("expected percentMissing metadata ~30, got %v", alert.Metadata["percentMissing"])
|
|
}
|
|
}
|
|
|
|
func TestNormalizeDockerIgnoredPrefixes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
input []string
|
|
expected []string
|
|
}{
|
|
{
|
|
name: "nil input",
|
|
input: nil,
|
|
expected: nil,
|
|
},
|
|
{
|
|
name: "blank entries removed",
|
|
input: []string{"", " ", "\t"},
|
|
expected: nil,
|
|
},
|
|
{
|
|
name: "trims and deduplicates preserving first occurrence casing",
|
|
input: []string{" Foo ", "foo", "Bar", " bar ", "Baz"},
|
|
expected: []string{"Foo", "Bar", "Baz"},
|
|
},
|
|
{
|
|
name: "already normalized list remains unchanged",
|
|
input: []string{"alpha", "beta"},
|
|
expected: []string{"alpha", "beta"},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := NormalizeDockerIgnoredPrefixes(tc.input)
|
|
if !reflect.DeepEqual(got, tc.expected) {
|
|
t.Fatalf("expected %v, got %v", tc.expected, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCheckDockerHostIgnoredPrefixClearsExistingAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
|
|
container := models.DockerContainer{
|
|
ID: "abc123456789",
|
|
Name: "runner-job-1",
|
|
State: "exited",
|
|
Status: "Exited (1) 10 seconds ago",
|
|
}
|
|
host := models.DockerHost{
|
|
ID: "docker-host",
|
|
DisplayName: "Docker Host",
|
|
Hostname: "docker-host.local",
|
|
Containers: []models.DockerContainer{container},
|
|
}
|
|
resourceID := dockerResourceID(host.ID, container.ID)
|
|
stateAlertID := fmt.Sprintf("docker-container-state-%s", resourceID)
|
|
healthAlertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
|
restartAlertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID)
|
|
|
|
m.mu.Lock()
|
|
m.config.Enabled = true
|
|
m.config.DockerIgnoredContainerPrefixes = []string{"runner-"}
|
|
m.activeAlerts[stateAlertID] = &Alert{ID: stateAlertID, ResourceID: resourceID}
|
|
m.activeAlerts[healthAlertID] = &Alert{ID: healthAlertID, ResourceID: resourceID}
|
|
m.activeAlerts[restartAlertID] = &Alert{ID: restartAlertID, ResourceID: resourceID}
|
|
m.dockerStateConfirm[resourceID] = 2
|
|
m.dockerRestartTracking[resourceID] = &dockerRestartRecord{}
|
|
m.dockerLastExitCode[resourceID] = 137
|
|
m.mu.Unlock()
|
|
|
|
m.CheckDockerHost(host)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
if _, exists := m.activeAlerts[stateAlertID]; exists {
|
|
t.Fatalf("expected state alert cleared for ignored container")
|
|
}
|
|
if _, exists := m.activeAlerts[healthAlertID]; exists {
|
|
t.Fatalf("expected health alert cleared for ignored container")
|
|
}
|
|
if _, exists := m.activeAlerts[restartAlertID]; exists {
|
|
t.Fatalf("expected restart alert cleared for ignored container")
|
|
}
|
|
if _, exists := m.dockerStateConfirm[resourceID]; exists {
|
|
t.Fatalf("expected state confirmation tracking cleared")
|
|
}
|
|
if _, exists := m.dockerRestartTracking[resourceID]; exists {
|
|
t.Fatalf("expected restart tracking cleared")
|
|
}
|
|
if _, exists := m.dockerLastExitCode[resourceID]; exists {
|
|
t.Fatalf("expected last exit code cleared")
|
|
}
|
|
}
|
|
|
|
func TestUpdateConfigNormalizesDockerIgnoredPrefixes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("nil input remains nil", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
m := NewManager()
|
|
m.UpdateConfig(AlertConfig{})
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
if m.config.DockerIgnoredContainerPrefixes != nil {
|
|
t.Fatalf("expected nil prefixes, got %v", m.config.DockerIgnoredContainerPrefixes)
|
|
}
|
|
})
|
|
|
|
t.Run("duplicates trimmed and deduplicated", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
m := NewManager()
|
|
cfg := AlertConfig{
|
|
DockerIgnoredContainerPrefixes: []string{
|
|
" Foo ",
|
|
"foo",
|
|
"Bar",
|
|
},
|
|
}
|
|
|
|
m.UpdateConfig(cfg)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
expected := []string{"Foo", "Bar"}
|
|
if !reflect.DeepEqual(m.config.DockerIgnoredContainerPrefixes, expected) {
|
|
t.Fatalf("expected normalized prefixes %v, got %v", expected, m.config.DockerIgnoredContainerPrefixes)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestMatchesDockerIgnoredPrefix(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
containerName string
|
|
containerID string
|
|
prefixes []string
|
|
want bool
|
|
}{
|
|
{name: "empty prefixes", containerName: "runner-123", containerID: "abc", prefixes: nil, want: false},
|
|
{name: "match with name", containerName: "runner-123", containerID: "abc", prefixes: []string{"runner-"}, want: true},
|
|
{name: "match with id", containerName: "app", containerID: "abc123", prefixes: []string{"abc"}, want: true},
|
|
{name: "trimmed comparison", containerName: "runner-job", containerID: "abc", prefixes: []string{" runner- "}, want: true},
|
|
{name: "case insensitive", containerName: "Runner-Job", containerID: "abc", prefixes: []string{"runner-"}, want: true},
|
|
{name: "no match", containerName: "service", containerID: "xyz", prefixes: []string{"runner-"}, want: false},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := matchesDockerIgnoredPrefix(tc.containerName, tc.containerID, tc.prefixes); got != tc.want {
|
|
t.Fatalf("matchesDockerIgnoredPrefix(%q, %q, %v) = %v, want %v", tc.containerName, tc.containerID, tc.prefixes, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDockerInstanceName(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
host models.DockerHost
|
|
want string
|
|
}{
|
|
{name: "uses display name", host: models.DockerHost{DisplayName: "Prod Host"}, want: "Docker:Prod Host"},
|
|
{name: "falls back to hostname", host: models.DockerHost{Hostname: "docker.local"}, want: "Docker:docker.local"},
|
|
{name: "defaults when empty", host: models.DockerHost{}, want: "Docker"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := dockerInstanceName(tc.host); got != tc.want {
|
|
t.Fatalf("dockerInstanceName(%+v) = %q, want %q", tc.host, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDockerContainerDisplayName(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
container models.DockerContainer
|
|
want string
|
|
}{
|
|
{name: "trims whitespace", container: models.DockerContainer{Name: " app "}, want: "app"},
|
|
{name: "strips leading slash", container: models.DockerContainer{Name: "/runner"}, want: "runner"},
|
|
{name: "falls back to id truncated", container: models.DockerContainer{ID: "0123456789abcdef"}, want: "0123456789ab"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := dockerContainerDisplayName(tc.container); got != tc.want {
|
|
t.Fatalf("dockerContainerDisplayName(%+v) = %q, want %q", tc.container, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDockerResourceID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
hostID string
|
|
containerID string
|
|
want string
|
|
}{
|
|
{name: "both ids present", hostID: "host1", containerID: "abc", want: "docker:host1/abc"},
|
|
{name: "missing host id", hostID: "", containerID: "abc", want: "docker:container/abc"},
|
|
{name: "missing container id", hostID: "host1", containerID: "", want: "docker:host1"},
|
|
{name: "both missing", hostID: "", containerID: "", want: "docker:unknown"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := dockerResourceID(tc.hostID, tc.containerID); got != tc.want {
|
|
t.Fatalf("dockerResourceID(%q, %q) = %q, want %q", tc.hostID, tc.containerID, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHasKnownFirmwareBug(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
model string
|
|
want bool
|
|
}{
|
|
{name: "Samsung 980 with SSD prefix", model: "Samsung SSD 980 1TB", want: true},
|
|
{name: "Samsung 980 without SSD prefix", model: "Samsung 980 PRO 2TB", want: true},
|
|
{name: "Samsung 990 with SSD prefix", model: "Samsung SSD 990 PRO 2TB", want: true},
|
|
{name: "Samsung 990 without SSD prefix", model: "Samsung 990 EVO 1TB", want: true},
|
|
{name: "Samsung 980 lowercase", model: "samsung ssd 980 1tb", want: true},
|
|
{name: "Samsung 990 mixed case", model: "SAMSUNG 990 PRO", want: true},
|
|
{name: "Samsung 970 (not affected)", model: "Samsung SSD 970 EVO Plus", want: false},
|
|
{name: "Samsung 870 (not affected)", model: "Samsung 870 QVO", want: false},
|
|
{name: "Other manufacturer", model: "WD Blue SN570", want: false},
|
|
{name: "Empty model", model: "", want: false},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
if got := hasKnownFirmwareBug(tc.model); got != tc.want {
|
|
t.Fatalf("hasKnownFirmwareBug(%q) = %v, want %v", tc.model, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCheckDiskHealthSkipsSamsung980FalseAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
// Samsung 980 reporting FAILED health (firmware bug) but actually healthy
|
|
disk := proxmox.Disk{
|
|
DevPath: "/dev/nvme0n1",
|
|
Model: "Samsung SSD 980 1TB",
|
|
Serial: "S649NF0R123456",
|
|
Type: "nvme",
|
|
Health: "FAILED", // False report due to firmware bug
|
|
Wearout: 99, // Drive is actually healthy with 99% life remaining
|
|
Size: 1000204886016,
|
|
}
|
|
|
|
// Should not create an alert for health status
|
|
m.CheckDiskHealth("test-instance", "pve-node1", disk)
|
|
|
|
m.mu.RLock()
|
|
healthAlertID := "disk-health-test-instance-pve-node1-/dev/nvme0n1"
|
|
if _, exists := m.activeAlerts[healthAlertID]; exists {
|
|
m.mu.RUnlock()
|
|
t.Fatalf("expected no health alert for Samsung 980 with known firmware bug")
|
|
}
|
|
m.mu.RUnlock()
|
|
|
|
// Now test that wearout alerts still work for these drives
|
|
disk.Wearout = 5 // Low wearout should still trigger alert
|
|
m.CheckDiskHealth("test-instance", "pve-node1", disk)
|
|
|
|
m.mu.RLock()
|
|
wearoutAlertID := "disk-wearout-test-instance-pve-node1-/dev/nvme0n1"
|
|
if _, exists := m.activeAlerts[wearoutAlertID]; !exists {
|
|
m.mu.RUnlock()
|
|
t.Fatalf("expected wearout alert to still work for Samsung 980")
|
|
}
|
|
m.mu.RUnlock()
|
|
}
|
|
|
|
func TestCheckDiskHealthClearsExistingSamsung980Alerts(t *testing.T) {
|
|
m := NewManager()
|
|
m.ClearActiveAlerts()
|
|
|
|
disk := proxmox.Disk{
|
|
DevPath: "/dev/nvme0n1",
|
|
Model: "Samsung SSD 990 PRO 2TB",
|
|
Serial: "S6Z0NF0R654321",
|
|
Type: "nvme",
|
|
Health: "FAILED",
|
|
Wearout: 98,
|
|
Size: 2000398934016,
|
|
}
|
|
|
|
alertID := "disk-health-test-instance-pve-node1-/dev/nvme0n1"
|
|
|
|
// Manually create an existing alert (simulating alert from before the fix)
|
|
m.mu.Lock()
|
|
m.activeAlerts[alertID] = &Alert{
|
|
ID: alertID,
|
|
Type: "disk-health",
|
|
Level: AlertLevelCritical,
|
|
ResourceID: "pve-node1-/dev/nvme0n1",
|
|
ResourceName: "Samsung SSD 990 PRO 2TB (/dev/nvme0n1)",
|
|
Node: "pve-node1",
|
|
Instance: "test-instance",
|
|
Message: "Disk health check failed: FAILED",
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
// Check disk health - should clear the existing false alert
|
|
m.CheckDiskHealth("test-instance", "pve-node1", disk)
|
|
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
if _, exists := m.activeAlerts[alertID]; exists {
|
|
t.Fatalf("expected existing Samsung 990 health alert to be cleared")
|
|
}
|
|
}
|
|
|
|
func TestDisableAllStorageClearsExistingAlerts(t *testing.T) {
|
|
m := NewManager()
|
|
|
|
storageID := "local-lvm"
|
|
|
|
// Start with configuration that allows storage alerts
|
|
initialConfig := AlertConfig{
|
|
Enabled: true,
|
|
DisableAllStorage: false,
|
|
StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
TimeThreshold: 0,
|
|
TimeThresholds: map[string]int{},
|
|
NodeDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
|
},
|
|
GuestDefaults: ThresholdConfig{
|
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
|
},
|
|
Overrides: make(map[string]ThresholdConfig),
|
|
}
|
|
m.UpdateConfig(initialConfig)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.ActivationState = ActivationActive
|
|
m.mu.Unlock()
|
|
|
|
var dispatched []*Alert
|
|
done := make(chan struct{}, 1)
|
|
var resolved []string
|
|
resolvedDone := make(chan struct{}, 1)
|
|
m.SetAlertCallback(func(alert *Alert) {
|
|
dispatched = append(dispatched, alert)
|
|
select {
|
|
case done <- struct{}{}:
|
|
default:
|
|
}
|
|
})
|
|
m.SetResolvedCallback(func(alertID string) {
|
|
resolved = append(resolved, alertID)
|
|
select {
|
|
case resolvedDone <- struct{}{}:
|
|
default:
|
|
}
|
|
})
|
|
|
|
storage := models.Storage{
|
|
ID: storageID,
|
|
Name: "local-lvm",
|
|
Usage: 90.0,
|
|
Status: "available",
|
|
}
|
|
|
|
// Initial check should trigger an alert
|
|
m.CheckStorage(storage)
|
|
select {
|
|
case <-done:
|
|
case <-time.After(100 * time.Millisecond):
|
|
t.Fatalf("did not receive initial alert dispatch")
|
|
}
|
|
if len(dispatched) != 1 {
|
|
t.Fatalf("expected 1 alert before disabling storage, got %d", len(dispatched))
|
|
}
|
|
|
|
// Apply config with DisableAllStorage enabled
|
|
disabledConfig := initialConfig
|
|
disabledConfig.DisableAllStorage = true
|
|
m.UpdateConfig(disabledConfig)
|
|
m.mu.Lock()
|
|
m.config.TimeThreshold = 0
|
|
m.config.TimeThresholds = map[string]int{}
|
|
m.config.ActivationState = ActivationActive
|
|
m.mu.Unlock()
|
|
|
|
// Clear dispatched slice to capture only post-disable notifications
|
|
dispatched = dispatched[:0]
|
|
done = make(chan struct{}, 1)
|
|
|
|
// Re-run CheckStorage with high usage; no alert should be dispatched
|
|
m.CheckStorage(storage)
|
|
select {
|
|
case <-done:
|
|
t.Fatalf("expected no alerts after disabling all storage, but callback fired")
|
|
case <-time.After(100 * time.Millisecond):
|
|
// No callback fired as expected
|
|
}
|
|
|
|
// Active alerts should be cleared by reevaluateActiveAlertsLocked
|
|
m.mu.RLock()
|
|
activeCount := len(m.activeAlerts)
|
|
m.mu.RUnlock()
|
|
if activeCount != 0 {
|
|
t.Fatalf("expected active alerts to be cleared after disabling all storage, got %d", activeCount)
|
|
}
|
|
|
|
// Resolved callback should have fired
|
|
select {
|
|
case <-resolvedDone:
|
|
case <-time.After(100 * time.Millisecond):
|
|
t.Fatalf("expected resolved callback to fire after disabling all storage")
|
|
}
|
|
expectedAlertID := fmt.Sprintf("%s-usage", storageID)
|
|
if len(resolved) != 1 || resolved[0] != expectedAlertID {
|
|
t.Fatalf("expected resolved callback for %s, got %v", expectedAlertID, resolved)
|
|
}
|
|
|
|
// Pending alert should be cleared
|
|
m.mu.RLock()
|
|
_, isPending := m.pendingAlerts[expectedAlertID]
|
|
m.mu.RUnlock()
|
|
if isPending {
|
|
t.Fatalf("expected pending alert entry to be cleared after disabling all storage")
|
|
}
|
|
}
|