parent
4eccf74bf6
commit
219fcc6de5
2 changed files with 139 additions and 0 deletions
|
|
@ -1037,6 +1037,14 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
|||
ResolvedTime: time.Now(),
|
||||
}
|
||||
|
||||
// Remove any pending notification tracking for this alert since it's no longer valid.
|
||||
if _, isPending := m.pendingAlerts[alertID]; isPending {
|
||||
delete(m.pendingAlerts, alertID)
|
||||
log.Debug().
|
||||
Str("alertID", alertID).
|
||||
Msg("Cleared pending alert after configuration update")
|
||||
}
|
||||
|
||||
// Remove from active alerts
|
||||
m.removeActiveAlertNoLock(alertID)
|
||||
|
||||
|
|
@ -1046,6 +1054,10 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
|||
log.Info().
|
||||
Str("alertID", alertID).
|
||||
Msg("Alert auto-resolved after configuration change")
|
||||
|
||||
if m.onResolved != nil {
|
||||
go m.onResolved(alertID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
|
@ -44,6 +45,132 @@ func TestAcknowledgePersistsThroughCheckMetric(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
||||
m := NewManager()
|
||||
|
||||
vmID := "instance-node-101"
|
||||
instanceName := "instance"
|
||||
|
||||
// Start with default configuration to allow CPU alerts.
|
||||
initialConfig := AlertConfig{
|
||||
Enabled: true,
|
||||
GuestDefaults: ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
},
|
||||
TimeThreshold: 0,
|
||||
TimeThresholds: map[string]int{},
|
||||
NodeDefaults: ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||
},
|
||||
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
Overrides: make(map[string]ThresholdConfig),
|
||||
}
|
||||
m.UpdateConfig(initialConfig)
|
||||
m.mu.Lock()
|
||||
m.config.TimeThreshold = 0
|
||||
m.config.TimeThresholds = map[string]int{}
|
||||
m.mu.Unlock()
|
||||
|
||||
var dispatched []*Alert
|
||||
done := make(chan struct{}, 1)
|
||||
var resolved []string
|
||||
resolvedDone := make(chan struct{}, 1)
|
||||
m.SetAlertCallback(func(alert *Alert) {
|
||||
dispatched = append(dispatched, alert)
|
||||
select {
|
||||
case done <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
m.SetResolvedCallback(func(alertID string) {
|
||||
resolved = append(resolved, alertID)
|
||||
select {
|
||||
case resolvedDone <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
|
||||
vm := models.VM{
|
||||
ID: vmID,
|
||||
Name: "test-vm",
|
||||
Node: "node",
|
||||
Instance: instanceName,
|
||||
Status: "running",
|
||||
CPU: 1.0, // 100% once multiplied by 100 inside CheckGuest
|
||||
Memory: models.Memory{
|
||||
Usage: 65,
|
||||
},
|
||||
Disk: models.Disk{
|
||||
Usage: 40,
|
||||
},
|
||||
}
|
||||
|
||||
// Initial check should trigger an alert with default thresholds.
|
||||
m.CheckGuest(vm, instanceName)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatalf("did not receive initial alert dispatch")
|
||||
}
|
||||
if len(dispatched) != 1 {
|
||||
t.Fatalf("expected 1 alert before disabling metric, got %d", len(dispatched))
|
||||
}
|
||||
|
||||
// Apply override disabling CPU alerts for this VM.
|
||||
disabledConfig := initialConfig
|
||||
disabledConfig.Overrides = map[string]ThresholdConfig{
|
||||
vmID: {
|
||||
CPU: &HysteresisThreshold{Trigger: -1, Clear: 0},
|
||||
},
|
||||
}
|
||||
disabledConfig.TimeThreshold = 0
|
||||
disabledConfig.TimeThresholds = map[string]int{}
|
||||
m.UpdateConfig(disabledConfig)
|
||||
m.mu.Lock()
|
||||
m.config.TimeThreshold = 0
|
||||
m.config.TimeThresholds = map[string]int{}
|
||||
m.mu.Unlock()
|
||||
|
||||
// Clear dispatched slice to capture only post-disable notifications.
|
||||
dispatched = dispatched[:0]
|
||||
done = make(chan struct{}, 1)
|
||||
|
||||
// Re-run evaluation with high CPU; no alert should be dispatched.
|
||||
m.CheckGuest(vm, instanceName)
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatalf("expected no alerts after disabling CPU metric, but callback fired")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// No callback fired as expected.
|
||||
}
|
||||
|
||||
// Active alerts should be cleared by the config update.
|
||||
m.mu.RLock()
|
||||
activeCount := len(m.activeAlerts)
|
||||
m.mu.RUnlock()
|
||||
if activeCount != 0 {
|
||||
t.Fatalf("expected active alerts to be cleared after disabling metric, got %d", activeCount)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-resolvedDone:
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatalf("expected resolved callback to fire after disabling metric")
|
||||
}
|
||||
if len(resolved) != 1 || resolved[0] != fmt.Sprintf("%s-cpu", vmID) {
|
||||
t.Fatalf("expected resolved callback for %s-cpu, got %v", vmID, resolved)
|
||||
}
|
||||
|
||||
m.mu.RLock()
|
||||
_, isPending := m.pendingAlerts[fmt.Sprintf("%s-cpu", vmID)]
|
||||
m.mu.RUnlock()
|
||||
if isPending {
|
||||
t.Fatalf("expected pending alert entry to be cleared after disabling metric")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
||||
m := NewManager()
|
||||
host := models.DockerHost{ID: "host1", DisplayName: "Host One", Hostname: "host-one"}
|
||||
|
|
|
|||
Loading…
Reference in a new issue