parent
4eccf74bf6
commit
219fcc6de5
2 changed files with 139 additions and 0 deletions
|
|
@ -1037,6 +1037,14 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
||||||
ResolvedTime: time.Now(),
|
ResolvedTime: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Remove any pending notification tracking for this alert since it's no longer valid.
|
||||||
|
if _, isPending := m.pendingAlerts[alertID]; isPending {
|
||||||
|
delete(m.pendingAlerts, alertID)
|
||||||
|
log.Debug().
|
||||||
|
Str("alertID", alertID).
|
||||||
|
Msg("Cleared pending alert after configuration update")
|
||||||
|
}
|
||||||
|
|
||||||
// Remove from active alerts
|
// Remove from active alerts
|
||||||
m.removeActiveAlertNoLock(alertID)
|
m.removeActiveAlertNoLock(alertID)
|
||||||
|
|
||||||
|
|
@ -1046,6 +1054,10 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alertID", alertID).
|
Str("alertID", alertID).
|
||||||
Msg("Alert auto-resolved after configuration change")
|
Msg("Alert auto-resolved after configuration change")
|
||||||
|
|
||||||
|
if m.onResolved != nil {
|
||||||
|
go m.onResolved(alertID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
)
|
)
|
||||||
|
|
@ -44,6 +45,132 @@ func TestAcknowledgePersistsThroughCheckMetric(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
|
||||||
|
vmID := "instance-node-101"
|
||||||
|
instanceName := "instance"
|
||||||
|
|
||||||
|
// Start with default configuration to allow CPU alerts.
|
||||||
|
initialConfig := AlertConfig{
|
||||||
|
Enabled: true,
|
||||||
|
GuestDefaults: ThresholdConfig{
|
||||||
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||||
|
},
|
||||||
|
TimeThreshold: 0,
|
||||||
|
TimeThresholds: map[string]int{},
|
||||||
|
NodeDefaults: ThresholdConfig{
|
||||||
|
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||||
|
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||||
|
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||||
|
},
|
||||||
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||||
|
Overrides: make(map[string]ThresholdConfig),
|
||||||
|
}
|
||||||
|
m.UpdateConfig(initialConfig)
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
var dispatched []*Alert
|
||||||
|
done := make(chan struct{}, 1)
|
||||||
|
var resolved []string
|
||||||
|
resolvedDone := make(chan struct{}, 1)
|
||||||
|
m.SetAlertCallback(func(alert *Alert) {
|
||||||
|
dispatched = append(dispatched, alert)
|
||||||
|
select {
|
||||||
|
case done <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
m.SetResolvedCallback(func(alertID string) {
|
||||||
|
resolved = append(resolved, alertID)
|
||||||
|
select {
|
||||||
|
case resolvedDone <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
vm := models.VM{
|
||||||
|
ID: vmID,
|
||||||
|
Name: "test-vm",
|
||||||
|
Node: "node",
|
||||||
|
Instance: instanceName,
|
||||||
|
Status: "running",
|
||||||
|
CPU: 1.0, // 100% once multiplied by 100 inside CheckGuest
|
||||||
|
Memory: models.Memory{
|
||||||
|
Usage: 65,
|
||||||
|
},
|
||||||
|
Disk: models.Disk{
|
||||||
|
Usage: 40,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initial check should trigger an alert with default thresholds.
|
||||||
|
m.CheckGuest(vm, instanceName)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
t.Fatalf("did not receive initial alert dispatch")
|
||||||
|
}
|
||||||
|
if len(dispatched) != 1 {
|
||||||
|
t.Fatalf("expected 1 alert before disabling metric, got %d", len(dispatched))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply override disabling CPU alerts for this VM.
|
||||||
|
disabledConfig := initialConfig
|
||||||
|
disabledConfig.Overrides = map[string]ThresholdConfig{
|
||||||
|
vmID: {
|
||||||
|
CPU: &HysteresisThreshold{Trigger: -1, Clear: 0},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
disabledConfig.TimeThreshold = 0
|
||||||
|
disabledConfig.TimeThresholds = map[string]int{}
|
||||||
|
m.UpdateConfig(disabledConfig)
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
// Clear dispatched slice to capture only post-disable notifications.
|
||||||
|
dispatched = dispatched[:0]
|
||||||
|
done = make(chan struct{}, 1)
|
||||||
|
|
||||||
|
// Re-run evaluation with high CPU; no alert should be dispatched.
|
||||||
|
m.CheckGuest(vm, instanceName)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatalf("expected no alerts after disabling CPU metric, but callback fired")
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
// No callback fired as expected.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active alerts should be cleared by the config update.
|
||||||
|
m.mu.RLock()
|
||||||
|
activeCount := len(m.activeAlerts)
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if activeCount != 0 {
|
||||||
|
t.Fatalf("expected active alerts to be cleared after disabling metric, got %d", activeCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-resolvedDone:
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
t.Fatalf("expected resolved callback to fire after disabling metric")
|
||||||
|
}
|
||||||
|
if len(resolved) != 1 || resolved[0] != fmt.Sprintf("%s-cpu", vmID) {
|
||||||
|
t.Fatalf("expected resolved callback for %s-cpu, got %v", vmID, resolved)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.mu.RLock()
|
||||||
|
_, isPending := m.pendingAlerts[fmt.Sprintf("%s-cpu", vmID)]
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if isPending {
|
||||||
|
t.Fatalf("expected pending alert entry to be cleared after disabling metric")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
||||||
m := NewManager()
|
m := NewManager()
|
||||||
host := models.DockerHost{ID: "host1", DisplayName: "Host One", Hostname: "host-one"}
|
host := models.DockerHost{ID: "host1", DisplayName: "Host One", Hostname: "host-one"}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue