Implement Pulse tag overrides and alert clear persistence
This commit is contained in:
parent
d643dcf0bc
commit
77282bd3a6
6 changed files with 500 additions and 36 deletions
|
|
@ -707,6 +707,15 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if isMonitorOnlyAlert(alert) {
|
||||||
|
log.Info().
|
||||||
|
Str("alertID", alert.ID).
|
||||||
|
Str("resource", alert.ResourceName).
|
||||||
|
Bool("monitorOnly", true).
|
||||||
|
Msg("Monitor-only alert detected, skipping alert dispatch")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
alertCopy := alert.Clone()
|
alertCopy := alert.Clone()
|
||||||
if async {
|
if async {
|
||||||
go func(a *Alert) {
|
go func(a *Alert) {
|
||||||
|
|
@ -739,6 +748,22 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isMonitorOnlyAlert(alert *Alert) bool {
|
||||||
|
if alert == nil || alert.Metadata == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if value, ok := alert.Metadata["monitorOnly"]; ok {
|
||||||
|
switch v := value.(type) {
|
||||||
|
case bool:
|
||||||
|
return v
|
||||||
|
case string:
|
||||||
|
return strings.EqualFold(v, "true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// ensureValidHysteresis ensures clear < trigger for hysteresis thresholds
|
// ensureValidHysteresis ensures clear < trigger for hysteresis thresholds
|
||||||
func ensureValidHysteresis(threshold *HysteresisThreshold, metricName string) {
|
func ensureValidHysteresis(threshold *HysteresisThreshold, metricName string) {
|
||||||
if threshold == nil {
|
if threshold == nil {
|
||||||
|
|
@ -1653,6 +1678,7 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
var cpu, memUsage, diskUsage float64
|
var cpu, memUsage, diskUsage float64
|
||||||
var diskRead, diskWrite, netIn, netOut int64
|
var diskRead, diskWrite, netIn, netOut int64
|
||||||
var disks []models.Disk
|
var disks []models.Disk
|
||||||
|
var tags []string
|
||||||
|
|
||||||
// Extract data based on guest type
|
// Extract data based on guest type
|
||||||
switch g := guest.(type) {
|
switch g := guest.(type) {
|
||||||
|
|
@ -1670,6 +1696,9 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
netIn = g.NetworkIn
|
netIn = g.NetworkIn
|
||||||
netOut = g.NetworkOut
|
netOut = g.NetworkOut
|
||||||
disks = g.Disks
|
disks = g.Disks
|
||||||
|
if len(g.Tags) > 0 {
|
||||||
|
tags = append(tags, g.Tags...)
|
||||||
|
}
|
||||||
|
|
||||||
// Debug logging for high memory VMs
|
// Debug logging for high memory VMs
|
||||||
if memUsage > 85 {
|
if memUsage > 85 {
|
||||||
|
|
@ -1693,6 +1722,9 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
netIn = g.NetworkIn
|
netIn = g.NetworkIn
|
||||||
netOut = g.NetworkOut
|
netOut = g.NetworkOut
|
||||||
disks = g.Disks
|
disks = g.Disks
|
||||||
|
if len(g.Tags) > 0 {
|
||||||
|
tags = append(tags, g.Tags...)
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("type", fmt.Sprintf("%T", guest)).
|
Str("type", fmt.Sprintf("%T", guest)).
|
||||||
|
|
@ -1700,6 +1732,25 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
settings := parsePulseTags(tags)
|
||||||
|
if settings.Suppress {
|
||||||
|
if cleared := m.suppressGuestAlerts(guestID); cleared {
|
||||||
|
m.saveActiveAlertsAsync("pulse-no-alerts")
|
||||||
|
}
|
||||||
|
log.Debug().
|
||||||
|
Str("guestID", guestID).
|
||||||
|
Msg("Pulse no-alerts tag active; suppressing guest alerts")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
monitorOnly := settings.MonitorOnly
|
||||||
|
if monitorOnly || m.guestHasMonitorOnlyAlerts(guestID) {
|
||||||
|
log.Info().
|
||||||
|
Str("guest", name).
|
||||||
|
Bool("monitorOnly", monitorOnly).
|
||||||
|
Msg("Pulse monitor-only status applied")
|
||||||
|
}
|
||||||
|
|
||||||
// Handle non-running guests
|
// Handle non-running guests
|
||||||
// Proxmox VM states: running, stopped, paused, suspended
|
// Proxmox VM states: running, stopped, paused, suspended
|
||||||
if status != "running" {
|
if status != "running" {
|
||||||
|
|
@ -1712,7 +1763,7 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
m.clearAlert(fmt.Sprintf("guest-powered-off-%s", guestID))
|
m.clearAlert(fmt.Sprintf("guest-powered-off-%s", guestID))
|
||||||
} else {
|
} else {
|
||||||
m.checkGuestPoweredOff(guestID, name, node, instanceName, guestType)
|
m.checkGuestPoweredOff(guestID, name, node, instanceName, guestType, monitorOnly)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// For paused/suspended, clear powered-off alert
|
// For paused/suspended, clear powered-off alert
|
||||||
|
|
@ -1754,6 +1805,14 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
thresholds := m.getGuestThresholds(guest, guestID)
|
thresholds := m.getGuestThresholds(guest, guestID)
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
|
if settings.Relaxed {
|
||||||
|
thresholds = applyRelaxedGuestThresholds(thresholds)
|
||||||
|
log.Info().
|
||||||
|
Str("guest", name).
|
||||||
|
Float64("trigger", thresholds.CPU.Trigger).
|
||||||
|
Msg("Applied relaxed thresholds for pulse-relaxed tag")
|
||||||
|
}
|
||||||
|
|
||||||
// If alerts are disabled for this guest, clear any existing alerts and return
|
// If alerts are disabled for this guest, clear any existing alerts and return
|
||||||
if thresholds.Disabled {
|
if thresholds.Disabled {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
@ -1780,9 +1839,19 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
Msg("Checking guest thresholds")
|
Msg("Checking guest thresholds")
|
||||||
|
|
||||||
// Check thresholds (checkMetric will skip if threshold is nil or <= 0)
|
// Check thresholds (checkMetric will skip if threshold is nil or <= 0)
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "cpu", cpu, thresholds.CPU, nil)
|
cpuOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "memory", memUsage, thresholds.Memory, nil)
|
memOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "disk", diskUsage, thresholds.Disk, nil)
|
diskOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
|
|
||||||
|
if !monitorOnly {
|
||||||
|
cpuOpts = nil
|
||||||
|
memOpts = nil
|
||||||
|
diskOpts = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "cpu", cpu, thresholds.CPU, cpuOpts)
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "memory", memUsage, thresholds.Memory, memOpts)
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "disk", diskUsage, thresholds.Disk, diskOpts)
|
||||||
|
|
||||||
if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 && len(disks) > 0 {
|
if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 && len(disks) > 0 {
|
||||||
seenDisks := make(map[string]struct{})
|
seenDisks := make(map[string]struct{})
|
||||||
|
|
@ -1840,24 +1909,41 @@ func (m *Manager) CheckGuest(guest interface{}, instanceName string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
m.checkMetric(perDiskResourceID, name, node, instanceName, guestType, "disk", disk.Usage, thresholds.Disk, &metricOptions{
|
m.checkMetric(perDiskResourceID, name, node, instanceName, guestType, "disk", disk.Usage, thresholds.Disk, &metricOptions{
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
Message: message,
|
Message: message,
|
||||||
|
MonitorOnly: monitorOnly,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check I/O metrics (convert bytes/s to MB/s) - checkMetric will skip if threshold is nil or <= 0
|
// Check I/O metrics (convert bytes/s to MB/s) - checkMetric will skip if threshold is nil or <= 0
|
||||||
if thresholds.DiskRead != nil && thresholds.DiskRead.Trigger > 0 {
|
if thresholds.DiskRead != nil && thresholds.DiskRead.Trigger > 0 {
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "diskRead", float64(diskRead)/1024/1024, thresholds.DiskRead, nil)
|
readOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
|
if !monitorOnly {
|
||||||
|
readOpts = nil
|
||||||
|
}
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "diskRead", float64(diskRead)/1024/1024, thresholds.DiskRead, readOpts)
|
||||||
}
|
}
|
||||||
if thresholds.DiskWrite != nil && thresholds.DiskWrite.Trigger > 0 {
|
if thresholds.DiskWrite != nil && thresholds.DiskWrite.Trigger > 0 {
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "diskWrite", float64(diskWrite)/1024/1024, thresholds.DiskWrite, nil)
|
writeOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
|
if !monitorOnly {
|
||||||
|
writeOpts = nil
|
||||||
|
}
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "diskWrite", float64(diskWrite)/1024/1024, thresholds.DiskWrite, writeOpts)
|
||||||
}
|
}
|
||||||
if thresholds.NetworkIn != nil && thresholds.NetworkIn.Trigger > 0 {
|
if thresholds.NetworkIn != nil && thresholds.NetworkIn.Trigger > 0 {
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "networkIn", float64(netIn)/1024/1024, thresholds.NetworkIn, nil)
|
netInOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
|
if !monitorOnly {
|
||||||
|
netInOpts = nil
|
||||||
|
}
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "networkIn", float64(netIn)/1024/1024, thresholds.NetworkIn, netInOpts)
|
||||||
}
|
}
|
||||||
if thresholds.NetworkOut != nil && thresholds.NetworkOut.Trigger > 0 {
|
if thresholds.NetworkOut != nil && thresholds.NetworkOut.Trigger > 0 {
|
||||||
m.checkMetric(guestID, name, node, instanceName, guestType, "networkOut", float64(netOut)/1024/1024, thresholds.NetworkOut, nil)
|
netOutOpts := &metricOptions{MonitorOnly: monitorOnly}
|
||||||
|
if !monitorOnly {
|
||||||
|
netOutOpts = nil
|
||||||
|
}
|
||||||
|
m.checkMetric(guestID, name, node, instanceName, guestType, "networkOut", float64(netOut)/1024/1024, thresholds.NetworkOut, netOutOpts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -4349,6 +4435,8 @@ func canonicalResourceTypeKeys(resourceType string) []string {
|
||||||
type metricOptions struct {
|
type metricOptions struct {
|
||||||
Metadata map[string]interface{}
|
Metadata map[string]interface{}
|
||||||
Message string
|
Message string
|
||||||
|
// MonitorOnly suppresses external notifications while still tracking the alert.
|
||||||
|
MonitorOnly bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resourceType, metricType string, value float64, threshold *HysteresisThreshold, opts *metricOptions) {
|
func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resourceType, metricType string, value float64, threshold *HysteresisThreshold, opts *metricOptions) {
|
||||||
|
|
@ -4371,6 +4459,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
existingAlert, exists := m.activeAlerts[alertID]
|
existingAlert, exists := m.activeAlerts[alertID]
|
||||||
|
monitorOnly := opts != nil && opts.MonitorOnly
|
||||||
|
|
||||||
// Check for suppression
|
// Check for suppression
|
||||||
if suppressUntil, suppressed := m.suppressedUntil[alertID]; suppressed && time.Now().Before(suppressUntil) {
|
if suppressUntil, suppressed := m.suppressedUntil[alertID]; suppressed && time.Now().Before(suppressUntil) {
|
||||||
|
|
@ -4477,6 +4566,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||||
alertMetadata[k] = v
|
alertMetadata[k] = v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
alertMetadata["monitorOnly"] = monitorOnly
|
||||||
|
|
||||||
alert := &Alert{
|
alert := &Alert{
|
||||||
ID: alertID,
|
ID: alertID,
|
||||||
|
|
@ -4565,6 +4655,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||||
}
|
}
|
||||||
existingAlert.Metadata["resourceType"] = resourceType
|
existingAlert.Metadata["resourceType"] = resourceType
|
||||||
existingAlert.Metadata["clearThreshold"] = threshold.Clear
|
existingAlert.Metadata["clearThreshold"] = threshold.Clear
|
||||||
|
existingAlert.Metadata["monitorOnly"] = monitorOnly
|
||||||
if opts != nil {
|
if opts != nil {
|
||||||
if opts.Message != "" {
|
if opts.Message != "" {
|
||||||
existingAlert.Message = opts.Message
|
existingAlert.Message = opts.Message
|
||||||
|
|
@ -6364,7 +6455,7 @@ func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkGuestPoweredOff creates an alert for powered-off guests
|
// checkGuestPoweredOff creates an alert for powered-off guests
|
||||||
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string) {
|
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) {
|
||||||
alertID := fmt.Sprintf("guest-powered-off-%s", guestID)
|
alertID := fmt.Sprintf("guest-powered-off-%s", guestID)
|
||||||
|
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
@ -6398,6 +6489,10 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT
|
||||||
// Alert already exists, just update LastSeen
|
// Alert already exists, just update LastSeen
|
||||||
alert.LastSeen = time.Now()
|
alert.LastSeen = time.Now()
|
||||||
alert.Level = severity
|
alert.Level = severity
|
||||||
|
if alert.Metadata == nil {
|
||||||
|
alert.Metadata = map[string]interface{}{}
|
||||||
|
}
|
||||||
|
alert.Metadata["monitorOnly"] = monitorOnly
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -6438,6 +6533,9 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT
|
||||||
StartTime: time.Now(),
|
StartTime: time.Now(),
|
||||||
LastSeen: time.Now(),
|
LastSeen: time.Now(),
|
||||||
Acknowledged: false,
|
Acknowledged: false,
|
||||||
|
Metadata: map[string]interface{}{
|
||||||
|
"monitorOnly": monitorOnly,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
m.preserveAlertState(alertID, alert)
|
m.preserveAlertState(alertID, alert)
|
||||||
|
|
@ -6504,16 +6602,22 @@ func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ClearAlert removes an alert from active alerts (but keeps in history)
|
// ClearAlert removes an alert from active alerts (but keeps in history)
|
||||||
func (m *Manager) ClearAlert(alertID string) {
|
func (m *Manager) ClearAlert(alertID string) bool {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
if _, exists := m.activeAlerts[alertID]; !exists {
|
||||||
|
m.mu.Unlock()
|
||||||
// Remove from active alerts only
|
return false
|
||||||
m.removeActiveAlertNoLock(alertID)
|
|
||||||
|
|
||||||
if m.onResolved != nil {
|
|
||||||
go m.onResolved(alertID)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
m.clearAlertNoLock(alertID)
|
||||||
|
delete(m.recentAlerts, alertID)
|
||||||
|
delete(m.pendingAlerts, alertID)
|
||||||
|
delete(m.suppressedUntil, alertID)
|
||||||
|
delete(m.alertRateLimit, alertID)
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
m.saveActiveAlertsAsync("manual-clear")
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleanup removes old acknowledged alerts and cleans up tracking maps
|
// Cleanup removes old acknowledged alerts and cleans up tracking maps
|
||||||
|
|
@ -6625,6 +6729,20 @@ func cloneThreshold(threshold *HysteresisThreshold) *HysteresisThreshold {
|
||||||
return &clone
|
return &clone
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneThresholdConfig(cfg ThresholdConfig) ThresholdConfig {
|
||||||
|
clone := cfg
|
||||||
|
clone.CPU = cloneThreshold(cfg.CPU)
|
||||||
|
clone.Memory = cloneThreshold(cfg.Memory)
|
||||||
|
clone.Disk = cloneThreshold(cfg.Disk)
|
||||||
|
clone.DiskRead = cloneThreshold(cfg.DiskRead)
|
||||||
|
clone.DiskWrite = cloneThreshold(cfg.DiskWrite)
|
||||||
|
clone.NetworkIn = cloneThreshold(cfg.NetworkIn)
|
||||||
|
clone.NetworkOut = cloneThreshold(cfg.NetworkOut)
|
||||||
|
clone.Temperature = cloneThreshold(cfg.Temperature)
|
||||||
|
clone.Usage = cloneThreshold(cfg.Usage)
|
||||||
|
return clone
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manager) applyThresholdOverride(base ThresholdConfig, override ThresholdConfig) ThresholdConfig {
|
func (m *Manager) applyThresholdOverride(base ThresholdConfig, override ThresholdConfig) ThresholdConfig {
|
||||||
result := base
|
result := base
|
||||||
|
|
||||||
|
|
@ -6699,6 +6817,120 @@ func ensureHysteresisThreshold(threshold *HysteresisThreshold) *HysteresisThresh
|
||||||
return threshold
|
return threshold
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type pulseTagSettings struct {
|
||||||
|
Suppress bool
|
||||||
|
MonitorOnly bool
|
||||||
|
Relaxed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePulseTags(tags []string) pulseTagSettings {
|
||||||
|
settings := pulseTagSettings{}
|
||||||
|
for _, raw := range tags {
|
||||||
|
tag := strings.TrimSpace(strings.ToLower(raw))
|
||||||
|
switch tag {
|
||||||
|
case "pulse-no-alerts":
|
||||||
|
settings.Suppress = true
|
||||||
|
case "pulse-monitor-only":
|
||||||
|
settings.MonitorOnly = true
|
||||||
|
case "pulse-relaxed":
|
||||||
|
settings.Relaxed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return settings
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyRelaxedGuestThresholds(cfg ThresholdConfig) ThresholdConfig {
|
||||||
|
relaxed := cloneThresholdConfig(cfg)
|
||||||
|
|
||||||
|
adjust := func(th **HysteresisThreshold, minTrigger float64) {
|
||||||
|
if *th == nil {
|
||||||
|
*th = &HysteresisThreshold{Trigger: minTrigger, Clear: minTrigger - 5}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ensureHysteresisThreshold(*th)
|
||||||
|
if (*th).Trigger < minTrigger {
|
||||||
|
(*th).Trigger = minTrigger
|
||||||
|
}
|
||||||
|
if (*th).Clear >= (*th).Trigger {
|
||||||
|
(*th).Clear = (*th).Trigger - 5
|
||||||
|
}
|
||||||
|
if (*th).Clear < 0 {
|
||||||
|
(*th).Clear = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
adjust(&relaxed.CPU, 95)
|
||||||
|
adjust(&relaxed.Memory, 92)
|
||||||
|
adjust(&relaxed.Disk, 95)
|
||||||
|
|
||||||
|
return relaxed
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Manager) suppressGuestAlerts(guestID string) bool {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
cleared := false
|
||||||
|
|
||||||
|
for alertID, alert := range m.activeAlerts {
|
||||||
|
if alert == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if alert.ResourceID == guestID || strings.HasPrefix(alert.ResourceID, guestID+"/") || strings.HasPrefix(alertID, guestID) {
|
||||||
|
m.clearAlertNoLock(alertID)
|
||||||
|
delete(m.recentAlerts, alertID)
|
||||||
|
delete(m.pendingAlerts, alertID)
|
||||||
|
delete(m.suppressedUntil, alertID)
|
||||||
|
delete(m.alertRateLimit, alertID)
|
||||||
|
cleared = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for key := range m.pendingAlerts {
|
||||||
|
if strings.HasPrefix(key, guestID) {
|
||||||
|
delete(m.pendingAlerts, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key := range m.recentAlerts {
|
||||||
|
if strings.HasPrefix(key, guestID) {
|
||||||
|
delete(m.recentAlerts, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key := range m.suppressedUntil {
|
||||||
|
if strings.HasPrefix(key, guestID) {
|
||||||
|
delete(m.suppressedUntil, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key := range m.alertRateLimit {
|
||||||
|
if strings.HasPrefix(key, guestID) {
|
||||||
|
delete(m.alertRateLimit, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(m.offlineConfirmations, guestID)
|
||||||
|
|
||||||
|
return cleared
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Manager) guestHasMonitorOnlyAlerts(guestID string) bool {
|
||||||
|
m.mu.RLock()
|
||||||
|
defer m.mu.RUnlock()
|
||||||
|
|
||||||
|
for _, alert := range m.activeAlerts {
|
||||||
|
if alert == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if alert.ResourceID != guestID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isMonitorOnlyAlert(alert) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// evaluateFilterCondition evaluates a single filter condition against a guest
|
// evaluateFilterCondition evaluates a single filter condition against a guest
|
||||||
func (m *Manager) evaluateFilterCondition(guest interface{}, condition FilterCondition) bool {
|
func (m *Manager) evaluateFilterCondition(guest interface{}, condition FilterCondition) bool {
|
||||||
switch g := guest.(type) {
|
switch g := guest.(type) {
|
||||||
|
|
@ -7150,6 +7382,25 @@ func (m *Manager) SaveActiveAlerts() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Manager) saveActiveAlertsAsync(context string) {
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Str("context", context).
|
||||||
|
Msg("Panic in SaveActiveAlerts goroutine")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := m.SaveActiveAlerts(); err != nil {
|
||||||
|
log.Error().
|
||||||
|
Err(err).
|
||||||
|
Str("context", context).
|
||||||
|
Msg("Failed to save active alerts")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
// LoadActiveAlerts restores active alerts from disk
|
// LoadActiveAlerts restores active alerts from disk
|
||||||
func (m *Manager) LoadActiveAlerts() error {
|
func (m *Manager) LoadActiveAlerts() error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,7 @@ func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
m.config.TimeThreshold = 0
|
m.config.TimeThreshold = 0
|
||||||
m.config.TimeThresholds = map[string]int{}
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.config.ActivationState = ActivationActive
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
var dispatched []*Alert
|
var dispatched []*Alert
|
||||||
|
|
@ -132,6 +133,7 @@ func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
m.config.TimeThreshold = 0
|
m.config.TimeThreshold = 0
|
||||||
m.config.TimeThresholds = map[string]int{}
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.config.ActivationState = ActivationActive
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
// Clear dispatched slice to capture only post-disable notifications.
|
// Clear dispatched slice to capture only post-disable notifications.
|
||||||
|
|
@ -172,6 +174,167 @@ func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPulseNoAlertsSuppressesGuestAlerts(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.config.ActivationState = ActivationActive
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
var dispatched int
|
||||||
|
m.SetAlertCallback(func(alert *Alert) {
|
||||||
|
dispatched++
|
||||||
|
})
|
||||||
|
|
||||||
|
vm := models.VM{
|
||||||
|
ID: "inst/qemu/101",
|
||||||
|
Name: "test-vm",
|
||||||
|
Node: "node1",
|
||||||
|
Instance: "inst",
|
||||||
|
Status: "running",
|
||||||
|
CPU: 1.0,
|
||||||
|
Memory: models.Memory{
|
||||||
|
Usage: 95,
|
||||||
|
},
|
||||||
|
Disk: models.Disk{
|
||||||
|
Usage: 95,
|
||||||
|
},
|
||||||
|
Tags: []string{"pulse-no-alerts"},
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckGuest(vm, "inst")
|
||||||
|
|
||||||
|
if dispatched != 0 {
|
||||||
|
t.Fatalf("expected no alert dispatch, got %d", dispatched)
|
||||||
|
}
|
||||||
|
|
||||||
|
if alerts := m.GetActiveAlerts(); len(alerts) != 0 {
|
||||||
|
t.Fatalf("expected no active alerts, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPulseMonitorOnlySkipsDispatchButRetainsAlert(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.config.ActivationState = ActivationActive
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
var dispatched int
|
||||||
|
m.SetAlertCallback(func(alert *Alert) {
|
||||||
|
dispatched++
|
||||||
|
})
|
||||||
|
|
||||||
|
vm := models.VM{
|
||||||
|
ID: "inst/qemu/102",
|
||||||
|
Name: "monitor-vm",
|
||||||
|
Node: "node1",
|
||||||
|
Instance: "inst",
|
||||||
|
Status: "running",
|
||||||
|
CPU: 1.0,
|
||||||
|
Memory: models.Memory{Usage: 90},
|
||||||
|
Disk: models.Disk{Usage: 50},
|
||||||
|
Tags: []string{"pulse-monitor-only"},
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckGuest(vm, "inst")
|
||||||
|
|
||||||
|
if dispatched != 0 {
|
||||||
|
t.Fatalf("expected monitor-only alert to skip dispatch, got %d callbacks", dispatched)
|
||||||
|
}
|
||||||
|
|
||||||
|
alerts := m.GetActiveAlerts()
|
||||||
|
if len(alerts) == 0 {
|
||||||
|
t.Fatalf("expected monitor-only alert to remain active")
|
||||||
|
}
|
||||||
|
|
||||||
|
if alerts[0].Metadata == nil || alerts[0].Metadata["monitorOnly"] != true {
|
||||||
|
t.Fatalf("expected alert metadata to mark monitorOnly, got %+v", alerts[0].Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPulseRelaxedThresholdsIncreaseCpuTrigger(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
vm := models.VM{
|
||||||
|
ID: "inst/qemu/103",
|
||||||
|
Name: "relaxed-vm",
|
||||||
|
Node: "node1",
|
||||||
|
Instance: "inst",
|
||||||
|
Status: "running",
|
||||||
|
CPU: 0.9, // 90%
|
||||||
|
Memory: models.Memory{Usage: 60},
|
||||||
|
Disk: models.Disk{Usage: 40},
|
||||||
|
Tags: []string{"pulse-relaxed"},
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckGuest(vm, "inst")
|
||||||
|
|
||||||
|
if alerts := m.GetActiveAlerts(); len(alerts) != 0 {
|
||||||
|
t.Fatalf("expected no alerts at 90%% CPU with relaxed thresholds, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
|
||||||
|
vm.CPU = 1.0
|
||||||
|
m.CheckGuest(vm, "inst")
|
||||||
|
|
||||||
|
if alerts := m.GetActiveAlerts(); len(alerts) == 0 {
|
||||||
|
t.Fatalf("expected alert once CPU exceeds relaxed threshold")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClearAlertMarksResolutionAndReturnsStatus(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
vm := models.VM{
|
||||||
|
ID: "inst/qemu/104",
|
||||||
|
Name: "clear-vm",
|
||||||
|
Node: "node1",
|
||||||
|
Instance: "inst",
|
||||||
|
Status: "running",
|
||||||
|
CPU: 1.0,
|
||||||
|
Memory: models.Memory{Usage: 80},
|
||||||
|
Disk: models.Disk{Usage: 80},
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckGuest(vm, "inst")
|
||||||
|
alerts := m.GetActiveAlerts()
|
||||||
|
if len(alerts) == 0 {
|
||||||
|
t.Fatalf("expected alert to be active before clearing")
|
||||||
|
}
|
||||||
|
|
||||||
|
alertID := alerts[0].ID
|
||||||
|
if ok := m.ClearAlert(alertID); !ok {
|
||||||
|
t.Fatalf("expected manual clear to succeed")
|
||||||
|
}
|
||||||
|
|
||||||
|
if remaining := m.GetActiveAlerts(); len(remaining) != 0 {
|
||||||
|
t.Fatalf("expected no active alerts after clear, found %d", len(remaining))
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved := m.GetRecentlyResolved()
|
||||||
|
if len(resolved) == 0 || resolved[0].Alert.ID != alertID {
|
||||||
|
t.Fatalf("expected alert %s to be tracked as recently resolved", alertID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok := m.ClearAlert(alertID); ok {
|
||||||
|
t.Fatalf("expected second clear to report missing alert")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
||||||
m := NewManager()
|
m := NewManager()
|
||||||
host := models.DockerHost{ID: "host1", DisplayName: "Host One", Hostname: "host-one"}
|
host := models.DockerHost{ID: "host1", DisplayName: "Host One", Hostname: "host-one"}
|
||||||
|
|
|
||||||
|
|
@ -80,14 +80,13 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request
|
||||||
h.monitor.GetAlertManager().UpdateConfig(config)
|
h.monitor.GetAlertManager().UpdateConfig(config)
|
||||||
|
|
||||||
// Update notification manager with schedule settings
|
// Update notification manager with schedule settings
|
||||||
if config.Schedule.Cooldown > 0 {
|
h.monitor.GetNotificationManager().SetCooldown(config.Schedule.Cooldown)
|
||||||
h.monitor.GetNotificationManager().SetCooldown(config.Schedule.Cooldown)
|
|
||||||
}
|
groupWindow := config.Schedule.Grouping.Window
|
||||||
if config.Schedule.GroupingWindow > 0 {
|
if groupWindow == 0 && config.Schedule.GroupingWindow != 0 {
|
||||||
h.monitor.GetNotificationManager().SetGroupingWindow(config.Schedule.GroupingWindow)
|
groupWindow = config.Schedule.GroupingWindow
|
||||||
} else if config.Schedule.Grouping.Window > 0 {
|
|
||||||
h.monitor.GetNotificationManager().SetGroupingWindow(config.Schedule.Grouping.Window)
|
|
||||||
}
|
}
|
||||||
|
h.monitor.GetNotificationManager().SetGroupingWindow(groupWindow)
|
||||||
h.monitor.GetNotificationManager().SetGroupingOptions(
|
h.monitor.GetNotificationManager().SetGroupingOptions(
|
||||||
config.Schedule.Grouping.ByNode,
|
config.Schedule.Grouping.ByNode,
|
||||||
config.Schedule.Grouping.ByGuest,
|
config.Schedule.Grouping.ByGuest,
|
||||||
|
|
@ -537,7 +536,10 @@ func (h *AlertHandlers) ClearAlert(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.monitor.GetAlertManager().ClearAlert(alertID)
|
if !h.monitor.GetAlertManager().ClearAlert(alertID) {
|
||||||
|
http.Error(w, "Alert not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
h.monitor.SyncAlertState()
|
h.monitor.SyncAlertState()
|
||||||
|
|
||||||
// Send response immediately
|
// Send response immediately
|
||||||
|
|
@ -643,8 +645,12 @@ func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request)
|
||||||
"alertId": alertID,
|
"alertId": alertID,
|
||||||
"success": true,
|
"success": true,
|
||||||
}
|
}
|
||||||
h.monitor.GetAlertManager().ClearAlert(alertID)
|
if h.monitor.GetAlertManager().ClearAlert(alertID) {
|
||||||
anySuccess = true
|
anySuccess = true
|
||||||
|
} else {
|
||||||
|
result["success"] = false
|
||||||
|
result["error"] = "alert not found"
|
||||||
|
}
|
||||||
results = append(results, result)
|
results = append(results, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2823,14 +2823,12 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||||
if alertConfig, err := m.configPersist.LoadAlertConfig(); err == nil {
|
if alertConfig, err := m.configPersist.LoadAlertConfig(); err == nil {
|
||||||
m.alertManager.UpdateConfig(*alertConfig)
|
m.alertManager.UpdateConfig(*alertConfig)
|
||||||
// Apply schedule settings to notification manager
|
// Apply schedule settings to notification manager
|
||||||
if alertConfig.Schedule.Cooldown > 0 {
|
m.notificationMgr.SetCooldown(alertConfig.Schedule.Cooldown)
|
||||||
m.notificationMgr.SetCooldown(alertConfig.Schedule.Cooldown)
|
groupWindow := alertConfig.Schedule.Grouping.Window
|
||||||
}
|
if groupWindow == 0 && alertConfig.Schedule.GroupingWindow != 0 {
|
||||||
if alertConfig.Schedule.GroupingWindow > 0 {
|
groupWindow = alertConfig.Schedule.GroupingWindow
|
||||||
m.notificationMgr.SetGroupingWindow(alertConfig.Schedule.GroupingWindow)
|
|
||||||
} else if alertConfig.Schedule.Grouping.Window > 0 {
|
|
||||||
m.notificationMgr.SetGroupingWindow(alertConfig.Schedule.Grouping.Window)
|
|
||||||
}
|
}
|
||||||
|
m.notificationMgr.SetGroupingWindow(groupWindow)
|
||||||
m.notificationMgr.SetGroupingOptions(
|
m.notificationMgr.SetGroupingOptions(
|
||||||
alertConfig.Schedule.Grouping.ByNode,
|
alertConfig.Schedule.Grouping.ByNode,
|
||||||
alertConfig.Schedule.Grouping.ByGuest,
|
alertConfig.Schedule.Grouping.ByGuest,
|
||||||
|
|
|
||||||
|
|
@ -390,6 +390,9 @@ func (n *NotificationManager) GetAppriseConfig() AppriseConfig {
|
||||||
func (n *NotificationManager) SetCooldown(minutes int) {
|
func (n *NotificationManager) SetCooldown(minutes int) {
|
||||||
n.mu.Lock()
|
n.mu.Lock()
|
||||||
defer n.mu.Unlock()
|
defer n.mu.Unlock()
|
||||||
|
if minutes < 0 {
|
||||||
|
minutes = 0
|
||||||
|
}
|
||||||
n.cooldown = time.Duration(minutes) * time.Minute
|
n.cooldown = time.Duration(minutes) * time.Minute
|
||||||
log.Info().Int("minutes", minutes).Msg("Updated notification cooldown")
|
log.Info().Int("minutes", minutes).Msg("Updated notification cooldown")
|
||||||
}
|
}
|
||||||
|
|
@ -398,6 +401,9 @@ func (n *NotificationManager) SetCooldown(minutes int) {
|
||||||
func (n *NotificationManager) SetGroupingWindow(seconds int) {
|
func (n *NotificationManager) SetGroupingWindow(seconds int) {
|
||||||
n.mu.Lock()
|
n.mu.Lock()
|
||||||
defer n.mu.Unlock()
|
defer n.mu.Unlock()
|
||||||
|
if seconds < 0 {
|
||||||
|
seconds = 0
|
||||||
|
}
|
||||||
n.groupWindow = time.Duration(seconds) * time.Second
|
n.groupWindow = time.Duration(seconds) * time.Second
|
||||||
log.Info().Int("seconds", seconds).Msg("Updated notification grouping window")
|
log.Info().Int("seconds", seconds).Msg("Updated notification grouping window")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -101,6 +101,46 @@ func TestNormalizeAppriseConfig(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSetCooldownClampsNegativeValues(t *testing.T) {
|
||||||
|
nm := NewNotificationManager("")
|
||||||
|
nm.SetCooldown(-10)
|
||||||
|
|
||||||
|
nm.mu.RLock()
|
||||||
|
if nm.cooldown != 0 {
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
t.Fatalf("expected cooldown to clamp to zero, got %s", nm.cooldown)
|
||||||
|
}
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
|
||||||
|
nm.SetCooldown(5)
|
||||||
|
nm.mu.RLock()
|
||||||
|
if nm.cooldown != 5*time.Minute {
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
t.Fatalf("expected cooldown of five minutes, got %s", nm.cooldown)
|
||||||
|
}
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetGroupingWindowClampsNegativeValues(t *testing.T) {
|
||||||
|
nm := NewNotificationManager("")
|
||||||
|
nm.SetGroupingWindow(-60)
|
||||||
|
|
||||||
|
nm.mu.RLock()
|
||||||
|
if nm.groupWindow != 0 {
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
t.Fatalf("expected grouping window to clamp to zero, got %s", nm.groupWindow)
|
||||||
|
}
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
|
||||||
|
nm.SetGroupingWindow(120)
|
||||||
|
nm.mu.RLock()
|
||||||
|
if nm.groupWindow != 120*time.Second {
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
t.Fatalf("expected grouping window of 120 seconds, got %s", nm.groupWindow)
|
||||||
|
}
|
||||||
|
nm.mu.RUnlock()
|
||||||
|
}
|
||||||
|
|
||||||
func TestSendGroupedAppriseInvokesExecutor(t *testing.T) {
|
func TestSendGroupedAppriseInvokesExecutor(t *testing.T) {
|
||||||
nm := NewNotificationManager("")
|
nm := NewNotificationManager("")
|
||||||
nm.SetGroupingWindow(0)
|
nm.SetGroupingWindow(0)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue