feat(ai): Wire alert history to pattern detector for event tracking
Connect alert system to failure prediction: 1. Add AlertCallback to HistoryManager: - OnAlert() method to register callbacks - Callbacks invoked when alerts are added - Called outside lock to prevent deadlocks 2. Expose OnAlertHistory() on alerts.Manager: - Pass-through to HistoryManager.OnAlert() - Enables external systems to track alerts 3. Wire pattern detector in router startup: - Register callback when pattern detector is created - Convert alert types to trackable events - Pattern detector now learns from production alerts Now every alert (memory_warning, cpu_critical, etc.) is recorded as a historical event for pattern analysis. The AI can predict: 'High memory usage typically occurs every ~3 days (next expected in ~1 day)' All tests passing.
This commit is contained in:
parent
d9d798084e
commit
6f1774f76a
3 changed files with 38 additions and 1 deletions
|
|
@ -5924,6 +5924,14 @@ func (m *Manager) ClearAlertHistory() error {
|
||||||
return m.historyManager.ClearAllHistory()
|
return m.historyManager.ClearAllHistory()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OnAlertHistory registers a callback to be called when alerts are added to history.
|
||||||
|
// This enables external systems like pattern detection to track alerts.
|
||||||
|
func (m *Manager) OnAlertHistory(cb AlertCallback) {
|
||||||
|
if m.historyManager != nil {
|
||||||
|
m.historyManager.OnAlert(cb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// checkNodeOffline creates an alert for offline nodes after confirmation
|
// checkNodeOffline creates an alert for offline nodes after confirmation
|
||||||
func (m *Manager) checkNodeOffline(node models.Node) {
|
func (m *Manager) checkNodeOffline(node models.Node) {
|
||||||
alertID := fmt.Sprintf("node-offline-%s", node.ID)
|
alertID := fmt.Sprintf("node-offline-%s", node.ID)
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,10 @@ type HistoryEntry struct {
|
||||||
Timestamp time.Time `json:"timestamp"`
|
Timestamp time.Time `json:"timestamp"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AlertCallback is called when an alert is added to history
|
||||||
|
// This enables external systems to track alerts (e.g., pattern detection)
|
||||||
|
type AlertCallback func(alert Alert)
|
||||||
|
|
||||||
// HistoryManager manages persistent alert history
|
// HistoryManager manages persistent alert history
|
||||||
type HistoryManager struct {
|
type HistoryManager struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
|
|
@ -38,6 +42,7 @@ type HistoryManager struct {
|
||||||
saveInterval time.Duration
|
saveInterval time.Duration
|
||||||
stopChan chan struct{}
|
stopChan chan struct{}
|
||||||
saveTicker *time.Ticker
|
saveTicker *time.Ticker
|
||||||
|
callbacks []AlertCallback // Called when alerts are added
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewHistoryManager creates a new history manager
|
// NewHistoryManager creates a new history manager
|
||||||
|
|
@ -74,10 +79,16 @@ func NewHistoryManager(dataDir string) *HistoryManager {
|
||||||
return hm
|
return hm
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OnAlert registers a callback to be called when alerts are added
|
||||||
|
func (hm *HistoryManager) OnAlert(cb AlertCallback) {
|
||||||
|
hm.mu.Lock()
|
||||||
|
defer hm.mu.Unlock()
|
||||||
|
hm.callbacks = append(hm.callbacks, cb)
|
||||||
|
}
|
||||||
|
|
||||||
// AddAlert adds an alert to history
|
// AddAlert adds an alert to history
|
||||||
func (hm *HistoryManager) AddAlert(alert Alert) {
|
func (hm *HistoryManager) AddAlert(alert Alert) {
|
||||||
hm.mu.Lock()
|
hm.mu.Lock()
|
||||||
defer hm.mu.Unlock()
|
|
||||||
|
|
||||||
entry := HistoryEntry{
|
entry := HistoryEntry{
|
||||||
Alert: *alert.Clone(),
|
Alert: *alert.Clone(),
|
||||||
|
|
@ -85,7 +96,15 @@ func (hm *HistoryManager) AddAlert(alert Alert) {
|
||||||
}
|
}
|
||||||
|
|
||||||
hm.history = append(hm.history, entry)
|
hm.history = append(hm.history, entry)
|
||||||
|
callbacks := hm.callbacks
|
||||||
|
hm.mu.Unlock()
|
||||||
|
|
||||||
log.Debug().Str("alertID", alert.ID).Msg("Added alert to history")
|
log.Debug().Str("alertID", alert.ID).Msg("Added alert to history")
|
||||||
|
|
||||||
|
// Call callbacks outside the lock
|
||||||
|
for _, cb := range callbacks {
|
||||||
|
cb(alert)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetHistory returns alert history within the specified time range
|
// GetHistory returns alert history within the specified time range
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,7 @@ import (
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentbinaries"
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentbinaries"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/auth"
|
"github.com/rcourtman/pulse-go-rewrite/internal/auth"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
|
|
@ -1469,6 +1470,15 @@ func (r *Router) StartPatrol(ctx context.Context) {
|
||||||
})
|
})
|
||||||
if patternDetector != nil {
|
if patternDetector != nil {
|
||||||
r.aiSettingsHandler.SetPatternDetector(patternDetector)
|
r.aiSettingsHandler.SetPatternDetector(patternDetector)
|
||||||
|
|
||||||
|
// Wire alert history to pattern detector for event tracking
|
||||||
|
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
|
||||||
|
alertManager.OnAlertHistory(func(alert alerts.Alert) {
|
||||||
|
// Convert alert type to trackable event
|
||||||
|
patternDetector.RecordFromAlert(alert.ResourceID, alert.Type+"_"+string(alert.Level), alert.StartTime)
|
||||||
|
})
|
||||||
|
log.Info().Msg("AI Pattern Detector: Wired to alert history for failure prediction")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
r.aiSettingsHandler.StartPatrol(ctx)
|
r.aiSettingsHandler.StartPatrol(ctx)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue