feat(ai): Wire alert history to pattern detector for event tracking
Connect alert system to failure prediction: 1. Add AlertCallback to HistoryManager: - OnAlert() method to register callbacks - Callbacks invoked when alerts are added - Called outside lock to prevent deadlocks 2. Expose OnAlertHistory() on alerts.Manager: - Pass-through to HistoryManager.OnAlert() - Enables external systems to track alerts 3. Wire pattern detector in router startup: - Register callback when pattern detector is created - Convert alert types to trackable events - Pattern detector now learns from production alerts Now every alert (memory_warning, cpu_critical, etc.) is recorded as a historical event for pattern analysis. The AI can predict: 'High memory usage typically occurs every ~3 days (next expected in ~1 day)' All tests passing.
This commit is contained in:
parent
d9d798084e
commit
6f1774f76a
3 changed files with 38 additions and 1 deletions
|
|
@ -5924,6 +5924,14 @@ func (m *Manager) ClearAlertHistory() error {
|
|||
return m.historyManager.ClearAllHistory()
|
||||
}
|
||||
|
||||
// OnAlertHistory registers a callback to be called when alerts are added to history.
|
||||
// This enables external systems like pattern detection to track alerts.
|
||||
func (m *Manager) OnAlertHistory(cb AlertCallback) {
|
||||
if m.historyManager != nil {
|
||||
m.historyManager.OnAlert(cb)
|
||||
}
|
||||
}
|
||||
|
||||
// checkNodeOffline creates an alert for offline nodes after confirmation
|
||||
func (m *Manager) checkNodeOffline(node models.Node) {
|
||||
alertID := fmt.Sprintf("node-offline-%s", node.ID)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ type HistoryEntry struct {
|
|||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// AlertCallback is called when an alert is added to history
|
||||
// This enables external systems to track alerts (e.g., pattern detection)
|
||||
type AlertCallback func(alert Alert)
|
||||
|
||||
// HistoryManager manages persistent alert history
|
||||
type HistoryManager struct {
|
||||
mu sync.RWMutex
|
||||
|
|
@ -38,6 +42,7 @@ type HistoryManager struct {
|
|||
saveInterval time.Duration
|
||||
stopChan chan struct{}
|
||||
saveTicker *time.Ticker
|
||||
callbacks []AlertCallback // Called when alerts are added
|
||||
}
|
||||
|
||||
// NewHistoryManager creates a new history manager
|
||||
|
|
@ -74,10 +79,16 @@ func NewHistoryManager(dataDir string) *HistoryManager {
|
|||
return hm
|
||||
}
|
||||
|
||||
// OnAlert registers a callback to be called when alerts are added
|
||||
func (hm *HistoryManager) OnAlert(cb AlertCallback) {
|
||||
hm.mu.Lock()
|
||||
defer hm.mu.Unlock()
|
||||
hm.callbacks = append(hm.callbacks, cb)
|
||||
}
|
||||
|
||||
// AddAlert adds an alert to history
|
||||
func (hm *HistoryManager) AddAlert(alert Alert) {
|
||||
hm.mu.Lock()
|
||||
defer hm.mu.Unlock()
|
||||
|
||||
entry := HistoryEntry{
|
||||
Alert: *alert.Clone(),
|
||||
|
|
@ -85,7 +96,15 @@ func (hm *HistoryManager) AddAlert(alert Alert) {
|
|||
}
|
||||
|
||||
hm.history = append(hm.history, entry)
|
||||
callbacks := hm.callbacks
|
||||
hm.mu.Unlock()
|
||||
|
||||
log.Debug().Str("alertID", alert.ID).Msg("Added alert to history")
|
||||
|
||||
// Call callbacks outside the lock
|
||||
for _, cb := range callbacks {
|
||||
cb(alert)
|
||||
}
|
||||
}
|
||||
|
||||
// GetHistory returns alert history within the specified time range
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import (
|
|||
"github.com/rcourtman/pulse-go-rewrite/internal/agentbinaries"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/auth"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
|
|
@ -1469,6 +1470,15 @@ func (r *Router) StartPatrol(ctx context.Context) {
|
|||
})
|
||||
if patternDetector != nil {
|
||||
r.aiSettingsHandler.SetPatternDetector(patternDetector)
|
||||
|
||||
// Wire alert history to pattern detector for event tracking
|
||||
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
|
||||
alertManager.OnAlertHistory(func(alert alerts.Alert) {
|
||||
// Convert alert type to trackable event
|
||||
patternDetector.RecordFromAlert(alert.ResourceID, alert.Type+"_"+string(alert.Level), alert.StartTime)
|
||||
})
|
||||
log.Info().Msg("AI Pattern Detector: Wired to alert history for failure prediction")
|
||||
}
|
||||
}
|
||||
|
||||
r.aiSettingsHandler.StartPatrol(ctx)
|
||||
|
|
|
|||
Loading…
Reference in a new issue