feat(ai): Wire alert history to pattern detector for event tracking

Connect alert system to failure prediction:

1. Add AlertCallback to HistoryManager:
   - OnAlert() method to register callbacks
   - Callbacks invoked when alerts are added
   - Called outside lock to prevent deadlocks

2. Expose OnAlertHistory() on alerts.Manager:
   - Pass-through to HistoryManager.OnAlert()
   - Enables external systems to track alerts

3. Wire pattern detector in router startup:
   - Register callback when pattern detector is created
   - Convert alert types to trackable events
   - Pattern detector now learns from production alerts

Now every alert (memory_warning, cpu_critical, etc.) is recorded as
a historical event for pattern analysis. The AI can predict:
'High memory usage typically occurs every ~3 days (next expected in ~1 day)'

All tests passing.
This commit is contained in:
rcourtman 2025-12-12 14:16:03 +00:00
parent d9d798084e
commit 6f1774f76a
3 changed files with 38 additions and 1 deletions

View file

@ -5924,6 +5924,14 @@ func (m *Manager) ClearAlertHistory() error {
return m.historyManager.ClearAllHistory()
}
// OnAlertHistory registers a callback to be called when alerts are added to history.
// This enables external systems like pattern detection to track alerts.
func (m *Manager) OnAlertHistory(cb AlertCallback) {
if m.historyManager != nil {
m.historyManager.OnAlert(cb)
}
}
// checkNodeOffline creates an alert for offline nodes after confirmation
func (m *Manager) checkNodeOffline(node models.Node) {
alertID := fmt.Sprintf("node-offline-%s", node.ID)

View file

@ -27,6 +27,10 @@ type HistoryEntry struct {
Timestamp time.Time `json:"timestamp"`
}
// AlertCallback is called when an alert is added to history
// This enables external systems to track alerts (e.g., pattern detection)
type AlertCallback func(alert Alert)
// HistoryManager manages persistent alert history
type HistoryManager struct {
mu sync.RWMutex
@ -38,6 +42,7 @@ type HistoryManager struct {
saveInterval time.Duration
stopChan chan struct{}
saveTicker *time.Ticker
callbacks []AlertCallback // Called when alerts are added
}
// NewHistoryManager creates a new history manager
@ -74,10 +79,16 @@ func NewHistoryManager(dataDir string) *HistoryManager {
return hm
}
// OnAlert registers a callback to be called when alerts are added
func (hm *HistoryManager) OnAlert(cb AlertCallback) {
hm.mu.Lock()
defer hm.mu.Unlock()
hm.callbacks = append(hm.callbacks, cb)
}
// AddAlert adds an alert to history
func (hm *HistoryManager) AddAlert(alert Alert) {
hm.mu.Lock()
defer hm.mu.Unlock()
entry := HistoryEntry{
Alert: *alert.Clone(),
@ -85,7 +96,15 @@ func (hm *HistoryManager) AddAlert(alert Alert) {
}
hm.history = append(hm.history, entry)
callbacks := hm.callbacks
hm.mu.Unlock()
log.Debug().Str("alertID", alert.ID).Msg("Added alert to history")
// Call callbacks outside the lock
for _, cb := range callbacks {
cb(alert)
}
}
// GetHistory returns alert history within the specified time range

View file

@ -28,6 +28,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/agentbinaries"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/auth"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
@ -1469,6 +1470,15 @@ func (r *Router) StartPatrol(ctx context.Context) {
})
if patternDetector != nil {
r.aiSettingsHandler.SetPatternDetector(patternDetector)
// Wire alert history to pattern detector for event tracking
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
alertManager.OnAlertHistory(func(alert alerts.Alert) {
// Convert alert type to trackable event
patternDetector.RecordFromAlert(alert.ResourceID, alert.Type+"_"+string(alert.Level), alert.StartTime)
})
log.Info().Msg("AI Pattern Detector: Wired to alert history for failure prediction")
}
}
r.aiSettingsHandler.StartPatrol(ctx)