feat(ai): Add failure pattern detection for predictive intelligence (Phase 5)
Create internal/ai/patterns package: 1. Pattern Detector (detector.go): - Records historical events (high memory, OOM, restarts, etc.) - Detects recurring failure patterns - Calculates average interval between occurrences - Computes confidence based on pattern consistency - Predicts when failures will occur again - Persists to ai_patterns.json 2. Event types tracked: - high_memory, high_cpu, disk_full - oom, restart, unresponsive - backup_failed 3. Integration: - Wire PatternDetector into router startup - Add to AI context in buildEnrichedContext - FormatForContext generates failure predictions Example AI context now includes: 'OOM events typically occurs every ~10 days (next expected in ~3 days)' This enables proactive alerts before problems recur. All tests passing.
This commit is contained in:
parent
dacdd48e28
commit
d9d798084e
7 changed files with 837 additions and 0 deletions
|
|
@ -214,6 +214,7 @@ type PatrolService struct {
|
||||||
baselineStore *baseline.Store // For anomaly detection via learned baselines
|
baselineStore *baseline.Store // For anomaly detection via learned baselines
|
||||||
changeDetector *ChangeDetector // For tracking infrastructure changes
|
changeDetector *ChangeDetector // For tracking infrastructure changes
|
||||||
remediationLog *RemediationLog // For tracking remediation actions
|
remediationLog *RemediationLog // For tracking remediation actions
|
||||||
|
patternDetector *PatternDetector // For failure prediction from historical patterns
|
||||||
|
|
||||||
// Cached thresholds (recalculated when thresholdProvider changes)
|
// Cached thresholds (recalculated when thresholdProvider changes)
|
||||||
thresholds PatrolThresholds
|
thresholds PatrolThresholds
|
||||||
|
|
@ -383,6 +384,21 @@ func (p *PatrolService) GetRemediationLog() *RemediationLog {
|
||||||
return p.remediationLog
|
return p.remediationLog
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
|
func (p *PatrolService) SetPatternDetector(detector *PatternDetector) {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
p.patternDetector = detector
|
||||||
|
log.Info().Msg("AI Patrol: Pattern detector set for failure prediction")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPatternDetector returns the pattern detector
|
||||||
|
func (p *PatrolService) GetPatternDetector() *PatternDetector {
|
||||||
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
return p.patternDetector
|
||||||
|
}
|
||||||
|
|
||||||
// GetConfig returns the current patrol configuration
|
// GetConfig returns the current patrol configuration
|
||||||
func (p *PatrolService) GetConfig() PatrolConfig {
|
func (p *PatrolService) GetConfig() PatrolConfig {
|
||||||
p.mu.RLock()
|
p.mu.RLock()
|
||||||
|
|
@ -1783,6 +1799,18 @@ func (p *PatrolService) buildEnrichedContext(state models.StateSnapshot) string
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Append failure predictions if pattern detector is available
|
||||||
|
p.mu.RLock()
|
||||||
|
patternDetector := p.patternDetector
|
||||||
|
p.mu.RUnlock()
|
||||||
|
|
||||||
|
if patternDetector != nil {
|
||||||
|
predictionsContext := patternDetector.FormatForContext("")
|
||||||
|
if predictionsContext != "" {
|
||||||
|
formatted += predictionsContext
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Int("resources", infraCtx.TotalResources).
|
Int("resources", infraCtx.TotalResources).
|
||||||
Int("predictions", len(infraCtx.Predictions)).
|
Int("predictions", len(infraCtx.Predictions)).
|
||||||
|
|
|
||||||
44
internal/ai/pattern_exports.go
Normal file
44
internal/ai/pattern_exports.go
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
package ai
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/patterns"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PatternDetector is an alias for patterns.Detector
|
||||||
|
type PatternDetector = patterns.Detector
|
||||||
|
|
||||||
|
// PatternDetectorConfig is an alias for patterns.DetectorConfig
|
||||||
|
type PatternDetectorConfig = patterns.DetectorConfig
|
||||||
|
|
||||||
|
// HistoricalEvent is an alias for patterns.HistoricalEvent
|
||||||
|
type HistoricalEvent = patterns.HistoricalEvent
|
||||||
|
|
||||||
|
// FailurePrediction is an alias for patterns.FailurePrediction
|
||||||
|
type FailurePrediction = patterns.FailurePrediction
|
||||||
|
|
||||||
|
// EventType is an alias for patterns.EventType
|
||||||
|
type EventType = patterns.EventType
|
||||||
|
|
||||||
|
// Pattern is an alias for patterns.Pattern
|
||||||
|
type Pattern = patterns.Pattern
|
||||||
|
|
||||||
|
// Event type constants
|
||||||
|
const (
|
||||||
|
EventHighMemory = patterns.EventHighMemory
|
||||||
|
EventHighCPU = patterns.EventHighCPU
|
||||||
|
EventDiskFull = patterns.EventDiskFull
|
||||||
|
EventOOM = patterns.EventOOM
|
||||||
|
EventRestart = patterns.EventRestart
|
||||||
|
EventUnresponsive = patterns.EventUnresponsive
|
||||||
|
EventBackupFailed = patterns.EventBackupFailed
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewPatternDetector creates a new pattern detector
|
||||||
|
func NewPatternDetector(cfg PatternDetectorConfig) *PatternDetector {
|
||||||
|
return patterns.NewDetector(cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultPatternConfig returns default pattern detector configuration
|
||||||
|
func DefaultPatternConfig() PatternDetectorConfig {
|
||||||
|
return patterns.DefaultConfig()
|
||||||
|
}
|
||||||
531
internal/ai/patterns/detector.go
Normal file
531
internal/ai/patterns/detector.go
Normal file
|
|
@ -0,0 +1,531 @@
|
||||||
|
// Package patterns provides failure pattern detection for predictive intelligence.
|
||||||
|
// It analyzes historical data to identify recurring issues and predict future failures.
|
||||||
|
package patterns
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EventType represents the type of event being tracked
|
||||||
|
type EventType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
EventHighMemory EventType = "high_memory" // Memory exceeded threshold
|
||||||
|
EventHighCPU EventType = "high_cpu" // CPU exceeded threshold
|
||||||
|
EventDiskFull EventType = "disk_full" // Disk space critical
|
||||||
|
EventOOM EventType = "oom" // Out of memory kill
|
||||||
|
EventRestart EventType = "restart" // Resource restarted
|
||||||
|
EventUnresponsive EventType = "unresponsive" // Resource became unresponsive
|
||||||
|
EventBackupFailed EventType = "backup_failed" // Backup job failed
|
||||||
|
)
|
||||||
|
|
||||||
|
// HistoricalEvent represents a recorded event
|
||||||
|
type HistoricalEvent struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
ResourceID string `json:"resource_id"`
|
||||||
|
EventType EventType `json:"event_type"`
|
||||||
|
Timestamp time.Time `json:"timestamp"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
Resolved bool `json:"resolved"`
|
||||||
|
ResolvedAt time.Time `json:"resolved_at,omitempty"`
|
||||||
|
Duration time.Duration `json:"duration,omitempty"` // How long it lasted
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pattern represents a detected recurring pattern
|
||||||
|
type Pattern struct {
|
||||||
|
ResourceID string `json:"resource_id"`
|
||||||
|
EventType EventType `json:"event_type"`
|
||||||
|
Occurrences int `json:"occurrences"` // Number of times event occurred
|
||||||
|
AverageInterval time.Duration `json:"average_interval"` // Average time between occurrences
|
||||||
|
StdDevInterval time.Duration `json:"stddev_interval"` // Standard deviation
|
||||||
|
LastOccurrence time.Time `json:"last_occurrence"`
|
||||||
|
NextPredicted time.Time `json:"next_predicted"` // When we expect it to happen again
|
||||||
|
Confidence float64 `json:"confidence"` // 0-1, based on consistency
|
||||||
|
AverageDuration time.Duration `json:"average_duration,omitempty"` // How long events typically last
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailurePrediction represents a predicted future failure
|
||||||
|
type FailurePrediction struct {
|
||||||
|
ResourceID string `json:"resource_id"`
|
||||||
|
EventType EventType `json:"event_type"`
|
||||||
|
PredictedAt time.Time `json:"predicted_at"`
|
||||||
|
DaysUntil float64 `json:"days_until"`
|
||||||
|
Confidence float64 `json:"confidence"`
|
||||||
|
Basis string `json:"basis"` // Human-readable explanation
|
||||||
|
Pattern *Pattern `json:"pattern,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detector tracks historical events and detects patterns
|
||||||
|
type Detector struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
events []HistoricalEvent
|
||||||
|
patterns map[string]*Pattern // resourceID:eventType -> pattern
|
||||||
|
|
||||||
|
// Configuration
|
||||||
|
maxEvents int
|
||||||
|
minOccurrences int // Minimum occurrences to form a pattern
|
||||||
|
patternWindow time.Duration // How far back to look for patterns
|
||||||
|
predictionLimit time.Duration // How far ahead to predict
|
||||||
|
|
||||||
|
// Persistence
|
||||||
|
dataDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectorConfig configures the pattern detector
|
||||||
|
type DetectorConfig struct {
|
||||||
|
MaxEvents int
|
||||||
|
MinOccurrences int // Default: 3
|
||||||
|
PatternWindow time.Duration // Default: 90 days
|
||||||
|
PredictionLimit time.Duration // Default: 30 days
|
||||||
|
DataDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns default detector configuration
|
||||||
|
func DefaultConfig() DetectorConfig {
|
||||||
|
return DetectorConfig{
|
||||||
|
MaxEvents: 5000,
|
||||||
|
MinOccurrences: 3,
|
||||||
|
PatternWindow: 90 * 24 * time.Hour,
|
||||||
|
PredictionLimit: 30 * 24 * time.Hour,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDetector creates a new pattern detector
|
||||||
|
func NewDetector(cfg DetectorConfig) *Detector {
|
||||||
|
if cfg.MaxEvents <= 0 {
|
||||||
|
cfg.MaxEvents = 5000
|
||||||
|
}
|
||||||
|
if cfg.MinOccurrences <= 0 {
|
||||||
|
cfg.MinOccurrences = 3
|
||||||
|
}
|
||||||
|
if cfg.PatternWindow <= 0 {
|
||||||
|
cfg.PatternWindow = 90 * 24 * time.Hour
|
||||||
|
}
|
||||||
|
if cfg.PredictionLimit <= 0 {
|
||||||
|
cfg.PredictionLimit = 30 * 24 * time.Hour
|
||||||
|
}
|
||||||
|
|
||||||
|
d := &Detector{
|
||||||
|
events: make([]HistoricalEvent, 0),
|
||||||
|
patterns: make(map[string]*Pattern),
|
||||||
|
maxEvents: cfg.MaxEvents,
|
||||||
|
minOccurrences: cfg.MinOccurrences,
|
||||||
|
patternWindow: cfg.PatternWindow,
|
||||||
|
predictionLimit: cfg.PredictionLimit,
|
||||||
|
dataDir: cfg.DataDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load existing data
|
||||||
|
if cfg.DataDir != "" {
|
||||||
|
if err := d.loadFromDisk(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to load pattern history from disk")
|
||||||
|
} else if len(d.events) > 0 {
|
||||||
|
log.Info().Int("events", len(d.events)).Int("patterns", len(d.patterns)).
|
||||||
|
Msg("Loaded pattern history from disk")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordEvent records a new event for pattern analysis
|
||||||
|
func (d *Detector) RecordEvent(event HistoricalEvent) {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
|
||||||
|
if event.ID == "" {
|
||||||
|
event.ID = generateEventID()
|
||||||
|
}
|
||||||
|
if event.Timestamp.IsZero() {
|
||||||
|
event.Timestamp = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
d.events = append(d.events, event)
|
||||||
|
d.trimEvents()
|
||||||
|
|
||||||
|
// Recompute pattern for this resource/event type
|
||||||
|
key := patternKey(event.ResourceID, event.EventType)
|
||||||
|
d.patterns[key] = d.computePattern(event.ResourceID, event.EventType)
|
||||||
|
|
||||||
|
// Persist asynchronously
|
||||||
|
go func() {
|
||||||
|
if err := d.saveToDisk(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to save pattern history")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordFromAlert records an event from an alert
|
||||||
|
func (d *Detector) RecordFromAlert(resourceID string, alertType string, timestamp time.Time) {
|
||||||
|
eventType := mapAlertToEventType(alertType)
|
||||||
|
if eventType == "" {
|
||||||
|
return // Not a trackable event type
|
||||||
|
}
|
||||||
|
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: resourceID,
|
||||||
|
EventType: eventType,
|
||||||
|
Timestamp: timestamp,
|
||||||
|
Description: alertType,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPredictions returns failure predictions for all tracked resources
|
||||||
|
func (d *Detector) GetPredictions() []FailurePrediction {
|
||||||
|
d.mu.RLock()
|
||||||
|
defer d.mu.RUnlock()
|
||||||
|
|
||||||
|
var predictions []FailurePrediction
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
for _, pattern := range d.patterns {
|
||||||
|
// Only predict if pattern has sufficient confidence
|
||||||
|
if pattern.Confidence < 0.3 || pattern.Occurrences < d.minOccurrences {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if prediction is within our limit
|
||||||
|
if pattern.NextPredicted.Before(now) || pattern.NextPredicted.After(now.Add(d.predictionLimit)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
daysUntil := pattern.NextPredicted.Sub(now).Hours() / 24
|
||||||
|
|
||||||
|
predictions = append(predictions, FailurePrediction{
|
||||||
|
ResourceID: pattern.ResourceID,
|
||||||
|
EventType: pattern.EventType,
|
||||||
|
PredictedAt: pattern.NextPredicted,
|
||||||
|
DaysUntil: daysUntil,
|
||||||
|
Confidence: pattern.Confidence,
|
||||||
|
Basis: formatPatternBasis(pattern),
|
||||||
|
Pattern: pattern,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by days until (soonest first)
|
||||||
|
sort.Slice(predictions, func(i, j int) bool {
|
||||||
|
return predictions[i].DaysUntil < predictions[j].DaysUntil
|
||||||
|
})
|
||||||
|
|
||||||
|
return predictions
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPredictionsForResource returns failure predictions for a specific resource
|
||||||
|
func (d *Detector) GetPredictionsForResource(resourceID string) []FailurePrediction {
|
||||||
|
all := d.GetPredictions()
|
||||||
|
var result []FailurePrediction
|
||||||
|
for _, p := range all {
|
||||||
|
if p.ResourceID == resourceID {
|
||||||
|
result = append(result, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPatterns returns all detected patterns
|
||||||
|
func (d *Detector) GetPatterns() map[string]*Pattern {
|
||||||
|
d.mu.RLock()
|
||||||
|
defer d.mu.RUnlock()
|
||||||
|
|
||||||
|
result := make(map[string]*Pattern)
|
||||||
|
for k, v := range d.patterns {
|
||||||
|
result[k] = v
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// computePattern analyzes events to find patterns for a resource/event type
|
||||||
|
func (d *Detector) computePattern(resourceID string, eventType EventType) *Pattern {
|
||||||
|
cutoff := time.Now().Add(-d.patternWindow)
|
||||||
|
|
||||||
|
// Get all events for this resource/type within the window
|
||||||
|
var events []HistoricalEvent
|
||||||
|
for _, e := range d.events {
|
||||||
|
if e.ResourceID == resourceID && e.EventType == eventType && e.Timestamp.After(cutoff) {
|
||||||
|
events = append(events, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(events) < d.minOccurrences {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by timestamp
|
||||||
|
sort.Slice(events, func(i, j int) bool {
|
||||||
|
return events[i].Timestamp.Before(events[j].Timestamp)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Calculate intervals between events
|
||||||
|
var intervals []time.Duration
|
||||||
|
var durations []time.Duration
|
||||||
|
|
||||||
|
for i := 1; i < len(events); i++ {
|
||||||
|
interval := events[i].Timestamp.Sub(events[i-1].Timestamp)
|
||||||
|
intervals = append(intervals, interval)
|
||||||
|
|
||||||
|
if events[i-1].Duration > 0 {
|
||||||
|
durations = append(durations, events[i-1].Duration)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(intervals) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate average and stddev of intervals
|
||||||
|
avgInterval := averageDuration(intervals)
|
||||||
|
stddevInterval := stddevDuration(intervals, avgInterval)
|
||||||
|
|
||||||
|
// Calculate confidence based on consistency
|
||||||
|
// If stddev is low relative to mean, pattern is more reliable
|
||||||
|
consistency := 1.0
|
||||||
|
if avgInterval > 0 {
|
||||||
|
cv := float64(stddevInterval) / float64(avgInterval) // Coefficient of variation
|
||||||
|
consistency = 1.0 - math.Min(cv, 1.0) // Higher consistency = lower CV
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adjust confidence based on number of occurrences
|
||||||
|
occurrenceBonus := math.Min(float64(len(events))/10.0, 0.3)
|
||||||
|
confidence := consistency*0.7 + occurrenceBonus
|
||||||
|
|
||||||
|
// Predict next occurrence
|
||||||
|
lastEvent := events[len(events)-1]
|
||||||
|
nextPredicted := lastEvent.Timestamp.Add(avgInterval)
|
||||||
|
|
||||||
|
// Calculate average duration if available
|
||||||
|
var avgDuration time.Duration
|
||||||
|
if len(durations) > 0 {
|
||||||
|
avgDuration = averageDuration(durations)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Pattern{
|
||||||
|
ResourceID: resourceID,
|
||||||
|
EventType: eventType,
|
||||||
|
Occurrences: len(events),
|
||||||
|
AverageInterval: avgInterval,
|
||||||
|
StdDevInterval: stddevInterval,
|
||||||
|
LastOccurrence: lastEvent.Timestamp,
|
||||||
|
NextPredicted: nextPredicted,
|
||||||
|
Confidence: confidence,
|
||||||
|
AverageDuration: avgDuration,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// trimEvents removes old events beyond maxEvents
|
||||||
|
func (d *Detector) trimEvents() {
|
||||||
|
if len(d.events) > d.maxEvents {
|
||||||
|
d.events = d.events[len(d.events)-d.maxEvents:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// saveToDisk persists events and patterns
|
||||||
|
func (d *Detector) saveToDisk() error {
|
||||||
|
if d.dataDir == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
d.mu.RLock()
|
||||||
|
data := struct {
|
||||||
|
Events []HistoricalEvent `json:"events"`
|
||||||
|
Patterns map[string]*Pattern `json:"patterns"`
|
||||||
|
}{
|
||||||
|
Events: d.events,
|
||||||
|
Patterns: d.patterns,
|
||||||
|
}
|
||||||
|
d.mu.RUnlock()
|
||||||
|
|
||||||
|
jsonData, err := json.MarshalIndent(data, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
path := filepath.Join(d.dataDir, "ai_patterns.json")
|
||||||
|
tmpPath := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmpPath, jsonData, 0600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.Rename(tmpPath, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadFromDisk loads events and patterns
|
||||||
|
func (d *Detector) loadFromDisk() error {
|
||||||
|
if d.dataDir == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
path := filepath.Join(d.dataDir, "ai_patterns.json")
|
||||||
|
jsonData, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var data struct {
|
||||||
|
Events []HistoricalEvent `json:"events"`
|
||||||
|
Patterns map[string]*Pattern `json:"patterns"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.Unmarshal(jsonData, &data); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
d.events = data.Events
|
||||||
|
d.patterns = data.Patterns
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForContext formats predictions for AI consumption
|
||||||
|
func (d *Detector) FormatForContext(resourceID string) string {
|
||||||
|
var predictions []FailurePrediction
|
||||||
|
if resourceID != "" {
|
||||||
|
predictions = d.GetPredictionsForResource(resourceID)
|
||||||
|
} else {
|
||||||
|
predictions = d.GetPredictions()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(predictions) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var result string
|
||||||
|
result = "\n## ⏰ Failure Predictions\n"
|
||||||
|
result += "Based on historical patterns:\n"
|
||||||
|
|
||||||
|
for _, p := range predictions {
|
||||||
|
if len(result) > 2000 { // Limit context size
|
||||||
|
result += "\n... and more\n"
|
||||||
|
break
|
||||||
|
}
|
||||||
|
result += "- " + p.Basis + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
|
||||||
|
var eventCounter int64
|
||||||
|
|
||||||
|
func generateEventID() string {
|
||||||
|
eventCounter++
|
||||||
|
return time.Now().Format("20060102150405") + "-" + intToStr(int(eventCounter%1000))
|
||||||
|
}
|
||||||
|
|
||||||
|
func intToStr(n int) string {
|
||||||
|
if n == 0 {
|
||||||
|
return "0"
|
||||||
|
}
|
||||||
|
var result string
|
||||||
|
for n > 0 {
|
||||||
|
result = string(rune('0'+n%10)) + result
|
||||||
|
n /= 10
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func patternKey(resourceID string, eventType EventType) string {
|
||||||
|
return resourceID + ":" + string(eventType)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapAlertToEventType(alertType string) EventType {
|
||||||
|
switch alertType {
|
||||||
|
case "memory_warning", "memory_critical":
|
||||||
|
return EventHighMemory
|
||||||
|
case "cpu_warning", "cpu_critical":
|
||||||
|
return EventHighCPU
|
||||||
|
case "disk_warning", "disk_critical":
|
||||||
|
return EventDiskFull
|
||||||
|
case "oom", "out_of_memory":
|
||||||
|
return EventOOM
|
||||||
|
case "restart", "restarted":
|
||||||
|
return EventRestart
|
||||||
|
case "unresponsive", "unreachable":
|
||||||
|
return EventUnresponsive
|
||||||
|
case "backup_failed":
|
||||||
|
return EventBackupFailed
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func averageDuration(durations []time.Duration) time.Duration {
|
||||||
|
if len(durations) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var sum int64
|
||||||
|
for _, d := range durations {
|
||||||
|
sum += int64(d)
|
||||||
|
}
|
||||||
|
return time.Duration(sum / int64(len(durations)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func stddevDuration(durations []time.Duration, mean time.Duration) time.Duration {
|
||||||
|
if len(durations) < 2 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var sumSquares float64
|
||||||
|
for _, d := range durations {
|
||||||
|
diff := float64(d - mean)
|
||||||
|
sumSquares += diff * diff
|
||||||
|
}
|
||||||
|
variance := sumSquares / float64(len(durations)-1)
|
||||||
|
return time.Duration(math.Sqrt(variance))
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatPatternBasis(p *Pattern) string {
|
||||||
|
daysInterval := p.AverageInterval.Hours() / 24
|
||||||
|
daysSinceLast := time.Since(p.LastOccurrence).Hours() / 24
|
||||||
|
daysUntilNext := p.NextPredicted.Sub(time.Now()).Hours() / 24
|
||||||
|
|
||||||
|
eventName := string(p.EventType)
|
||||||
|
switch p.EventType {
|
||||||
|
case EventHighMemory:
|
||||||
|
eventName = "high memory usage"
|
||||||
|
case EventHighCPU:
|
||||||
|
eventName = "high CPU usage"
|
||||||
|
case EventDiskFull:
|
||||||
|
eventName = "disk space critical"
|
||||||
|
case EventOOM:
|
||||||
|
eventName = "OOM events"
|
||||||
|
case EventRestart:
|
||||||
|
eventName = "restarts"
|
||||||
|
case EventUnresponsive:
|
||||||
|
eventName = "unresponsive periods"
|
||||||
|
case EventBackupFailed:
|
||||||
|
eventName = "backup failures"
|
||||||
|
}
|
||||||
|
|
||||||
|
if daysUntilNext < 0 {
|
||||||
|
return eventName + " typically occurs every ~" + formatDays(daysInterval) +
|
||||||
|
" (last: " + formatDays(daysSinceLast) + " ago, overdue)"
|
||||||
|
}
|
||||||
|
|
||||||
|
return eventName + " typically occurs every ~" + formatDays(daysInterval) +
|
||||||
|
" (next expected in ~" + formatDays(daysUntilNext) + ")"
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatDays(days float64) string {
|
||||||
|
if days < 1 {
|
||||||
|
hours := days * 24
|
||||||
|
if hours < 1 {
|
||||||
|
return "less than an hour"
|
||||||
|
}
|
||||||
|
return intToStr(int(hours)) + " hours"
|
||||||
|
}
|
||||||
|
if days < 2 {
|
||||||
|
return "1 day"
|
||||||
|
}
|
||||||
|
return intToStr(int(days)) + " days"
|
||||||
|
}
|
||||||
206
internal/ai/patterns/detector_test.go
Normal file
206
internal/ai/patterns/detector_test.go
Normal file
|
|
@ -0,0 +1,206 @@
|
||||||
|
package patterns
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDetector_RecordEvent(t *testing.T) {
|
||||||
|
d := NewDetector(DetectorConfig{MinOccurrences: 2})
|
||||||
|
|
||||||
|
// Record first event
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "vm-100",
|
||||||
|
EventType: EventHighMemory,
|
||||||
|
Timestamp: time.Now().Add(-10 * 24 * time.Hour),
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(d.events) != 1 {
|
||||||
|
t.Errorf("Expected 1 event, got %d", len(d.events))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetector_PatternDetection(t *testing.T) {
|
||||||
|
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||||
|
|
||||||
|
// Record events with 10-day interval
|
||||||
|
now := time.Now()
|
||||||
|
for i := 5; i >= 0; i-- {
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "vm-100",
|
||||||
|
EventType: EventHighMemory,
|
||||||
|
Timestamp: now.Add(-time.Duration(i*10) * 24 * time.Hour),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that pattern was detected
|
||||||
|
patterns := d.GetPatterns()
|
||||||
|
key := patternKey("vm-100", EventHighMemory)
|
||||||
|
pattern, ok := patterns[key]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Expected pattern to be detected")
|
||||||
|
}
|
||||||
|
|
||||||
|
if pattern.Occurrences != 6 {
|
||||||
|
t.Errorf("Expected 6 occurrences, got %d", pattern.Occurrences)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Average interval should be ~10 days
|
||||||
|
avgDays := pattern.AverageInterval.Hours() / 24
|
||||||
|
if avgDays < 9 || avgDays > 11 {
|
||||||
|
t.Errorf("Expected ~10 day interval, got %.1f days", avgDays)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetector_GetPredictions(t *testing.T) {
|
||||||
|
d := NewDetector(DetectorConfig{
|
||||||
|
MinOccurrences: 3,
|
||||||
|
PatternWindow: 365 * 24 * time.Hour,
|
||||||
|
PredictionLimit: 30 * 24 * time.Hour,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Record events with regular interval
|
||||||
|
now := time.Now()
|
||||||
|
for i := 3; i >= 0; i-- {
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "vm-100",
|
||||||
|
EventType: EventOOM,
|
||||||
|
Timestamp: now.Add(-time.Duration(i*7) * 24 * time.Hour), // 7-day interval
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
predictions := d.GetPredictions()
|
||||||
|
|
||||||
|
// Should have a prediction for OOM
|
||||||
|
found := false
|
||||||
|
for _, p := range predictions {
|
||||||
|
if p.ResourceID == "vm-100" && p.EventType == EventOOM {
|
||||||
|
found = true
|
||||||
|
// Should predict in ~7 days
|
||||||
|
if p.DaysUntil < 5 || p.DaysUntil > 9 {
|
||||||
|
t.Errorf("Expected prediction in ~7 days, got %.1f days", p.DaysUntil)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !found {
|
||||||
|
t.Error("Expected OOM prediction for vm-100")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetector_GetPredictionsForResource(t *testing.T) {
|
||||||
|
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
// Add pattern for vm-100
|
||||||
|
for i := 3; i >= 0; i-- {
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "vm-100",
|
||||||
|
EventType: EventRestart,
|
||||||
|
Timestamp: now.Add(-time.Duration(i*14) * 24 * time.Hour),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Add pattern for vm-200
|
||||||
|
for i := 3; i >= 0; i-- {
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "vm-200",
|
||||||
|
EventType: EventHighCPU,
|
||||||
|
Timestamp: now.Add(-time.Duration(i*5) * 24 * time.Hour),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get predictions for vm-100 only
|
||||||
|
predictions := d.GetPredictionsForResource("vm-100")
|
||||||
|
for _, p := range predictions {
|
||||||
|
if p.ResourceID != "vm-100" {
|
||||||
|
t.Errorf("Got prediction for wrong resource: %s", p.ResourceID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetector_Confidence(t *testing.T) {
|
||||||
|
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
// Add very consistent pattern (every 7 days exactly)
|
||||||
|
for i := 5; i >= 0; i-- {
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "consistent-vm",
|
||||||
|
EventType: EventHighMemory,
|
||||||
|
Timestamp: now.Add(-time.Duration(i*7*24) * time.Hour),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
patterns := d.GetPatterns()
|
||||||
|
pattern := patterns[patternKey("consistent-vm", EventHighMemory)]
|
||||||
|
if pattern == nil {
|
||||||
|
t.Fatal("Expected pattern")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consistent pattern should have high confidence
|
||||||
|
if pattern.Confidence < 0.5 {
|
||||||
|
t.Errorf("Expected high confidence for consistent pattern, got %.2f", pattern.Confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetector_FormatForContext(t *testing.T) {
|
||||||
|
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
for i := 3; i >= 0; i-- {
|
||||||
|
d.RecordEvent(HistoricalEvent{
|
||||||
|
ResourceID: "vm-100",
|
||||||
|
EventType: EventOOM,
|
||||||
|
Timestamp: now.Add(-time.Duration(i*10) * 24 * time.Hour),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
context := d.FormatForContext("vm-100")
|
||||||
|
if context == "" {
|
||||||
|
t.Error("Expected non-empty context")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !contains(context, "OOM") && !contains(context, "oom") {
|
||||||
|
t.Errorf("Expected context to mention OOM: %s", context)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapAlertToEventType(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
alertType string
|
||||||
|
expected EventType
|
||||||
|
}{
|
||||||
|
{"memory_warning", EventHighMemory},
|
||||||
|
{"memory_critical", EventHighMemory},
|
||||||
|
{"cpu_warning", EventHighCPU},
|
||||||
|
{"cpu_critical", EventHighCPU},
|
||||||
|
{"disk_warning", EventDiskFull},
|
||||||
|
{"disk_critical", EventDiskFull},
|
||||||
|
{"oom", EventOOM},
|
||||||
|
{"restart", EventRestart},
|
||||||
|
{"unresponsive", EventUnresponsive},
|
||||||
|
{"backup_failed", EventBackupFailed},
|
||||||
|
{"unknown_alert", ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
result := mapAlertToEventType(tc.alertType)
|
||||||
|
if result != tc.expected {
|
||||||
|
t.Errorf("mapAlertToEventType(%q) = %q, want %q", tc.alertType, result, tc.expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(s, substr string) bool {
|
||||||
|
return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsHelper(s, substr))
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsHelper(s, substr string) bool {
|
||||||
|
for i := 0; i <= len(s)-len(substr); i++ {
|
||||||
|
if s[i:i+len(substr)] == substr {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
@ -241,6 +241,17 @@ func (s *Service) SetRemediationLog(remLog *RemediationLog) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
|
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
||||||
|
s.mu.RLock()
|
||||||
|
patrol := s.patrolService
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
if patrol != nil {
|
||||||
|
patrol.SetPatternDetector(detector)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// StartPatrol starts the background patrol service
|
// StartPatrol starts the background patrol service
|
||||||
func (s *Service) StartPatrol(ctx context.Context) {
|
func (s *Service) StartPatrol(ctx context.Context) {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
|
|
|
||||||
|
|
@ -115,6 +115,11 @@ func (h *AISettingsHandler) SetRemediationLog(remLog *ai.RemediationLog) {
|
||||||
h.aiService.SetRemediationLog(remLog)
|
h.aiService.SetRemediationLog(remLog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
|
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
|
||||||
|
h.aiService.SetPatternDetector(detector)
|
||||||
|
}
|
||||||
|
|
||||||
// StopPatrol stops the background AI patrol service
|
// StopPatrol stops the background AI patrol service
|
||||||
func (h *AISettingsHandler) StopPatrol() {
|
func (h *AISettingsHandler) StopPatrol() {
|
||||||
h.aiService.StopPatrol()
|
h.aiService.StopPatrol()
|
||||||
|
|
|
||||||
|
|
@ -1459,6 +1459,18 @@ func (r *Router) StartPatrol(ctx context.Context) {
|
||||||
r.aiSettingsHandler.SetRemediationLog(remediationLog)
|
r.aiSettingsHandler.SetRemediationLog(remediationLog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Initialize pattern detector for failure prediction
|
||||||
|
patternDetector := ai.NewPatternDetector(ai.PatternDetectorConfig{
|
||||||
|
MaxEvents: 5000,
|
||||||
|
MinOccurrences: 3,
|
||||||
|
PatternWindow: 90 * 24 * time.Hour,
|
||||||
|
PredictionLimit: 30 * 24 * time.Hour,
|
||||||
|
DataDir: dataDir,
|
||||||
|
})
|
||||||
|
if patternDetector != nil {
|
||||||
|
r.aiSettingsHandler.SetPatternDetector(patternDetector)
|
||||||
|
}
|
||||||
|
|
||||||
r.aiSettingsHandler.StartPatrol(ctx)
|
r.aiSettingsHandler.StartPatrol(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue