feat(ai): Add failure pattern detection for predictive intelligence (Phase 5)
Create internal/ai/patterns package: 1. Pattern Detector (detector.go): - Records historical events (high memory, OOM, restarts, etc.) - Detects recurring failure patterns - Calculates average interval between occurrences - Computes confidence based on pattern consistency - Predicts when failures will occur again - Persists to ai_patterns.json 2. Event types tracked: - high_memory, high_cpu, disk_full - oom, restart, unresponsive - backup_failed 3. Integration: - Wire PatternDetector into router startup - Add to AI context in buildEnrichedContext - FormatForContext generates failure predictions Example AI context now includes: 'OOM events typically occurs every ~10 days (next expected in ~3 days)' This enables proactive alerts before problems recur. All tests passing.
This commit is contained in:
parent
dacdd48e28
commit
d9d798084e
7 changed files with 837 additions and 0 deletions
|
|
@ -214,6 +214,7 @@ type PatrolService struct {
|
|||
baselineStore *baseline.Store // For anomaly detection via learned baselines
|
||||
changeDetector *ChangeDetector // For tracking infrastructure changes
|
||||
remediationLog *RemediationLog // For tracking remediation actions
|
||||
patternDetector *PatternDetector // For failure prediction from historical patterns
|
||||
|
||||
// Cached thresholds (recalculated when thresholdProvider changes)
|
||||
thresholds PatrolThresholds
|
||||
|
|
@ -383,6 +384,21 @@ func (p *PatrolService) GetRemediationLog() *RemediationLog {
|
|||
return p.remediationLog
|
||||
}
|
||||
|
||||
// SetPatternDetector sets the pattern detector for failure prediction
|
||||
func (p *PatrolService) SetPatternDetector(detector *PatternDetector) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.patternDetector = detector
|
||||
log.Info().Msg("AI Patrol: Pattern detector set for failure prediction")
|
||||
}
|
||||
|
||||
// GetPatternDetector returns the pattern detector
|
||||
func (p *PatrolService) GetPatternDetector() *PatternDetector {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.patternDetector
|
||||
}
|
||||
|
||||
// GetConfig returns the current patrol configuration
|
||||
func (p *PatrolService) GetConfig() PatrolConfig {
|
||||
p.mu.RLock()
|
||||
|
|
@ -1782,6 +1798,18 @@ func (p *PatrolService) buildEnrichedContext(state models.StateSnapshot) string
|
|||
log.Debug().Int("new_changes", len(newChanges)).Msg("AI Patrol: Detected infrastructure changes")
|
||||
}
|
||||
}
|
||||
|
||||
// Append failure predictions if pattern detector is available
|
||||
p.mu.RLock()
|
||||
patternDetector := p.patternDetector
|
||||
p.mu.RUnlock()
|
||||
|
||||
if patternDetector != nil {
|
||||
predictionsContext := patternDetector.FormatForContext("")
|
||||
if predictionsContext != "" {
|
||||
formatted += predictionsContext
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Int("resources", infraCtx.TotalResources).
|
||||
|
|
|
|||
44
internal/ai/pattern_exports.go
Normal file
44
internal/ai/pattern_exports.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/patterns"
|
||||
)
|
||||
|
||||
// PatternDetector is an alias for patterns.Detector
|
||||
type PatternDetector = patterns.Detector
|
||||
|
||||
// PatternDetectorConfig is an alias for patterns.DetectorConfig
|
||||
type PatternDetectorConfig = patterns.DetectorConfig
|
||||
|
||||
// HistoricalEvent is an alias for patterns.HistoricalEvent
|
||||
type HistoricalEvent = patterns.HistoricalEvent
|
||||
|
||||
// FailurePrediction is an alias for patterns.FailurePrediction
|
||||
type FailurePrediction = patterns.FailurePrediction
|
||||
|
||||
// EventType is an alias for patterns.EventType
|
||||
type EventType = patterns.EventType
|
||||
|
||||
// Pattern is an alias for patterns.Pattern
|
||||
type Pattern = patterns.Pattern
|
||||
|
||||
// Event type constants
|
||||
const (
|
||||
EventHighMemory = patterns.EventHighMemory
|
||||
EventHighCPU = patterns.EventHighCPU
|
||||
EventDiskFull = patterns.EventDiskFull
|
||||
EventOOM = patterns.EventOOM
|
||||
EventRestart = patterns.EventRestart
|
||||
EventUnresponsive = patterns.EventUnresponsive
|
||||
EventBackupFailed = patterns.EventBackupFailed
|
||||
)
|
||||
|
||||
// NewPatternDetector creates a new pattern detector
|
||||
func NewPatternDetector(cfg PatternDetectorConfig) *PatternDetector {
|
||||
return patterns.NewDetector(cfg)
|
||||
}
|
||||
|
||||
// DefaultPatternConfig returns default pattern detector configuration
|
||||
func DefaultPatternConfig() PatternDetectorConfig {
|
||||
return patterns.DefaultConfig()
|
||||
}
|
||||
531
internal/ai/patterns/detector.go
Normal file
531
internal/ai/patterns/detector.go
Normal file
|
|
@ -0,0 +1,531 @@
|
|||
// Package patterns provides failure pattern detection for predictive intelligence.
|
||||
// It analyzes historical data to identify recurring issues and predict future failures.
|
||||
package patterns
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// EventType represents the type of event being tracked
|
||||
type EventType string
|
||||
|
||||
const (
|
||||
EventHighMemory EventType = "high_memory" // Memory exceeded threshold
|
||||
EventHighCPU EventType = "high_cpu" // CPU exceeded threshold
|
||||
EventDiskFull EventType = "disk_full" // Disk space critical
|
||||
EventOOM EventType = "oom" // Out of memory kill
|
||||
EventRestart EventType = "restart" // Resource restarted
|
||||
EventUnresponsive EventType = "unresponsive" // Resource became unresponsive
|
||||
EventBackupFailed EventType = "backup_failed" // Backup job failed
|
||||
)
|
||||
|
||||
// HistoricalEvent represents a recorded event
|
||||
type HistoricalEvent struct {
|
||||
ID string `json:"id"`
|
||||
ResourceID string `json:"resource_id"`
|
||||
EventType EventType `json:"event_type"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Resolved bool `json:"resolved"`
|
||||
ResolvedAt time.Time `json:"resolved_at,omitempty"`
|
||||
Duration time.Duration `json:"duration,omitempty"` // How long it lasted
|
||||
}
|
||||
|
||||
// Pattern represents a detected recurring pattern
|
||||
type Pattern struct {
|
||||
ResourceID string `json:"resource_id"`
|
||||
EventType EventType `json:"event_type"`
|
||||
Occurrences int `json:"occurrences"` // Number of times event occurred
|
||||
AverageInterval time.Duration `json:"average_interval"` // Average time between occurrences
|
||||
StdDevInterval time.Duration `json:"stddev_interval"` // Standard deviation
|
||||
LastOccurrence time.Time `json:"last_occurrence"`
|
||||
NextPredicted time.Time `json:"next_predicted"` // When we expect it to happen again
|
||||
Confidence float64 `json:"confidence"` // 0-1, based on consistency
|
||||
AverageDuration time.Duration `json:"average_duration,omitempty"` // How long events typically last
|
||||
}
|
||||
|
||||
// FailurePrediction represents a predicted future failure
|
||||
type FailurePrediction struct {
|
||||
ResourceID string `json:"resource_id"`
|
||||
EventType EventType `json:"event_type"`
|
||||
PredictedAt time.Time `json:"predicted_at"`
|
||||
DaysUntil float64 `json:"days_until"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Basis string `json:"basis"` // Human-readable explanation
|
||||
Pattern *Pattern `json:"pattern,omitempty"`
|
||||
}
|
||||
|
||||
// Detector tracks historical events and detects patterns
|
||||
type Detector struct {
|
||||
mu sync.RWMutex
|
||||
events []HistoricalEvent
|
||||
patterns map[string]*Pattern // resourceID:eventType -> pattern
|
||||
|
||||
// Configuration
|
||||
maxEvents int
|
||||
minOccurrences int // Minimum occurrences to form a pattern
|
||||
patternWindow time.Duration // How far back to look for patterns
|
||||
predictionLimit time.Duration // How far ahead to predict
|
||||
|
||||
// Persistence
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// DetectorConfig configures the pattern detector
|
||||
type DetectorConfig struct {
|
||||
MaxEvents int
|
||||
MinOccurrences int // Default: 3
|
||||
PatternWindow time.Duration // Default: 90 days
|
||||
PredictionLimit time.Duration // Default: 30 days
|
||||
DataDir string
|
||||
}
|
||||
|
||||
// DefaultConfig returns default detector configuration
|
||||
func DefaultConfig() DetectorConfig {
|
||||
return DetectorConfig{
|
||||
MaxEvents: 5000,
|
||||
MinOccurrences: 3,
|
||||
PatternWindow: 90 * 24 * time.Hour,
|
||||
PredictionLimit: 30 * 24 * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
// NewDetector creates a new pattern detector
|
||||
func NewDetector(cfg DetectorConfig) *Detector {
|
||||
if cfg.MaxEvents <= 0 {
|
||||
cfg.MaxEvents = 5000
|
||||
}
|
||||
if cfg.MinOccurrences <= 0 {
|
||||
cfg.MinOccurrences = 3
|
||||
}
|
||||
if cfg.PatternWindow <= 0 {
|
||||
cfg.PatternWindow = 90 * 24 * time.Hour
|
||||
}
|
||||
if cfg.PredictionLimit <= 0 {
|
||||
cfg.PredictionLimit = 30 * 24 * time.Hour
|
||||
}
|
||||
|
||||
d := &Detector{
|
||||
events: make([]HistoricalEvent, 0),
|
||||
patterns: make(map[string]*Pattern),
|
||||
maxEvents: cfg.MaxEvents,
|
||||
minOccurrences: cfg.MinOccurrences,
|
||||
patternWindow: cfg.PatternWindow,
|
||||
predictionLimit: cfg.PredictionLimit,
|
||||
dataDir: cfg.DataDir,
|
||||
}
|
||||
|
||||
// Load existing data
|
||||
if cfg.DataDir != "" {
|
||||
if err := d.loadFromDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to load pattern history from disk")
|
||||
} else if len(d.events) > 0 {
|
||||
log.Info().Int("events", len(d.events)).Int("patterns", len(d.patterns)).
|
||||
Msg("Loaded pattern history from disk")
|
||||
}
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
// RecordEvent records a new event for pattern analysis
|
||||
func (d *Detector) RecordEvent(event HistoricalEvent) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
if event.ID == "" {
|
||||
event.ID = generateEventID()
|
||||
}
|
||||
if event.Timestamp.IsZero() {
|
||||
event.Timestamp = time.Now()
|
||||
}
|
||||
|
||||
d.events = append(d.events, event)
|
||||
d.trimEvents()
|
||||
|
||||
// Recompute pattern for this resource/event type
|
||||
key := patternKey(event.ResourceID, event.EventType)
|
||||
d.patterns[key] = d.computePattern(event.ResourceID, event.EventType)
|
||||
|
||||
// Persist asynchronously
|
||||
go func() {
|
||||
if err := d.saveToDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to save pattern history")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// RecordFromAlert records an event from an alert
|
||||
func (d *Detector) RecordFromAlert(resourceID string, alertType string, timestamp time.Time) {
|
||||
eventType := mapAlertToEventType(alertType)
|
||||
if eventType == "" {
|
||||
return // Not a trackable event type
|
||||
}
|
||||
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: resourceID,
|
||||
EventType: eventType,
|
||||
Timestamp: timestamp,
|
||||
Description: alertType,
|
||||
})
|
||||
}
|
||||
|
||||
// GetPredictions returns failure predictions for all tracked resources
|
||||
func (d *Detector) GetPredictions() []FailurePrediction {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
var predictions []FailurePrediction
|
||||
now := time.Now()
|
||||
|
||||
for _, pattern := range d.patterns {
|
||||
// Only predict if pattern has sufficient confidence
|
||||
if pattern.Confidence < 0.3 || pattern.Occurrences < d.minOccurrences {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if prediction is within our limit
|
||||
if pattern.NextPredicted.Before(now) || pattern.NextPredicted.After(now.Add(d.predictionLimit)) {
|
||||
continue
|
||||
}
|
||||
|
||||
daysUntil := pattern.NextPredicted.Sub(now).Hours() / 24
|
||||
|
||||
predictions = append(predictions, FailurePrediction{
|
||||
ResourceID: pattern.ResourceID,
|
||||
EventType: pattern.EventType,
|
||||
PredictedAt: pattern.NextPredicted,
|
||||
DaysUntil: daysUntil,
|
||||
Confidence: pattern.Confidence,
|
||||
Basis: formatPatternBasis(pattern),
|
||||
Pattern: pattern,
|
||||
})
|
||||
}
|
||||
|
||||
// Sort by days until (soonest first)
|
||||
sort.Slice(predictions, func(i, j int) bool {
|
||||
return predictions[i].DaysUntil < predictions[j].DaysUntil
|
||||
})
|
||||
|
||||
return predictions
|
||||
}
|
||||
|
||||
// GetPredictionsForResource returns failure predictions for a specific resource
|
||||
func (d *Detector) GetPredictionsForResource(resourceID string) []FailurePrediction {
|
||||
all := d.GetPredictions()
|
||||
var result []FailurePrediction
|
||||
for _, p := range all {
|
||||
if p.ResourceID == resourceID {
|
||||
result = append(result, p)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetPatterns returns all detected patterns
|
||||
func (d *Detector) GetPatterns() map[string]*Pattern {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
result := make(map[string]*Pattern)
|
||||
for k, v := range d.patterns {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// computePattern analyzes events to find patterns for a resource/event type
|
||||
func (d *Detector) computePattern(resourceID string, eventType EventType) *Pattern {
|
||||
cutoff := time.Now().Add(-d.patternWindow)
|
||||
|
||||
// Get all events for this resource/type within the window
|
||||
var events []HistoricalEvent
|
||||
for _, e := range d.events {
|
||||
if e.ResourceID == resourceID && e.EventType == eventType && e.Timestamp.After(cutoff) {
|
||||
events = append(events, e)
|
||||
}
|
||||
}
|
||||
|
||||
if len(events) < d.minOccurrences {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort by timestamp
|
||||
sort.Slice(events, func(i, j int) bool {
|
||||
return events[i].Timestamp.Before(events[j].Timestamp)
|
||||
})
|
||||
|
||||
// Calculate intervals between events
|
||||
var intervals []time.Duration
|
||||
var durations []time.Duration
|
||||
|
||||
for i := 1; i < len(events); i++ {
|
||||
interval := events[i].Timestamp.Sub(events[i-1].Timestamp)
|
||||
intervals = append(intervals, interval)
|
||||
|
||||
if events[i-1].Duration > 0 {
|
||||
durations = append(durations, events[i-1].Duration)
|
||||
}
|
||||
}
|
||||
|
||||
if len(intervals) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate average and stddev of intervals
|
||||
avgInterval := averageDuration(intervals)
|
||||
stddevInterval := stddevDuration(intervals, avgInterval)
|
||||
|
||||
// Calculate confidence based on consistency
|
||||
// If stddev is low relative to mean, pattern is more reliable
|
||||
consistency := 1.0
|
||||
if avgInterval > 0 {
|
||||
cv := float64(stddevInterval) / float64(avgInterval) // Coefficient of variation
|
||||
consistency = 1.0 - math.Min(cv, 1.0) // Higher consistency = lower CV
|
||||
}
|
||||
|
||||
// Adjust confidence based on number of occurrences
|
||||
occurrenceBonus := math.Min(float64(len(events))/10.0, 0.3)
|
||||
confidence := consistency*0.7 + occurrenceBonus
|
||||
|
||||
// Predict next occurrence
|
||||
lastEvent := events[len(events)-1]
|
||||
nextPredicted := lastEvent.Timestamp.Add(avgInterval)
|
||||
|
||||
// Calculate average duration if available
|
||||
var avgDuration time.Duration
|
||||
if len(durations) > 0 {
|
||||
avgDuration = averageDuration(durations)
|
||||
}
|
||||
|
||||
return &Pattern{
|
||||
ResourceID: resourceID,
|
||||
EventType: eventType,
|
||||
Occurrences: len(events),
|
||||
AverageInterval: avgInterval,
|
||||
StdDevInterval: stddevInterval,
|
||||
LastOccurrence: lastEvent.Timestamp,
|
||||
NextPredicted: nextPredicted,
|
||||
Confidence: confidence,
|
||||
AverageDuration: avgDuration,
|
||||
}
|
||||
}
|
||||
|
||||
// trimEvents removes old events beyond maxEvents
|
||||
func (d *Detector) trimEvents() {
|
||||
if len(d.events) > d.maxEvents {
|
||||
d.events = d.events[len(d.events)-d.maxEvents:]
|
||||
}
|
||||
}
|
||||
|
||||
// saveToDisk persists events and patterns
|
||||
func (d *Detector) saveToDisk() error {
|
||||
if d.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.mu.RLock()
|
||||
data := struct {
|
||||
Events []HistoricalEvent `json:"events"`
|
||||
Patterns map[string]*Pattern `json:"patterns"`
|
||||
}{
|
||||
Events: d.events,
|
||||
Patterns: d.patterns,
|
||||
}
|
||||
d.mu.RUnlock()
|
||||
|
||||
jsonData, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
path := filepath.Join(d.dataDir, "ai_patterns.json")
|
||||
tmpPath := path + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, jsonData, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(tmpPath, path)
|
||||
}
|
||||
|
||||
// loadFromDisk loads events and patterns
|
||||
func (d *Detector) loadFromDisk() error {
|
||||
if d.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
path := filepath.Join(d.dataDir, "ai_patterns.json")
|
||||
jsonData, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var data struct {
|
||||
Events []HistoricalEvent `json:"events"`
|
||||
Patterns map[string]*Pattern `json:"patterns"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(jsonData, &data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.events = data.Events
|
||||
d.patterns = data.Patterns
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FormatForContext formats predictions for AI consumption
|
||||
func (d *Detector) FormatForContext(resourceID string) string {
|
||||
var predictions []FailurePrediction
|
||||
if resourceID != "" {
|
||||
predictions = d.GetPredictionsForResource(resourceID)
|
||||
} else {
|
||||
predictions = d.GetPredictions()
|
||||
}
|
||||
|
||||
if len(predictions) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var result string
|
||||
result = "\n## ⏰ Failure Predictions\n"
|
||||
result += "Based on historical patterns:\n"
|
||||
|
||||
for _, p := range predictions {
|
||||
if len(result) > 2000 { // Limit context size
|
||||
result += "\n... and more\n"
|
||||
break
|
||||
}
|
||||
result += "- " + p.Basis + "\n"
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
var eventCounter int64
|
||||
|
||||
func generateEventID() string {
|
||||
eventCounter++
|
||||
return time.Now().Format("20060102150405") + "-" + intToStr(int(eventCounter%1000))
|
||||
}
|
||||
|
||||
func intToStr(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var result string
|
||||
for n > 0 {
|
||||
result = string(rune('0'+n%10)) + result
|
||||
n /= 10
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func patternKey(resourceID string, eventType EventType) string {
|
||||
return resourceID + ":" + string(eventType)
|
||||
}
|
||||
|
||||
func mapAlertToEventType(alertType string) EventType {
|
||||
switch alertType {
|
||||
case "memory_warning", "memory_critical":
|
||||
return EventHighMemory
|
||||
case "cpu_warning", "cpu_critical":
|
||||
return EventHighCPU
|
||||
case "disk_warning", "disk_critical":
|
||||
return EventDiskFull
|
||||
case "oom", "out_of_memory":
|
||||
return EventOOM
|
||||
case "restart", "restarted":
|
||||
return EventRestart
|
||||
case "unresponsive", "unreachable":
|
||||
return EventUnresponsive
|
||||
case "backup_failed":
|
||||
return EventBackupFailed
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func averageDuration(durations []time.Duration) time.Duration {
|
||||
if len(durations) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum int64
|
||||
for _, d := range durations {
|
||||
sum += int64(d)
|
||||
}
|
||||
return time.Duration(sum / int64(len(durations)))
|
||||
}
|
||||
|
||||
func stddevDuration(durations []time.Duration, mean time.Duration) time.Duration {
|
||||
if len(durations) < 2 {
|
||||
return 0
|
||||
}
|
||||
var sumSquares float64
|
||||
for _, d := range durations {
|
||||
diff := float64(d - mean)
|
||||
sumSquares += diff * diff
|
||||
}
|
||||
variance := sumSquares / float64(len(durations)-1)
|
||||
return time.Duration(math.Sqrt(variance))
|
||||
}
|
||||
|
||||
func formatPatternBasis(p *Pattern) string {
|
||||
daysInterval := p.AverageInterval.Hours() / 24
|
||||
daysSinceLast := time.Since(p.LastOccurrence).Hours() / 24
|
||||
daysUntilNext := p.NextPredicted.Sub(time.Now()).Hours() / 24
|
||||
|
||||
eventName := string(p.EventType)
|
||||
switch p.EventType {
|
||||
case EventHighMemory:
|
||||
eventName = "high memory usage"
|
||||
case EventHighCPU:
|
||||
eventName = "high CPU usage"
|
||||
case EventDiskFull:
|
||||
eventName = "disk space critical"
|
||||
case EventOOM:
|
||||
eventName = "OOM events"
|
||||
case EventRestart:
|
||||
eventName = "restarts"
|
||||
case EventUnresponsive:
|
||||
eventName = "unresponsive periods"
|
||||
case EventBackupFailed:
|
||||
eventName = "backup failures"
|
||||
}
|
||||
|
||||
if daysUntilNext < 0 {
|
||||
return eventName + " typically occurs every ~" + formatDays(daysInterval) +
|
||||
" (last: " + formatDays(daysSinceLast) + " ago, overdue)"
|
||||
}
|
||||
|
||||
return eventName + " typically occurs every ~" + formatDays(daysInterval) +
|
||||
" (next expected in ~" + formatDays(daysUntilNext) + ")"
|
||||
}
|
||||
|
||||
func formatDays(days float64) string {
|
||||
if days < 1 {
|
||||
hours := days * 24
|
||||
if hours < 1 {
|
||||
return "less than an hour"
|
||||
}
|
||||
return intToStr(int(hours)) + " hours"
|
||||
}
|
||||
if days < 2 {
|
||||
return "1 day"
|
||||
}
|
||||
return intToStr(int(days)) + " days"
|
||||
}
|
||||
206
internal/ai/patterns/detector_test.go
Normal file
206
internal/ai/patterns/detector_test.go
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
package patterns
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDetector_RecordEvent(t *testing.T) {
|
||||
d := NewDetector(DetectorConfig{MinOccurrences: 2})
|
||||
|
||||
// Record first event
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "vm-100",
|
||||
EventType: EventHighMemory,
|
||||
Timestamp: time.Now().Add(-10 * 24 * time.Hour),
|
||||
})
|
||||
|
||||
if len(d.events) != 1 {
|
||||
t.Errorf("Expected 1 event, got %d", len(d.events))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_PatternDetection(t *testing.T) {
|
||||
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||
|
||||
// Record events with 10-day interval
|
||||
now := time.Now()
|
||||
for i := 5; i >= 0; i-- {
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "vm-100",
|
||||
EventType: EventHighMemory,
|
||||
Timestamp: now.Add(-time.Duration(i*10) * 24 * time.Hour),
|
||||
})
|
||||
}
|
||||
|
||||
// Check that pattern was detected
|
||||
patterns := d.GetPatterns()
|
||||
key := patternKey("vm-100", EventHighMemory)
|
||||
pattern, ok := patterns[key]
|
||||
if !ok {
|
||||
t.Fatal("Expected pattern to be detected")
|
||||
}
|
||||
|
||||
if pattern.Occurrences != 6 {
|
||||
t.Errorf("Expected 6 occurrences, got %d", pattern.Occurrences)
|
||||
}
|
||||
|
||||
// Average interval should be ~10 days
|
||||
avgDays := pattern.AverageInterval.Hours() / 24
|
||||
if avgDays < 9 || avgDays > 11 {
|
||||
t.Errorf("Expected ~10 day interval, got %.1f days", avgDays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_GetPredictions(t *testing.T) {
|
||||
d := NewDetector(DetectorConfig{
|
||||
MinOccurrences: 3,
|
||||
PatternWindow: 365 * 24 * time.Hour,
|
||||
PredictionLimit: 30 * 24 * time.Hour,
|
||||
})
|
||||
|
||||
// Record events with regular interval
|
||||
now := time.Now()
|
||||
for i := 3; i >= 0; i-- {
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "vm-100",
|
||||
EventType: EventOOM,
|
||||
Timestamp: now.Add(-time.Duration(i*7) * 24 * time.Hour), // 7-day interval
|
||||
})
|
||||
}
|
||||
|
||||
predictions := d.GetPredictions()
|
||||
|
||||
// Should have a prediction for OOM
|
||||
found := false
|
||||
for _, p := range predictions {
|
||||
if p.ResourceID == "vm-100" && p.EventType == EventOOM {
|
||||
found = true
|
||||
// Should predict in ~7 days
|
||||
if p.DaysUntil < 5 || p.DaysUntil > 9 {
|
||||
t.Errorf("Expected prediction in ~7 days, got %.1f days", p.DaysUntil)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Error("Expected OOM prediction for vm-100")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_GetPredictionsForResource(t *testing.T) {
|
||||
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||
|
||||
now := time.Now()
|
||||
// Add pattern for vm-100
|
||||
for i := 3; i >= 0; i-- {
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "vm-100",
|
||||
EventType: EventRestart,
|
||||
Timestamp: now.Add(-time.Duration(i*14) * 24 * time.Hour),
|
||||
})
|
||||
}
|
||||
// Add pattern for vm-200
|
||||
for i := 3; i >= 0; i-- {
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "vm-200",
|
||||
EventType: EventHighCPU,
|
||||
Timestamp: now.Add(-time.Duration(i*5) * 24 * time.Hour),
|
||||
})
|
||||
}
|
||||
|
||||
// Get predictions for vm-100 only
|
||||
predictions := d.GetPredictionsForResource("vm-100")
|
||||
for _, p := range predictions {
|
||||
if p.ResourceID != "vm-100" {
|
||||
t.Errorf("Got prediction for wrong resource: %s", p.ResourceID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_Confidence(t *testing.T) {
|
||||
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||
|
||||
now := time.Now()
|
||||
// Add very consistent pattern (every 7 days exactly)
|
||||
for i := 5; i >= 0; i-- {
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "consistent-vm",
|
||||
EventType: EventHighMemory,
|
||||
Timestamp: now.Add(-time.Duration(i*7*24) * time.Hour),
|
||||
})
|
||||
}
|
||||
|
||||
patterns := d.GetPatterns()
|
||||
pattern := patterns[patternKey("consistent-vm", EventHighMemory)]
|
||||
if pattern == nil {
|
||||
t.Fatal("Expected pattern")
|
||||
}
|
||||
|
||||
// Consistent pattern should have high confidence
|
||||
if pattern.Confidence < 0.5 {
|
||||
t.Errorf("Expected high confidence for consistent pattern, got %.2f", pattern.Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetector_FormatForContext(t *testing.T) {
|
||||
d := NewDetector(DetectorConfig{MinOccurrences: 3, PatternWindow: 365 * 24 * time.Hour})
|
||||
|
||||
now := time.Now()
|
||||
for i := 3; i >= 0; i-- {
|
||||
d.RecordEvent(HistoricalEvent{
|
||||
ResourceID: "vm-100",
|
||||
EventType: EventOOM,
|
||||
Timestamp: now.Add(-time.Duration(i*10) * 24 * time.Hour),
|
||||
})
|
||||
}
|
||||
|
||||
context := d.FormatForContext("vm-100")
|
||||
if context == "" {
|
||||
t.Error("Expected non-empty context")
|
||||
}
|
||||
|
||||
if !contains(context, "OOM") && !contains(context, "oom") {
|
||||
t.Errorf("Expected context to mention OOM: %s", context)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapAlertToEventType(t *testing.T) {
|
||||
tests := []struct {
|
||||
alertType string
|
||||
expected EventType
|
||||
}{
|
||||
{"memory_warning", EventHighMemory},
|
||||
{"memory_critical", EventHighMemory},
|
||||
{"cpu_warning", EventHighCPU},
|
||||
{"cpu_critical", EventHighCPU},
|
||||
{"disk_warning", EventDiskFull},
|
||||
{"disk_critical", EventDiskFull},
|
||||
{"oom", EventOOM},
|
||||
{"restart", EventRestart},
|
||||
{"unresponsive", EventUnresponsive},
|
||||
{"backup_failed", EventBackupFailed},
|
||||
{"unknown_alert", ""},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
result := mapAlertToEventType(tc.alertType)
|
||||
if result != tc.expected {
|
||||
t.Errorf("mapAlertToEventType(%q) = %q, want %q", tc.alertType, result, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsHelper(s, substr))
|
||||
}
|
||||
|
||||
func containsHelper(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -241,6 +241,17 @@ func (s *Service) SetRemediationLog(remLog *RemediationLog) {
|
|||
}
|
||||
}
|
||||
|
||||
// SetPatternDetector sets the pattern detector for failure prediction
|
||||
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
||||
s.mu.RLock()
|
||||
patrol := s.patrolService
|
||||
s.mu.RUnlock()
|
||||
|
||||
if patrol != nil {
|
||||
patrol.SetPatternDetector(detector)
|
||||
}
|
||||
}
|
||||
|
||||
// StartPatrol starts the background patrol service
|
||||
func (s *Service) StartPatrol(ctx context.Context) {
|
||||
s.mu.RLock()
|
||||
|
|
|
|||
|
|
@ -115,6 +115,11 @@ func (h *AISettingsHandler) SetRemediationLog(remLog *ai.RemediationLog) {
|
|||
h.aiService.SetRemediationLog(remLog)
|
||||
}
|
||||
|
||||
// SetPatternDetector sets the pattern detector for failure prediction
|
||||
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
|
||||
h.aiService.SetPatternDetector(detector)
|
||||
}
|
||||
|
||||
// StopPatrol stops the background AI patrol service
|
||||
func (h *AISettingsHandler) StopPatrol() {
|
||||
h.aiService.StopPatrol()
|
||||
|
|
|
|||
|
|
@ -1458,6 +1458,18 @@ func (r *Router) StartPatrol(ctx context.Context) {
|
|||
if remediationLog != nil {
|
||||
r.aiSettingsHandler.SetRemediationLog(remediationLog)
|
||||
}
|
||||
|
||||
// Initialize pattern detector for failure prediction
|
||||
patternDetector := ai.NewPatternDetector(ai.PatternDetectorConfig{
|
||||
MaxEvents: 5000,
|
||||
MinOccurrences: 3,
|
||||
PatternWindow: 90 * 24 * time.Hour,
|
||||
PredictionLimit: 30 * 24 * time.Hour,
|
||||
DataDir: dataDir,
|
||||
})
|
||||
if patternDetector != nil {
|
||||
r.aiSettingsHandler.SetPatternDetector(patternDetector)
|
||||
}
|
||||
|
||||
r.aiSettingsHandler.StartPatrol(ctx)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue