Pulse/internal/ai/correlation/detector_test.go
rcourtman f6d31d166a feat(ai): Add multi-resource correlation detection (Phase 6)
Create internal/ai/correlation package:

1. Correlation Detector (detector.go):
   - Tracks events across resources
   - Detects when events on one resource follow events on another
   - Calculates average delay between correlated events
   - Confidence scoring based on occurrence count
   - Persists to ai_correlations.json

2. Features:
   - GetCorrelations() - All detected relationships
   - GetCorrelationsForResource() - Relationships for one resource
   - GetDependencies() - What resources depend on this one
   - GetDependsOn() - What this resource depends on
   - PredictCascade() - Predict what will be affected
   - FormatForContext() - AI-consumable summary

3. Integration:
   - Wire to alert history in router startup
   - Map alert types to correlation event types
   - Add correlation context to enriched AI context

Example AI context now includes:
'When local-zfs experiences high usage, database often follows within 5 minutes'

This enables the AI to understand infrastructure dependencies
and predict cascade failures.

All tests passing.
2025-12-12 14:26:10 +00:00

246 lines
5.5 KiB
Go

package correlation
import (
"testing"
"time"
)
func TestDetector_RecordEvent(t *testing.T) {
d := NewDetector(DefaultConfig())
d.RecordEvent(Event{
ResourceID: "vm-100",
ResourceName: "web-server",
EventType: EventHighMem,
Timestamp: time.Now(),
})
if len(d.events) != 1 {
t.Errorf("Expected 1 event, got %d", len(d.events))
}
}
func TestDetector_CorrelationDetection(t *testing.T) {
d := NewDetector(Config{
MaxEvents: 1000,
CorrelationWindow: 10 * time.Minute,
MinOccurrences: 2, // Lower threshold for testing
RetentionWindow: 30 * 24 * time.Hour,
})
now := time.Now()
// Simulate pattern: when storage has high usage, database VM restarts
// Occurrence 1
d.RecordEvent(Event{
ResourceID: "storage-1",
ResourceName: "local-zfs",
ResourceType: "storage",
EventType: EventDiskFull,
Timestamp: now.Add(-2 * time.Hour),
})
d.RecordEvent(Event{
ResourceID: "vm-100",
ResourceName: "database",
ResourceType: "vm",
EventType: EventRestart,
Timestamp: now.Add(-2*time.Hour + 5*time.Minute),
})
// Occurrence 2
d.RecordEvent(Event{
ResourceID: "storage-1",
ResourceName: "local-zfs",
ResourceType: "storage",
EventType: EventDiskFull,
Timestamp: now.Add(-1 * time.Hour),
})
d.RecordEvent(Event{
ResourceID: "vm-100",
ResourceName: "database",
ResourceType: "vm",
EventType: EventRestart,
Timestamp: now.Add(-1*time.Hour + 5*time.Minute),
})
// Occurrence 3
d.RecordEvent(Event{
ResourceID: "storage-1",
ResourceName: "local-zfs",
ResourceType: "storage",
EventType: EventDiskFull,
Timestamp: now,
})
d.RecordEvent(Event{
ResourceID: "vm-100",
ResourceName: "database",
ResourceType: "vm",
EventType: EventRestart,
Timestamp: now.Add(5 * time.Minute),
})
// Check correlations
correlations := d.GetCorrelations()
found := false
for _, c := range correlations {
if c.SourceID == "storage-1" && c.TargetID == "vm-100" {
found = true
if c.Occurrences < 2 {
t.Errorf("Expected at least 2 occurrences, got %d", c.Occurrences)
}
break
}
}
if !found {
t.Error("Expected correlation between storage-1 and vm-100")
}
}
func TestDetector_GetCorrelationsForResource(t *testing.T) {
d := NewDetector(Config{
MaxEvents: 1000,
CorrelationWindow: 10 * time.Minute,
MinOccurrences: 2,
RetentionWindow: 30 * 24 * time.Hour,
})
now := time.Now()
// Create correlation pattern
for i := 0; i < 3; i++ {
offset := time.Duration(-i) * time.Hour
d.RecordEvent(Event{
ResourceID: "node-1",
EventType: EventHighCPU,
Timestamp: now.Add(offset),
})
d.RecordEvent(Event{
ResourceID: "vm-100",
EventType: EventHighMem,
Timestamp: now.Add(offset + 2*time.Minute),
})
}
// Get correlations for node-1
correlations := d.GetCorrelationsForResource("node-1")
if len(correlations) == 0 {
t.Error("Expected correlations for node-1")
}
}
func TestDetector_GetDependencies(t *testing.T) {
d := NewDetector(Config{
MaxEvents: 1000,
CorrelationWindow: 10 * time.Minute,
MinOccurrences: 3,
RetentionWindow: 30 * 24 * time.Hour,
})
now := time.Now()
// When storage is full, both VM and container have issues
for i := 0; i < 4; i++ {
offset := time.Duration(-i) * time.Hour
d.RecordEvent(Event{
ResourceID: "storage-1",
EventType: EventDiskFull,
Timestamp: now.Add(offset),
})
d.RecordEvent(Event{
ResourceID: "vm-100",
EventType: EventRestart,
Timestamp: now.Add(offset + 3*time.Minute),
})
d.RecordEvent(Event{
ResourceID: "ct-200",
EventType: EventRestart,
Timestamp: now.Add(offset + 5*time.Minute),
})
}
deps := d.GetDependencies("storage-1")
if len(deps) < 2 {
t.Errorf("Expected at least 2 dependencies, got %d", len(deps))
}
}
func TestDetector_PredictCascade(t *testing.T) {
d := NewDetector(Config{
MaxEvents: 1000,
CorrelationWindow: 10 * time.Minute,
MinOccurrences: 3,
RetentionWindow: 30 * 24 * time.Hour,
})
now := time.Now()
// Create strong correlation
for i := 0; i < 5; i++ {
offset := time.Duration(-i) * time.Hour
d.RecordEvent(Event{
ResourceID: "node-1",
ResourceName: "pve-main",
EventType: EventHighMem,
Timestamp: now.Add(offset),
})
d.RecordEvent(Event{
ResourceID: "vm-100",
ResourceName: "database",
EventType: EventRestart,
Timestamp: now.Add(offset + 5*time.Minute),
})
}
predictions := d.PredictCascade("node-1", EventHighMem)
if len(predictions) == 0 {
t.Error("Expected cascade predictions")
}
}
func TestDetector_FormatForContext(t *testing.T) {
d := NewDetector(Config{
MaxEvents: 1000,
CorrelationWindow: 10 * time.Minute,
MinOccurrences: 2,
RetentionWindow: 30 * 24 * time.Hour,
})
now := time.Now()
// Create correlations
for i := 0; i < 3; i++ {
offset := time.Duration(-i) * time.Hour
d.RecordEvent(Event{
ResourceID: "storage-1",
ResourceName: "local-zfs",
EventType: EventDiskFull,
Timestamp: now.Add(offset),
})
d.RecordEvent(Event{
ResourceID: "vm-100",
ResourceName: "database",
EventType: EventRestart,
Timestamp: now.Add(offset + 5*time.Minute),
})
}
context := d.FormatForContext("")
if context == "" {
t.Error("Expected non-empty context")
}
if !contains(context, "Correlation") {
t.Errorf("Expected context to mention correlations: %s", context)
}
}
func contains(s, substr string) bool {
for i := 0; i <= len(s)-len(substr); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}