feat(ai): Add baseline learning and anomaly detection (Phase 2)
Phase 2 of Pulse AI differentiation: - Create internal/ai/baseline package for learned baselines - Implement statistical baseline learning with mean, stddev, percentiles - Add z-score based anomaly detection with severity classification (low, medium, high, critical based on standard deviations) - Integrate baseline provider into context builder - Wire baseline store into patrol service with adapters - Add anomaly enrichment to resource contexts Key features: - Learn computes baseline from historical metric data points - IsAnomaly and CheckAnomaly detect deviations from normal - Persists baselines to disk as JSON for durability - Formatted anomaly descriptions for AI consumption Example: 'Memory is high above normal (85.2% vs typical 42.1% ± 8.3%)' The baseline store needs to be initialized and triggered to learn from metrics history. Next step is adding the learning loop. All tests passing.
This commit is contained in:
rcourtman
parent
64dad395a2
commit
f3e95c24ae
7 changed files with 793 additions and 9 deletions
|
|
@ -38,7 +38,11 @@ export const AICostDashboard: Component = () => {
|
|||
|
||||
const loadSummary = async (rangeDays: number) => {
|
||||
const seq = ++requestSeq;
|
||||
setLoading(true);
|
||||
const isInitialLoad = summary() === null;
|
||||
// Only show loading indicator on initial load to prevent flicker on range changes
|
||||
if (isInitialLoad) {
|
||||
setLoading(true);
|
||||
}
|
||||
setLoadError(null);
|
||||
try {
|
||||
const data = await AIAPI.getCostSummary(rangeDays);
|
||||
|
|
@ -47,7 +51,10 @@ export const AICostDashboard: Component = () => {
|
|||
} catch (err) {
|
||||
if (seq !== requestSeq) return;
|
||||
logger.error('[AICostDashboard] Failed to load cost summary:', err);
|
||||
notificationStore.error('Failed to load AI cost summary');
|
||||
// Only show notification on refresh failures, not initial load
|
||||
if (!isInitialLoad) {
|
||||
notificationStore.error('Failed to refresh AI cost summary');
|
||||
}
|
||||
const message =
|
||||
err instanceof Error && err.message ? err.message : 'Failed to load usage data';
|
||||
setLoadError(message);
|
||||
|
|
|
|||
395
internal/ai/baseline/store.go
Normal file
395
internal/ai/baseline/store.go
Normal file
|
|
@ -0,0 +1,395 @@
|
|||
// Package baseline provides learned baseline metrics for anomaly detection.
|
||||
// It learns what "normal" looks like for each resource by analyzing historical
|
||||
// metrics and can then flag current values that deviate significantly from the baseline.
|
||||
package baseline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// MetricBaseline represents learned "normal" behavior for a single metric
|
||||
type MetricBaseline struct {
|
||||
Mean float64 `json:"mean"` // Average value
|
||||
StdDev float64 `json:"stddev"` // Standard deviation
|
||||
Percentiles map[int]float64 `json:"percentiles"` // 5, 25, 50, 75, 95
|
||||
SampleCount int `json:"sample_count"` // Number of samples used
|
||||
|
||||
// Time-of-day patterns (future enhancement)
|
||||
HourlyMeans [24]float64 `json:"hourly_means,omitempty"`
|
||||
}
|
||||
|
||||
// ResourceBaseline contains baselines for all metrics of a resource
|
||||
type ResourceBaseline struct {
|
||||
ResourceID string `json:"resource_id"`
|
||||
ResourceType string `json:"resource_type"` // node, vm, container, storage
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
Metrics map[string]*MetricBaseline `json:"metrics"` // cpu, memory, disk
|
||||
}
|
||||
|
||||
// Store manages baseline storage and learning
|
||||
type Store struct {
|
||||
mu sync.RWMutex
|
||||
baselines map[string]*ResourceBaseline // resourceID -> baseline
|
||||
|
||||
// Configuration
|
||||
learningWindow time.Duration // How far back to learn from (default: 7 days)
|
||||
minSamples int // Minimum samples needed (default: 50)
|
||||
updateInterval time.Duration // How often to recompute (default: 1 hour)
|
||||
|
||||
// Persistence
|
||||
dataDir string
|
||||
persistence Persistence
|
||||
}
|
||||
|
||||
// Persistence interface for saving/loading baselines
|
||||
type Persistence interface {
|
||||
Save(baselines map[string]*ResourceBaseline) error
|
||||
Load() (map[string]*ResourceBaseline, error)
|
||||
}
|
||||
|
||||
// StoreConfig configures the baseline store
|
||||
type StoreConfig struct {
|
||||
LearningWindow time.Duration
|
||||
MinSamples int
|
||||
UpdateInterval time.Duration
|
||||
DataDir string
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults
|
||||
func DefaultConfig() StoreConfig {
|
||||
return StoreConfig{
|
||||
LearningWindow: 7 * 24 * time.Hour, // 7 days
|
||||
MinSamples: 50,
|
||||
UpdateInterval: 1 * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
// NewStore creates a new baseline store
|
||||
func NewStore(cfg StoreConfig) *Store {
|
||||
if cfg.LearningWindow == 0 {
|
||||
cfg.LearningWindow = 7 * 24 * time.Hour
|
||||
}
|
||||
if cfg.MinSamples == 0 {
|
||||
cfg.MinSamples = 50
|
||||
}
|
||||
if cfg.UpdateInterval == 0 {
|
||||
cfg.UpdateInterval = 1 * time.Hour
|
||||
}
|
||||
|
||||
s := &Store{
|
||||
baselines: make(map[string]*ResourceBaseline),
|
||||
learningWindow: cfg.LearningWindow,
|
||||
minSamples: cfg.MinSamples,
|
||||
updateInterval: cfg.UpdateInterval,
|
||||
dataDir: cfg.DataDir,
|
||||
}
|
||||
|
||||
// Try to load existing baselines from disk
|
||||
if cfg.DataDir != "" {
|
||||
if err := s.loadFromDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to load baselines from disk, starting fresh")
|
||||
} else {
|
||||
log.Info().Int("count", len(s.baselines)).Msg("Loaded baselines from disk")
|
||||
}
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// MetricPoint represents a single metric value at a point in time
|
||||
type MetricPoint struct {
|
||||
Value float64
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// Learn computes baseline from historical data points
|
||||
func (s *Store) Learn(resourceID, resourceType, metric string, points []MetricPoint) error {
|
||||
if len(points) < s.minSamples {
|
||||
log.Debug().
|
||||
Str("resource", resourceID).
|
||||
Str("metric", metric).
|
||||
Int("samples", len(points)).
|
||||
Int("required", s.minSamples).
|
||||
Msg("Insufficient data for baseline learning")
|
||||
return nil // Not an error, just not enough data yet
|
||||
}
|
||||
|
||||
// Extract values
|
||||
values := make([]float64, len(points))
|
||||
for i, p := range points {
|
||||
values[i] = p.Value
|
||||
}
|
||||
|
||||
// Compute statistics
|
||||
baseline := &MetricBaseline{
|
||||
Mean: computeMean(values),
|
||||
StdDev: computeStdDev(values),
|
||||
Percentiles: computePercentiles(values),
|
||||
SampleCount: len(values),
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Get or create resource baseline
|
||||
rb, exists := s.baselines[resourceID]
|
||||
if !exists {
|
||||
rb = &ResourceBaseline{
|
||||
ResourceID: resourceID,
|
||||
ResourceType: resourceType,
|
||||
Metrics: make(map[string]*MetricBaseline),
|
||||
}
|
||||
s.baselines[resourceID] = rb
|
||||
}
|
||||
|
||||
rb.Metrics[metric] = baseline
|
||||
rb.LastUpdated = time.Now()
|
||||
|
||||
log.Debug().
|
||||
Str("resource", resourceID).
|
||||
Str("metric", metric).
|
||||
Float64("mean", baseline.Mean).
|
||||
Float64("stddev", baseline.StdDev).
|
||||
Int("samples", baseline.SampleCount).
|
||||
Msg("Baseline learned")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBaseline returns the baseline for a resource/metric
|
||||
func (s *Store) GetBaseline(resourceID, metric string) (*MetricBaseline, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
rb, exists := s.baselines[resourceID]
|
||||
if !exists {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
mb, exists := rb.Metrics[metric]
|
||||
return mb, exists
|
||||
}
|
||||
|
||||
// GetResourceBaseline returns all baselines for a resource
|
||||
func (s *Store) GetResourceBaseline(resourceID string) (*ResourceBaseline, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
rb, exists := s.baselines[resourceID]
|
||||
if !exists {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Return a copy to prevent mutation
|
||||
copy := &ResourceBaseline{
|
||||
ResourceID: rb.ResourceID,
|
||||
ResourceType: rb.ResourceType,
|
||||
LastUpdated: rb.LastUpdated,
|
||||
Metrics: make(map[string]*MetricBaseline),
|
||||
}
|
||||
for k, v := range rb.Metrics {
|
||||
copy.Metrics[k] = v
|
||||
}
|
||||
return copy, true
|
||||
}
|
||||
|
||||
// IsAnomaly checks if a value is anomalous for the given resource/metric
|
||||
// Returns: isAnomaly, zScore (number of standard deviations from mean)
|
||||
func (s *Store) IsAnomaly(resourceID, metric string, value float64) (bool, float64) {
|
||||
baseline, ok := s.GetBaseline(resourceID, metric)
|
||||
if !ok || baseline.SampleCount < s.minSamples {
|
||||
return false, 0 // Not enough data to determine
|
||||
}
|
||||
|
||||
if baseline.StdDev == 0 {
|
||||
// No variance - any different value is anomalous
|
||||
if value != baseline.Mean {
|
||||
return true, math.Inf(1)
|
||||
}
|
||||
return false, 0
|
||||
}
|
||||
|
||||
zScore := (value - baseline.Mean) / baseline.StdDev
|
||||
|
||||
// Consider anything > 2 standard deviations as anomalous
|
||||
// (covers ~95% of normal distribution)
|
||||
isAnomaly := math.Abs(zScore) > 2.0
|
||||
|
||||
return isAnomaly, zScore
|
||||
}
|
||||
|
||||
// AnomalySeverity categorizes how severe an anomaly is
|
||||
type AnomalySeverity string
|
||||
|
||||
const (
|
||||
AnomalyNone AnomalySeverity = ""
|
||||
AnomalyLow AnomalySeverity = "low" // 2-2.5 std devs
|
||||
AnomalyMedium AnomalySeverity = "medium" // 2.5-3 std devs
|
||||
AnomalyHigh AnomalySeverity = "high" // 3-4 std devs
|
||||
AnomalyCritical AnomalySeverity = "critical" // > 4 std devs
|
||||
)
|
||||
|
||||
// CheckAnomaly performs a detailed anomaly check with severity classification
|
||||
func (s *Store) CheckAnomaly(resourceID, metric string, value float64) (AnomalySeverity, float64, *MetricBaseline) {
|
||||
baseline, ok := s.GetBaseline(resourceID, metric)
|
||||
if !ok || baseline.SampleCount < s.minSamples {
|
||||
return AnomalyNone, 0, nil
|
||||
}
|
||||
|
||||
if baseline.StdDev == 0 {
|
||||
if value != baseline.Mean {
|
||||
return AnomalyCritical, math.Inf(1), baseline
|
||||
}
|
||||
return AnomalyNone, 0, baseline
|
||||
}
|
||||
|
||||
zScore := (value - baseline.Mean) / baseline.StdDev
|
||||
absZ := math.Abs(zScore)
|
||||
|
||||
var severity AnomalySeverity
|
||||
switch {
|
||||
case absZ < 2.0:
|
||||
severity = AnomalyNone
|
||||
case absZ < 2.5:
|
||||
severity = AnomalyLow
|
||||
case absZ < 3.0:
|
||||
severity = AnomalyMedium
|
||||
case absZ < 4.0:
|
||||
severity = AnomalyHigh
|
||||
default:
|
||||
severity = AnomalyCritical
|
||||
}
|
||||
|
||||
return severity, zScore, baseline
|
||||
}
|
||||
|
||||
// ResourceCount returns the number of resources with baselines
|
||||
func (s *Store) ResourceCount() int {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return len(s.baselines)
|
||||
}
|
||||
|
||||
// Save persists baselines to disk
|
||||
func (s *Store) Save() error {
|
||||
if s.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
return s.saveToDisk()
|
||||
}
|
||||
|
||||
// saveToDisk writes baselines to JSON file
|
||||
func (s *Store) saveToDisk() error {
|
||||
if s.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
path := filepath.Join(s.dataDir, "baselines.json")
|
||||
|
||||
data, err := json.MarshalIndent(s.baselines, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write to temp file first, then rename for atomicity
|
||||
tmpPath := path + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(tmpPath, path)
|
||||
}
|
||||
|
||||
// loadFromDisk reads baselines from JSON file
|
||||
func (s *Store) loadFromDisk() error {
|
||||
path := filepath.Join(s.dataDir, "baselines.json")
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil // No saved baselines yet
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return json.Unmarshal(data, &s.baselines)
|
||||
}
|
||||
|
||||
// Helper functions for statistics
|
||||
|
||||
func computeMean(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, v := range values {
|
||||
sum += v
|
||||
}
|
||||
return sum / float64(len(values))
|
||||
}
|
||||
|
||||
func computeStdDev(values []float64) float64 {
|
||||
if len(values) < 2 {
|
||||
return 0
|
||||
}
|
||||
mean := computeMean(values)
|
||||
sumSqDiff := 0.0
|
||||
for _, v := range values {
|
||||
diff := v - mean
|
||||
sumSqDiff += diff * diff
|
||||
}
|
||||
variance := sumSqDiff / float64(len(values)-1) // Sample standard deviation
|
||||
return math.Sqrt(variance)
|
||||
}
|
||||
|
||||
func computePercentiles(values []float64) map[int]float64 {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort a copy
|
||||
sorted := make([]float64, len(values))
|
||||
copy(sorted, values)
|
||||
sort.Float64s(sorted)
|
||||
|
||||
percentiles := map[int]float64{
|
||||
5: percentile(sorted, 5),
|
||||
25: percentile(sorted, 25),
|
||||
50: percentile(sorted, 50),
|
||||
75: percentile(sorted, 75),
|
||||
95: percentile(sorted, 95),
|
||||
}
|
||||
|
||||
return percentiles
|
||||
}
|
||||
|
||||
func percentile(sorted []float64, p int) float64 {
|
||||
if len(sorted) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Use linear interpolation
|
||||
rank := float64(p) / 100.0 * float64(len(sorted)-1)
|
||||
lower := int(rank)
|
||||
upper := lower + 1
|
||||
|
||||
if upper >= len(sorted) {
|
||||
return sorted[len(sorted)-1]
|
||||
}
|
||||
|
||||
// Interpolate
|
||||
weight := rank - float64(lower)
|
||||
return sorted[lower]*(1-weight) + sorted[upper]*weight
|
||||
}
|
||||
209
internal/ai/baseline/store_test.go
Normal file
209
internal/ai/baseline/store_test.go
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
package baseline
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLearn_Basic(t *testing.T) {
|
||||
store := NewStore(StoreConfig{MinSamples: 10})
|
||||
|
||||
// Create 50 data points with mean ~50 and some variance
|
||||
points := make([]MetricPoint, 50)
|
||||
now := time.Now()
|
||||
for i := 0; i < 50; i++ {
|
||||
points[i] = MetricPoint{
|
||||
Value: 50 + float64(i%10) - 5, // Values from 45-54
|
||||
Timestamp: now.Add(-time.Duration(50-i) * time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
err := store.Learn("test-vm", "vm", "cpu", points)
|
||||
if err != nil {
|
||||
t.Fatalf("Learn failed: %v", err)
|
||||
}
|
||||
|
||||
baseline, ok := store.GetBaseline("test-vm", "cpu")
|
||||
if !ok {
|
||||
t.Fatal("Baseline not found after learning")
|
||||
}
|
||||
|
||||
// Check mean is around 50
|
||||
if math.Abs(baseline.Mean-50) > 1 {
|
||||
t.Errorf("Expected mean ~50, got %f", baseline.Mean)
|
||||
}
|
||||
|
||||
// Check stddev is reasonable (should be ~3 for our data)
|
||||
if baseline.StdDev < 1 || baseline.StdDev > 5 {
|
||||
t.Errorf("Expected stddev ~3, got %f", baseline.StdDev)
|
||||
}
|
||||
|
||||
if baseline.SampleCount != 50 {
|
||||
t.Errorf("Expected 50 samples, got %d", baseline.SampleCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLearn_InsufficientData(t *testing.T) {
|
||||
store := NewStore(StoreConfig{MinSamples: 50})
|
||||
|
||||
// Only 10 points, not enough
|
||||
points := make([]MetricPoint, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
points[i] = MetricPoint{Value: float64(i)}
|
||||
}
|
||||
|
||||
err := store.Learn("test-vm", "vm", "cpu", points)
|
||||
if err != nil {
|
||||
t.Fatalf("Learn should not error on insufficient data: %v", err)
|
||||
}
|
||||
|
||||
_, ok := store.GetBaseline("test-vm", "cpu")
|
||||
if ok {
|
||||
t.Error("Should not have baseline with insufficient data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsAnomaly(t *testing.T) {
|
||||
store := NewStore(StoreConfig{MinSamples: 10})
|
||||
|
||||
// Create stable data around 50 with low variance
|
||||
points := make([]MetricPoint, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
points[i] = MetricPoint{
|
||||
Value: 50 + float64(i%3) - 1, // Values 49, 50, 51
|
||||
}
|
||||
}
|
||||
|
||||
store.Learn("test-vm", "vm", "cpu", points)
|
||||
|
||||
// Test normal value
|
||||
isAnomaly, zScore := store.IsAnomaly("test-vm", "cpu", 50)
|
||||
if isAnomaly {
|
||||
t.Errorf("50 should not be anomaly, zScore=%f", zScore)
|
||||
}
|
||||
|
||||
// Test slightly high - with stddev ~0.82, 51 is within 2 std devs
|
||||
isAnomaly, zScore = store.IsAnomaly("test-vm", "cpu", 51)
|
||||
if isAnomaly {
|
||||
t.Errorf("51 should not be anomaly with this variance, zScore=%f", zScore)
|
||||
}
|
||||
|
||||
// Test very high (should be anomaly)
|
||||
isAnomaly, zScore = store.IsAnomaly("test-vm", "cpu", 60)
|
||||
if !isAnomaly {
|
||||
t.Errorf("60 should be anomaly, zScore=%f", zScore)
|
||||
}
|
||||
|
||||
// Test very low (should be anomaly)
|
||||
isAnomaly, zScore = store.IsAnomaly("test-vm", "cpu", 40)
|
||||
if !isAnomaly {
|
||||
t.Errorf("40 should be anomaly, zScore=%f", zScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckAnomaly_Severity(t *testing.T) {
|
||||
store := NewStore(StoreConfig{MinSamples: 10})
|
||||
|
||||
// Create very stable data with known statistics
|
||||
// Mean = 50, StdDev = 1
|
||||
points := make([]MetricPoint, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
// Alternate between 49, 50, 51 for stddev ~1
|
||||
points[i] = MetricPoint{Value: 50 + float64(i%3) - 1}
|
||||
}
|
||||
|
||||
store.Learn("test-vm", "vm", "cpu", points)
|
||||
baseline, _ := store.GetBaseline("test-vm", "cpu")
|
||||
|
||||
testCases := []struct {
|
||||
value float64
|
||||
expectedSeverity AnomalySeverity
|
||||
}{
|
||||
{50, AnomalyNone}, // Mean
|
||||
{50 + baseline.StdDev*1.5, AnomalyNone}, // 1.5 std devs - normal
|
||||
{50 + baseline.StdDev*2.2, AnomalyLow}, // 2.2 std devs
|
||||
{50 + baseline.StdDev*2.7, AnomalyMedium}, // 2.7 std devs
|
||||
{50 + baseline.StdDev*3.5, AnomalyHigh}, // 3.5 std devs
|
||||
{50 + baseline.StdDev*4.5, AnomalyCritical}, // 4.5 std devs
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
severity, _, _ := store.CheckAnomaly("test-vm", "cpu", tc.value)
|
||||
if severity != tc.expectedSeverity {
|
||||
t.Errorf("Value %f: expected severity %s, got %s", tc.value, tc.expectedSeverity, severity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetResourceBaseline(t *testing.T) {
|
||||
store := NewStore(StoreConfig{MinSamples: 10})
|
||||
|
||||
// Learn multiple metrics
|
||||
cpuPoints := make([]MetricPoint, 50)
|
||||
memPoints := make([]MetricPoint, 50)
|
||||
for i := 0; i < 50; i++ {
|
||||
cpuPoints[i] = MetricPoint{Value: 30}
|
||||
memPoints[i] = MetricPoint{Value: 70}
|
||||
}
|
||||
|
||||
store.Learn("test-vm", "vm", "cpu", cpuPoints)
|
||||
store.Learn("test-vm", "vm", "memory", memPoints)
|
||||
|
||||
rb, ok := store.GetResourceBaseline("test-vm")
|
||||
if !ok {
|
||||
t.Fatal("Resource baseline not found")
|
||||
}
|
||||
|
||||
if rb.ResourceType != "vm" {
|
||||
t.Errorf("Expected resource type 'vm', got '%s'", rb.ResourceType)
|
||||
}
|
||||
|
||||
if len(rb.Metrics) != 2 {
|
||||
t.Errorf("Expected 2 metrics, got %d", len(rb.Metrics))
|
||||
}
|
||||
|
||||
if rb.Metrics["cpu"] == nil {
|
||||
t.Error("CPU metric baseline missing")
|
||||
}
|
||||
|
||||
if rb.Metrics["memory"] == nil {
|
||||
t.Error("Memory metric baseline missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercentiles(t *testing.T) {
|
||||
values := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
|
||||
percentiles := computePercentiles(values)
|
||||
|
||||
// P50 should be ~5.5 for 1-10
|
||||
if percentiles[50] < 5 || percentiles[50] > 6 {
|
||||
t.Errorf("P50 should be ~5.5, got %f", percentiles[50])
|
||||
}
|
||||
|
||||
// P5 should be close to 1
|
||||
if percentiles[5] < 1 || percentiles[5] > 2 {
|
||||
t.Errorf("P5 should be ~1, got %f", percentiles[5])
|
||||
}
|
||||
|
||||
// P95 should be close to 10
|
||||
if percentiles[95] < 9 || percentiles[95] > 10 {
|
||||
t.Errorf("P95 should be ~10, got %f", percentiles[95])
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeStats(t *testing.T) {
|
||||
// Test mean and stddev with known values
|
||||
values := []float64{2, 4, 4, 4, 5, 5, 7, 9} // Mean = 5, Stddev = 2 (sample)
|
||||
|
||||
mean := computeMean(values)
|
||||
if mean != 5 {
|
||||
t.Errorf("Expected mean 5, got %f", mean)
|
||||
}
|
||||
|
||||
stddev := computeStdDev(values)
|
||||
// Sample stddev of [2,4,4,4,5,5,7,9] is approximately 2.14, not exactly 2
|
||||
if math.Abs(stddev-2.14) > 0.1 {
|
||||
t.Errorf("Expected stddev ~2.14, got %f", stddev)
|
||||
}
|
||||
}
|
||||
46
internal/ai/baseline_adapter.go
Normal file
46
internal/ai/baseline_adapter.go
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||
)
|
||||
|
||||
// BaselineStoreAdapter adapts baseline.Store to the context.BaselineProvider interface
|
||||
type BaselineStoreAdapter struct {
|
||||
store *baseline.Store
|
||||
}
|
||||
|
||||
// NewBaselineStoreAdapter creates an adapter for baseline.Store
|
||||
func NewBaselineStoreAdapter(store *baseline.Store) *BaselineStoreAdapter {
|
||||
if store == nil {
|
||||
return nil
|
||||
}
|
||||
return &BaselineStoreAdapter{store: store}
|
||||
}
|
||||
|
||||
// CheckAnomaly implements context.BaselineProvider
|
||||
func (a *BaselineStoreAdapter) CheckAnomaly(resourceID, metric string, value float64) (severity string, zScore float64, mean float64, stddev float64, ok bool) {
|
||||
if a.store == nil {
|
||||
return "", 0, 0, 0, false
|
||||
}
|
||||
|
||||
s, z, b := a.store.CheckAnomaly(resourceID, metric, value)
|
||||
if b == nil {
|
||||
return "", 0, 0, 0, false
|
||||
}
|
||||
|
||||
return string(s), z, b.Mean, b.StdDev, true
|
||||
}
|
||||
|
||||
// GetBaseline implements context.BaselineProvider
|
||||
func (a *BaselineStoreAdapter) GetBaseline(resourceID, metric string) (mean float64, stddev float64, sampleCount int, ok bool) {
|
||||
if a.store == nil {
|
||||
return 0, 0, 0, false
|
||||
}
|
||||
|
||||
b, exists := a.store.GetBaseline(resourceID, metric)
|
||||
if !exists || b == nil {
|
||||
return 0, 0, 0, false
|
||||
}
|
||||
|
||||
return b.Mean, b.StdDev, b.SampleCount, true
|
||||
}
|
||||
|
|
@ -29,12 +29,22 @@ type FindingsProvider interface {
|
|||
GetPastFindingsForResource(resourceID string) []string
|
||||
}
|
||||
|
||||
// BaselineProvider provides learned baselines for anomaly detection
|
||||
type BaselineProvider interface {
|
||||
// CheckAnomaly returns severity, z-score, and baseline data
|
||||
// Severity is "", "low", "medium", "high", or "critical"
|
||||
CheckAnomaly(resourceID, metric string, value float64) (severity string, zScore float64, mean float64, stddev float64, ok bool)
|
||||
// GetBaseline returns the baseline for a resource/metric
|
||||
GetBaseline(resourceID, metric string) (mean float64, stddev float64, sampleCount int, ok bool)
|
||||
}
|
||||
|
||||
// Builder constructs enriched AI context from multiple data sources
|
||||
type Builder struct {
|
||||
// Data sources
|
||||
metricsHistory MetricsHistoryProvider
|
||||
knowledge KnowledgeProvider
|
||||
findings FindingsProvider
|
||||
baseline BaselineProvider
|
||||
|
||||
// Configuration
|
||||
trendWindow24h time.Duration
|
||||
|
|
@ -51,7 +61,7 @@ func NewBuilder() *Builder {
|
|||
trendWindow7d: 7 * 24 * time.Hour,
|
||||
includeHistory: true,
|
||||
includeTrends: true,
|
||||
includeBaseline: false, // Disabled until baseline store is implemented
|
||||
includeBaseline: true, // Enable when baseline provider is set
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -73,6 +83,12 @@ func (b *Builder) WithFindings(f FindingsProvider) *Builder {
|
|||
return b
|
||||
}
|
||||
|
||||
// WithBaseline sets the baseline provider for anomaly detection
|
||||
func (b *Builder) WithBaseline(bp BaselineProvider) *Builder {
|
||||
b.baseline = bp
|
||||
return b
|
||||
}
|
||||
|
||||
// BuildForInfrastructure creates comprehensive context for the entire infrastructure
|
||||
func (b *Builder) BuildForInfrastructure(state models.StateSnapshot) *InfrastructureContext {
|
||||
ctx := &InfrastructureContext{
|
||||
|
|
@ -84,6 +100,7 @@ func (b *Builder) BuildForInfrastructure(state models.StateSnapshot) *Infrastruc
|
|||
trends := b.computeNodeTrends(node.ID)
|
||||
resourceCtx := FormatNodeForContext(node, trends)
|
||||
b.enrichWithNotes(&resourceCtx)
|
||||
b.enrichWithAnomalies(&resourceCtx)
|
||||
ctx.Nodes = append(ctx.Nodes, resourceCtx)
|
||||
}
|
||||
|
||||
|
|
@ -99,6 +116,7 @@ func (b *Builder) BuildForInfrastructure(state models.StateSnapshot) *Infrastruc
|
|||
vm.Uptime, vm.LastBackup, trends,
|
||||
)
|
||||
b.enrichWithNotes(&resourceCtx)
|
||||
b.enrichWithAnomalies(&resourceCtx)
|
||||
ctx.VMs = append(ctx.VMs, resourceCtx)
|
||||
}
|
||||
|
||||
|
|
@ -114,6 +132,7 @@ func (b *Builder) BuildForInfrastructure(state models.StateSnapshot) *Infrastruc
|
|||
ct.Uptime, ct.LastBackup, trends,
|
||||
)
|
||||
b.enrichWithNotes(&resourceCtx)
|
||||
b.enrichWithAnomalies(&resourceCtx)
|
||||
ctx.Containers = append(ctx.Containers, resourceCtx)
|
||||
}
|
||||
|
||||
|
|
@ -368,6 +387,65 @@ func (b *Builder) enrichWithNotes(ctx *ResourceContext) {
|
|||
}
|
||||
}
|
||||
|
||||
// enrichWithAnomalies checks current values against baselines and adds anomalies
|
||||
func (b *Builder) enrichWithAnomalies(ctx *ResourceContext) {
|
||||
if b.baseline == nil || !b.includeBaseline {
|
||||
return
|
||||
}
|
||||
|
||||
// Check each metric type for anomalies
|
||||
metrics := map[string]float64{
|
||||
"cpu": ctx.CurrentCPU,
|
||||
"memory": ctx.CurrentMemory,
|
||||
"disk": ctx.CurrentDisk,
|
||||
}
|
||||
|
||||
for metric, value := range metrics {
|
||||
if value == 0 {
|
||||
continue // Skip zeroes (usually means not reported)
|
||||
}
|
||||
|
||||
severity, zScore, mean, stddev, ok := b.baseline.CheckAnomaly(ctx.ResourceID, metric, value)
|
||||
if !ok || severity == "" {
|
||||
continue // No anomaly or no baseline
|
||||
}
|
||||
|
||||
direction := "above"
|
||||
if zScore < 0 {
|
||||
direction = "below"
|
||||
}
|
||||
|
||||
anomaly := Anomaly{
|
||||
Metric: metric,
|
||||
Current: value,
|
||||
Expected: mean,
|
||||
Deviation: zScore,
|
||||
Severity: severity,
|
||||
Since: time.Now(), // We don't track onset time yet
|
||||
Description: formatAnomalyDescription(metric, value, mean, stddev, severity, direction),
|
||||
}
|
||||
ctx.Anomalies = append(ctx.Anomalies, anomaly)
|
||||
}
|
||||
}
|
||||
|
||||
// formatAnomalyDescription creates human-readable anomaly description
|
||||
func formatAnomalyDescription(metric string, current, mean, stddev float64, severity, direction string) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(strings.Title(metric))
|
||||
sb.WriteString(" is ")
|
||||
sb.WriteString(severity)
|
||||
sb.WriteString(" ")
|
||||
sb.WriteString(direction)
|
||||
sb.WriteString(" normal (")
|
||||
sb.WriteString(formatFloat(current, 1))
|
||||
sb.WriteString("% vs typical ")
|
||||
sb.WriteString(formatFloat(mean, 1))
|
||||
sb.WriteString("% ± ")
|
||||
sb.WriteString(formatFloat(stddev, 1))
|
||||
sb.WriteString("%)")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// filterRecentPoints filters points to only include those within duration
|
||||
func filterRecentPoints(points []MetricPoint, duration time.Duration) []MetricPoint {
|
||||
cutoff := time.Now().Add(-duration)
|
||||
|
|
|
|||
|
|
@ -385,6 +385,7 @@ func FormatNodeForContext(node models.Node, trends map[string]Trend) ResourceCon
|
|||
}
|
||||
|
||||
// FormatGuestForContext creates context for a VM or container
|
||||
// Note: cpu is 0-1 ratio from Proxmox API, memUsage and diskUsage are already 0-100 percentages
|
||||
func FormatGuestForContext(
|
||||
id, name, node, guestType, status string,
|
||||
cpu, memUsage, diskUsage float64,
|
||||
|
|
@ -397,9 +398,9 @@ func FormatGuestForContext(
|
|||
ResourceType: guestType,
|
||||
ResourceName: name,
|
||||
Node: node,
|
||||
CurrentCPU: cpu * 100, // Convert from 0-1 to percentage
|
||||
CurrentMemory: memUsage * 100,
|
||||
CurrentDisk: diskUsage * 100,
|
||||
CurrentCPU: cpu * 100, // Convert from 0-1 to percentage
|
||||
CurrentMemory: memUsage, // Already 0-100 percentage from Memory.Usage
|
||||
CurrentDisk: diskUsage, // Already 0-100 percentage from Disk.Usage
|
||||
Status: status,
|
||||
Uptime: time.Duration(uptime) * time.Second,
|
||||
Trends: trends,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
|
|
@ -210,6 +211,7 @@ type PatrolService struct {
|
|||
findings *FindingsStore
|
||||
knowledgeStore *knowledge.Store // For per-resource notes in patrol context
|
||||
metricsHistory MetricsHistoryProvider // For trend analysis and predictions
|
||||
baselineStore *baseline.Store // For anomaly detection via learned baselines
|
||||
|
||||
// Cached thresholds (recalculated when thresholdProvider changes)
|
||||
thresholds PatrolThresholds
|
||||
|
|
@ -340,6 +342,22 @@ func (p *PatrolService) SetMetricsHistoryProvider(provider MetricsHistoryProvide
|
|||
log.Info().Msg("AI Patrol: Metrics history provider set for enriched context")
|
||||
}
|
||||
|
||||
// SetBaselineStore sets the baseline store for anomaly detection
|
||||
// This enables the patrol service to detect anomalies based on learned normal behavior
|
||||
func (p *PatrolService) SetBaselineStore(store *baseline.Store) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.baselineStore = store
|
||||
log.Info().Msg("AI Patrol: Baseline store set for anomaly detection")
|
||||
}
|
||||
|
||||
// GetBaselineStore returns the baseline store (for external baseline learning)
|
||||
func (p *PatrolService) GetBaselineStore() *baseline.Store {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.baselineStore
|
||||
}
|
||||
|
||||
// GetConfig returns the current patrol configuration
|
||||
func (p *PatrolService) GetConfig() PatrolConfig {
|
||||
p.mu.RLock()
|
||||
|
|
@ -828,6 +846,7 @@ func (p *PatrolService) analyzeNode(node models.Node) []*Finding {
|
|||
}
|
||||
|
||||
// analyzeGuest checks a VM or container for issues
|
||||
// Note: cpu is 0-1 ratio, memUsage and diskUsage are already 0-100 percentages from Memory.Usage/Disk.Usage
|
||||
func (p *PatrolService) analyzeGuest(id, name, guestType, node, status string,
|
||||
cpu, memUsage, diskUsage float64, lastBackup *time.Time, template bool) []*Finding {
|
||||
var findings []*Finding
|
||||
|
|
@ -837,9 +856,9 @@ func (p *PatrolService) analyzeGuest(id, name, guestType, node, status string,
|
|||
return findings
|
||||
}
|
||||
|
||||
// Convert ratios to percentages for comparison with thresholds
|
||||
memPct := memUsage * 100
|
||||
diskPct := diskUsage * 100
|
||||
// memUsage and diskUsage are already percentages (0-100)
|
||||
memPct := memUsage
|
||||
diskPct := diskUsage
|
||||
|
||||
// High memory (sustained) - use dynamic thresholds
|
||||
if memPct > p.thresholds.GuestMemWatch {
|
||||
|
|
@ -1683,6 +1702,7 @@ func (p *PatrolService) buildEnrichedContext(state models.StateSnapshot) string
|
|||
p.mu.RLock()
|
||||
metricsHistory := p.metricsHistory
|
||||
knowledgeStore := p.knowledgeStore
|
||||
baselineStore := p.baselineStore
|
||||
p.mu.RUnlock()
|
||||
|
||||
// If no metrics history, fall back to basic summary
|
||||
|
|
@ -1699,6 +1719,14 @@ func (p *PatrolService) buildEnrichedContext(state models.StateSnapshot) string
|
|||
if knowledgeStore != nil {
|
||||
builder = builder.WithKnowledge(&knowledgeShim{store: knowledgeStore})
|
||||
}
|
||||
|
||||
// Add baseline provider for anomaly detection if available
|
||||
if baselineStore != nil {
|
||||
adapter := NewBaselineStoreAdapter(baselineStore)
|
||||
if adapter != nil {
|
||||
builder = builder.WithBaseline(&baselineShim{adapter: adapter})
|
||||
}
|
||||
}
|
||||
|
||||
// Build full infrastructure context with trends
|
||||
infraCtx := builder.BuildForInfrastructure(state)
|
||||
|
|
@ -1713,6 +1741,7 @@ func (p *PatrolService) buildEnrichedContext(state models.StateSnapshot) string
|
|||
log.Debug().
|
||||
Int("resources", infraCtx.TotalResources).
|
||||
Int("predictions", len(infraCtx.Predictions)).
|
||||
Int("anomalies", len(infraCtx.Anomalies)).
|
||||
Msg("AI Patrol: Built enriched context with trends")
|
||||
|
||||
return formatted
|
||||
|
|
@ -1783,6 +1812,25 @@ func (k *knowledgeShim) FormatAllForContext() string {
|
|||
return k.store.FormatAllForContext()
|
||||
}
|
||||
|
||||
// baselineShim adapts BaselineStoreAdapter to aicontext.BaselineProvider
|
||||
type baselineShim struct {
|
||||
adapter *BaselineStoreAdapter
|
||||
}
|
||||
|
||||
func (b *baselineShim) CheckAnomaly(resourceID, metric string, value float64) (severity string, zScore float64, mean float64, stddev float64, ok bool) {
|
||||
if b.adapter == nil {
|
||||
return "", 0, 0, 0, false
|
||||
}
|
||||
return b.adapter.CheckAnomaly(resourceID, metric, value)
|
||||
}
|
||||
|
||||
func (b *baselineShim) GetBaseline(resourceID, metric string) (mean float64, stddev float64, sampleCount int, ok bool) {
|
||||
if b.adapter == nil {
|
||||
return 0, 0, 0, false
|
||||
}
|
||||
return b.adapter.GetBaseline(resourceID, metric)
|
||||
}
|
||||
|
||||
// convertToContextPoints converts ai.MetricPoint to aicontext.MetricPoint
|
||||
// Since both are aliases for types.MetricPoint, this is just a type assertion
|
||||
func convertToContextPoints(points []MetricPoint) []aicontext.MetricPoint {
|
||||
|
|
|
|||
Loading…
Reference in a new issue