feat(ai): implement metric-specific anomaly thresholds
Smarter anomaly detection to reduce false positives: **Learning Window:** 7 days → 14 days - Captures weekly patterns (weekday vs weekend) **Metric-Specific Thresholds:** CPU: - Only report if usage >70% AND >2x baseline - Low CPU variance (5% vs 10%) is not actionable Memory: - Report if >80% OR (>1.5x baseline AND >60%) - Memory is more stable, lower threshold makes sense Disk: - Report if >85% usage OR +15 percentage points growth - Disk problems are critical, use absolute thresholds Other metrics: - Use 2x threshold as default This dramatically reduces 'noise' anomalies while catching actual problems that need operator attention.
This commit is contained in:
parent
4780dd2f83
commit
f7bb6d5446
1 changed files with 41 additions and 7 deletions
|
|
@ -66,12 +66,13 @@ type StoreConfig struct {
|
||||||
// DefaultConfig returns sensible defaults
|
// DefaultConfig returns sensible defaults
|
||||||
func DefaultConfig() StoreConfig {
|
func DefaultConfig() StoreConfig {
|
||||||
return StoreConfig{
|
return StoreConfig{
|
||||||
LearningWindow: 7 * 24 * time.Hour, // 7 days
|
LearningWindow: 14 * 24 * time.Hour, // 14 days to capture weekly patterns
|
||||||
MinSamples: 50,
|
MinSamples: 50,
|
||||||
UpdateInterval: 1 * time.Hour,
|
UpdateInterval: 1 * time.Hour,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// NewStore creates a new baseline store
|
// NewStore creates a new baseline store
|
||||||
func NewStore(cfg StoreConfig) *Store {
|
func NewStore(cfg StoreConfig) *Store {
|
||||||
if cfg.LearningWindow == 0 {
|
if cfg.LearningWindow == 0 {
|
||||||
|
|
@ -295,13 +296,45 @@ func (s *Store) CheckResourceAnomalies(resourceID string, metrics map[string]flo
|
||||||
// Compute ratio: current value / baseline mean
|
// Compute ratio: current value / baseline mean
|
||||||
ratio := value / baseline.Mean
|
ratio := value / baseline.Mean
|
||||||
|
|
||||||
// Filter out statistically significant but practically meaningless anomalies
|
// Apply metric-specific filters to reduce noise
|
||||||
// Users don't care about small deviations from baseline
|
// Different metrics have different thresholds for what's "actionable"
|
||||||
// Require at least 2x above baseline OR 0.5x below to be truly actionable
|
shouldReport := false
|
||||||
if ratio >= 0.5 && ratio <= 2.0 {
|
|
||||||
continue // Too close to baseline to be actionable
|
switch metric {
|
||||||
|
case "disk":
|
||||||
|
// Disk is critical - report if:
|
||||||
|
// 1. Usage is above 85% (absolute threshold), OR
|
||||||
|
// 2. Usage increased by more than 15 percentage points from baseline
|
||||||
|
if value >= 85.0 || (value - baseline.Mean) >= 15.0 {
|
||||||
|
shouldReport = true
|
||||||
|
}
|
||||||
|
|
||||||
|
case "cpu":
|
||||||
|
// CPU fluctuates a lot - only report if:
|
||||||
|
// 1. Current usage is above 70% (actually busy), AND
|
||||||
|
// 2. It's at least 2x above baseline
|
||||||
|
if value >= 70.0 && ratio >= 2.0 {
|
||||||
|
shouldReport = true
|
||||||
|
}
|
||||||
|
|
||||||
|
case "memory":
|
||||||
|
// Memory is more stable - report if:
|
||||||
|
// 1. Current usage is above 80% (getting tight), OR
|
||||||
|
// 2. It's at least 1.5x above baseline AND above 60%
|
||||||
|
if value >= 80.0 || (ratio >= 1.5 && value >= 60.0) {
|
||||||
|
shouldReport = true
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
// For other metrics (network, etc), use 2x threshold
|
||||||
|
if ratio >= 2.0 || ratio <= 0.5 {
|
||||||
|
shouldReport = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !shouldReport {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
report := AnomalyReport{
|
report := AnomalyReport{
|
||||||
ResourceID: resourceID,
|
ResourceID: resourceID,
|
||||||
|
|
@ -328,6 +361,7 @@ func (s *Store) CheckResourceAnomalies(resourceID string, metrics map[string]flo
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// formatAnomalyDescription generates a human-readable anomaly description
|
// formatAnomalyDescription generates a human-readable anomaly description
|
||||||
func formatAnomalyDescription(metric string, ratio float64, direction string, severity AnomalySeverity) string {
|
func formatAnomalyDescription(metric string, ratio float64, direction string, severity AnomalySeverity) string {
|
||||||
metricLabel := metric
|
metricLabel := metric
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue