feat(ai): implement metric-specific anomaly thresholds
Smarter anomaly detection to reduce false positives: **Learning Window:** 7 days → 14 days - Captures weekly patterns (weekday vs weekend) **Metric-Specific Thresholds:** CPU: - Only report if usage >70% AND >2x baseline - Low CPU variance (5% vs 10%) is not actionable Memory: - Report if >80% OR (>1.5x baseline AND >60%) - Memory is more stable, lower threshold makes sense Disk: - Report if >85% usage OR +15 percentage points growth - Disk problems are critical, use absolute thresholds Other metrics: - Use 2x threshold as default This dramatically reduces 'noise' anomalies while catching actual problems that need operator attention.
This commit is contained in:
parent
4780dd2f83
commit
f7bb6d5446
1 changed files with 41 additions and 7 deletions
|
|
@ -66,12 +66,13 @@ type StoreConfig struct {
|
|||
// DefaultConfig returns sensible defaults
|
||||
func DefaultConfig() StoreConfig {
|
||||
return StoreConfig{
|
||||
LearningWindow: 7 * 24 * time.Hour, // 7 days
|
||||
LearningWindow: 14 * 24 * time.Hour, // 14 days to capture weekly patterns
|
||||
MinSamples: 50,
|
||||
UpdateInterval: 1 * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// NewStore creates a new baseline store
|
||||
func NewStore(cfg StoreConfig) *Store {
|
||||
if cfg.LearningWindow == 0 {
|
||||
|
|
@ -295,13 +296,45 @@ func (s *Store) CheckResourceAnomalies(resourceID string, metrics map[string]flo
|
|||
// Compute ratio: current value / baseline mean
|
||||
ratio := value / baseline.Mean
|
||||
|
||||
// Filter out statistically significant but practically meaningless anomalies
|
||||
// Users don't care about small deviations from baseline
|
||||
// Require at least 2x above baseline OR 0.5x below to be truly actionable
|
||||
if ratio >= 0.5 && ratio <= 2.0 {
|
||||
continue // Too close to baseline to be actionable
|
||||
// Apply metric-specific filters to reduce noise
|
||||
// Different metrics have different thresholds for what's "actionable"
|
||||
shouldReport := false
|
||||
|
||||
switch metric {
|
||||
case "disk":
|
||||
// Disk is critical - report if:
|
||||
// 1. Usage is above 85% (absolute threshold), OR
|
||||
// 2. Usage increased by more than 15 percentage points from baseline
|
||||
if value >= 85.0 || (value - baseline.Mean) >= 15.0 {
|
||||
shouldReport = true
|
||||
}
|
||||
|
||||
case "cpu":
|
||||
// CPU fluctuates a lot - only report if:
|
||||
// 1. Current usage is above 70% (actually busy), AND
|
||||
// 2. It's at least 2x above baseline
|
||||
if value >= 70.0 && ratio >= 2.0 {
|
||||
shouldReport = true
|
||||
}
|
||||
|
||||
case "memory":
|
||||
// Memory is more stable - report if:
|
||||
// 1. Current usage is above 80% (getting tight), OR
|
||||
// 2. It's at least 1.5x above baseline AND above 60%
|
||||
if value >= 80.0 || (ratio >= 1.5 && value >= 60.0) {
|
||||
shouldReport = true
|
||||
}
|
||||
|
||||
default:
|
||||
// For other metrics (network, etc), use 2x threshold
|
||||
if ratio >= 2.0 || ratio <= 0.5 {
|
||||
shouldReport = true
|
||||
}
|
||||
}
|
||||
|
||||
if !shouldReport {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
report := AnomalyReport{
|
||||
ResourceID: resourceID,
|
||||
|
|
@ -328,6 +361,7 @@ func (s *Store) CheckResourceAnomalies(resourceID string, metrics map[string]flo
|
|||
|
||||
}
|
||||
|
||||
|
||||
// formatAnomalyDescription generates a human-readable anomaly description
|
||||
func formatAnomalyDescription(metric string, ratio float64, direction string, severity AnomalySeverity) string {
|
||||
metricLabel := metric
|
||||
|
|
|
|||
Loading…
Reference in a new issue