Pulse/internal/ai/alert_triggered.go
rcourtman 82e5b28840 feat: enhance AI baseline context visibility and incident timeline improvements
Backend:
- Enhanced buildEnrichedResourceContext to ALWAYS show learned baselines with
  status indicators (normal/elevated/anomaly) instead of only when anomalous
- This makes Pulse Pro's 'moat' visible - users can see the AI understands
  their infrastructure's normal behavior patterns
- Added baseline import to service.go

Frontend (user changes):
- Added incident event type filtering with toggle buttons
- Added resource incident panel to view all incidents for a resource
- Added timeline expand/collapse functionality in alert history
- Added incident note saving with proper incidentId tracking
- Added startedAt parameter for proper incident timeline loading
2025-12-21 00:14:20 +00:00

393 lines
11 KiB
Go

package ai
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rs/zerolog/log"
)
// AlertTriggeredAnalyzer handles AI analysis triggered by firing alerts
// This provides token-efficient, real-time AI insights on specific resources
type AlertTriggeredAnalyzer struct {
mu sync.RWMutex
patrolService *PatrolService
stateProvider StateProvider
enabled bool
// Cooldown to prevent analyzing the same resource repeatedly
lastAnalyzed map[string]time.Time
cooldown time.Duration
// Track pending analyses to deduplicate concurrent alerts
pending map[string]bool
}
// NewAlertTriggeredAnalyzer creates a new alert-triggered analyzer
func NewAlertTriggeredAnalyzer(patrolService *PatrolService, stateProvider StateProvider) *AlertTriggeredAnalyzer {
return &AlertTriggeredAnalyzer{
patrolService: patrolService,
stateProvider: stateProvider,
enabled: false,
lastAnalyzed: make(map[string]time.Time),
cooldown: 5 * time.Minute, // Don't re-analyze the same resource within 5 minutes
pending: make(map[string]bool),
}
}
// SetEnabled enables or disables alert-triggered analysis
func (a *AlertTriggeredAnalyzer) SetEnabled(enabled bool) {
a.mu.Lock()
defer a.mu.Unlock()
a.enabled = enabled
log.Info().Bool("enabled", enabled).Msg("Alert-triggered AI analysis setting updated")
}
// IsEnabled returns whether alert-triggered analysis is enabled
func (a *AlertTriggeredAnalyzer) IsEnabled() bool {
a.mu.RLock()
defer a.mu.RUnlock()
return a.enabled
}
// OnAlertFired is called when an alert fires - triggers AI analysis of the affected resource
func (a *AlertTriggeredAnalyzer) OnAlertFired(alert *alerts.Alert) {
if alert == nil {
return
}
a.mu.Lock()
if !a.enabled {
a.mu.Unlock()
return
}
// Create a resource key for deduplication
resourceKey := a.resourceKeyFromAlert(alert)
if resourceKey == "" {
a.mu.Unlock()
log.Debug().
Str("alertID", alert.ID).
Str("type", alert.Type).
Msg("Cannot determine resource key for alert, skipping AI analysis")
return
}
// Check cooldown
if lastTime, exists := a.lastAnalyzed[resourceKey]; exists {
if time.Since(lastTime) < a.cooldown {
a.mu.Unlock()
log.Debug().
Str("resourceKey", resourceKey).
Str("alertID", alert.ID).
Dur("cooldownRemaining", a.cooldown-time.Since(lastTime)).
Msg("Resource recently analyzed, skipping due to cooldown")
return
}
}
// Check for pending analysis
if a.pending[resourceKey] {
a.mu.Unlock()
log.Debug().
Str("resourceKey", resourceKey).
Str("alertID", alert.ID).
Msg("Analysis already pending for resource, skipping duplicate")
return
}
// Mark as pending
a.pending[resourceKey] = true
a.mu.Unlock()
// Run analysis in background
go a.analyzeResource(alert, resourceKey)
}
// analyzeResource performs AI analysis on the resource associated with an alert
func (a *AlertTriggeredAnalyzer) analyzeResource(alert *alerts.Alert, resourceKey string) {
defer func() {
a.mu.Lock()
delete(a.pending, resourceKey)
a.lastAnalyzed[resourceKey] = time.Now()
a.mu.Unlock()
}()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
log.Info().
Str("alertID", alert.ID).
Str("type", alert.Type).
Str("resource", alert.ResourceName).
Str("resourceKey", resourceKey).
Float64("value", alert.Value).
Float64("threshold", alert.Threshold).
Msg("Starting AI analysis triggered by alert")
startTime := time.Now()
// Determine what type of resource this is and analyze it
findings := a.analyzeResourceByAlert(ctx, alert)
duration := time.Since(startTime)
if len(findings) > 0 {
log.Info().
Str("alertID", alert.ID).
Str("resourceKey", resourceKey).
Int("findingsCount", len(findings)).
Dur("duration", duration).
Msg("Alert-triggered AI analysis completed with findings")
// Add findings to the patrol service's findings store
if a.patrolService != nil && a.patrolService.findings != nil {
for _, finding := range findings {
// Link finding to the triggering alert
finding.AlertID = alert.ID
a.patrolService.findings.Add(finding)
}
}
} else {
log.Debug().
Str("alertID", alert.ID).
Str("resourceKey", resourceKey).
Dur("duration", duration).
Msg("Alert-triggered AI analysis completed with no additional findings")
}
if a.patrolService != nil && a.patrolService.aiService != nil {
summary := "Alert-triggered AI analysis completed"
if len(findings) > 0 {
summary = fmt.Sprintf("Alert-triggered AI analysis found %d findings", len(findings))
}
a.patrolService.aiService.RecordIncidentAnalysis(alert.ID, summary, map[string]interface{}{
"findings": len(findings),
"duration": duration.String(),
})
}
}
// analyzeResourceByAlert determines the resource type from the alert and analyzes it
func (a *AlertTriggeredAnalyzer) analyzeResourceByAlert(ctx context.Context, alert *alerts.Alert) []*Finding {
if a.patrolService == nil {
return nil
}
// Parse alert type to determine what kind of resource this is
alertType := strings.ToLower(alert.Type)
switch {
// Node alerts
case strings.HasPrefix(alertType, "node") ||
alertType == "cpu" && strings.Contains(alert.ResourceID, "/node/") ||
alertType == "memory" && strings.Contains(alert.ResourceID, "/node/"):
return a.analyzeNodeFromAlert(ctx, alert)
// Guest (VM/Container) alerts
case strings.Contains(alertType, "container") ||
strings.Contains(alertType, "vm") ||
strings.HasPrefix(alertType, "qemu") ||
strings.HasPrefix(alertType, "lxc") ||
strings.Contains(alert.ResourceID, "/qemu/") ||
strings.Contains(alert.ResourceID, "/lxc/"):
return a.analyzeGuestFromAlert(ctx, alert)
// Docker alerts
case strings.Contains(alertType, "docker"):
return a.analyzeDockerFromAlert(ctx, alert)
// Storage alerts
case strings.Contains(alertType, "storage") ||
strings.HasSuffix(alertType, "-usage"):
return a.analyzeStorageFromAlert(ctx, alert)
// Generic CPU/Memory/Disk alerts - try to determine from resource ID
case alertType == "cpu" || alertType == "memory" || alertType == "disk":
return a.analyzeGenericResourceFromAlert(ctx, alert)
default:
log.Debug().
Str("alertType", alertType).
Str("resourceID", alert.ResourceID).
Msg("Unknown alert type for targeted AI analysis, skipping")
return nil
}
}
// analyzeNodeFromAlert analyzes a Proxmox node triggered by an alert
func (a *AlertTriggeredAnalyzer) analyzeNodeFromAlert(_ context.Context, alert *alerts.Alert) []*Finding {
if a.stateProvider == nil {
return nil
}
state := a.stateProvider.GetState()
// Find the node - first try ResourceID, then ResourceName
var targetNode *models.Node
for i := range state.Nodes {
node := &state.Nodes[i]
if node.ID == alert.ResourceID || node.Name == alert.ResourceName || node.Name == alert.Node {
targetNode = node
break
}
}
if targetNode == nil {
log.Warn().
Str("alertID", alert.ID).
Str("resourceID", alert.ResourceID).
Str("resourceName", alert.ResourceName).
Msg("Could not find node for alert-triggered analysis")
return nil
}
// Use patrol service's node analysis
return a.patrolService.analyzeNode(*targetNode)
}
// analyzeGuestFromAlert analyzes a VM/Container triggered by an alert
func (a *AlertTriggeredAnalyzer) analyzeGuestFromAlert(_ context.Context, alert *alerts.Alert) []*Finding {
if a.stateProvider == nil {
return nil
}
state := a.stateProvider.GetState()
// Check VMs
for _, vm := range state.VMs {
if vm.ID == alert.ResourceID || vm.Name == alert.ResourceName {
return a.patrolService.analyzeGuest(
vm.ID, vm.Name, "vm", vm.Node, vm.Status,
vm.CPU, vm.Memory.Usage, vm.Disk.Usage,
nil, vm.Template,
)
}
}
// Check containers
for _, ct := range state.Containers {
if ct.ID == alert.ResourceID || ct.Name == alert.ResourceName {
return a.patrolService.analyzeGuest(
ct.ID, ct.Name, "container", ct.Node, ct.Status,
ct.CPU, ct.Memory.Usage, ct.Disk.Usage,
nil, ct.Template,
)
}
}
log.Warn().
Str("alertID", alert.ID).
Str("resourceID", alert.ResourceID).
Msg("Could not find guest for alert-triggered analysis")
return nil
}
// analyzeDockerFromAlert analyzes a Docker container/host triggered by an alert
func (a *AlertTriggeredAnalyzer) analyzeDockerFromAlert(_ context.Context, alert *alerts.Alert) []*Finding {
if a.stateProvider == nil {
return nil
}
state := a.stateProvider.GetState()
// Try to find the Docker host or container
// Containers are nested inside DockerHosts
for _, dh := range state.DockerHosts {
// Check if this is a host alert
if dh.ID == alert.ResourceID || dh.Hostname == alert.ResourceName {
return a.patrolService.analyzeDockerHost(dh)
}
// Check containers within this host
for _, container := range dh.Containers {
if container.ID == alert.ResourceID || container.Name == alert.ResourceName {
return a.patrolService.analyzeDockerHost(dh)
}
}
}
log.Warn().
Str("alertID", alert.ID).
Str("resourceID", alert.ResourceID).
Msg("Could not find Docker resource for alert-triggered analysis")
return nil
}
// analyzeStorageFromAlert analyzes a storage resource triggered by an alert
func (a *AlertTriggeredAnalyzer) analyzeStorageFromAlert(_ context.Context, alert *alerts.Alert) []*Finding {
if a.stateProvider == nil {
return nil
}
state := a.stateProvider.GetState()
// Storage is at the top level of StateSnapshot
for _, storage := range state.Storage {
if storage.ID == alert.ResourceID || storage.Name == alert.ResourceName {
return a.patrolService.analyzeStorage(storage)
}
}
log.Warn().
Str("alertID", alert.ID).
Str("resourceID", alert.ResourceID).
Msg("Could not find storage for alert-triggered analysis")
return nil
}
// analyzeGenericResourceFromAlert tries to determine resource type and analyze
func (a *AlertTriggeredAnalyzer) analyzeGenericResourceFromAlert(ctx context.Context, alert *alerts.Alert) []*Finding {
// Try each resource type in order of likelihood
resourceID := alert.ResourceID
switch {
case strings.Contains(resourceID, "/node/"):
return a.analyzeNodeFromAlert(ctx, alert)
case strings.Contains(resourceID, "/qemu/") || strings.Contains(resourceID, "/lxc/"):
return a.analyzeGuestFromAlert(ctx, alert)
case strings.Contains(resourceID, "docker"):
return a.analyzeDockerFromAlert(ctx, alert)
default:
// Try guest first (most common), then node
findings := a.analyzeGuestFromAlert(ctx, alert)
if len(findings) > 0 {
return findings
}
return a.analyzeNodeFromAlert(ctx, alert)
}
}
// resourceKeyFromAlert creates a unique key for the resource in an alert
func (a *AlertTriggeredAnalyzer) resourceKeyFromAlert(alert *alerts.Alert) string {
if alert.ResourceID != "" {
return alert.ResourceID
}
if alert.ResourceName != "" && alert.Instance != "" {
return fmt.Sprintf("%s/%s", alert.Instance, alert.ResourceName)
}
if alert.ResourceName != "" {
return alert.ResourceName
}
return ""
}
// CleanupOldCooldowns removes expired cooldown entries to prevent memory growth
func (a *AlertTriggeredAnalyzer) CleanupOldCooldowns() {
a.mu.Lock()
defer a.mu.Unlock()
now := time.Now()
for key, lastTime := range a.lastAnalyzed {
// Remove entries older than 1 hour
if now.Sub(lastTime) > time.Hour {
delete(a.lastAnalyzed, key)
}
}
}