Pulse/internal/ai/findings.go
rcourtman 7ef96919d3 fix(ai): Make LLM finding IDs stable across patrol runs
The main issue was that finding IDs included the title, which the LLM
generates differently each time. 'High CPU on minipc' vs 'Node minipc
experiencing high CPU load' got different IDs, making dismissals useless.

Changes:
1. LLM findings now get IDs based on resource+category only, not title
2. Add() now checks if finding is suppressed before adding as new
3. Add() now checks dismissed findings and only reactivates on severity escalation
4. IsSuppressed() now matches by resource+category only, not title
5. Added isSuppressedInternal() for use when lock is already held

Now when you dismiss 'performance issues on minipc', any future patrol finding
about performance on minipc will be recognized as the same issue and stay dismissed.
2025-12-11 00:03:17 +00:00

658 lines
18 KiB
Go

// Package ai provides AI-powered infrastructure monitoring and investigation.
package ai
import (
"fmt"
"strings"
"sync"
"time"
)
// FindingSeverity represents how urgent a finding is
type FindingSeverity string
const (
// FindingSeverityInfo is informational - user may want to know
FindingSeverityInfo FindingSeverity = "info"
// FindingSeverityWatch means something to keep an eye on
FindingSeverityWatch FindingSeverity = "watch"
// FindingSeverityWarning means action should be taken soon
FindingSeverityWarning FindingSeverity = "warning"
// FindingSeverityCritical means immediate action needed
FindingSeverityCritical FindingSeverity = "critical"
)
// FindingCategory groups findings by type
type FindingCategory string
const (
FindingCategoryPerformance FindingCategory = "performance"
FindingCategoryCapacity FindingCategory = "capacity"
FindingCategoryReliability FindingCategory = "reliability"
FindingCategoryBackup FindingCategory = "backup"
FindingCategorySecurity FindingCategory = "security"
FindingCategoryGeneral FindingCategory = "general"
)
// Finding represents an AI-discovered insight about infrastructure
type Finding struct {
ID string `json:"id"`
Severity FindingSeverity `json:"severity"`
Category FindingCategory `json:"category"`
ResourceID string `json:"resource_id"`
ResourceName string `json:"resource_name"`
ResourceType string `json:"resource_type"` // node, vm, container, docker, storage, host, pbs, host_raid
Node string `json:"node,omitempty"`
Title string `json:"title"`
Description string `json:"description"`
Recommendation string `json:"recommendation,omitempty"`
Evidence string `json:"evidence,omitempty"` // data/commands that led to this finding
Source string `json:"source,omitempty"` // "ai-analysis" for LLM findings, empty for rule-based
DetectedAt time.Time `json:"detected_at"`
LastSeenAt time.Time `json:"last_seen_at"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
AutoResolved bool `json:"auto_resolved"`
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
SnoozedUntil *time.Time `json:"snoozed_until,omitempty"` // Finding hidden until this time
// Link to alert if this finding was triggered by or attached to an alert
AlertID string `json:"alert_id,omitempty"`
// User feedback fields - enables LLM "memory" by tracking how users respond
// This helps prevent the LLM from repeatedly raising the same dismissed issues
DismissedReason string `json:"dismissed_reason,omitempty"` // "not_an_issue", "expected_behavior", "will_fix_later"
UserNote string `json:"user_note,omitempty"` // Freeform user explanation, included in LLM context
TimesRaised int `json:"times_raised"` // How many times this finding has been detected
Suppressed bool `json:"suppressed"` // Permanently suppress similar findings for this resource
}
// IsActive returns true if the finding is still active (not resolved, not snoozed, not suppressed)
func (f *Finding) IsActive() bool {
return f.ResolvedAt == nil && !f.IsSnoozed() && !f.Suppressed
}
// IsDismissed returns true if the user has dismissed this finding with a reason
func (f *Finding) IsDismissed() bool {
return f.DismissedReason != ""
}
// IsSnoozed returns true if the finding is currently snoozed
func (f *Finding) IsSnoozed() bool {
return f.SnoozedUntil != nil && time.Now().Before(*f.SnoozedUntil)
}
// IsResolved returns true if the finding has been resolved (ignores snooze)
func (f *Finding) IsResolved() bool {
return f.ResolvedAt != nil
}
// FindingsPersistence interface for saving/loading findings (avoids circular imports)
type FindingsPersistence interface {
SaveFindings(findings map[string]*Finding) error
LoadFindings() (map[string]*Finding, error)
}
// FindingsStore provides thread-safe storage for AI findings with optional persistence
type FindingsStore struct {
mu sync.RWMutex
findings map[string]*Finding // keyed by ID
// Index by resource for quick lookups
byResource map[string][]string // resource_id -> []finding_id
// Keep track of active findings count by severity (cached, but GetSummary calculates dynamically)
activeCounts map[FindingSeverity]int
// Persistence layer (optional)
persistence FindingsPersistence
// Debounce save operations
saveTimer *time.Timer
savePending bool
saveDebounce time.Duration
}
// NewFindingsStore creates a new findings store
func NewFindingsStore() *FindingsStore {
return &FindingsStore{
findings: make(map[string]*Finding),
byResource: make(map[string][]string),
activeCounts: make(map[FindingSeverity]int),
saveDebounce: 5 * time.Second, // Debounce saves by 5 seconds
}
}
// SetPersistence sets the persistence layer and loads existing findings
func (s *FindingsStore) SetPersistence(p FindingsPersistence) error {
s.mu.Lock()
s.persistence = p
s.mu.Unlock()
// Load existing findings from disk
if p != nil {
findings, err := p.LoadFindings()
if err != nil {
return err
}
if len(findings) > 0 {
s.mu.Lock()
for id, f := range findings {
s.findings[id] = f
s.byResource[f.ResourceID] = append(s.byResource[f.ResourceID], id)
if f.IsActive() {
s.activeCounts[f.Severity]++
}
}
s.mu.Unlock()
}
}
return nil
}
// scheduleSave schedules a debounced save operation
func (s *FindingsStore) scheduleSave() {
if s.persistence == nil {
return
}
// Already have a save pending
if s.savePending {
return
}
s.savePending = true
s.saveTimer = time.AfterFunc(s.saveDebounce, func() {
s.mu.Lock()
s.savePending = false
// Make a copy for saving
findingsCopy := make(map[string]*Finding, len(s.findings))
for id, f := range s.findings {
copy := *f
findingsCopy[id] = &copy
}
persistence := s.persistence
s.mu.Unlock()
if persistence != nil {
if err := persistence.SaveFindings(findingsCopy); err != nil {
// Log error but don't fail - persistence is best-effort
// (log import would create circular dep, so we silently fail)
}
}
})
}
// ForceSave immediately saves findings (useful for shutdown)
func (s *FindingsStore) ForceSave() error {
s.mu.Lock()
if s.saveTimer != nil {
s.saveTimer.Stop()
}
s.savePending = false
findingsCopy := make(map[string]*Finding, len(s.findings))
for id, f := range s.findings {
copy := *f
findingsCopy[id] = &copy
}
persistence := s.persistence
s.mu.Unlock()
if persistence != nil {
return persistence.SaveFindings(findingsCopy)
}
return nil
}
// Add adds or updates a finding
// If a finding with the same ID exists, it updates LastSeenAt and increments TimesRaised
// If the finding is suppressed or dismissed, it may be skipped
// Returns true if this is a new finding
func (s *FindingsStore) Add(f *Finding) bool {
s.mu.Lock()
existing, exists := s.findings[f.ID]
if exists {
// Check if it's permanently suppressed - don't update at all
if existing.Suppressed {
s.mu.Unlock()
return false
}
// Check if dismissed - only update if severity has escalated
if existing.DismissedReason != "" {
severityOrder := map[FindingSeverity]int{
FindingSeverityInfo: 0,
FindingSeverityWatch: 1,
FindingSeverityWarning: 2,
FindingSeverityCritical: 3,
}
// If new severity is same or lower, don't reactivate
if severityOrder[f.Severity] <= severityOrder[existing.Severity] {
existing.LastSeenAt = time.Now()
existing.TimesRaised++
s.mu.Unlock()
s.scheduleSave()
return false
}
// Severity escalated - clear dismissal and reactivate
existing.DismissedReason = ""
existing.UserNote = "" // Clear note since situation changed
existing.AcknowledgedAt = nil
}
// Update existing finding
existing.LastSeenAt = time.Now()
existing.Description = f.Description
existing.Recommendation = f.Recommendation
existing.Evidence = f.Evidence
existing.Title = f.Title // Update title in case LLM phrased it better
existing.Severity = f.Severity
existing.TimesRaised++ // Track recurrence
s.mu.Unlock()
s.scheduleSave()
return false
}
// New finding - check if resource+category is suppressed
if s.isSuppressedInternal(f.ResourceID, f.Category) {
s.mu.Unlock()
return false
}
// New finding
if f.DetectedAt.IsZero() {
f.DetectedAt = time.Now()
}
f.LastSeenAt = time.Now()
s.findings[f.ID] = f
s.byResource[f.ResourceID] = append(s.byResource[f.ResourceID], f.ID)
if f.IsActive() {
s.activeCounts[f.Severity]++
}
s.mu.Unlock()
s.scheduleSave()
return true
}
// Resolve marks a finding as resolved
func (s *FindingsStore) Resolve(id string, auto bool) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || !f.IsActive() {
s.mu.Unlock()
return false
}
now := time.Now()
f.ResolvedAt = &now
f.AutoResolved = auto
s.activeCounts[f.Severity]--
s.mu.Unlock()
s.scheduleSave()
return true
}
// Acknowledge marks a finding as acknowledged
func (s *FindingsStore) Acknowledge(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
now := time.Now()
f.AcknowledgedAt = &now
s.mu.Unlock()
s.scheduleSave()
return true
}
// Snooze hides a finding for the specified duration
// Common durations: 1h, 24h, 7d (168h)
func (s *FindingsStore) Snooze(id string, duration time.Duration) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || f.IsResolved() {
s.mu.Unlock()
return false
}
// If was previously active (not snoozed), decrement count
if f.SnoozedUntil == nil || time.Now().After(*f.SnoozedUntil) {
s.activeCounts[f.Severity]--
}
until := time.Now().Add(duration)
f.SnoozedUntil = &until
s.mu.Unlock()
s.scheduleSave()
return true
}
// Unsnooze removes the snooze from a finding, making it active again
func (s *FindingsStore) Unsnooze(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || f.IsResolved() {
s.mu.Unlock()
return false
}
if f.SnoozedUntil != nil {
f.SnoozedUntil = nil
s.activeCounts[f.Severity]++
}
s.mu.Unlock()
s.scheduleSave()
return true
}
// Dismiss marks a finding as dismissed with a reason and optional note
// Reasons: "not_an_issue", "expected_behavior", "will_fix_later"
func (s *FindingsStore) Dismiss(id, reason, note string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
f.DismissedReason = reason
if note != "" {
f.UserNote = note
}
// Also mark as acknowledged
now := time.Now()
f.AcknowledgedAt = &now
s.mu.Unlock()
s.scheduleSave()
return true
}
// SetUserNote updates the user note on a finding
func (s *FindingsStore) SetUserNote(id, note string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
f.UserNote = note
s.mu.Unlock()
s.scheduleSave()
return true
}
// Suppress marks a finding type as permanently suppressed for a resource
// Future findings with the same resource+category will be auto-dismissed
func (s *FindingsStore) Suppress(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
f.Suppressed = true
f.DismissedReason = "suppressed"
now := time.Now()
f.AcknowledgedAt = &now
s.mu.Unlock()
s.scheduleSave()
return true
}
// IsSuppressed checks if findings of this type for this resource are suppressed
// Checks by resource+category only (not title, since LLM titles vary)
func (s *FindingsStore) IsSuppressed(resourceID string, category FindingCategory) bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.isSuppressedInternal(resourceID, category)
}
// isSuppressedInternal checks suppression without locking (caller must hold lock)
func (s *FindingsStore) isSuppressedInternal(resourceID string, category FindingCategory) bool {
ids := s.byResource[resourceID]
for _, id := range ids {
if f, exists := s.findings[id]; exists {
// Match by resource+category only
if f.Suppressed && f.Category == category {
return true
}
}
}
return false
}
// Get returns a finding by ID
func (s *FindingsStore) Get(id string) *Finding {
s.mu.RLock()
defer s.mu.RUnlock()
if f, exists := s.findings[id]; exists {
// Return a copy to prevent mutations
copy := *f
return &copy
}
return nil
}
// GetByResource returns all active findings for a resource
func (s *FindingsStore) GetByResource(resourceID string) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
ids := s.byResource[resourceID]
result := make([]*Finding, 0, len(ids))
for _, id := range ids {
if f, exists := s.findings[id]; exists && f.IsActive() {
copy := *f
result = append(result, &copy)
}
}
return result
}
// GetActive returns all active findings, optionally filtered by severity
func (s *FindingsStore) GetActive(minSeverity FindingSeverity) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
severityOrder := map[FindingSeverity]int{
FindingSeverityInfo: 0,
FindingSeverityWatch: 1,
FindingSeverityWarning: 2,
FindingSeverityCritical: 3,
}
minOrder := severityOrder[minSeverity]
result := make([]*Finding, 0)
for _, f := range s.findings {
if f.IsActive() && severityOrder[f.Severity] >= minOrder {
copy := *f
result = append(result, &copy)
}
}
return result
}
// GetSummary returns a summary of active findings
// Note: This calculates counts dynamically to handle time-based snooze expiration
func (s *FindingsStore) GetSummary() FindingsSummary {
s.mu.RLock()
defer s.mu.RUnlock()
summary := FindingsSummary{
Total: len(s.findings),
}
// Calculate active counts dynamically since IsActive() checks time-based snooze
for _, f := range s.findings {
if f.IsActive() {
switch f.Severity {
case FindingSeverityCritical:
summary.Critical++
case FindingSeverityWarning:
summary.Warning++
case FindingSeverityWatch:
summary.Watch++
case FindingSeverityInfo:
summary.Info++
}
}
}
return summary
}
// GetAll returns all findings including resolved ones (for history)
// Results can be filtered by time range using startTime parameter
func (s *FindingsStore) GetAll(startTime *time.Time) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
result := make([]*Finding, 0, len(s.findings))
for _, f := range s.findings {
// If startTime specified, only include findings detected after it
if startTime != nil && f.DetectedAt.Before(*startTime) {
continue
}
copy := *f
result = append(result, &copy)
}
return result
}
// Cleanup removes old resolved findings
func (s *FindingsStore) Cleanup(maxAge time.Duration) int {
s.mu.Lock()
defer s.mu.Unlock()
cutoff := time.Now().Add(-maxAge)
removed := 0
for id, f := range s.findings {
if f.ResolvedAt != nil && f.ResolvedAt.Before(cutoff) {
delete(s.findings, id)
// Clean up resource index
ids := s.byResource[f.ResourceID]
for i, fid := range ids {
if fid == id {
s.byResource[f.ResourceID] = append(ids[:i], ids[i+1:]...)
break
}
}
removed++
}
}
return removed
}
// GetDismissedForContext returns findings that the user has dismissed/acknowledged,
// formatted for injection into LLM prompts. This is the core of the "memory" system -
// it tells the LLM what not to re-raise.
func (s *FindingsStore) GetDismissedForContext() string {
s.mu.RLock()
defer s.mu.RUnlock()
var suppressed, dismissed, snoozed []string
for _, f := range s.findings {
// Skip very old findings (more than 30 days)
if time.Since(f.LastSeenAt) > 30*24*time.Hour {
continue
}
// Collect suppressed findings
if f.Suppressed {
note := ""
if f.UserNote != "" {
note = " - User note: " + f.UserNote
}
suppressed = append(suppressed,
fmt.Sprintf("- %s on %s: %s%s", f.Title, f.ResourceName, f.DismissedReason, note))
continue
}
// Collect dismissed/acknowledged findings
if f.DismissedReason != "" {
note := ""
if f.UserNote != "" {
note = " - User note: " + f.UserNote
}
dismissed = append(dismissed,
fmt.Sprintf("- %s on %s (%s)%s", f.Title, f.ResourceName, f.DismissedReason, note))
continue
}
// Collect snoozed findings
if f.IsSnoozed() {
snoozed = append(snoozed,
fmt.Sprintf("- %s on %s (snoozed until %s)",
f.Title, f.ResourceName, f.SnoozedUntil.Format("Jan 2")))
}
}
if len(suppressed) == 0 && len(dismissed) == 0 && len(snoozed) == 0 {
return ""
}
var result strings.Builder
result.WriteString("\n## Previous Findings - User Feedback\n")
result.WriteString("The following findings have been addressed by the user. Do NOT re-raise these unless the situation has significantly worsened:\n\n")
if len(suppressed) > 0 {
result.WriteString("### Permanently Suppressed (never re-raise):\n")
for _, s := range suppressed {
result.WriteString(s + "\n")
}
result.WriteString("\n")
}
if len(dismissed) > 0 {
result.WriteString("### Dismissed by User:\n")
for _, d := range dismissed {
result.WriteString(d + "\n")
}
result.WriteString("\n")
}
if len(snoozed) > 0 {
result.WriteString("### Temporarily Snoozed:\n")
for _, s := range snoozed {
result.WriteString(s + "\n")
}
}
return result.String()
}
// FindingsSummary provides a quick count of findings by severity
type FindingsSummary struct {
Critical int `json:"critical"`
Warning int `json:"warning"`
Watch int `json:"watch"`
Info int `json:"info"`
Total int `json:"total"`
}
// HasIssues returns true if there are any warning or critical findings
func (s FindingsSummary) HasIssues() bool {
return s.Critical > 0 || s.Warning > 0
}
// IsHealthy returns true if there are no watch, warning, or critical findings
func (s FindingsSummary) IsHealthy() bool {
return s.Critical == 0 && s.Warning == 0 && s.Watch == 0
}