Pulse/internal/ai/patrol.go

package ai

import (
	"context"
	"crypto/sha256"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rs/zerolog/log"
)

// ThresholdProvider provides user-configured alert thresholds for patrol to use
type ThresholdProvider interface {
	// GetNodeCPUThreshold returns the CPU alert trigger threshold for nodes (0-100%)
	GetNodeCPUThreshold() float64
	// GetNodeMemoryThreshold returns the memory alert trigger threshold for nodes (0-100%)
	GetNodeMemoryThreshold() float64
	// GetGuestMemoryThreshold returns the memory alert trigger threshold for guests (0-100%)
	GetGuestMemoryThreshold() float64
	// GetGuestDiskThreshold returns the disk alert trigger threshold for guests (0-100%)
	GetGuestDiskThreshold() float64
	// GetStorageThreshold returns the usage alert trigger threshold for storage (0-100%)
	GetStorageThreshold() float64
}

// PatrolThresholds holds calculated thresholds for patrol (derived from alert thresholds)
type PatrolThresholds struct {
	// Node thresholds
	NodeCPUWatch    float64 // CPU % to flag as "watch" (typically alertThreshold - 15)
	NodeCPUWarning  float64 // CPU % to flag as "warning" (typically alertThreshold - 5)
	NodeMemWatch    float64
	NodeMemWarning  float64
	// Guest thresholds (VMs/containers)
	GuestMemWatch   float64
	GuestMemWarning float64
	GuestDiskWatch  float64
	GuestDiskWarn   float64
	GuestDiskCrit   float64
	// Storage thresholds
	StorageWatch    float64
	StorageWarning  float64
	StorageCritical float64
}

// DefaultPatrolThresholds returns fallback thresholds when no provider is set
func DefaultPatrolThresholds() PatrolThresholds {
	return PatrolThresholds{
		NodeCPUWatch:    75, NodeCPUWarning: 85,
		NodeMemWatch:    75, NodeMemWarning: 85,
		GuestMemWatch:   80, GuestMemWarning: 88,
		GuestDiskWatch:  75, GuestDiskWarn: 85, GuestDiskCrit: 92,
		StorageWatch:    70, StorageWarning: 80, StorageCritical: 90,
	}
}

// CalculatePatrolThresholds derives patrol thresholds from alert thresholds
// Patrol warns ~10% BEFORE the alert fires so users can take action early
func CalculatePatrolThresholds(provider ThresholdProvider) PatrolThresholds {
	if provider == nil {
		return DefaultPatrolThresholds()
	}

	// Get user's alert thresholds
	nodeCPU := provider.GetNodeCPUThreshold()
	nodeMem := provider.GetNodeMemoryThreshold()
	guestMem := provider.GetGuestMemoryThreshold()
	guestDisk := provider.GetGuestDiskThreshold()
	storage := provider.GetStorageThreshold()

	// Calculate patrol thresholds (watch = alert-15%, warning = alert-5%)
	return PatrolThresholds{
		NodeCPUWatch:    clampThreshold(nodeCPU - 15),
		NodeCPUWarning:  clampThreshold(nodeCPU - 5),
		NodeMemWatch:    clampThreshold(nodeMem - 15),
		NodeMemWarning:  clampThreshold(nodeMem - 5),
		GuestMemWatch:   clampThreshold(guestMem - 12),
		GuestMemWarning: clampThreshold(guestMem - 5),
		GuestDiskWatch:  clampThreshold(guestDisk - 15),
		GuestDiskWarn:   clampThreshold(guestDisk - 8),
		GuestDiskCrit:   clampThreshold(guestDisk - 3),
		StorageWatch:    clampThreshold(storage - 15),
		StorageWarning:  clampThreshold(storage - 8),
		StorageCritical: clampThreshold(storage - 3),
	}
}

// clampThreshold ensures a threshold is within valid range
func clampThreshold(v float64) float64 {
	if v < 10 {
		return 10 // Never go below 10%
	}
	if v > 99 {
		return 99
	}
	return v
}

// PatrolConfig holds configuration for the AI patrol service
type PatrolConfig struct {
	// Enabled controls whether background patrol runs
	Enabled bool `json:"enabled"`
	// QuickCheckInterval is how often to do quick health checks
	QuickCheckInterval time.Duration `json:"quick_check_interval"`
	// DeepAnalysisInterval is how often to do thorough analysis
	DeepAnalysisInterval time.Duration `json:"deep_analysis_interval"`
	// AnalyzeNodes controls whether to analyze Proxmox nodes
	AnalyzeNodes bool `json:"analyze_nodes"`
	// AnalyzeGuests controls whether to analyze VMs/containers
	AnalyzeGuests bool `json:"analyze_guests"`
	// AnalyzeDocker controls whether to analyze Docker hosts
	AnalyzeDocker bool `json:"analyze_docker"`
	// AnalyzeStorage controls whether to analyze storage
	AnalyzeStorage bool `json:"analyze_storage"`
	// AnalyzePBS controls whether to analyze PBS backup servers
	AnalyzePBS bool `json:"analyze_pbs"`
	// AnalyzeHosts controls whether to analyze agent hosts (RAID, sensors)
	AnalyzeHosts bool `json:"analyze_hosts"`
}

// DefaultPatrolConfig returns sensible defaults
func DefaultPatrolConfig() PatrolConfig {
	return PatrolConfig{
		Enabled:              true,
		QuickCheckInterval:   15 * time.Minute,
		DeepAnalysisInterval: 6 * time.Hour,
		AnalyzeNodes:         true,
		AnalyzeGuests:        true,
		AnalyzeDocker:        true,
		AnalyzeStorage:       true,
		AnalyzePBS:           true,
		AnalyzeHosts:         true,
	}
}

// PatrolStatus represents the current state of the patrol service
type PatrolStatus struct {
	Running          bool          `json:"running"`
	LastPatrolAt     *time.Time    `json:"last_patrol_at,omitempty"`
	LastDeepAnalysis *time.Time    `json:"last_deep_analysis_at,omitempty"`
	NextPatrolAt     *time.Time    `json:"next_patrol_at,omitempty"`
	LastDuration     time.Duration `json:"last_duration_ms"`
	ResourcesChecked int           `json:"resources_checked"`
	FindingsCount    int           `json:"findings_count"`
	ErrorCount       int           `json:"error_count"`
	Healthy          bool          `json:"healthy"`
}

// PatrolService runs background AI analysis of infrastructure
type PatrolService struct {
	mu sync.RWMutex

	aiService         *Service
	stateProvider     StateProvider
	thresholdProvider ThresholdProvider
	config            PatrolConfig
	findings          *FindingsStore

	// Cached thresholds (recalculated when thresholdProvider changes)
	thresholds PatrolThresholds

	// Runtime state
	running          bool
	stopCh           chan struct{}
	lastPatrol       time.Time
	lastDeepAnalysis time.Time
	lastDuration     time.Duration
	resourcesChecked int
	errorCount       int
}

// NewPatrolService creates a new patrol service
func NewPatrolService(aiService *Service, stateProvider StateProvider) *PatrolService {
	return &PatrolService{
		aiService:     aiService,
		stateProvider: stateProvider,
		config:        DefaultPatrolConfig(),
		findings:      NewFindingsStore(),
		thresholds:    DefaultPatrolThresholds(),
		stopCh:        make(chan struct{}),
	}
}

// SetConfig updates the patrol configuration
func (p *PatrolService) SetConfig(cfg PatrolConfig) {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.config = cfg
}

// SetThresholdProvider sets the provider for user-configured alert thresholds
// This allows patrol to warn BEFORE alerts fire
func (p *PatrolService) SetThresholdProvider(provider ThresholdProvider) {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.thresholdProvider = provider
	p.thresholds = CalculatePatrolThresholds(provider)
	log.Debug().
		Float64("storageWatch", p.thresholds.StorageWatch).
		Float64("storageWarning", p.thresholds.StorageWarning).
		Float64("storageCritical", p.thresholds.StorageCritical).
		Msg("Patrol thresholds updated from alert config")
}

// SetFindingsPersistence enables findings persistence (load from and save to disk)
// This should be called before Start() to load any existing findings
func (p *PatrolService) SetFindingsPersistence(persistence FindingsPersistence) error {
	p.mu.Lock()
	findings := p.findings
	p.mu.Unlock()

	if findings != nil && persistence != nil {
		if err := findings.SetPersistence(persistence); err != nil {
			return err
		}
		log.Info().Msg("AI Patrol findings persistence enabled")
	}
	return nil
}

// GetConfig returns the current patrol configuration
func (p *PatrolService) GetConfig() PatrolConfig {
	p.mu.RLock()
	defer p.mu.RUnlock()
	return p.config
}

// GetFindings returns the findings store
func (p *PatrolService) GetFindings() *FindingsStore {
	return p.findings
}

// GetStatus returns the current patrol status
func (p *PatrolService) GetStatus() PatrolStatus {
	p.mu.RLock()
	defer p.mu.RUnlock()

	status := PatrolStatus{
		Running:          p.running,
		LastDuration:     p.lastDuration,
		ResourcesChecked: p.resourcesChecked,
		FindingsCount:    len(p.findings.GetActive(FindingSeverityInfo)),
		ErrorCount:       p.errorCount,
	}

	if !p.lastPatrol.IsZero() {
		status.LastPatrolAt = &p.lastPatrol
	}
	if !p.lastDeepAnalysis.IsZero() {
		status.LastDeepAnalysis = &p.lastDeepAnalysis
	}

	if p.running && p.config.QuickCheckInterval > 0 {
		next := p.lastPatrol.Add(p.config.QuickCheckInterval)
		status.NextPatrolAt = &next
	}

	summary := p.findings.GetSummary()
	status.Healthy = summary.IsHealthy()

	return status
}

// Start begins the background patrol loop
func (p *PatrolService) Start(ctx context.Context) {
	p.mu.Lock()
	if p.running {
		p.mu.Unlock()
		return
	}
	p.running = true
	p.stopCh = make(chan struct{})
	p.mu.Unlock()

	log.Info().
		Dur("quick_interval", p.config.QuickCheckInterval).
		Dur("deep_interval", p.config.DeepAnalysisInterval).
		Msg("Starting AI Patrol Service")

	go p.patrolLoop(ctx)
}

// Stop stops the patrol service
func (p *PatrolService) Stop() {
	p.mu.Lock()
	if !p.running {
		p.mu.Unlock()
		return
	}
	p.running = false
	close(p.stopCh)
	p.mu.Unlock()

	log.Info().Msg("Stopping AI Patrol Service")
}

// patrolLoop is the main background loop
func (p *PatrolService) patrolLoop(ctx context.Context) {
	// Run initial quick patrol shortly after startup
	initialDelay := 30 * time.Second
	select {
	case <-time.After(initialDelay):
		p.runPatrol(ctx, false)
	case <-p.stopCh:
		return
	case <-ctx.Done():
		return
	}

	quickTicker := time.NewTicker(p.config.QuickCheckInterval)
	defer quickTicker.Stop()

	// Deep analysis ticker (if configured)
	var deepTicker *time.Ticker
	if p.config.DeepAnalysisInterval > 0 {
		deepTicker = time.NewTicker(p.config.DeepAnalysisInterval)
		defer deepTicker.Stop()
	}

	for {
		select {
		case <-quickTicker.C:
			p.runPatrol(ctx, false)

		case <-func() <-chan time.Time {
			if deepTicker != nil {
				return deepTicker.C
			}
			return make(chan time.Time) // never fires
		}():
			p.runPatrol(ctx, true)

		case <-p.stopCh:
			return

		case <-ctx.Done():
			return
		}
	}
}

// runPatrol executes a patrol run
func (p *PatrolService) runPatrol(ctx context.Context, deep bool) {
	p.mu.RLock()
	cfg := p.config
	p.mu.RUnlock()

	if !cfg.Enabled {
		return
	}

	// Check if AI service is enabled
	if p.aiService == nil || !p.aiService.IsEnabled() {
		log.Debug().Msg("AI Patrol: AI service not enabled, skipping patrol")
		return
	}

	start := time.Now()
	patrolType := "quick"
	if deep {
		patrolType = "deep"
	}

	log.Debug().Str("type", patrolType).Msg("AI Patrol: Starting patrol run")

	var resourceCount int
	var errors int

	// Get current state
	if p.stateProvider == nil {
		log.Warn().Msg("AI Patrol: No state provider available")
		return
	}

	state := p.stateProvider.GetState()

	// Analyze nodes
	if cfg.AnalyzeNodes {
		for _, node := range state.Nodes {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			findings := p.analyzeNode(node, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}
	}

	// Analyze VMs and containers
	if cfg.AnalyzeGuests {
		for _, vm := range state.VMs {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			// Calculate usage percentages from Memory/Disk structs
			var memUsage, diskUsage float64
			if vm.Memory.Total > 0 {
				memUsage = float64(vm.Memory.Used) / float64(vm.Memory.Total)
			}
			if vm.Disk.Total > 0 {
				diskUsage = float64(vm.Disk.Used) / float64(vm.Disk.Total)
			}
			// Handle LastBackup - pass nil if zero time
			var lastBackup *time.Time
			if !vm.LastBackup.IsZero() {
				t := vm.LastBackup
				lastBackup = &t
			}
			findings := p.analyzeGuest(vm.ID, vm.Name, "vm", vm.Node, vm.Status,
				vm.CPU, memUsage, diskUsage, lastBackup, vm.Template, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}

		for _, ct := range state.Containers {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			// Calculate usage percentages from Memory/Disk structs
			var memUsage, diskUsage float64
			if ct.Memory.Total > 0 {
				memUsage = float64(ct.Memory.Used) / float64(ct.Memory.Total)
			}
			if ct.Disk.Total > 0 {
				diskUsage = float64(ct.Disk.Used) / float64(ct.Disk.Total)
			}
			// Handle LastBackup - pass nil if zero time
			var lastBackup *time.Time
			if !ct.LastBackup.IsZero() {
				t := ct.LastBackup
				lastBackup = &t
			}
			findings := p.analyzeGuest(ct.ID, ct.Name, "container", ct.Node, ct.Status,
				ct.CPU, memUsage, diskUsage, lastBackup, ct.Template, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}
	}

	// Analyze Docker hosts
	if cfg.AnalyzeDocker {
		for _, dh := range state.DockerHosts {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			findings := p.analyzeDockerHost(dh, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}
	}

	// Analyze storage
	if cfg.AnalyzeStorage {
		for _, st := range state.Storage {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			findings := p.analyzeStorage(st, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}
	}

	// Analyze PBS instances (backup servers)
	if cfg.AnalyzePBS {
		for _, pbs := range state.PBSInstances {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			findings := p.analyzePBSInstance(pbs, state.PBSBackups, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}
	}

	// Analyze agent hosts (RAID, sensors)
	if cfg.AnalyzeHosts {
		for _, host := range state.Hosts {
			select {
			case <-ctx.Done():
				return
			default:
			}
			resourceCount++
			findings := p.analyzeHost(host, deep)
			for _, f := range findings {
				if p.findings.Add(f) {
					log.Info().
						Str("finding_id", f.ID).
						Str("severity", string(f.Severity)).
						Str("resource", f.ResourceName).
						Str("title", f.Title).
						Msg("AI Patrol: New finding")
				}
			}
		}
	}

	// Auto-resolve findings that weren't seen in this patrol run
	p.autoResolveStaleFindings(start)

	// Cleanup old resolved findings
	cleaned := p.findings.Cleanup(24 * time.Hour)
	if cleaned > 0 {
		log.Debug().Int("cleaned", cleaned).Msg("AI Patrol: Cleaned up old findings")
	}

	duration := time.Since(start)

	p.mu.Lock()
	p.lastPatrol = time.Now()
	p.lastDuration = duration
	p.resourcesChecked = resourceCount
	p.errorCount = errors
	if deep {
		p.lastDeepAnalysis = time.Now()
	}
	p.mu.Unlock()

	summary := p.findings.GetSummary()
	log.Info().
		Str("type", patrolType).
		Dur("duration", duration).
		Int("resources", resourceCount).
		Int("critical", summary.Critical).
		Int("warning", summary.Warning).
		Int("watch", summary.Watch).
		Msg("AI Patrol: Completed patrol run")
}

// generateFindingID creates a stable ID for a finding based on resource and issue
func generateFindingID(resourceID, category, issue string) string {
	hash := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", resourceID, category, issue)))
	return fmt.Sprintf("%x", hash[:8])
}

// analyzeNode checks a Proxmox node for issues
func (p *PatrolService) analyzeNode(node models.Node, deep bool) []*Finding {
	var findings []*Finding

	// Calculate memory usage from Memory struct (as percentage 0-100)
	var memUsagePct float64
	if node.Memory.Total > 0 {
		memUsagePct = float64(node.Memory.Used) / float64(node.Memory.Total) * 100
	}

	// CPU as percentage (node.CPU is 0-1 ratio from Proxmox)
	cpuPct := node.CPU * 100

	// Check for offline nodes
	if node.Status == "offline" || node.Status == "unknown" {
		findings = append(findings, &Finding{
			ID:             generateFindingID(node.ID, "reliability", "offline"),
			Severity:       FindingSeverityCritical,
			Category:       FindingCategoryReliability,
			ResourceID:     node.ID,
			ResourceName:   node.Name,
			ResourceType:   "node",
			Title:          "Node offline",
			Description:    fmt.Sprintf("Node '%s' is not responding", node.Name),
			Recommendation: "Check network connectivity, SSH access, and Proxmox services on the node",
		})
	}

	// High CPU - use dynamic thresholds from user's alert config
	if cpuPct > p.thresholds.NodeCPUWatch {
		severity := FindingSeverityWatch
		if cpuPct > p.thresholds.NodeCPUWarning {
			severity = FindingSeverityWarning
		}
		findings = append(findings, &Finding{
			ID:             generateFindingID(node.ID, "performance", "high-cpu"),
			Severity:       severity,
			Category:       FindingCategoryPerformance,
			ResourceID:     node.ID,
			ResourceName:   node.Name,
			ResourceType:   "node",
			Title:          "High CPU usage",
			Description:    fmt.Sprintf("Node '%s' CPU at %.0f%%", node.Name, cpuPct),
			Recommendation: "Check which VMs/containers are consuming CPU. Consider load balancing.",
			Evidence:       fmt.Sprintf("CPU: %.1f%%", cpuPct),
		})
	}

	// High memory - use dynamic thresholds
	if memUsagePct > p.thresholds.NodeMemWatch {
		severity := FindingSeverityWatch
		if memUsagePct > p.thresholds.NodeMemWarning {
			severity = FindingSeverityWarning
		}
		findings = append(findings, &Finding{
			ID:             generateFindingID(node.ID, "performance", "high-memory"),
			Severity:       severity,
			Category:       FindingCategoryPerformance,
			ResourceID:     node.ID,
			ResourceName:   node.Name,
			ResourceType:   "node",
			Title:          "High memory usage",
			Description:    fmt.Sprintf("Node '%s' memory at %.0f%%", node.Name, memUsagePct),
			Recommendation: "Consider migrating some VMs to other nodes or increasing node RAM",
			Evidence:       fmt.Sprintf("Memory: %.1f%%", memUsagePct),
		})
	}

	return findings
}

// analyzeGuest checks a VM or container for issues
func (p *PatrolService) analyzeGuest(id, name, guestType, node, status string,
	cpu, memUsage, diskUsage float64, lastBackup *time.Time, template, deep bool) []*Finding {
	var findings []*Finding

	// Skip templates
	if template {
		return findings
	}

	// Convert ratios to percentages for comparison with thresholds
	memPct := memUsage * 100
	diskPct := diskUsage * 100

	// High memory (sustained) - use dynamic thresholds
	if memPct > p.thresholds.GuestMemWatch {
		severity := FindingSeverityWatch
		if memPct > p.thresholds.GuestMemWarning {
			severity = FindingSeverityWarning
		}
		findings = append(findings, &Finding{
			ID:             generateFindingID(id, "performance", "high-memory"),
			Severity:       severity,
			Category:       FindingCategoryPerformance,
			ResourceID:     id,
			ResourceName:   name,
			ResourceType:   guestType,
			Node:           node,
			Title:          "High memory usage",
			Description:    fmt.Sprintf("'%s' memory at %.0f%% - risk of OOM", name, memPct),
			Recommendation: "Consider increasing allocated RAM or investigating memory-hungry processes",
			Evidence:       fmt.Sprintf("Memory: %.1f%%", memPct),
		})
	}

	// High disk usage - use dynamic thresholds
	if diskPct > p.thresholds.GuestDiskWatch {
		severity := FindingSeverityWatch
		if diskPct > p.thresholds.GuestDiskWarn {
			severity = FindingSeverityWarning
		}
		if diskPct > p.thresholds.GuestDiskCrit {
			severity = FindingSeverityCritical
		}
		findings = append(findings, &Finding{
			ID:             generateFindingID(id, "capacity", "high-disk"),
			Severity:       severity,
			Category:       FindingCategoryCapacity,
			ResourceID:     id,
			ResourceName:   name,
			ResourceType:   guestType,
			Node:           node,
			Title:          "High disk usage",
			Description:    fmt.Sprintf("'%s' disk at %.0f%%", name, diskPct),
			Recommendation: "Clean up old files, logs, or docker images. Consider expanding disk.",
			Evidence:       fmt.Sprintf("Disk: %.1f%%", diskPct),
		})
	}

	// Backup check (only for running guests)
	if status == "running" && lastBackup != nil {
		daysSinceBackup := time.Since(*lastBackup).Hours() / 24
		if daysSinceBackup > 14 {
			severity := FindingSeverityWatch
			if daysSinceBackup > 30 {
				severity = FindingSeverityWarning
			}
			findings = append(findings, &Finding{
				ID:             generateFindingID(id, "backup", "stale"),
				Severity:       severity,
				Category:       FindingCategoryBackup,
				ResourceID:     id,
				ResourceName:   name,
				ResourceType:   guestType,
				Node:           node,
				Title:          "Backup overdue",
				Description:    fmt.Sprintf("'%s' hasn't been backed up in %.0f days", name, daysSinceBackup),
				Recommendation: "Check backup job configuration or run a manual backup",
				Evidence:       fmt.Sprintf("Last backup: %s", lastBackup.Format("2006-01-02")),
			})
		}
	} else if status == "running" && lastBackup == nil {
		findings = append(findings, &Finding{
			ID:             generateFindingID(id, "backup", "never"),
			Severity:       FindingSeverityWarning,
			Category:       FindingCategoryBackup,
			ResourceID:     id,
			ResourceName:   name,
			ResourceType:   guestType,
			Node:           node,
			Title:          "Never backed up",
			Description:    fmt.Sprintf("'%s' has no backup history", name),
			Recommendation: "Configure backup job for this guest",
		})
	}

	return findings
}

// analyzeDockerHost checks a Docker host for issues
func (p *PatrolService) analyzeDockerHost(host models.DockerHost, deep bool) []*Finding {
	var findings []*Finding

	hostName := host.Hostname
	if host.DisplayName != "" {
		hostName = host.DisplayName
	}

	// Host offline
	if host.Status != "online" && host.Status != "connected" {
		findings = append(findings, &Finding{
			ID:             generateFindingID(host.ID, "reliability", "offline"),
			Severity:       FindingSeverityCritical,
			Category:       FindingCategoryReliability,
			ResourceID:     host.ID,
			ResourceName:   hostName,
			ResourceType:   "docker_host",
			Title:          "Docker host offline",
			Description:    fmt.Sprintf("Docker host '%s' is not responding", hostName),
			Recommendation: "Check network connectivity and docker-agent service",
		})
	}

	// Check individual containers
	for _, c := range host.Containers {
		// Restarting containers
		if c.State == "restarting" || c.RestartCount > 3 {
			findings = append(findings, &Finding{
				ID:             generateFindingID(c.ID, "reliability", "restart-loop"),
				Severity:       FindingSeverityWarning,
				Category:       FindingCategoryReliability,
				ResourceID:     c.ID,
				ResourceName:   c.Name,
				ResourceType:   "docker_container",
				Node:           hostName,
				Title:          "Container restart loop",
				Description:    fmt.Sprintf("Container '%s' has restarted %d times", c.Name, c.RestartCount),
				Recommendation: "Check container logs: docker logs " + c.Name,
				Evidence:       fmt.Sprintf("State: %s, Restarts: %d", c.State, c.RestartCount),
			})
		}

		// High memory containers
		if c.MemoryPercent > 90 {
			findings = append(findings, &Finding{
				ID:             generateFindingID(c.ID, "performance", "high-memory"),
				Severity:       FindingSeverityWatch,
				Category:       FindingCategoryPerformance,
				ResourceID:     c.ID,
				ResourceName:   c.Name,
				ResourceType:   "docker_container",
				Node:           hostName,
				Title:          "High memory usage",
				Description:    fmt.Sprintf("Container '%s' using %.0f%% of allocated memory", c.Name, c.MemoryPercent),
				Recommendation: "Consider increasing container memory limit",
				Evidence:       fmt.Sprintf("Memory: %.1f%%", c.MemoryPercent),
			})
		}
	}

	return findings
}

// analyzeStorage checks storage for issues
func (p *PatrolService) analyzeStorage(storage models.Storage, deep bool) []*Finding {
	var findings []*Finding

	// Note: storage.Usage is already a percentage (0-100, e.g. 85.5 means 85.5%)
	// If Usage is 0 but we have bytes data, calculate it as percentage
	usage := storage.Usage
	if usage == 0 && storage.Total > 0 {
		usage = float64(storage.Used) / float64(storage.Total) * 100
	}

	// High storage usage - use dynamic thresholds from user's alert config
	if usage > p.thresholds.StorageWatch {
		severity := FindingSeverityWatch
		if usage > p.thresholds.StorageWarning {
			severity = FindingSeverityWarning
		}
		if usage > p.thresholds.StorageCritical {
			severity = FindingSeverityCritical
		}

		findings = append(findings, &Finding{
			ID:             generateFindingID(storage.ID, "capacity", "high-usage"),
			Severity:       severity,
			Category:       FindingCategoryCapacity,
			ResourceID:     storage.ID,
			ResourceName:   storage.Name,
			ResourceType:   "storage",
			Title:          "Storage filling up",
			Description:    fmt.Sprintf("Storage '%s' at %.0f%% capacity", storage.Name, usage),
			Recommendation: "Clean up old backups, snapshots, or unused disk images",
			Evidence:       fmt.Sprintf("Usage: %.1f%%", usage),
		})
	}

	return findings
}

// autoResolveHealthyResources marks findings as resolved when they weren't seen in the current patrol
// patrolStartTime is used to determine which findings are stale (LastSeenAt < patrolStartTime)
func (p *PatrolService) autoResolveStaleFindings(patrolStartTime time.Time) {
	// Get all active findings and check if they're stale
	activeFindings := p.findings.GetActive(FindingSeverityInfo)
	
	for _, f := range activeFindings {
		// If the finding wasn't updated during this patrol (LastSeenAt is before patrol started),
		// it means the condition that caused it has been resolved
		if f.LastSeenAt.Before(patrolStartTime) {
			if p.findings.Resolve(f.ID, true) {
				log.Info().
					Str("finding_id", f.ID).
					Str("resource", f.ResourceName).
					Str("title", f.Title).
					Msg("AI Patrol: Auto-resolved finding")
			}
		}
	}
}

// GetFindingsForResource returns active findings for a specific resource
func (p *PatrolService) GetFindingsForResource(resourceID string) []*Finding {
	return p.findings.GetByResource(resourceID)
}

// GetFindingsSummary returns a summary of all findings
func (p *PatrolService) GetFindingsSummary() FindingsSummary {
	return p.findings.GetSummary()
}

// GetAllFindings returns all active findings sorted by severity
func (p *PatrolService) GetAllFindings() []*Finding {
	findings := p.findings.GetActive(FindingSeverityInfo)
	
	// Sort by severity (critical first) then by time
	severityOrder := map[FindingSeverity]int{
		FindingSeverityCritical: 0,
		FindingSeverityWarning:  1,
		FindingSeverityWatch:    2,
		FindingSeverityInfo:     3,
	}
	
	sort.Slice(findings, func(i, j int) bool {
		if severityOrder[findings[i].Severity] != severityOrder[findings[j].Severity] {
			return severityOrder[findings[i].Severity] < severityOrder[findings[j].Severity]
		}
		return findings[i].DetectedAt.After(findings[j].DetectedAt)
	})
	
	return findings
}

// GetFindingsHistory returns all findings including resolved ones for history display
// Optionally filter by startTime
func (p *PatrolService) GetFindingsHistory(startTime *time.Time) []*Finding {
	findings := p.findings.GetAll(startTime)
	
	// Sort by detected time (newest first)
	sort.Slice(findings, func(i, j int) bool {
		return findings[i].DetectedAt.After(findings[j].DetectedAt)
	})
	
	return findings
}

// ForcePatrol triggers an immediate patrol run
func (p *PatrolService) ForcePatrol(ctx context.Context, deep bool) {
	go p.runPatrol(ctx, deep)
}

// analyzePBSInstance checks a PBS backup server for issues
func (p *PatrolService) analyzePBSInstance(pbs models.PBSInstance, allBackups []models.PBSBackup, deep bool) []*Finding {
	var findings []*Finding

	pbsName := pbs.Name
	if pbsName == "" {
		pbsName = pbs.Host
	}

	// Check PBS connectivity
	if pbs.Status != "online" && pbs.Status != "connected" && pbs.Status != "" {
		findings = append(findings, &Finding{
			ID:             generateFindingID(pbs.ID, "reliability", "offline"),
			Severity:       FindingSeverityCritical,
			Category:       FindingCategoryReliability,
			ResourceID:     pbs.ID,
			ResourceName:   pbsName,
			ResourceType:   "pbs",
			Title:          "PBS server offline",
			Description:    fmt.Sprintf("Proxmox Backup Server '%s' is not responding", pbsName),
			Recommendation: "Check network connectivity and PBS service status",
		})
	}

	// Check each datastore capacity
	for _, ds := range pbs.Datastores {
		usage := ds.Usage
		if usage == 0 && ds.Total > 0 {
			usage = float64(ds.Used) / float64(ds.Total) * 100
		}

		// PBS datastores should trigger earlier than regular storage
		// since running out of backup space is critical
		if usage > p.thresholds.StorageWatch {
			severity := FindingSeverityWatch
			if usage > p.thresholds.StorageWarning {
				severity = FindingSeverityWarning
			}
			if usage > p.thresholds.StorageCritical {
				severity = FindingSeverityCritical
			}

			findings = append(findings, &Finding{
				ID:             generateFindingID(pbs.ID+":"+ds.Name, "capacity", "high-usage"),
				Severity:       severity,
				Category:       FindingCategoryCapacity,
				ResourceID:     pbs.ID + ":" + ds.Name,
				ResourceName:   fmt.Sprintf("%s/%s", pbsName, ds.Name),
				ResourceType:   "pbs_datastore",
				Title:          "PBS datastore filling up",
				Description:    fmt.Sprintf("Datastore '%s' on PBS '%s' at %.0f%% capacity", ds.Name, pbsName, usage),
				Recommendation: "Run garbage collection, prune old backups, or expand storage",
				Evidence:       fmt.Sprintf("Usage: %.1f%%", usage),
			})
		}

		// Check for datastore errors
		if ds.Error != "" {
			findings = append(findings, &Finding{
				ID:             generateFindingID(pbs.ID+":"+ds.Name, "reliability", "error"),
				Severity:       FindingSeverityCritical,
				Category:       FindingCategoryReliability,
				ResourceID:     pbs.ID + ":" + ds.Name,
				ResourceName:   fmt.Sprintf("%s/%s", pbsName, ds.Name),
				ResourceType:   "pbs_datastore",
				Title:          "PBS datastore error",
				Description:    fmt.Sprintf("Datastore '%s' has an error: %s", ds.Name, ds.Error),
				Recommendation: "Check PBS server logs and datastore configuration",
				Evidence:       ds.Error,
			})
		}
	}

	// Check for backup staleness per datastore
	// Build a map of latest backup time per datastore
	datastoreLastBackup := make(map[string]time.Time)
	for _, backup := range allBackups {
		if backup.Instance != pbs.ID && backup.Instance != pbs.Name {
			continue
		}
		dsKey := backup.Datastore
		if backup.BackupTime.After(datastoreLastBackup[dsKey]) {
			datastoreLastBackup[dsKey] = backup.BackupTime
		}
	}

	for _, ds := range pbs.Datastores {
		lastBackup, hasBackups := datastoreLastBackup[ds.Name]
		
		if !hasBackups {
			// No backups found for this datastore - might be intentional (empty datastore)
			// Only warn if datastore has actual content
			if ds.Used > 0 {
				findings = append(findings, &Finding{
					ID:             generateFindingID(pbs.ID+":"+ds.Name, "backup", "no-recent"),
					Severity:       FindingSeverityWatch,
					Category:       FindingCategoryBackup,
					ResourceID:     pbs.ID + ":" + ds.Name,
					ResourceName:   fmt.Sprintf("%s/%s", pbsName, ds.Name),
					ResourceType:   "pbs_datastore",
					Title:          "No recent backup metadata",
					Description:    fmt.Sprintf("Datastore '%s' has content but no recent backup entries visible", ds.Name),
					Recommendation: "Verify backup jobs are configured and running",
				})
			}
			continue
		}

		hoursSinceBackup := time.Since(lastBackup).Hours()
		if hoursSinceBackup > 48 {
			severity := FindingSeverityWatch
			if hoursSinceBackup > 72 {
				severity = FindingSeverityWarning
			}
			if hoursSinceBackup > 168 { // 7 days
				severity = FindingSeverityCritical
			}

			findings = append(findings, &Finding{
				ID:             generateFindingID(pbs.ID+":"+ds.Name, "backup", "stale"),
				Severity:       severity,
				Category:       FindingCategoryBackup,
				ResourceID:     pbs.ID + ":" + ds.Name,
				ResourceName:   fmt.Sprintf("%s/%s", pbsName, ds.Name),
				ResourceType:   "pbs_datastore",
				Title:          "Stale backups",
				Description:    fmt.Sprintf("No backups to '%s/%s' in %.0f hours", pbsName, ds.Name, hoursSinceBackup),
				Recommendation: "Check backup job schedule and logs for failures",
				Evidence:       fmt.Sprintf("Last backup: %s", lastBackup.Format("2006-01-02 15:04")),
			})
		}
	}

	// Check backup jobs for failures (only during deep analysis)
	if deep {
		for _, job := range pbs.BackupJobs {
			if job.Status == "error" || job.Error != "" {
				findings = append(findings, &Finding{
					ID:             generateFindingID(pbs.ID+":job:"+job.ID, "backup", "job-failed"),
					Severity:       FindingSeverityWarning,
					Category:       FindingCategoryBackup,
					ResourceID:     pbs.ID + ":job:" + job.ID,
					ResourceName:   fmt.Sprintf("%s/job/%s", pbsName, job.ID),
					ResourceType:   "pbs_job",
					Title:          "Backup job failed",
					Description:    fmt.Sprintf("Backup job '%s' on PBS '%s' is failing", job.ID, pbsName),
					Recommendation: "Check PBS task logs for error details",
					Evidence:       job.Error,
				})
			}
		}

		for _, job := range pbs.VerifyJobs {
			if job.Status == "error" || job.Error != "" {
				findings = append(findings, &Finding{
					ID:             generateFindingID(pbs.ID+":verify:"+job.ID, "backup", "verify-failed"),
					Severity:       FindingSeverityWarning,
					Category:       FindingCategoryBackup,
					ResourceID:     pbs.ID + ":verify:" + job.ID,
					ResourceName:   fmt.Sprintf("%s/verify/%s", pbsName, job.ID),
					ResourceType:   "pbs_job",
					Title:          "Verify job failed",
					Description:    fmt.Sprintf("Verify job '%s' on PBS '%s' is failing", job.ID, pbsName),
					Recommendation: "Check PBS task logs - verify failures may indicate backup corruption",
					Evidence:       job.Error,
				})
			}
		}
	}

	return findings
}

// analyzeHost checks an agent host for issues (RAID, sensors, connectivity)
func (p *PatrolService) analyzeHost(host models.Host, deep bool) []*Finding {
	var findings []*Finding

	hostName := host.DisplayName
	if hostName == "" {
		hostName = host.Hostname
	}

	// Check host connectivity
	if host.Status != "online" && host.Status != "connected" && host.Status != "" {
		findings = append(findings, &Finding{
			ID:             generateFindingID(host.ID, "reliability", "offline"),
			Severity:       FindingSeverityCritical,
			Category:       FindingCategoryReliability,
			ResourceID:     host.ID,
			ResourceName:   hostName,
			ResourceType:   "host",
			Title:          "Host agent offline",
			Description:    fmt.Sprintf("Host '%s' agent is not reporting", hostName),
			Recommendation: "Check network connectivity and pulse-agent service status",
		})
	}

	// Check RAID arrays
	for _, raid := range host.RAID {
		raidName := raid.Device
		if raid.Name != "" {
			raidName = raid.Name
		}

		// Check for degraded/failed state
		switch raid.State {
		case "degraded", "DEGRADED":
			findings = append(findings, &Finding{
				ID:             generateFindingID(host.ID+":"+raid.Device, "reliability", "raid-degraded"),
				Severity:       FindingSeverityCritical,
				Category:       FindingCategoryReliability,
				ResourceID:     host.ID + ":" + raid.Device,
				ResourceName:   fmt.Sprintf("%s/%s", hostName, raidName),
				ResourceType:   "host_raid",
				Title:          "RAID array degraded",
				Description:    fmt.Sprintf("RAID array '%s' on '%s' is degraded (%d/%d devices active)", raidName, hostName, raid.ActiveDevices, raid.TotalDevices),
				Recommendation: "Replace failed drive and initiate rebuild. Check dmesg for drive errors.",
				Evidence:       fmt.Sprintf("State: %s, Active: %d/%d, Failed: %d", raid.State, raid.ActiveDevices, raid.TotalDevices, raid.FailedDevices),
			})

		case "recovering", "rebuilding", "resyncing", "RECOVERING":
			severity := FindingSeverityWarning
			if raid.RebuildPercent < 50 {
				severity = FindingSeverityWatch // Early in rebuild, less urgent
			}
			findings = append(findings, &Finding{
				ID:             generateFindingID(host.ID+":"+raid.Device, "reliability", "raid-rebuilding"),
				Severity:       severity,
				Category:       FindingCategoryReliability,
				ResourceID:     host.ID + ":" + raid.Device,
				ResourceName:   fmt.Sprintf("%s/%s", hostName, raidName),
				ResourceType:   "host_raid",
				Title:          "RAID array rebuilding",
				Description:    fmt.Sprintf("RAID array '%s' on '%s' is rebuilding (%.1f%% complete)", raidName, hostName, raid.RebuildPercent),
				Recommendation: "Monitor rebuild progress. Avoid heavy I/O if possible. Array is vulnerable until rebuild completes.",
				Evidence:       fmt.Sprintf("State: %s, Progress: %.1f%%, Speed: %s", raid.State, raid.RebuildPercent, raid.RebuildSpeed),
			})

		case "inactive", "INACTIVE":
			findings = append(findings, &Finding{
				ID:             generateFindingID(host.ID+":"+raid.Device, "reliability", "raid-inactive"),
				Severity:       FindingSeverityCritical,
				Category:       FindingCategoryReliability,
				ResourceID:     host.ID + ":" + raid.Device,
				ResourceName:   fmt.Sprintf("%s/%s", hostName, raidName),
				ResourceType:   "host_raid",
				Title:          "RAID array inactive",
				Description:    fmt.Sprintf("RAID array '%s' on '%s' is inactive", raidName, hostName),
				Recommendation: "RAID array is not running. Check mdadm status and system logs.",
				Evidence:       fmt.Sprintf("State: %s", raid.State),
			})
		}

		// Check for failed devices even if array state is "clean"
		if raid.FailedDevices > 0 && raid.State != "degraded" {
			findings = append(findings, &Finding{
				ID:             generateFindingID(host.ID+":"+raid.Device, "reliability", "raid-failed-devices"),
				Severity:       FindingSeverityWarning,
				Category:       FindingCategoryReliability,
				ResourceID:     host.ID + ":" + raid.Device,
				ResourceName:   fmt.Sprintf("%s/%s", hostName, raidName),
				ResourceType:   "host_raid",
				Title:          "RAID has failed devices",
				Description:    fmt.Sprintf("RAID array '%s' on '%s' has %d failed device(s)", raidName, hostName, raid.FailedDevices),
				Recommendation: "Replace failed drives. Array may still be operational due to spares.",
				Evidence:       fmt.Sprintf("Failed: %d, Spare: %d", raid.FailedDevices, raid.SpareDevices),
			})
		}
	}

	// Check high temperature (during deep analysis)
	if deep && len(host.Sensors.TemperatureCelsius) > 0 {
		for sensorName, temp := range host.Sensors.TemperatureCelsius {
			if temp > 85 {
				severity := FindingSeverityWarning
				if temp > 95 {
					severity = FindingSeverityCritical
				}
				findings = append(findings, &Finding{
					ID:             generateFindingID(host.ID+":temp:"+sensorName, "reliability", "high-temp"),
					Severity:       severity,
					Category:       FindingCategoryReliability,
					ResourceID:     host.ID + ":temp:" + sensorName,
					ResourceName:   fmt.Sprintf("%s/%s", hostName, sensorName),
					ResourceType:   "host_sensor",
					Title:          "High temperature",
					Description:    fmt.Sprintf("Sensor '%s' on '%s' reading %.0f°C", sensorName, hostName, temp),
					Recommendation: "Check cooling, fans, and airflow. High temps can cause hardware damage.",
					Evidence:       fmt.Sprintf("Temperature: %.1f°C", temp),
				})
			}
		}
	}

	return findings
}