feat(ai): pass raw metric samples to LLM for pattern interpretation
Instead of relying on pre-computed trend heuristics (which can be misleading for edge cases like step changes vs continuous growth), we now pass downsampled raw data points to the LLM so it can interpret patterns directly. Changes: - Add MetricSamples field to ResourceContext - Add DownsampleMetrics() to reduce data points for LLM consumption - Add formatMetricSamples() to format data compactly (e.g., 'Disk: 26→26→31%') - Add computeGuestMetricSamples() to gather 7-day sampled history - Populate MetricSamples for VMs and containers during context build - Add History section to formatted context output The LLM now sees actual patterns like 'stable for 6 days then jumped' rather than just '45.8%/day growth rate' - allowing for much more nuanced interpretation. This approach: - Leverages LLM's pattern recognition instead of hard-coded heuristics - Provides 7 days of data (~24 samples) for context on normal behavior - Uses minimal tokens due to compact formatting with deduplication - Is more future-proof as LLMs improve Example output: **History (7d sampled, oldest→newest)**: Disk: 26→26→26→26→26→31% Refs: Frigate disk usage false positive investigation
This commit is contained in:
rcourtman
parent
17c36ed124
commit
4d7d2e42dc
5 changed files with 283 additions and 12 deletions
|
|
@ -32,7 +32,7 @@ import History from 'lucide-solid/icons/history';
|
|||
import Gauge from 'lucide-solid/icons/gauge';
|
||||
import Send from 'lucide-solid/icons/send';
|
||||
import Calendar from 'lucide-solid/icons/calendar';
|
||||
import { getPatrolStatus, getFindings, getFindingsHistory, getPatrolRunHistory, forcePatrol, subscribeToPatrolStream, dismissFinding, suppressFinding, getSuppressionRules, addSuppressionRule, deleteSuppressionRule, type Finding, type PatrolStatus, type PatrolRunRecord, type SuppressionRule, severityColors, formatTimestamp, categoryLabels } from '@/api/patrol';
|
||||
import { getPatrolStatus, getFindings, getFindingsHistory, getPatrolRunHistory, forcePatrol, subscribeToPatrolStream, dismissFinding, suppressFinding, resolveFinding, getSuppressionRules, addSuppressionRule, deleteSuppressionRule, type Finding, type PatrolStatus, type PatrolRunRecord, type SuppressionRule, severityColors, formatTimestamp, categoryLabels } from '@/api/patrol';
|
||||
import { aiChatStore } from '@/stores/aiChat';
|
||||
|
||||
type AlertTab = 'overview' | 'thresholds' | 'destinations' | 'schedule' | 'history';
|
||||
|
|
@ -2180,6 +2180,8 @@ function OverviewTab(props: {
|
|||
const [licenseLoading, setLicenseLoading] = createSignal(false);
|
||||
const [remediationsByFinding, setRemediationsByFinding] = createSignal<Record<string, RemediationRecord[]>>({});
|
||||
const [remediationLoadingByFinding, setRemediationLoadingByFinding] = createSignal<Record<string, boolean>>({});
|
||||
// Track which findings are expanded - lifted to parent to persist across API updates
|
||||
const [expandedFindingIds, setExpandedFindingIds] = createSignal<Set<string>>(new Set());
|
||||
const hasAIAlertsFeature = createMemo(() => {
|
||||
const status = licenseFeatures();
|
||||
if (!status) return true;
|
||||
|
|
@ -2822,7 +2824,22 @@ function OverviewTab(props: {
|
|||
<For each={aiFindings().filter(f => !pendingFixFindings().has(f.id))}>
|
||||
{(finding) => {
|
||||
const colors = severityColors[finding.severity];
|
||||
const [isExpanded, setIsExpanded] = createSignal(false);
|
||||
const isExpanded = () => expandedFindingIds().has(finding.id);
|
||||
const toggleExpanded = () => {
|
||||
setExpandedFindingIds((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(finding.id)) {
|
||||
next.delete(finding.id);
|
||||
} else {
|
||||
next.add(finding.id);
|
||||
// Load remediations when expanding (if not already loaded)
|
||||
if (remediationsByFinding()[finding.id] === undefined && !remediationLoadingByFinding()[finding.id]) {
|
||||
void loadRemediationsForFinding(finding.id);
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
return (
|
||||
<div
|
||||
class="border rounded-lg transition-all"
|
||||
|
|
@ -2834,13 +2851,7 @@ function OverviewTab(props: {
|
|||
{/* Compact header - always visible, clickable */}
|
||||
<div
|
||||
class="flex items-center gap-3 p-3 cursor-pointer hover:opacity-80"
|
||||
onClick={() => {
|
||||
const nextExpanded = !isExpanded();
|
||||
setIsExpanded(nextExpanded);
|
||||
if (nextExpanded && remediationsByFinding()[finding.id] === undefined && !remediationLoadingByFinding()[finding.id]) {
|
||||
void loadRemediationsForFinding(finding.id);
|
||||
}
|
||||
}}
|
||||
onClick={toggleExpanded}
|
||||
>
|
||||
{/* Expand chevron */}
|
||||
<svg
|
||||
|
|
@ -2961,15 +2972,24 @@ function OverviewTab(props: {
|
|||
|
||||
<button
|
||||
class="px-3 py-1.5 text-xs font-medium border rounded-lg transition-all bg-green-50 dark:bg-green-900/30 text-green-700 dark:text-green-300 border-green-300 dark:border-green-700 hover:bg-green-100 dark:hover:bg-green-900/50 flex items-center gap-1.5"
|
||||
onClick={(e) => {
|
||||
onClick={async (e) => {
|
||||
e.stopPropagation();
|
||||
// Immediately hide the finding locally
|
||||
setPendingFixFindings(prev => {
|
||||
const next = new Set(prev);
|
||||
next.add(finding.id);
|
||||
return next;
|
||||
});
|
||||
try {
|
||||
await resolveFinding(finding.id);
|
||||
showSuccess('Marked as fixed - the next patrol will verify');
|
||||
fetchAiData();
|
||||
} catch (_err) {
|
||||
// Still keep it hidden locally since user said they fixed it
|
||||
showError('Failed to mark as fixed on server');
|
||||
}
|
||||
}}
|
||||
title="Hide until next patrol verifies the fix"
|
||||
title="Mark as fixed - the next patrol will verify"
|
||||
>
|
||||
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7" />
|
||||
|
|
|
|||
|
|
@ -115,6 +115,8 @@ func (b *Builder) BuildForInfrastructure(state models.StateSnapshot) *Infrastruc
|
|||
vm.CPU, vm.Memory.Usage, vm.Disk.Usage,
|
||||
vm.Uptime, vm.LastBackup, trends,
|
||||
)
|
||||
// Add raw metric samples for LLM interpretation
|
||||
resourceCtx.MetricSamples = b.computeGuestMetricSamples(vm.ID)
|
||||
b.enrichWithNotes(&resourceCtx)
|
||||
b.enrichWithAnomalies(&resourceCtx)
|
||||
ctx.VMs = append(ctx.VMs, resourceCtx)
|
||||
|
|
@ -139,6 +141,10 @@ func (b *Builder) BuildForInfrastructure(state models.StateSnapshot) *Infrastruc
|
|||
ct.Uptime, ct.LastBackup, trends,
|
||||
)
|
||||
|
||||
// Add raw metric samples for LLM interpretation
|
||||
// This lets the LLM see actual patterns without pre-computed heuristics
|
||||
resourceCtx.MetricSamples = b.computeGuestMetricSamples(ct.ID)
|
||||
|
||||
// Add OCI image info for AI context
|
||||
if ct.IsOCI && ct.OSTemplate != "" {
|
||||
if resourceCtx.Metadata == nil {
|
||||
|
|
@ -462,6 +468,34 @@ func formatAnomalyDescription(metric string, current, mean, stddev float64, seve
|
|||
return sb.String()
|
||||
}
|
||||
|
||||
// computeGuestMetricSamples gets downsampled raw metrics for LLM interpretation
|
||||
// Returns ~24 samples from the last 7 days, letting the LLM see patterns and determine if behavior is normal
|
||||
// With modern context windows (128k+ tokens), this is a small cost for much better insights
|
||||
func (b *Builder) computeGuestMetricSamples(guestID string) map[string][]MetricPoint {
|
||||
samples := make(map[string][]MetricPoint)
|
||||
|
||||
if b.metricsHistory == nil {
|
||||
return samples
|
||||
}
|
||||
|
||||
// Get 7 days of data - enough to see weekly patterns and determine normalcy
|
||||
allMetrics := b.metricsHistory.GetAllGuestMetrics(guestID, b.trendWindow7d)
|
||||
|
||||
for metric, points := range allMetrics {
|
||||
if len(points) < 3 {
|
||||
continue
|
||||
}
|
||||
// Downsample to ~24 points (roughly 3 per day over 7 days)
|
||||
// This lets the LLM see: daily patterns, weekly cycles, and recent changes
|
||||
sampled := DownsampleMetrics(points, 24)
|
||||
if len(sampled) >= 3 {
|
||||
samples[metric] = sampled
|
||||
}
|
||||
}
|
||||
|
||||
return samples
|
||||
}
|
||||
|
||||
// filterRecentPoints filters points to only include those within duration
|
||||
func filterRecentPoints(points []MetricPoint, duration time.Duration) []MetricPoint {
|
||||
cutoff := time.Now().Add(-duration)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func FormatResourceContext(ctx ResourceContext) string {
|
|||
sb.WriteString("**Current**: " + strings.Join(metrics, " | ") + "\n")
|
||||
}
|
||||
|
||||
// Trends section (the differentiating context)
|
||||
// Trends section (computed summaries - kept for backwards compatibility)
|
||||
if len(ctx.Trends) > 0 {
|
||||
var trendLines []string
|
||||
for metric, trend := range ctx.Trends {
|
||||
|
|
@ -61,6 +61,25 @@ func FormatResourceContext(ctx ResourceContext) string {
|
|||
}
|
||||
}
|
||||
|
||||
// Raw metric samples - let the LLM interpret patterns directly
|
||||
// This is more reliable than pre-computed trends for edge cases
|
||||
if len(ctx.MetricSamples) > 0 {
|
||||
sb.WriteString("**History (7d sampled, oldest→newest)**: ")
|
||||
var sampleLines []string
|
||||
for metric, points := range ctx.MetricSamples {
|
||||
if len(points) >= 3 {
|
||||
line := formatMetricSamples(metric, points)
|
||||
if line != "" {
|
||||
sampleLines = append(sampleLines, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(sampleLines) > 0 {
|
||||
sb.WriteString(strings.Join(sampleLines, " | "))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
// Anomalies (high value - what's unusual)
|
||||
if len(ctx.Anomalies) > 0 {
|
||||
sb.WriteString("**ANOMALIES**: ")
|
||||
|
|
@ -157,6 +176,68 @@ func formatRate(ratePerDay float64) string {
|
|||
return "slow"
|
||||
}
|
||||
|
||||
// formatMetricSamples creates a compact representation of sampled values
|
||||
// Example output: "Disk: 26→26→26→31→31→31" (shows step change visually)
|
||||
// This lets the LLM interpret patterns directly rather than relying on computed rates
|
||||
func formatMetricSamples(metric string, points []MetricPoint) string {
|
||||
if len(points) < 3 {
|
||||
return ""
|
||||
}
|
||||
|
||||
metricLabel := strings.Title(metric)
|
||||
|
||||
// Build compact arrow-separated value list
|
||||
var values []string
|
||||
prevValue := -1.0
|
||||
for _, p := range points {
|
||||
roundedValue := float64(int(p.Value + 0.5)) // Round to nearest integer
|
||||
// Skip consecutive duplicates for compactness
|
||||
if roundedValue == prevValue && len(values) > 0 {
|
||||
continue
|
||||
}
|
||||
values = append(values, fmt.Sprintf("%.0f", roundedValue))
|
||||
prevValue = roundedValue
|
||||
}
|
||||
|
||||
// If all values are the same, just show "stable at X%"
|
||||
if len(values) == 1 {
|
||||
return fmt.Sprintf("%s: stable at %.0f%%", metricLabel, prevValue)
|
||||
}
|
||||
|
||||
// Join with arrows to show progression
|
||||
return fmt.Sprintf("%s: %s%%", metricLabel, strings.Join(values, "→"))
|
||||
}
|
||||
|
||||
// DownsampleMetrics takes raw metric points and returns a smaller set for LLM consumption
|
||||
// It aims for about 10-15 samples across the time range, picking representative values
|
||||
func DownsampleMetrics(points []MetricPoint, targetSamples int) []MetricPoint {
|
||||
if len(points) <= targetSamples {
|
||||
return points
|
||||
}
|
||||
|
||||
if targetSamples < 3 {
|
||||
targetSamples = 3
|
||||
}
|
||||
|
||||
// Calculate step size
|
||||
step := len(points) / targetSamples
|
||||
if step < 1 {
|
||||
step = 1
|
||||
}
|
||||
|
||||
var sampled []MetricPoint
|
||||
for i := 0; i < len(points); i += step {
|
||||
sampled = append(sampled, points[i])
|
||||
}
|
||||
|
||||
// Always include the last point (current value)
|
||||
if len(sampled) > 0 && sampled[len(sampled)-1].Timestamp != points[len(points)-1].Timestamp {
|
||||
sampled = append(sampled, points[len(points)-1])
|
||||
}
|
||||
|
||||
return sampled
|
||||
}
|
||||
|
||||
// FormatInfrastructureContext formats full infrastructure context for AI
|
||||
func FormatInfrastructureContext(ctx *InfrastructureContext) string {
|
||||
var sb strings.Builder
|
||||
|
|
|
|||
|
|
@ -663,3 +663,134 @@ func containsStr(s, substr string) bool {
|
|||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestFormatMetricSamples_StepChange(t *testing.T) {
|
||||
// Simulate a step change: stable at 26%, then jump to 31%, then stable at 31%
|
||||
now := time.Now()
|
||||
points := []MetricPoint{
|
||||
{Value: 26.2, Timestamp: now.Add(-6 * time.Hour)},
|
||||
{Value: 26.1, Timestamp: now.Add(-5 * time.Hour)},
|
||||
{Value: 26.3, Timestamp: now.Add(-4 * time.Hour)},
|
||||
{Value: 30.7, Timestamp: now.Add(-2 * time.Hour)}, // Jump
|
||||
{Value: 30.8, Timestamp: now.Add(-1 * time.Hour)},
|
||||
{Value: 30.7, Timestamp: now},
|
||||
}
|
||||
|
||||
result := formatMetricSamples("disk", points)
|
||||
|
||||
// Should show the step change: 26→31 (deduped)
|
||||
if !containsStr(result, "Disk:") {
|
||||
t.Error("Expected result to contain 'Disk:'")
|
||||
}
|
||||
// Should show the progression, not just the rate
|
||||
if !containsStr(result, "26") || !containsStr(result, "31") {
|
||||
t.Errorf("Expected result to show both values (26 and 31), got: %s", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatMetricSamples_Stable(t *testing.T) {
|
||||
// All values the same
|
||||
now := time.Now()
|
||||
points := []MetricPoint{
|
||||
{Value: 50.0, Timestamp: now.Add(-3 * time.Hour)},
|
||||
{Value: 50.1, Timestamp: now.Add(-2 * time.Hour)},
|
||||
{Value: 49.9, Timestamp: now.Add(-1 * time.Hour)},
|
||||
{Value: 50.0, Timestamp: now},
|
||||
}
|
||||
|
||||
result := formatMetricSamples("memory", points)
|
||||
|
||||
// All values round to 50, should show "stable at 50%"
|
||||
if !containsStr(result, "stable at 50%") {
|
||||
t.Errorf("Expected 'stable at 50%%' for consistent values, got: %s", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatMetricSamples_InsufficientData(t *testing.T) {
|
||||
points := []MetricPoint{
|
||||
{Value: 50.0, Timestamp: time.Now()},
|
||||
}
|
||||
|
||||
result := formatMetricSamples("cpu", points)
|
||||
|
||||
if result != "" {
|
||||
t.Errorf("Expected empty string for insufficient data, got: %s", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownsampleMetrics(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
// Create 100 points
|
||||
points := make([]MetricPoint, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
points[i] = MetricPoint{
|
||||
Value: float64(i),
|
||||
Timestamp: now.Add(time.Duration(-100+i) * time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
// Downsample to 10
|
||||
sampled := DownsampleMetrics(points, 10)
|
||||
|
||||
// Should have roughly 10-11 points (plus potentially the last one)
|
||||
if len(sampled) < 10 || len(sampled) > 15 {
|
||||
t.Errorf("Expected ~10-15 samples, got %d", len(sampled))
|
||||
}
|
||||
|
||||
// Last point should be included
|
||||
if sampled[len(sampled)-1].Timestamp != points[99].Timestamp {
|
||||
t.Error("Expected last point to be included")
|
||||
}
|
||||
|
||||
// First point should be included
|
||||
if sampled[0].Timestamp != points[0].Timestamp {
|
||||
t.Error("Expected first point to be included")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownsampleMetrics_SmallInput(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
// Create 5 points - less than target
|
||||
points := []MetricPoint{
|
||||
{Value: 10, Timestamp: now.Add(-4 * time.Minute)},
|
||||
{Value: 20, Timestamp: now.Add(-3 * time.Minute)},
|
||||
{Value: 30, Timestamp: now.Add(-2 * time.Minute)},
|
||||
{Value: 40, Timestamp: now.Add(-1 * time.Minute)},
|
||||
{Value: 50, Timestamp: now},
|
||||
}
|
||||
|
||||
// Downsample to 10 should return all 5
|
||||
sampled := DownsampleMetrics(points, 10)
|
||||
|
||||
if len(sampled) != 5 {
|
||||
t.Errorf("Expected all 5 points when target > input, got %d", len(sampled))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatResourceContext_WithMetricSamples(t *testing.T) {
|
||||
now := time.Now()
|
||||
ctx := ResourceContext{
|
||||
ResourceID: "ct-105",
|
||||
ResourceType: "container",
|
||||
ResourceName: "frigate",
|
||||
Status: "running",
|
||||
CurrentDisk: 30.7,
|
||||
MetricSamples: map[string][]MetricPoint{
|
||||
"disk": {
|
||||
{Value: 26.2, Timestamp: now.Add(-3 * time.Hour)},
|
||||
{Value: 30.7, Timestamp: now.Add(-1 * time.Hour)},
|
||||
{Value: 30.7, Timestamp: now},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := FormatResourceContext(ctx)
|
||||
|
||||
// Should contain the History section with sampled data
|
||||
if !containsStr(result, "History") {
|
||||
t.Error("Expected result to contain History section with metric samples")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -129,6 +129,11 @@ type ResourceContext struct {
|
|||
Baselines map[string]Baseline // metric -> baseline
|
||||
Anomalies []Anomaly // Current anomalies
|
||||
|
||||
// Raw metric samples - downsampled for LLM interpretation
|
||||
// Key is metric name (cpu, memory, disk), value is sampled points
|
||||
// This lets the LLM see actual patterns without pre-computed heuristics
|
||||
MetricSamples map[string][]MetricPoint
|
||||
|
||||
// Predictions
|
||||
Predictions []Prediction
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue