feat: enhance AI baseline context visibility and incident timeline improvements
Backend: - Enhanced buildEnrichedResourceContext to ALWAYS show learned baselines with status indicators (normal/elevated/anomaly) instead of only when anomalous - This makes Pulse Pro's 'moat' visible - users can see the AI understands their infrastructure's normal behavior patterns - Added baseline import to service.go Frontend (user changes): - Added incident event type filtering with toggle buttons - Added resource incident panel to view all incidents for a resource - Added timeline expand/collapse functionality in alert history - Added incident note saving with proper incidentId tracking - Added startedAt parameter for proper incident timeline loading
This commit is contained in:
rcourtman
parent
f0b983667c
commit
82e5b28840
14 changed files with 2397 additions and 226 deletions
|
|
@ -1,4 +1,4 @@
|
||||||
import type { Alert } from '@/types/api';
|
import type { Alert, Incident } from '@/types/api';
|
||||||
import type { AlertConfig } from '@/types/alerts';
|
import type { AlertConfig } from '@/types/alerts';
|
||||||
import { apiFetchJSON } from '@/utils/apiClient';
|
import { apiFetchJSON } from '@/utils/apiClient';
|
||||||
|
|
||||||
|
|
@ -29,6 +29,37 @@ export class AlertsAPI {
|
||||||
return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`);
|
return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async getIncidentTimeline(alertId: string, startedAt?: string): Promise<Incident | null> {
|
||||||
|
const query = new URLSearchParams({ alert_id: alertId });
|
||||||
|
if (startedAt) {
|
||||||
|
query.set('started_at', startedAt);
|
||||||
|
}
|
||||||
|
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident | null>;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async getIncidentsForResource(resourceId: string, limit?: number): Promise<Incident[]> {
|
||||||
|
const query = new URLSearchParams({ resource_id: resourceId });
|
||||||
|
if (limit) query.set('limit', String(limit));
|
||||||
|
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async addIncidentNote(params: {
|
||||||
|
alertId?: string;
|
||||||
|
incidentId?: string;
|
||||||
|
note: string;
|
||||||
|
user?: string;
|
||||||
|
}): Promise<{ success: boolean }> {
|
||||||
|
return apiFetchJSON(`${this.baseUrl}/incidents/note`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({
|
||||||
|
alert_id: params.alertId,
|
||||||
|
incident_id: params.incidentId,
|
||||||
|
note: params.note,
|
||||||
|
user: params.user,
|
||||||
|
}),
|
||||||
|
}) as Promise<{ success: boolean }>;
|
||||||
|
}
|
||||||
|
|
||||||
static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> {
|
static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> {
|
||||||
return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, {
|
return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ import { AIAPI } from '@/api/ai';
|
||||||
import { LicenseAPI, type LicenseFeatureStatus } from '@/api/license';
|
import { LicenseAPI, type LicenseFeatureStatus } from '@/api/license';
|
||||||
import type { EmailConfig, AppriseConfig } from '@/api/notifications';
|
import type { EmailConfig, AppriseConfig } from '@/api/notifications';
|
||||||
import type { HysteresisThreshold } from '@/types/alerts';
|
import type { HysteresisThreshold } from '@/types/alerts';
|
||||||
import type { Alert, State, VM, Container, DockerHost, DockerContainer, Host } from '@/types/api';
|
import type { Alert, Incident, IncidentEvent, State, VM, Container, DockerHost, DockerContainer, Host } from '@/types/api';
|
||||||
import type { RemediationRecord } from '@/types/aiIntelligence';
|
import type { RemediationRecord } from '@/types/aiIntelligence';
|
||||||
import { useNavigate, useLocation } from '@solidjs/router';
|
import { useNavigate, useLocation } from '@solidjs/router';
|
||||||
import { useAlertsActivation } from '@/stores/alertsActivation';
|
import { useAlertsActivation } from '@/stores/alertsActivation';
|
||||||
|
|
@ -106,6 +106,93 @@ export const tabFromPath = (
|
||||||
return 'overview';
|
return 'overview';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const INCIDENT_EVENT_TYPES = [
|
||||||
|
'alert_fired',
|
||||||
|
'alert_acknowledged',
|
||||||
|
'alert_unacknowledged',
|
||||||
|
'alert_resolved',
|
||||||
|
'ai_analysis',
|
||||||
|
'command',
|
||||||
|
'runbook',
|
||||||
|
'note',
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const INCIDENT_EVENT_LABELS: Record<(typeof INCIDENT_EVENT_TYPES)[number], string> = {
|
||||||
|
alert_fired: 'Fired',
|
||||||
|
alert_acknowledged: 'Ack',
|
||||||
|
alert_unacknowledged: 'Unack',
|
||||||
|
alert_resolved: 'Resolved',
|
||||||
|
ai_analysis: 'AI',
|
||||||
|
command: 'Cmd',
|
||||||
|
runbook: 'Runbook',
|
||||||
|
note: 'Note',
|
||||||
|
};
|
||||||
|
|
||||||
|
const filterIncidentEvents = (
|
||||||
|
events: IncidentEvent[] | undefined,
|
||||||
|
filters: Set<string>,
|
||||||
|
): IncidentEvent[] => {
|
||||||
|
if (!events || events.length === 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
if (filters.size === 0 || filters.size === INCIDENT_EVENT_TYPES.length) {
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
return events.filter((event) => filters.has(event.type));
|
||||||
|
};
|
||||||
|
|
||||||
|
function IncidentEventFilters(props: {
|
||||||
|
filters: () => Set<string>;
|
||||||
|
setFilters: (next: Set<string>) => void;
|
||||||
|
}) {
|
||||||
|
const toggleFilter = (type: (typeof INCIDENT_EVENT_TYPES)[number]) => {
|
||||||
|
const next = new Set(props.filters());
|
||||||
|
if (next.has(type)) {
|
||||||
|
next.delete(type);
|
||||||
|
} else {
|
||||||
|
next.add(type);
|
||||||
|
}
|
||||||
|
props.setFilters(next);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-[10px] text-gray-500 dark:text-gray-400">
|
||||||
|
<span class="uppercase tracking-wide text-[9px] text-gray-400 dark:text-gray-500">Filters</span>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-0.5 rounded border border-gray-300 dark:border-gray-600 text-gray-500 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
|
||||||
|
onClick={() => props.setFilters(new Set(INCIDENT_EVENT_TYPES))}
|
||||||
|
>
|
||||||
|
All
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-0.5 rounded border border-gray-300 dark:border-gray-600 text-gray-500 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
|
||||||
|
onClick={() => props.setFilters(new Set())}
|
||||||
|
>
|
||||||
|
None
|
||||||
|
</button>
|
||||||
|
<For each={INCIDENT_EVENT_TYPES}>
|
||||||
|
{(type) => {
|
||||||
|
const selected = () => props.filters().has(type);
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class={`px-2 py-0.5 rounded border text-[10px] ${selected()
|
||||||
|
? 'border-blue-300 bg-blue-100 text-blue-700 dark:border-blue-800 dark:bg-blue-900/40 dark:text-blue-300'
|
||||||
|
: 'border-gray-300 text-gray-500 dark:border-gray-600 dark:text-gray-300'
|
||||||
|
}`}
|
||||||
|
onClick={() => toggleFilter(type)}
|
||||||
|
>
|
||||||
|
{INCIDENT_EVENT_LABELS[type]}
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
}}
|
||||||
|
</For>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Store reference interfaces
|
// Store reference interfaces
|
||||||
interface DestinationsRef {
|
interface DestinationsRef {
|
||||||
emailConfig?: () => EmailConfig;
|
emailConfig?: () => EmailConfig;
|
||||||
|
|
@ -2062,6 +2149,14 @@ function OverviewTab(props: {
|
||||||
}) {
|
}) {
|
||||||
// Loading states for buttons
|
// Loading states for buttons
|
||||||
const [processingAlerts, setProcessingAlerts] = createSignal<Set<string>>(new Set());
|
const [processingAlerts, setProcessingAlerts] = createSignal<Set<string>>(new Set());
|
||||||
|
const [incidentTimelines, setIncidentTimelines] = createSignal<Record<string, Incident | null>>({});
|
||||||
|
const [incidentLoading, setIncidentLoading] = createSignal<Record<string, boolean>>({});
|
||||||
|
const [expandedIncidents, setExpandedIncidents] = createSignal<Set<string>>(new Set());
|
||||||
|
const [incidentNoteDrafts, setIncidentNoteDrafts] = createSignal<Record<string, string>>({});
|
||||||
|
const [incidentNoteSaving, setIncidentNoteSaving] = createSignal<Set<string>>(new Set());
|
||||||
|
const [incidentEventFilters, setIncidentEventFilters] = createSignal<Set<string>>(
|
||||||
|
new Set(INCIDENT_EVENT_TYPES),
|
||||||
|
);
|
||||||
|
|
||||||
// AI Patrol findings state
|
// AI Patrol findings state
|
||||||
const [aiFindings, setAiFindings] = createSignal<Finding[]>([]);
|
const [aiFindings, setAiFindings] = createSignal<Finding[]>([]);
|
||||||
|
|
@ -2142,6 +2237,58 @@ function OverviewTab(props: {
|
||||||
return 'AI Patrol insights require Pulse Pro.';
|
return 'AI Patrol insights require Pulse Pro.';
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const loadIncidentTimeline = async (alertId: string, startedAt?: string) => {
|
||||||
|
setIncidentLoading((prev) => ({ ...prev, [alertId]: true }));
|
||||||
|
try {
|
||||||
|
const timeline = await AlertsAPI.getIncidentTimeline(alertId, startedAt);
|
||||||
|
setIncidentTimelines((prev) => ({ ...prev, [alertId]: timeline }));
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to load incident timeline', error);
|
||||||
|
showError('Failed to load incident timeline');
|
||||||
|
} finally {
|
||||||
|
setIncidentLoading((prev) => ({ ...prev, [alertId]: false }));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const toggleIncidentTimeline = async (alertId: string, startedAt?: string) => {
|
||||||
|
const expanded = expandedIncidents();
|
||||||
|
const next = new Set(expanded);
|
||||||
|
if (next.has(alertId)) {
|
||||||
|
next.delete(alertId);
|
||||||
|
setExpandedIncidents(next);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
next.add(alertId);
|
||||||
|
setExpandedIncidents(next);
|
||||||
|
if (!(alertId in incidentTimelines())) {
|
||||||
|
await loadIncidentTimeline(alertId, startedAt);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const saveIncidentNote = async (alertId: string, startedAt?: string) => {
|
||||||
|
const note = (incidentNoteDrafts()[alertId] || '').trim();
|
||||||
|
if (!note) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setIncidentNoteSaving((prev) => new Set(prev).add(alertId));
|
||||||
|
try {
|
||||||
|
const incidentId = incidentTimelines()[alertId]?.id;
|
||||||
|
await AlertsAPI.addIncidentNote({ alertId, incidentId, note });
|
||||||
|
setIncidentNoteDrafts((prev) => ({ ...prev, [alertId]: '' }));
|
||||||
|
await loadIncidentTimeline(alertId, startedAt);
|
||||||
|
showSuccess('Incident note saved');
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to save incident note', error);
|
||||||
|
showError('Failed to save incident note');
|
||||||
|
} finally {
|
||||||
|
setIncidentNoteSaving((prev) => {
|
||||||
|
const next = new Set(prev);
|
||||||
|
next.delete(alertId);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Effect to manage live stream subscription when expanded
|
// Effect to manage live stream subscription when expanded
|
||||||
createEffect(() => {
|
createEffect(() => {
|
||||||
const isExpanded = expandedLiveStream();
|
const isExpanded = expandedLiveStream();
|
||||||
|
|
@ -4058,6 +4205,14 @@ function OverviewTab(props: {
|
||||||
? 'Unacknowledge'
|
? 'Unacknowledge'
|
||||||
: 'Acknowledge'}
|
: 'Acknowledge'}
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
class="px-3 py-1.5 text-xs font-medium border rounded-lg transition-all bg-white dark:bg-gray-700 text-gray-700 dark:text-gray-300 border-gray-300 dark:border-gray-600 hover:bg-gray-50 dark:hover:bg-gray-600"
|
||||||
|
onClick={() => {
|
||||||
|
void toggleIncidentTimeline(alert.id, alert.startTime);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{expandedIncidents().has(alert.id) ? 'Hide Timeline' : 'Timeline'}
|
||||||
|
</button>
|
||||||
<InvestigateAlertButton
|
<InvestigateAlertButton
|
||||||
alert={alert}
|
alert={alert}
|
||||||
variant="text"
|
variant="text"
|
||||||
|
|
@ -4066,6 +4221,115 @@ function OverviewTab(props: {
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<Show when={expandedIncidents().has(alert.id)}>
|
||||||
|
<div class="mt-3 border-t border-gray-200 dark:border-gray-700 pt-3">
|
||||||
|
<Show when={incidentLoading()[alert.id]}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">Loading timeline...</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={!incidentLoading()[alert.id]}>
|
||||||
|
<Show when={incidentTimelines()[alert.id]}>
|
||||||
|
{(timeline) => (
|
||||||
|
<div class="space-y-3">
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-xs text-gray-600 dark:text-gray-400">
|
||||||
|
<span class="font-medium text-gray-700 dark:text-gray-200">Incident</span>
|
||||||
|
<span>{timeline().status}</span>
|
||||||
|
<Show when={timeline().acknowledged}>
|
||||||
|
<span class="px-2 py-0.5 rounded bg-emerald-100 text-emerald-700 dark:bg-emerald-900/40 dark:text-emerald-300">
|
||||||
|
acknowledged
|
||||||
|
</span>
|
||||||
|
</Show>
|
||||||
|
<Show when={timeline().openedAt}>
|
||||||
|
<span>opened {new Date(timeline().openedAt).toLocaleString()}</span>
|
||||||
|
</Show>
|
||||||
|
<Show when={timeline().closedAt}>
|
||||||
|
<span>closed {new Date(timeline().closedAt as string).toLocaleString()}</span>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
{(() => {
|
||||||
|
const events = timeline().events || [];
|
||||||
|
const filteredEvents = filterIncidentEvents(events, incidentEventFilters());
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Show when={events.length > 0}>
|
||||||
|
<IncidentEventFilters
|
||||||
|
filters={incidentEventFilters}
|
||||||
|
setFilters={setIncidentEventFilters}
|
||||||
|
/>
|
||||||
|
</Show>
|
||||||
|
<Show when={filteredEvents.length > 0}>
|
||||||
|
<div class="space-y-2">
|
||||||
|
<For each={filteredEvents}>
|
||||||
|
{(event) => (
|
||||||
|
<div class="rounded border border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-900/30 p-2">
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-xs text-gray-600 dark:text-gray-400">
|
||||||
|
<span class="font-medium text-gray-800 dark:text-gray-200">
|
||||||
|
{event.summary}
|
||||||
|
</span>
|
||||||
|
<span>{new Date(event.timestamp).toLocaleString()}</span>
|
||||||
|
</div>
|
||||||
|
<Show when={event.details && (event.details as { note?: string }).note}>
|
||||||
|
<p class="text-xs text-gray-700 dark:text-gray-300 mt-1">
|
||||||
|
{(event.details as { note?: string }).note}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={event.details && (event.details as { command?: string }).command}>
|
||||||
|
<p class="text-xs text-gray-700 dark:text-gray-300 mt-1 font-mono">
|
||||||
|
{(event.details as { command?: string }).command}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={event.details && (event.details as { output_excerpt?: string }).output_excerpt}>
|
||||||
|
<p class="text-xs text-gray-600 dark:text-gray-400 mt-1">
|
||||||
|
{(event.details as { output_excerpt?: string }).output_excerpt}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</For>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
<Show when={events.length > 0 && filteredEvents.length === 0}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
No timeline events match the selected filters.
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={events.length === 0}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">No timeline events yet.</p>
|
||||||
|
</Show>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
})()}
|
||||||
|
<div class="flex flex-col gap-2">
|
||||||
|
<textarea
|
||||||
|
class="w-full rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-900 p-2 text-xs text-gray-800 dark:text-gray-200"
|
||||||
|
rows={2}
|
||||||
|
placeholder="Add a note for this incident..."
|
||||||
|
value={incidentNoteDrafts()[alert.id] || ''}
|
||||||
|
onInput={(e) => {
|
||||||
|
const value = e.currentTarget.value;
|
||||||
|
setIncidentNoteDrafts((prev) => ({ ...prev, [alert.id]: value }));
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<div class="flex justify-end">
|
||||||
|
<button
|
||||||
|
class="px-3 py-1.5 text-xs font-medium border rounded-lg transition-all bg-white dark:bg-gray-700 text-gray-700 dark:text-gray-300 border-gray-300 dark:border-gray-600 hover:bg-gray-50 dark:hover:bg-gray-600 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
|
disabled={incidentNoteSaving().has(alert.id) || !(incidentNoteDrafts()[alert.id] || '').trim()}
|
||||||
|
onClick={() => {
|
||||||
|
void saveIncidentNote(alert.id, alert.startTime);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{incidentNoteSaving().has(alert.id) ? 'Saving...' : 'Save Note'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Show>
|
||||||
|
<Show when={!incidentTimelines()[alert.id]}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">No incident timeline available.</p>
|
||||||
|
</Show>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</For>
|
</For>
|
||||||
|
|
@ -5601,6 +5865,21 @@ function HistoryTab() {
|
||||||
const [aiFindingsHistory, setAiFindingsHistory] = createSignal<Finding[]>([]);
|
const [aiFindingsHistory, setAiFindingsHistory] = createSignal<Finding[]>([]);
|
||||||
const [loading, setLoading] = createSignal(true);
|
const [loading, setLoading] = createSignal(true);
|
||||||
const [selectedBarIndex, setSelectedBarIndex] = createSignal<number | null>(null);
|
const [selectedBarIndex, setSelectedBarIndex] = createSignal<number | null>(null);
|
||||||
|
const [resourceIncidentPanel, setResourceIncidentPanel] = createSignal<{ resourceId: string; resourceName: string } | null>(null);
|
||||||
|
const [resourceIncidents, setResourceIncidents] = createSignal<Record<string, Incident[]>>({});
|
||||||
|
const [resourceIncidentLoading, setResourceIncidentLoading] = createSignal<Record<string, boolean>>({});
|
||||||
|
const [expandedResourceIncidentIds, setExpandedResourceIncidentIds] = createSignal<Set<string>>(new Set());
|
||||||
|
const [historyIncidentEventFilters, setHistoryIncidentEventFilters] = createSignal<Set<string>>(
|
||||||
|
new Set(INCIDENT_EVENT_TYPES),
|
||||||
|
);
|
||||||
|
const [resourceIncidentEventFilters, setResourceIncidentEventFilters] = createSignal<Set<string>>(
|
||||||
|
new Set(INCIDENT_EVENT_TYPES),
|
||||||
|
);
|
||||||
|
const [incidentTimelines, setIncidentTimelines] = createSignal<Record<string, Incident | null>>({});
|
||||||
|
const [incidentLoading, setIncidentLoading] = createSignal<Record<string, boolean>>({});
|
||||||
|
const [expandedIncidents, setExpandedIncidents] = createSignal<Set<string>>(new Set());
|
||||||
|
const [incidentNoteDrafts, setIncidentNoteDrafts] = createSignal<Record<string, string>>({});
|
||||||
|
const [incidentNoteSaving, setIncidentNoteSaving] = createSignal<Set<string>>(new Set());
|
||||||
const MS_PER_HOUR = 60 * 60 * 1000;
|
const MS_PER_HOUR = 60 * 60 * 1000;
|
||||||
const userLocale =
|
const userLocale =
|
||||||
Intl.DateTimeFormat().resolvedOptions().locale ||
|
Intl.DateTimeFormat().resolvedOptions().locale ||
|
||||||
|
|
@ -5770,6 +6049,53 @@ function HistoryTab() {
|
||||||
return `${minutes}m`;
|
return `${minutes}m`;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const loadResourceIncidents = async (resourceId: string, limit = 10) => {
|
||||||
|
if (!resourceId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setResourceIncidentLoading((prev) => ({ ...prev, [resourceId]: true }));
|
||||||
|
try {
|
||||||
|
const incidents = await AlertsAPI.getIncidentsForResource(resourceId, limit);
|
||||||
|
setResourceIncidents((prev) => ({ ...prev, [resourceId]: incidents }));
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to load resource incidents', error);
|
||||||
|
showError('Failed to load resource incidents');
|
||||||
|
} finally {
|
||||||
|
setResourceIncidentLoading((prev) => ({ ...prev, [resourceId]: false }));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const openResourceIncidentPanel = async (resourceId: string, resourceName: string) => {
|
||||||
|
if (!resourceId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setResourceIncidentPanel({ resourceId, resourceName });
|
||||||
|
setExpandedResourceIncidentIds(new Set());
|
||||||
|
if (!(resourceId in resourceIncidents())) {
|
||||||
|
await loadResourceIncidents(resourceId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const refreshResourceIncidentPanel = async () => {
|
||||||
|
const selection = resourceIncidentPanel();
|
||||||
|
if (!selection) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await loadResourceIncidents(selection.resourceId);
|
||||||
|
};
|
||||||
|
|
||||||
|
const toggleResourceIncidentDetails = (incidentId: string) => {
|
||||||
|
setExpandedResourceIncidentIds((prev) => {
|
||||||
|
const next = new Set(prev);
|
||||||
|
if (next.has(incidentId)) {
|
||||||
|
next.delete(incidentId);
|
||||||
|
} else {
|
||||||
|
next.add(incidentId);
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const formatBucketRange = (startMs: number, endMs: number) => {
|
const formatBucketRange = (startMs: number, endMs: number) => {
|
||||||
const start = new Date(startMs);
|
const start = new Date(startMs);
|
||||||
const end = new Date(endMs);
|
const end = new Date(endMs);
|
||||||
|
|
@ -6103,6 +6429,7 @@ function HistoryTab() {
|
||||||
};
|
};
|
||||||
|
|
||||||
type AlertHistoryRow = ReturnType<typeof alertData>[number];
|
type AlertHistoryRow = ReturnType<typeof alertData>[number];
|
||||||
|
const getIncidentRowKey = (alert: AlertHistoryRow) => `${alert.id}::${alert.startTime}`;
|
||||||
|
|
||||||
// Group alerts by day for display
|
// Group alerts by day for display
|
||||||
const groupedAlerts = createMemo(() => {
|
const groupedAlerts = createMemo(() => {
|
||||||
|
|
@ -6238,6 +6565,58 @@ function HistoryTab() {
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const loadIncidentTimeline = async (rowKey: string, alertId: string, startedAt?: string) => {
|
||||||
|
setIncidentLoading((prev) => ({ ...prev, [rowKey]: true }));
|
||||||
|
try {
|
||||||
|
const timeline = await AlertsAPI.getIncidentTimeline(alertId, startedAt);
|
||||||
|
setIncidentTimelines((prev) => ({ ...prev, [rowKey]: timeline }));
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to load incident timeline', error);
|
||||||
|
showError('Failed to load incident timeline');
|
||||||
|
} finally {
|
||||||
|
setIncidentLoading((prev) => ({ ...prev, [rowKey]: false }));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const toggleIncidentTimeline = async (rowKey: string, alertId: string, startedAt?: string) => {
|
||||||
|
const expanded = expandedIncidents();
|
||||||
|
const next = new Set(expanded);
|
||||||
|
if (next.has(rowKey)) {
|
||||||
|
next.delete(rowKey);
|
||||||
|
setExpandedIncidents(next);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
next.add(rowKey);
|
||||||
|
setExpandedIncidents(next);
|
||||||
|
if (!(rowKey in incidentTimelines())) {
|
||||||
|
await loadIncidentTimeline(rowKey, alertId, startedAt);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const saveIncidentNote = async (rowKey: string, alertId: string, startedAt?: string) => {
|
||||||
|
const note = (incidentNoteDrafts()[rowKey] || '').trim();
|
||||||
|
if (!note) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setIncidentNoteSaving((prev) => new Set(prev).add(rowKey));
|
||||||
|
try {
|
||||||
|
const incidentId = incidentTimelines()[rowKey]?.id;
|
||||||
|
await AlertsAPI.addIncidentNote({ alertId, incidentId, note });
|
||||||
|
setIncidentNoteDrafts((prev) => ({ ...prev, [rowKey]: '' }));
|
||||||
|
await loadIncidentTimeline(rowKey, alertId, startedAt);
|
||||||
|
showSuccess('Incident note saved');
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to save incident note', error);
|
||||||
|
showError('Failed to save incident note');
|
||||||
|
} finally {
|
||||||
|
setIncidentNoteSaving((prev) => {
|
||||||
|
const next = new Set(prev);
|
||||||
|
next.delete(rowKey);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const bucketDurationLabel = createMemo(() => {
|
const bucketDurationLabel = createMemo(() => {
|
||||||
const bucketHours = alertTrends().bucketSize;
|
const bucketHours = alertTrends().bucketSize;
|
||||||
if (!Number.isFinite(bucketHours) || bucketHours <= 0) {
|
if (!Number.isFinite(bucketHours) || bucketHours <= 0) {
|
||||||
|
|
@ -6602,6 +6981,190 @@ function HistoryTab() {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<Show when={resourceIncidentPanel()}>
|
||||||
|
{(selection) => {
|
||||||
|
const resourceId = selection().resourceId;
|
||||||
|
const incidents = () => resourceIncidents()[resourceId] || [];
|
||||||
|
const isLoading = () => resourceIncidentLoading()[resourceId];
|
||||||
|
return (
|
||||||
|
<Card padding="md">
|
||||||
|
<div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
||||||
|
<div>
|
||||||
|
<h3 class="text-sm font-semibold text-gray-800 dark:text-gray-100">Resource incidents</h3>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
{selection().resourceName}
|
||||||
|
<Show when={incidents().length > 0}>
|
||||||
|
<span> · {incidents().length} incident{incidents().length === 1 ? '' : 's'}</span>
|
||||||
|
</Show>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-1 text-xs border rounded-md border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700 disabled:opacity-50"
|
||||||
|
disabled={isLoading()}
|
||||||
|
onClick={() => {
|
||||||
|
void refreshResourceIncidentPanel();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{isLoading() ? 'Refreshing...' : 'Refresh'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-1 text-xs border rounded-md border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
|
||||||
|
onClick={() => setResourceIncidentPanel(null)}
|
||||||
|
>
|
||||||
|
Close
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<Show when={isLoading()}>
|
||||||
|
<p class="mt-2 text-xs text-gray-500 dark:text-gray-400">Loading incidents...</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={!isLoading()}>
|
||||||
|
<Show when={incidents().length > 0}>
|
||||||
|
<div class="mt-2">
|
||||||
|
<IncidentEventFilters
|
||||||
|
filters={resourceIncidentEventFilters}
|
||||||
|
setFilters={setResourceIncidentEventFilters}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
<Show
|
||||||
|
when={incidents().length > 0}
|
||||||
|
fallback={
|
||||||
|
<p class="mt-2 text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
No incidents recorded for this resource yet.
|
||||||
|
</p>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<div class="mt-3 space-y-3">
|
||||||
|
<For each={incidents()}>
|
||||||
|
{(incident) => {
|
||||||
|
const statusLabel =
|
||||||
|
incident.status === 'open' && incident.acknowledged
|
||||||
|
? 'acknowledged'
|
||||||
|
: incident.status;
|
||||||
|
const statusClasses =
|
||||||
|
statusLabel === 'acknowledged'
|
||||||
|
? 'bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300'
|
||||||
|
: statusLabel === 'open'
|
||||||
|
? 'bg-red-100 dark:bg-red-900/40 text-red-700 dark:text-red-300'
|
||||||
|
: 'bg-gray-100 dark:bg-gray-700 text-gray-700 dark:text-gray-300';
|
||||||
|
const levelClasses =
|
||||||
|
incident.level === 'critical'
|
||||||
|
? 'bg-red-100 dark:bg-red-900/40 text-red-700 dark:text-red-300'
|
||||||
|
: 'bg-yellow-100 dark:bg-yellow-900/40 text-yellow-700 dark:text-yellow-300';
|
||||||
|
const isExpanded = expandedResourceIncidentIds().has(incident.id);
|
||||||
|
const events = incident.events || [];
|
||||||
|
const filteredEvents = filterIncidentEvents(events, resourceIncidentEventFilters());
|
||||||
|
const recentEvents =
|
||||||
|
filteredEvents.length > 6 ? filteredEvents.slice(filteredEvents.length - 6) : filteredEvents;
|
||||||
|
const lastEvent =
|
||||||
|
filteredEvents.length > 0 ? filteredEvents[filteredEvents.length - 1] : undefined;
|
||||||
|
const filteredLabel =
|
||||||
|
filteredEvents.length !== events.length
|
||||||
|
? `${filteredEvents.length}/${events.length}`
|
||||||
|
: `${events.length}`;
|
||||||
|
return (
|
||||||
|
<div class="rounded border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-900/40 p-3">
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-xs text-gray-600 dark:text-gray-400">
|
||||||
|
<span class="font-medium text-gray-800 dark:text-gray-200">
|
||||||
|
{incident.alertType}
|
||||||
|
</span>
|
||||||
|
<span class={`px-2 py-0.5 rounded ${levelClasses}`}>{incident.level}</span>
|
||||||
|
<span class={`px-2 py-0.5 rounded ${statusClasses}`}>{statusLabel}</span>
|
||||||
|
<span>opened {new Date(incident.openedAt).toLocaleString()}</span>
|
||||||
|
<Show when={incident.closedAt}>
|
||||||
|
<span>closed {new Date(incident.closedAt as string).toLocaleString()}</span>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
<Show when={incident.message}>
|
||||||
|
<p class="mt-1 text-xs text-gray-600 dark:text-gray-300">
|
||||||
|
{incident.message}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={incident.acknowledged && incident.ackUser}>
|
||||||
|
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
Acknowledged by {incident.ackUser}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={events.length > 0}>
|
||||||
|
<div class="mt-2 flex flex-wrap items-center justify-between gap-2 text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
<span>
|
||||||
|
<Show
|
||||||
|
when={filteredEvents.length > 0}
|
||||||
|
fallback={<span>No events match filters</span>}
|
||||||
|
>
|
||||||
|
Last event: {lastEvent?.summary}
|
||||||
|
</Show>
|
||||||
|
</span>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-1 text-[10px] border rounded-md border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
|
||||||
|
onClick={() => toggleResourceIncidentDetails(incident.id)}
|
||||||
|
>
|
||||||
|
{isExpanded ? 'Hide events' : `Events (${filteredLabel})`}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
<Show when={isExpanded}>
|
||||||
|
<div class="mt-2 space-y-2">
|
||||||
|
<Show
|
||||||
|
when={filteredEvents.length > 0}
|
||||||
|
fallback={
|
||||||
|
<p class="text-[10px] text-gray-400 dark:text-gray-500">
|
||||||
|
No events match the selected filters.
|
||||||
|
</p>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<For each={recentEvents}>
|
||||||
|
{(event) => (
|
||||||
|
<div class="rounded border border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-800/40 p-2">
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-xs text-gray-600 dark:text-gray-400">
|
||||||
|
<span class="font-medium text-gray-800 dark:text-gray-200">
|
||||||
|
{event.summary}
|
||||||
|
</span>
|
||||||
|
<span>{new Date(event.timestamp).toLocaleString()}</span>
|
||||||
|
</div>
|
||||||
|
<Show when={event.details && (event.details as { note?: string }).note}>
|
||||||
|
<p class="text-xs text-gray-700 dark:text-gray-300 mt-1">
|
||||||
|
{(event.details as { note?: string }).note}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={event.details && (event.details as { command?: string }).command}>
|
||||||
|
<p class="text-xs text-gray-700 dark:text-gray-300 mt-1 font-mono">
|
||||||
|
{(event.details as { command?: string }).command}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={event.details && (event.details as { output_excerpt?: string }).output_excerpt}>
|
||||||
|
<p class="text-xs text-gray-600 dark:text-gray-400 mt-1">
|
||||||
|
{(event.details as { output_excerpt?: string }).output_excerpt}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</For>
|
||||||
|
<Show when={filteredEvents.length > recentEvents.length}>
|
||||||
|
<p class="text-[10px] text-gray-400 dark:text-gray-500">
|
||||||
|
Showing last {recentEvents.length} events
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}}
|
||||||
|
</For>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
</Show>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}}
|
||||||
|
</Show>
|
||||||
|
|
||||||
{/* Alert History Table */}
|
{/* Alert History Table */}
|
||||||
<Show
|
<Show
|
||||||
when={loading()}
|
when={loading()}
|
||||||
|
|
@ -6683,7 +7246,10 @@ function HistoryTab() {
|
||||||
|
|
||||||
{/* Alerts for this day */}
|
{/* Alerts for this day */}
|
||||||
<For each={group.alerts}>
|
<For each={group.alerts}>
|
||||||
{(alert) => (
|
{(alert) => {
|
||||||
|
const rowKey = getIncidentRowKey(alert);
|
||||||
|
return (
|
||||||
|
<>
|
||||||
<tr
|
<tr
|
||||||
class={`border-b border-gray-200 dark:border-gray-600 hover:bg-gray-50 dark:hover:bg-gray-700 ${alert.status === 'active' ? 'bg-red-50 dark:bg-red-900/10' : ''
|
class={`border-b border-gray-200 dark:border-gray-600 hover:bg-gray-50 dark:hover:bg-gray-700 ${alert.status === 'active' ? 'bg-red-50 dark:bg-red-900/10' : ''
|
||||||
}`}
|
}`}
|
||||||
|
|
@ -6777,7 +7343,31 @@ function HistoryTab() {
|
||||||
|
|
||||||
{/* Actions */}
|
{/* Actions */}
|
||||||
<td class="p-1 px-2 text-center">
|
<td class="p-1 px-2 text-center">
|
||||||
<Show when={alert.status === 'active' || alert.status === 'acknowledged'}>
|
<div class="flex items-center justify-center gap-1">
|
||||||
|
<Show when={alert.source === 'alert'}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-1 text-[10px] border rounded-md border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
|
||||||
|
onClick={() => {
|
||||||
|
void toggleIncidentTimeline(rowKey, alert.id, alert.startTime);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{expandedIncidents().has(rowKey) ? 'Hide' : 'Timeline'}
|
||||||
|
</button>
|
||||||
|
</Show>
|
||||||
|
<Show when={alert.source === 'alert' && alert.resourceId}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="px-2 py-1 text-[10px] border rounded-md border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-700"
|
||||||
|
title="View incidents for this resource"
|
||||||
|
onClick={() => {
|
||||||
|
void openResourceIncidentPanel(alert.resourceId as string, alert.resourceName);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Resource
|
||||||
|
</button>
|
||||||
|
</Show>
|
||||||
|
<Show when={alert.source === 'alert' && (alert.status === 'active' || alert.status === 'acknowledged')}>
|
||||||
<InvestigateAlertButton
|
<InvestigateAlertButton
|
||||||
alert={{
|
alert={{
|
||||||
id: alert.id,
|
id: alert.id,
|
||||||
|
|
@ -6799,10 +7389,124 @@ function HistoryTab() {
|
||||||
licenseLocked={!hasAIAlertsFeature() && !licenseLoading()}
|
licenseLocked={!hasAIAlertsFeature() && !licenseLoading()}
|
||||||
/>
|
/>
|
||||||
</Show>
|
</Show>
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<Show when={alert.source === 'alert' && expandedIncidents().has(rowKey)}>
|
||||||
|
<tr class="bg-gray-50 dark:bg-gray-900/40 border-b border-gray-200 dark:border-gray-700">
|
||||||
|
<td colspan="11" class="p-3">
|
||||||
|
<Show when={incidentLoading()[rowKey]}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">Loading timeline...</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={!incidentLoading()[rowKey]}>
|
||||||
|
<Show when={incidentTimelines()[rowKey]}>
|
||||||
|
{(timeline) => (
|
||||||
|
<div class="space-y-3">
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-xs text-gray-600 dark:text-gray-400">
|
||||||
|
<span class="font-medium text-gray-700 dark:text-gray-200">Incident</span>
|
||||||
|
<span>{timeline().status}</span>
|
||||||
|
<Show when={timeline().acknowledged}>
|
||||||
|
<span class="px-2 py-0.5 rounded bg-emerald-100 text-emerald-700 dark:bg-emerald-900/40 dark:text-emerald-300">
|
||||||
|
acknowledged
|
||||||
|
</span>
|
||||||
|
</Show>
|
||||||
|
<Show when={timeline().openedAt}>
|
||||||
|
<span>opened {new Date(timeline().openedAt).toLocaleString()}</span>
|
||||||
|
</Show>
|
||||||
|
<Show when={timeline().closedAt}>
|
||||||
|
<span>closed {new Date(timeline().closedAt as string).toLocaleString()}</span>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
{(() => {
|
||||||
|
const events = timeline().events || [];
|
||||||
|
const filteredEvents = filterIncidentEvents(events, historyIncidentEventFilters());
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Show when={events.length > 0}>
|
||||||
|
<IncidentEventFilters
|
||||||
|
filters={historyIncidentEventFilters}
|
||||||
|
setFilters={setHistoryIncidentEventFilters}
|
||||||
|
/>
|
||||||
|
</Show>
|
||||||
|
<Show when={filteredEvents.length > 0}>
|
||||||
|
<div class="space-y-2">
|
||||||
|
<For each={filteredEvents}>
|
||||||
|
{(event) => (
|
||||||
|
<div class="rounded border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-900/30 p-2">
|
||||||
|
<div class="flex flex-wrap items-center gap-2 text-xs text-gray-600 dark:text-gray-400">
|
||||||
|
<span class="font-medium text-gray-800 dark:text-gray-200">
|
||||||
|
{event.summary}
|
||||||
|
</span>
|
||||||
|
<span>{new Date(event.timestamp).toLocaleString()}</span>
|
||||||
|
</div>
|
||||||
|
<Show when={event.details && (event.details as { note?: string }).note}>
|
||||||
|
<p class="text-xs text-gray-700 dark:text-gray-300 mt-1">
|
||||||
|
{(event.details as { note?: string }).note}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={event.details && (event.details as { command?: string }).command}>
|
||||||
|
<p class="text-xs text-gray-700 dark:text-gray-300 mt-1 font-mono">
|
||||||
|
{(event.details as { command?: string }).command}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={event.details && (event.details as { output_excerpt?: string }).output_excerpt}>
|
||||||
|
<p class="text-xs text-gray-600 dark:text-gray-400 mt-1">
|
||||||
|
{(event.details as { output_excerpt?: string }).output_excerpt}
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
</For>
|
</For>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
<Show when={events.length > 0 && filteredEvents.length === 0}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
No timeline events match the selected filters.
|
||||||
|
</p>
|
||||||
|
</Show>
|
||||||
|
<Show when={events.length === 0}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">No timeline events yet.</p>
|
||||||
|
</Show>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
})()}
|
||||||
|
<div class="flex flex-col gap-2">
|
||||||
|
<textarea
|
||||||
|
class="w-full rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-900 p-2 text-xs text-gray-800 dark:text-gray-200"
|
||||||
|
rows={2}
|
||||||
|
placeholder="Add a note for this incident..."
|
||||||
|
value={incidentNoteDrafts()[rowKey] || ''}
|
||||||
|
onInput={(e) => {
|
||||||
|
const value = e.currentTarget.value;
|
||||||
|
setIncidentNoteDrafts((prev) => ({ ...prev, [rowKey]: value }));
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<div class="flex justify-end">
|
||||||
|
<button
|
||||||
|
class="px-3 py-1.5 text-xs font-medium border rounded-lg transition-all bg-white dark:bg-gray-700 text-gray-700 dark:text-gray-300 border-gray-300 dark:border-gray-600 hover:bg-gray-50 dark:hover:bg-gray-600 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
|
disabled={incidentNoteSaving().has(rowKey) || !(incidentNoteDrafts()[rowKey] || '').trim()}
|
||||||
|
onClick={() => {
|
||||||
|
void saveIncidentNote(rowKey, alert.id, alert.startTime);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{incidentNoteSaving().has(rowKey) ? 'Saving...' : 'Save Note'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Show>
|
||||||
|
<Show when={!incidentTimelines()[rowKey]}>
|
||||||
|
<p class="text-xs text-gray-500 dark:text-gray-400">No incident timeline available.</p>
|
||||||
|
</Show>
|
||||||
|
</Show>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</Show>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}}
|
||||||
|
</For>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</For>
|
</For>
|
||||||
|
|
|
||||||
|
|
@ -1019,6 +1019,34 @@ export interface ResolvedAlert extends Alert {
|
||||||
resolvedTime: string;
|
resolvedTime: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface IncidentEvent {
|
||||||
|
id: string;
|
||||||
|
type: string;
|
||||||
|
timestamp: string;
|
||||||
|
summary: string;
|
||||||
|
details?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Incident {
|
||||||
|
id: string;
|
||||||
|
alertId: string;
|
||||||
|
alertType: string;
|
||||||
|
level: string;
|
||||||
|
resourceId: string;
|
||||||
|
resourceName: string;
|
||||||
|
resourceType?: string;
|
||||||
|
node?: string;
|
||||||
|
instance?: string;
|
||||||
|
message?: string;
|
||||||
|
status: string;
|
||||||
|
openedAt: string;
|
||||||
|
closedAt?: string;
|
||||||
|
acknowledged: boolean;
|
||||||
|
ackUser?: string;
|
||||||
|
ackTime?: string;
|
||||||
|
events?: IncidentEvent[];
|
||||||
|
}
|
||||||
|
|
||||||
// WebSocket message types
|
// WebSocket message types
|
||||||
export type WSMessage =
|
export type WSMessage =
|
||||||
| { type: 'initialState'; data: State }
|
| { type: 'initialState'; data: State }
|
||||||
|
|
|
||||||
|
|
@ -161,6 +161,17 @@ func (a *AlertTriggeredAnalyzer) analyzeResource(alert *alerts.Alert, resourceKe
|
||||||
Dur("duration", duration).
|
Dur("duration", duration).
|
||||||
Msg("Alert-triggered AI analysis completed with no additional findings")
|
Msg("Alert-triggered AI analysis completed with no additional findings")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if a.patrolService != nil && a.patrolService.aiService != nil {
|
||||||
|
summary := "Alert-triggered AI analysis completed"
|
||||||
|
if len(findings) > 0 {
|
||||||
|
summary = fmt.Sprintf("Alert-triggered AI analysis found %d findings", len(findings))
|
||||||
|
}
|
||||||
|
a.patrolService.aiService.RecordIncidentAnalysis(alert.ID, summary, map[string]interface{}{
|
||||||
|
"findings": len(findings),
|
||||||
|
"duration": duration.String(),
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// analyzeResourceByAlert determines the resource type from the alert and analyzes it
|
// analyzeResourceByAlert determines the resource type from the alert and analyzes it
|
||||||
|
|
|
||||||
848
internal/ai/memory/incidents.go
Normal file
848
internal/ai/memory/incidents.go
Normal file
|
|
@ -0,0 +1,848 @@
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncidentStatus represents the current state of an incident.
|
||||||
|
type IncidentStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
IncidentStatusOpen IncidentStatus = "open"
|
||||||
|
IncidentStatusResolved IncidentStatus = "resolved"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncidentEventType describes a timeline event type.
|
||||||
|
type IncidentEventType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
IncidentEventAlertFired IncidentEventType = "alert_fired"
|
||||||
|
IncidentEventAlertAcknowledged IncidentEventType = "alert_acknowledged"
|
||||||
|
IncidentEventAlertUnacknowledged IncidentEventType = "alert_unacknowledged"
|
||||||
|
IncidentEventAlertResolved IncidentEventType = "alert_resolved"
|
||||||
|
IncidentEventAnalysis IncidentEventType = "ai_analysis"
|
||||||
|
IncidentEventCommand IncidentEventType = "command"
|
||||||
|
IncidentEventRunbook IncidentEventType = "runbook"
|
||||||
|
IncidentEventNote IncidentEventType = "note"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncidentEvent represents a single timeline entry for an incident.
|
||||||
|
type IncidentEvent struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Type IncidentEventType `json:"type"`
|
||||||
|
Timestamp time.Time `json:"timestamp"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
Details map[string]interface{} `json:"details,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Incident captures an alert occurrence and its timeline.
|
||||||
|
type Incident struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
AlertID string `json:"alertId"`
|
||||||
|
AlertType string `json:"alertType"`
|
||||||
|
Level string `json:"level"`
|
||||||
|
ResourceID string `json:"resourceId"`
|
||||||
|
ResourceName string `json:"resourceName"`
|
||||||
|
ResourceType string `json:"resourceType,omitempty"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
|
Instance string `json:"instance,omitempty"`
|
||||||
|
Message string `json:"message,omitempty"`
|
||||||
|
Status IncidentStatus `json:"status"`
|
||||||
|
OpenedAt time.Time `json:"openedAt"`
|
||||||
|
ClosedAt *time.Time `json:"closedAt,omitempty"`
|
||||||
|
Acknowledged bool `json:"acknowledged"`
|
||||||
|
AckUser string `json:"ackUser,omitempty"`
|
||||||
|
AckTime *time.Time `json:"ackTime,omitempty"`
|
||||||
|
Events []IncidentEvent `json:"events,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncidentStoreConfig configures incident retention and persistence.
|
||||||
|
type IncidentStoreConfig struct {
|
||||||
|
DataDir string
|
||||||
|
MaxIncidents int
|
||||||
|
MaxEventsPerIncident int
|
||||||
|
MaxAgeDays int
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncidentStore maintains incident timelines and persistence.
|
||||||
|
type IncidentStore struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
saveMu sync.Mutex
|
||||||
|
incidents []*Incident
|
||||||
|
maxIncidents int
|
||||||
|
maxEvents int
|
||||||
|
maxAge time.Duration
|
||||||
|
dataDir string
|
||||||
|
filePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultIncidentMaxIncidents = 500
|
||||||
|
defaultIncidentMaxEvents = 120
|
||||||
|
defaultIncidentMaxAgeDays = 90
|
||||||
|
incidentFileName = "ai_incidents.json"
|
||||||
|
maxIncidentFileSize = 20 * 1024 * 1024 // 20MB
|
||||||
|
incidentStartMatchTolerance = 10 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewIncidentStore creates a new incident store with persistence.
|
||||||
|
func NewIncidentStore(cfg IncidentStoreConfig) *IncidentStore {
|
||||||
|
maxIncidents := cfg.MaxIncidents
|
||||||
|
if maxIncidents <= 0 {
|
||||||
|
maxIncidents = defaultIncidentMaxIncidents
|
||||||
|
}
|
||||||
|
maxEvents := cfg.MaxEventsPerIncident
|
||||||
|
if maxEvents <= 0 {
|
||||||
|
maxEvents = defaultIncidentMaxEvents
|
||||||
|
}
|
||||||
|
maxAgeDays := cfg.MaxAgeDays
|
||||||
|
if maxAgeDays <= 0 {
|
||||||
|
maxAgeDays = defaultIncidentMaxAgeDays
|
||||||
|
}
|
||||||
|
|
||||||
|
store := &IncidentStore{
|
||||||
|
incidents: make([]*Incident, 0),
|
||||||
|
maxIncidents: maxIncidents,
|
||||||
|
maxEvents: maxEvents,
|
||||||
|
maxAge: time.Duration(maxAgeDays) * 24 * time.Hour,
|
||||||
|
dataDir: cfg.DataDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
if store.dataDir != "" {
|
||||||
|
store.filePath = filepath.Join(store.dataDir, incidentFileName)
|
||||||
|
if err := store.loadFromDisk(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to load incident history from disk")
|
||||||
|
} else if len(store.incidents) > 0 {
|
||||||
|
log.Info().Int("count", len(store.incidents)).Msg("Loaded incident history from disk")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return store
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertFired opens or updates an incident for a fired alert.
|
||||||
|
func (s *IncidentStore) RecordAlertFired(alert *alerts.Alert) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = newIncidentFromAlert(alert)
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertFired, formatAlertSummary(alert), map[string]interface{}{
|
||||||
|
"type": alert.Type,
|
||||||
|
"level": string(alert.Level),
|
||||||
|
"value": alert.Value,
|
||||||
|
"threshold": alert.Threshold,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
updateIncidentFromAlert(incident, alert)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertAcknowledged records an acknowledgement event for an alert.
|
||||||
|
func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.ensureIncidentForAlertLocked(alert)
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
incident.Acknowledged = true
|
||||||
|
if alert.AckTime != nil {
|
||||||
|
incident.AckTime = alert.AckTime
|
||||||
|
} else {
|
||||||
|
now := time.Now()
|
||||||
|
incident.AckTime = &now
|
||||||
|
}
|
||||||
|
incident.AckUser = user
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{
|
||||||
|
"user": user,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertUnacknowledged records an unacknowledge event for an alert.
|
||||||
|
func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.ensureIncidentForAlertLocked(alert)
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
incident.Acknowledged = false
|
||||||
|
incident.AckTime = nil
|
||||||
|
incident.AckUser = ""
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{
|
||||||
|
"user": user,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertResolved records a resolved event and closes the incident.
|
||||||
|
func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time.Time) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = newIncidentFromAlert(alert)
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
incident.Status = IncidentStatusResolved
|
||||||
|
if resolvedAt.IsZero() {
|
||||||
|
now := time.Now()
|
||||||
|
resolvedAt = now
|
||||||
|
}
|
||||||
|
incident.ClosedAt = &resolvedAt
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{
|
||||||
|
"resolved_at": resolvedAt.Format(time.RFC3339),
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAnalysis adds an AI analysis event to the incident for an alert.
|
||||||
|
func (s *IncidentStore) RecordAnalysis(alertID, summary string, details map[string]interface{}) {
|
||||||
|
if alertID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alertID,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
if summary == "" {
|
||||||
|
summary = "AI analysis completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAnalysis, summary, details)
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordCommand adds a command execution event to the incident for an alert.
|
||||||
|
func (s *IncidentStore) RecordCommand(alertID, command string, success bool, output string, details map[string]interface{}) {
|
||||||
|
if alertID == "" || command == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alertID,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
if details == nil {
|
||||||
|
details = make(map[string]interface{})
|
||||||
|
}
|
||||||
|
details["command"] = command
|
||||||
|
details["success"] = success
|
||||||
|
if output != "" {
|
||||||
|
details["output_excerpt"] = truncateOutput(output, 500)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := "failed"
|
||||||
|
if success {
|
||||||
|
status = "succeeded"
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("Command %s: %s", status, command)
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventCommand, summary, details)
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordRunbook adds a runbook execution event to the incident for an alert.
|
||||||
|
func (s *IncidentStore) RecordRunbook(alertID, runbookID, title string, outcome string, automatic bool, message string) {
|
||||||
|
if alertID == "" || runbookID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alertID,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := fmt.Sprintf("Runbook %s (%s)", title, outcome)
|
||||||
|
details := map[string]interface{}{
|
||||||
|
"runbook_id": runbookID,
|
||||||
|
"outcome": outcome,
|
||||||
|
"automatic": automatic,
|
||||||
|
}
|
||||||
|
if message != "" {
|
||||||
|
details["message"] = message
|
||||||
|
}
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventRunbook, summary, details)
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordNote appends a user note to an incident identified by alert ID or incident ID.
|
||||||
|
func (s *IncidentStore) RecordNote(alertID, incidentID, note, user string) bool {
|
||||||
|
note = strings.TrimSpace(note)
|
||||||
|
if note == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
var incident *Incident
|
||||||
|
if incidentID != "" {
|
||||||
|
incident = s.findIncidentByIDLocked(incidentID)
|
||||||
|
} else if alertID != "" {
|
||||||
|
incident = s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
}
|
||||||
|
if incident == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := "Note added"
|
||||||
|
if user != "" {
|
||||||
|
summary = fmt.Sprintf("Note added by %s", user)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventNote, summary, map[string]interface{}{
|
||||||
|
"note": note,
|
||||||
|
"user": user,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTimelineByAlertID returns the most recent incident for the alert.
|
||||||
|
func (s *IncidentStore) GetTimelineByAlertID(alertID string) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return cloneIncident(incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTimelineByAlertAt returns the incident closest to the provided start time for an alert.
|
||||||
|
func (s *IncidentStore) GetTimelineByAlertAt(alertID string, startedAt time.Time) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if startedAt.IsZero() {
|
||||||
|
return s.GetTimelineByAlertID(alertID)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
var best *Incident
|
||||||
|
var bestDelta time.Duration
|
||||||
|
for _, incident := range s.incidents {
|
||||||
|
if incident == nil || incident.AlertID != alertID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delta := incident.OpenedAt.Sub(startedAt)
|
||||||
|
if delta < 0 {
|
||||||
|
delta = -delta
|
||||||
|
}
|
||||||
|
if best == nil || delta < bestDelta {
|
||||||
|
best = incident
|
||||||
|
bestDelta = delta
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if best == nil || bestDelta > incidentStartMatchTolerance {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return cloneIncident(best)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListIncidentsByResource returns recent incidents for a resource.
|
||||||
|
func (s *IncidentStore) ListIncidentsByResource(resourceID string, limit int) []*Incident {
|
||||||
|
if resourceID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
var matches []*Incident
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.ResourceID == resourceID {
|
||||||
|
matches = append(matches, cloneIncident(incident))
|
||||||
|
if limit > 0 && len(matches) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return matches
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForAlert returns a condensed incident timeline for prompt injection.
|
||||||
|
func (s *IncidentStore) FormatForAlert(alertID string, maxEvents int) string {
|
||||||
|
incident := s.GetTimelineByAlertID(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("\n\n## Incident Memory\n")
|
||||||
|
b.WriteString(fmt.Sprintf("Alert incident for %s (%s, %s)\n",
|
||||||
|
incident.ResourceName, incident.AlertType, incident.Level))
|
||||||
|
b.WriteString(fmt.Sprintf("Status: %s\n", incident.Status))
|
||||||
|
|
||||||
|
events := incident.Events
|
||||||
|
if maxEvents > 0 && len(events) > maxEvents {
|
||||||
|
events = events[len(events)-maxEvents:]
|
||||||
|
}
|
||||||
|
for _, evt := range events {
|
||||||
|
b.WriteString("- ")
|
||||||
|
b.WriteString(evt.Timestamp.Format(time.RFC3339))
|
||||||
|
b.WriteString(": ")
|
||||||
|
b.WriteString(evt.Summary)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForResource returns a condensed incident summary for a resource.
|
||||||
|
func (s *IncidentStore) FormatForResource(resourceID string, limit int) string {
|
||||||
|
incidents := s.ListIncidentsByResource(resourceID, limit)
|
||||||
|
if len(incidents) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("\n\n## Incident Memory\n")
|
||||||
|
b.WriteString("Recent incidents for this resource:\n")
|
||||||
|
for _, incident := range incidents {
|
||||||
|
status := string(incident.Status)
|
||||||
|
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
|
||||||
|
status = "acknowledged"
|
||||||
|
}
|
||||||
|
b.WriteString("- ")
|
||||||
|
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
|
||||||
|
b.WriteString(": ")
|
||||||
|
b.WriteString(incident.AlertType)
|
||||||
|
if incident.Level != "" {
|
||||||
|
b.WriteString(" (")
|
||||||
|
b.WriteString(incident.Level)
|
||||||
|
b.WriteString(")")
|
||||||
|
}
|
||||||
|
b.WriteString(" - ")
|
||||||
|
b.WriteString(status)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForPatrol returns a condensed incident summary for infrastructure-wide patrol analysis.
|
||||||
|
func (s *IncidentStore) FormatForPatrol(limit int) string {
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 8
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
if len(s.incidents) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("\n\n## Incident Memory\n")
|
||||||
|
b.WriteString("Recent incidents across infrastructure:\n")
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
for i := len(s.incidents) - 1; i >= 0 && count < limit; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
status := string(incident.Status)
|
||||||
|
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
|
||||||
|
status = "acknowledged"
|
||||||
|
}
|
||||||
|
|
||||||
|
lastSummary := ""
|
||||||
|
if len(incident.Events) > 0 {
|
||||||
|
lastSummary = incident.Events[len(incident.Events)-1].Summary
|
||||||
|
}
|
||||||
|
|
||||||
|
b.WriteString("- ")
|
||||||
|
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
|
||||||
|
b.WriteString(": ")
|
||||||
|
if incident.ResourceName != "" {
|
||||||
|
b.WriteString(incident.ResourceName)
|
||||||
|
b.WriteString(" - ")
|
||||||
|
}
|
||||||
|
if incident.AlertType != "" {
|
||||||
|
b.WriteString(incident.AlertType)
|
||||||
|
}
|
||||||
|
if incident.Level != "" {
|
||||||
|
b.WriteString(" (")
|
||||||
|
b.WriteString(incident.Level)
|
||||||
|
b.WriteString(")")
|
||||||
|
}
|
||||||
|
b.WriteString(" - ")
|
||||||
|
b.WriteString(status)
|
||||||
|
if lastSummary != "" {
|
||||||
|
b.WriteString(" - last: ")
|
||||||
|
b.WriteString(truncateOutput(lastSummary, 80))
|
||||||
|
} else if incident.Message != "" {
|
||||||
|
b.WriteString(" - ")
|
||||||
|
b.WriteString(truncateOutput(incident.Message, 80))
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIncidentFromAlert(alert *alerts.Alert) *Incident {
|
||||||
|
openedAt := alert.StartTime
|
||||||
|
if openedAt.IsZero() {
|
||||||
|
openedAt = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alert.ID,
|
||||||
|
AlertType: alert.Type,
|
||||||
|
Level: string(alert.Level),
|
||||||
|
ResourceID: alert.ResourceID,
|
||||||
|
ResourceName: alert.ResourceName,
|
||||||
|
Node: alert.Node,
|
||||||
|
Instance: alert.Instance,
|
||||||
|
Message: alert.Message,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: openedAt,
|
||||||
|
Acknowledged: alert.Acknowledged,
|
||||||
|
AckUser: alert.AckUser,
|
||||||
|
AckTime: alert.AckTime,
|
||||||
|
Events: make([]IncidentEvent, 0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateIncidentFromAlert(incident *Incident, alert *alerts.Alert) {
|
||||||
|
if incident == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
incident.AlertType = alert.Type
|
||||||
|
incident.Level = string(alert.Level)
|
||||||
|
incident.ResourceID = alert.ResourceID
|
||||||
|
incident.ResourceName = alert.ResourceName
|
||||||
|
incident.Node = alert.Node
|
||||||
|
incident.Instance = alert.Instance
|
||||||
|
incident.Message = alert.Message
|
||||||
|
incident.Acknowledged = alert.Acknowledged
|
||||||
|
incident.AckUser = alert.AckUser
|
||||||
|
incident.AckTime = alert.AckTime
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident {
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alert.ID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = newIncidentFromAlert(alert)
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
updateIncidentFromAlert(incident, alert)
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) addEventLocked(incident *Incident, eventType IncidentEventType, summary string, details map[string]interface{}) {
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if summary == "" {
|
||||||
|
summary = string(eventType)
|
||||||
|
}
|
||||||
|
|
||||||
|
event := IncidentEvent{
|
||||||
|
ID: generateIncidentEventID(),
|
||||||
|
Type: eventType,
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
Summary: summary,
|
||||||
|
Details: details,
|
||||||
|
}
|
||||||
|
incident.Events = append(incident.Events, event)
|
||||||
|
if s.maxEvents > 0 && len(incident.Events) > s.maxEvents {
|
||||||
|
incident.Events = incident.Events[len(incident.Events)-s.maxEvents:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) findOpenIncidentByAlertIDLocked(alertID string) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.AlertID == alertID && incident.Status == IncidentStatusOpen {
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) findLatestIncidentByAlertIDLocked(alertID string) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.AlertID == alertID {
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) findIncidentByIDLocked(incidentID string) *Incident {
|
||||||
|
if incidentID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.ID == incidentID {
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) trimLocked() {
|
||||||
|
if s.maxAge > 0 {
|
||||||
|
cutoff := time.Now().Add(-s.maxAge)
|
||||||
|
filtered := make([]*Incident, 0, len(s.incidents))
|
||||||
|
for _, incident := range s.incidents {
|
||||||
|
if incident == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
compareTime := incident.OpenedAt
|
||||||
|
if incident.ClosedAt != nil {
|
||||||
|
compareTime = *incident.ClosedAt
|
||||||
|
}
|
||||||
|
if compareTime.After(cutoff) {
|
||||||
|
filtered = append(filtered, incident)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.incidents = filtered
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.maxIncidents > 0 && len(s.incidents) > s.maxIncidents {
|
||||||
|
sort.Slice(s.incidents, func(i, j int) bool {
|
||||||
|
return s.incidents[i].OpenedAt.Before(s.incidents[j].OpenedAt)
|
||||||
|
})
|
||||||
|
if len(s.incidents) > s.maxIncidents {
|
||||||
|
s.incidents = s.incidents[len(s.incidents)-s.maxIncidents:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) saveAsync() {
|
||||||
|
if s.dataDir == "" || s.filePath == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
if err := s.saveToDisk(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to save incident history")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) saveToDisk() error {
|
||||||
|
s.saveMu.Lock()
|
||||||
|
defer s.saveMu.Unlock()
|
||||||
|
|
||||||
|
if s.dataDir == "" || s.filePath == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(s.dataDir, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
snapshot := make([]*Incident, 0, len(s.incidents))
|
||||||
|
for _, incident := range s.incidents {
|
||||||
|
snapshot = append(snapshot, cloneIncident(incident))
|
||||||
|
}
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
data, err := json.Marshal(snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpFile := s.filePath + ".tmp"
|
||||||
|
if err := os.WriteFile(tmpFile, data, 0644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpFile, s.filePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) loadFromDisk() error {
|
||||||
|
if s.filePath == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Stat(s.filePath)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Size() > maxIncidentFileSize {
|
||||||
|
return fmt.Errorf("incident history file too large (%d bytes)", info.Size())
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(s.filePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var incidents []*Incident
|
||||||
|
if err := json.Unmarshal(data, &incidents); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.incidents = incidents
|
||||||
|
s.trimLocked()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneIncident(src *Incident) *Incident {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
clone := *src
|
||||||
|
if src.AckTime != nil {
|
||||||
|
t := *src.AckTime
|
||||||
|
clone.AckTime = &t
|
||||||
|
}
|
||||||
|
if src.ClosedAt != nil {
|
||||||
|
t := *src.ClosedAt
|
||||||
|
clone.ClosedAt = &t
|
||||||
|
}
|
||||||
|
if len(src.Events) > 0 {
|
||||||
|
clone.Events = make([]IncidentEvent, len(src.Events))
|
||||||
|
for i, event := range src.Events {
|
||||||
|
cloneEvent := event
|
||||||
|
if event.Details != nil {
|
||||||
|
detailsCopy := make(map[string]interface{}, len(event.Details))
|
||||||
|
for key, value := range event.Details {
|
||||||
|
detailsCopy[key] = value
|
||||||
|
}
|
||||||
|
cloneEvent.Details = detailsCopy
|
||||||
|
}
|
||||||
|
clone.Events[i] = cloneEvent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &clone
|
||||||
|
}
|
||||||
|
|
||||||
|
var incidentCounter int64
|
||||||
|
|
||||||
|
func generateIncidentID() string {
|
||||||
|
incidentCounter++
|
||||||
|
return "inc-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentCounter%1000))
|
||||||
|
}
|
||||||
|
|
||||||
|
var incidentEventCounter int64
|
||||||
|
|
||||||
|
func generateIncidentEventID() string {
|
||||||
|
incidentEventCounter++
|
||||||
|
return "inc-evt-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentEventCounter%1000))
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatAlertSummary(alert *alerts.Alert) string {
|
||||||
|
if alert == nil {
|
||||||
|
return "Alert triggered"
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("Alert triggered: %s (%s)", alert.Type, alert.Level)
|
||||||
|
if alert.Value > 0 || alert.Threshold > 0 {
|
||||||
|
summary = fmt.Sprintf("Alert triggered: %s (%s %.1f >= %.1f)", alert.Type, alert.Level, alert.Value, alert.Threshold)
|
||||||
|
}
|
||||||
|
return summary
|
||||||
|
}
|
||||||
108
internal/ai/memory/incidents_test.go
Normal file
108
internal/ai/memory/incidents_test.go
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIncidentStore_RecordTimeline(t *testing.T) {
|
||||||
|
store := NewIncidentStore(IncidentStoreConfig{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
MaxIncidents: 10,
|
||||||
|
MaxEventsPerIncident: 10,
|
||||||
|
MaxAgeDays: 30,
|
||||||
|
})
|
||||||
|
|
||||||
|
alert := &alerts.Alert{
|
||||||
|
ID: "alert-1",
|
||||||
|
Type: "cpu",
|
||||||
|
Level: alerts.AlertLevelWarning,
|
||||||
|
ResourceID: "res-1",
|
||||||
|
ResourceName: "vm-1",
|
||||||
|
StartTime: time.Now().Add(-5 * time.Minute),
|
||||||
|
Value: 92,
|
||||||
|
Threshold: 85,
|
||||||
|
}
|
||||||
|
|
||||||
|
store.RecordAlertFired(alert)
|
||||||
|
store.RecordAlertAcknowledged(alert, "admin")
|
||||||
|
store.RecordAnalysis(alert.ID, "analysis complete", map[string]interface{}{
|
||||||
|
"findings": 1,
|
||||||
|
})
|
||||||
|
store.RecordCommand(alert.ID, "systemctl restart nginx", true, "ok", nil)
|
||||||
|
store.RecordAlertResolved(alert, time.Now())
|
||||||
|
|
||||||
|
timeline := store.GetTimelineByAlertID(alert.ID)
|
||||||
|
if timeline == nil {
|
||||||
|
t.Fatalf("expected timeline, got nil")
|
||||||
|
}
|
||||||
|
if timeline.Status != IncidentStatusResolved {
|
||||||
|
t.Fatalf("expected status %q, got %q", IncidentStatusResolved, timeline.Status)
|
||||||
|
}
|
||||||
|
if timeline.AckUser != "admin" {
|
||||||
|
t.Fatalf("expected ack user admin, got %q", timeline.AckUser)
|
||||||
|
}
|
||||||
|
if len(timeline.Events) < 4 {
|
||||||
|
t.Fatalf("expected events recorded, got %d", len(timeline.Events))
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok := store.RecordNote(alert.ID, "", "note text", ""); !ok {
|
||||||
|
t.Fatalf("expected note to be saved")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) {
|
||||||
|
store := NewIncidentStore(IncidentStoreConfig{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
MaxIncidents: 10,
|
||||||
|
MaxEventsPerIncident: 10,
|
||||||
|
MaxAgeDays: 30,
|
||||||
|
})
|
||||||
|
|
||||||
|
base := time.Now().UTC()
|
||||||
|
|
||||||
|
first := &alerts.Alert{
|
||||||
|
ID: "alert-2",
|
||||||
|
Type: "cpu",
|
||||||
|
Level: alerts.AlertLevelWarning,
|
||||||
|
ResourceID: "res-2",
|
||||||
|
ResourceName: "vm-2",
|
||||||
|
StartTime: base.Add(-2 * time.Hour),
|
||||||
|
}
|
||||||
|
|
||||||
|
second := &alerts.Alert{
|
||||||
|
ID: "alert-2",
|
||||||
|
Type: "cpu",
|
||||||
|
Level: alerts.AlertLevelWarning,
|
||||||
|
ResourceID: "res-2",
|
||||||
|
ResourceName: "vm-2",
|
||||||
|
StartTime: base.Add(-10 * time.Minute),
|
||||||
|
}
|
||||||
|
|
||||||
|
store.RecordAlertFired(first)
|
||||||
|
store.RecordAlertResolved(first, base.Add(-90*time.Minute))
|
||||||
|
store.RecordAlertFired(second)
|
||||||
|
|
||||||
|
timeline := store.GetTimelineByAlertAt(first.ID, first.StartTime)
|
||||||
|
if timeline == nil {
|
||||||
|
t.Fatalf("expected timeline for first incident, got nil")
|
||||||
|
}
|
||||||
|
if !timeline.OpenedAt.Equal(first.StartTime) {
|
||||||
|
t.Fatalf("expected openedAt %s, got %s", first.StartTime, timeline.OpenedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeline = store.GetTimelineByAlertAt(second.ID, second.StartTime)
|
||||||
|
if timeline == nil {
|
||||||
|
t.Fatalf("expected timeline for second incident, got nil")
|
||||||
|
}
|
||||||
|
if !timeline.OpenedAt.Equal(second.StartTime) {
|
||||||
|
t.Fatalf("expected openedAt %s, got %s", second.StartTime, timeline.OpenedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeline = store.GetTimelineByAlertAt(second.ID, base.Add(-45*time.Minute))
|
||||||
|
if timeline != nil {
|
||||||
|
t.Fatalf("expected no timeline for mismatched start time")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||||
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
|
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
@ -217,6 +218,7 @@ type PatrolService struct {
|
||||||
remediationLog *RemediationLog // For tracking remediation actions
|
remediationLog *RemediationLog // For tracking remediation actions
|
||||||
patternDetector *PatternDetector // For failure prediction from historical patterns
|
patternDetector *PatternDetector // For failure prediction from historical patterns
|
||||||
correlationDetector *CorrelationDetector // For multi-resource correlation
|
correlationDetector *CorrelationDetector // For multi-resource correlation
|
||||||
|
incidentStore *memory.IncidentStore // For incident timeline capture
|
||||||
|
|
||||||
// Cached thresholds (recalculated when thresholdProvider changes)
|
// Cached thresholds (recalculated when thresholdProvider changes)
|
||||||
thresholds PatrolThresholds
|
thresholds PatrolThresholds
|
||||||
|
|
@ -263,6 +265,20 @@ func NewPatrolService(aiService *Service, stateProvider StateProvider) *PatrolSe
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetIncidentStore attaches an incident store for alert timeline capture.
|
||||||
|
func (p *PatrolService) SetIncidentStore(store *memory.IncidentStore) {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
p.incidentStore = store
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIncidentStore returns the incident store if configured.
|
||||||
|
func (p *PatrolService) GetIncidentStore() *memory.IncidentStore {
|
||||||
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
return p.incidentStore
|
||||||
|
}
|
||||||
|
|
||||||
// SetConfig updates the patrol configuration
|
// SetConfig updates the patrol configuration
|
||||||
func (p *PatrolService) SetConfig(cfg PatrolConfig) {
|
func (p *PatrolService) SetConfig(cfg PatrolConfig) {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
|
|
@ -2185,12 +2201,17 @@ func (p *PatrolService) buildPatrolPrompt(summary string) string {
|
||||||
|
|
||||||
// Get resource notes from knowledge store (per-resource user notes)
|
// Get resource notes from knowledge store (per-resource user notes)
|
||||||
var knowledgeContext string
|
var knowledgeContext string
|
||||||
|
var incidentContext string
|
||||||
p.mu.RLock()
|
p.mu.RLock()
|
||||||
knowledgeStore := p.knowledgeStore
|
knowledgeStore := p.knowledgeStore
|
||||||
|
incidentStore := p.incidentStore
|
||||||
p.mu.RUnlock()
|
p.mu.RUnlock()
|
||||||
if knowledgeStore != nil {
|
if knowledgeStore != nil {
|
||||||
knowledgeContext = knowledgeStore.FormatAllForContext()
|
knowledgeContext = knowledgeStore.FormatAllForContext()
|
||||||
}
|
}
|
||||||
|
if incidentStore != nil {
|
||||||
|
incidentContext = incidentStore.FormatForPatrol(8)
|
||||||
|
}
|
||||||
|
|
||||||
basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities.
|
basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities.
|
||||||
|
|
||||||
|
|
@ -2235,6 +2256,12 @@ IMPORTANT: Respect the user's feedback above. Do NOT re-raise findings that are:
|
||||||
Only report NEW issues or issues where the severity has clearly escalated.`)
|
Only report NEW issues or issues where the severity has clearly escalated.`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if incidentContext != "" {
|
||||||
|
contextAdditions.WriteString("\n\n")
|
||||||
|
contextAdditions.WriteString(incidentContext)
|
||||||
|
contextAdditions.WriteString("\nIMPORTANT: Use incident memory to avoid repeating known issues and to build on successful past investigations.")
|
||||||
|
}
|
||||||
|
|
||||||
if contextAdditions.Len() > 0 {
|
if contextAdditions.Len() > 0 {
|
||||||
return basePrompt + contextAdditions.String()
|
return basePrompt + contextAdditions.String()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -323,6 +323,10 @@ func (p *PatrolService) logRunbookExecution(finding *Finding, runbook Runbook, s
|
||||||
if err := p.remediationLog.Log(record); err != nil {
|
if err := p.remediationLog.Log(record); err != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to log runbook execution")
|
log.Warn().Err(err).Msg("Failed to log runbook execution")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if p.aiService != nil && finding != nil && finding.AlertID != "" {
|
||||||
|
p.aiService.RecordIncidentRunbook(finding.AlertID, runbook.ID, runbook.Title, outcome, automatic, message)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type runbookContext struct {
|
type runbookContext struct {
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,10 @@ import (
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
|
|
@ -76,6 +78,7 @@ type Service struct {
|
||||||
resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
|
resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
|
||||||
patrolService *PatrolService // Background AI monitoring service
|
patrolService *PatrolService // Background AI monitoring service
|
||||||
metadataProvider MetadataProvider // Enables AI to update resource URLs
|
metadataProvider MetadataProvider // Enables AI to update resource URLs
|
||||||
|
incidentStore *memory.IncidentStore // Incident timelines for alert memory
|
||||||
|
|
||||||
// Alert-triggered analysis - token-efficient real-time AI insights
|
// Alert-triggered analysis - token-efficient real-time AI insights
|
||||||
alertTriggeredAnalyzer *AlertTriggeredAnalyzer
|
alertTriggeredAnalyzer *AlertTriggeredAnalyzer
|
||||||
|
|
@ -197,6 +200,9 @@ func (s *Service) SetStateProvider(sp StateProvider) {
|
||||||
if s.knowledgeStore != nil {
|
if s.knowledgeStore != nil {
|
||||||
s.patrolService.SetKnowledgeStore(s.knowledgeStore)
|
s.patrolService.SetKnowledgeStore(s.knowledgeStore)
|
||||||
}
|
}
|
||||||
|
if s.incidentStore != nil {
|
||||||
|
s.patrolService.SetIncidentStore(s.incidentStore)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize alert-triggered analyzer if not already done
|
// Initialize alert-triggered analyzer if not already done
|
||||||
|
|
@ -350,6 +356,17 @@ func (s *Service) SetRemediationLog(remLog *RemediationLog) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetIncidentStore sets the incident store for alert timeline memory.
|
||||||
|
func (s *Service) SetIncidentStore(store *memory.IncidentStore) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.incidentStore = store
|
||||||
|
|
||||||
|
if s.patrolService != nil {
|
||||||
|
s.patrolService.SetIncidentStore(store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// SetPatternDetector sets the pattern detector for failure prediction
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
|
|
@ -2183,11 +2200,34 @@ func (s *Service) executeTool(ctx context.Context, req ExecuteRequest, tc provid
|
||||||
|
|
||||||
// Execute via agent
|
// Execute via agent
|
||||||
result, err := s.executeOnAgent(ctx, execReq, command)
|
result, err := s.executeOnAgent(ctx, execReq, command)
|
||||||
|
recordIncident := func(success bool, output string) {
|
||||||
|
alertID := extractAlertID(req.Context)
|
||||||
|
if alertID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
incidentStore := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
if incidentStore == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
details := map[string]interface{}{
|
||||||
|
"resource_id": req.TargetID,
|
||||||
|
"resource_type": req.TargetType,
|
||||||
|
"run_on_host": runOnHost,
|
||||||
|
}
|
||||||
|
if targetHost != "" {
|
||||||
|
details["target_host"] = targetHost
|
||||||
|
}
|
||||||
|
incidentStore.RecordCommand(alertID, command, success, output, details)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
recordIncident(false, result)
|
||||||
execution.Output = fmt.Sprintf("Error executing command: %s", err)
|
execution.Output = fmt.Sprintf("Error executing command: %s", err)
|
||||||
return execution.Output, execution
|
return execution.Output, execution
|
||||||
}
|
}
|
||||||
|
|
||||||
|
recordIncident(true, result)
|
||||||
execution.Output = result
|
execution.Output = result
|
||||||
execution.Success = true
|
execution.Success = true
|
||||||
return result, execution
|
return result, execution
|
||||||
|
|
@ -2842,6 +2882,31 @@ This is a 3-command job. Don't over-investigate.`
|
||||||
// Add current alert status - this gives AI awareness of active issues
|
// Add current alert status - this gives AI awareness of active issues
|
||||||
prompt += s.buildAlertContext()
|
prompt += s.buildAlertContext()
|
||||||
|
|
||||||
|
// Add incident memory for alert or resource context
|
||||||
|
alertID := ""
|
||||||
|
resourceID := req.TargetID
|
||||||
|
if req.Context != nil {
|
||||||
|
if val, ok := req.Context["alertId"].(string); ok && val != "" {
|
||||||
|
alertID = val
|
||||||
|
} else if val, ok := req.Context["alert_id"].(string); ok && val != "" {
|
||||||
|
alertID = val
|
||||||
|
}
|
||||||
|
|
||||||
|
if resourceID == "" {
|
||||||
|
if val, ok := req.Context["resourceId"].(string); ok && val != "" {
|
||||||
|
resourceID = val
|
||||||
|
} else if val, ok := req.Context["resource_id"].(string); ok && val != "" {
|
||||||
|
resourceID = val
|
||||||
|
} else if val, ok := req.Context["guest_id"].(string); ok && val != "" {
|
||||||
|
resourceID = val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if incidentContext := s.buildIncidentContext(resourceID, alertID); incidentContext != "" {
|
||||||
|
prompt += incidentContext
|
||||||
|
}
|
||||||
|
|
||||||
// Add all saved knowledge when no specific target is selected
|
// Add all saved knowledge when no specific target is selected
|
||||||
// This gives the AI context about everything learned from previous sessions
|
// This gives the AI context about everything learned from previous sessions
|
||||||
if req.TargetType == "" && s.knowledgeStore != nil {
|
if req.TargetType == "" && s.knowledgeStore != nil {
|
||||||
|
|
@ -2877,6 +2942,7 @@ This is a 3-command job. Don't over-investigate.`
|
||||||
|
|
||||||
// Add past remediation history for this resource
|
// Add past remediation history for this resource
|
||||||
prompt += s.buildRemediationContext(req.TargetID, req.Prompt)
|
prompt += s.buildRemediationContext(req.TargetID, req.Prompt)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add any provided context in a structured way
|
// Add any provided context in a structured way
|
||||||
|
|
@ -3258,6 +3324,66 @@ func (s *Service) buildRemediationContext(resourceID, currentProblem string) str
|
||||||
return context
|
return context
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildIncidentContext adds incident timeline context for alerts/resources.
|
||||||
|
func (s *Service) buildIncidentContext(resourceID, alertID string) string {
|
||||||
|
s.mu.RLock()
|
||||||
|
store := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
if store == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if alertID != "" {
|
||||||
|
return store.FormatForAlert(alertID, 8)
|
||||||
|
}
|
||||||
|
if resourceID != "" {
|
||||||
|
return store.FormatForResource(resourceID, 4)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordIncidentAnalysis stores an AI analysis event for an alert.
|
||||||
|
func (s *Service) RecordIncidentAnalysis(alertID, summary string, details map[string]interface{}) {
|
||||||
|
if alertID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
store := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
if store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
store.RecordAnalysis(alertID, summary, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordIncidentRunbook stores a runbook execution event for an alert.
|
||||||
|
func (s *Service) RecordIncidentRunbook(alertID, runbookID, title string, outcome memory.Outcome, automatic bool, message string) {
|
||||||
|
if alertID == "" || runbookID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
store := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
if store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
store.RecordRunbook(alertID, runbookID, title, string(outcome), automatic, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractAlertID(ctx map[string]interface{}) string {
|
||||||
|
if ctx == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if alertID, ok := ctx["alertId"].(string); ok && alertID != "" {
|
||||||
|
return alertID
|
||||||
|
}
|
||||||
|
if alertID, ok := ctx["alert_id"].(string); ok && alertID != "" {
|
||||||
|
return alertID
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// truncateString truncates a string to maxLen characters
|
// truncateString truncates a string to maxLen characters
|
||||||
func truncateString(s string, maxLen int) string {
|
func truncateString(s string, maxLen int) string {
|
||||||
if len(s) <= maxLen {
|
if len(s) <= maxLen {
|
||||||
|
|
@ -3302,40 +3428,51 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check CPU baseline
|
// Helper to format baseline comparison with status
|
||||||
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
|
// ALWAYS shows baseline info when available - this is Pulse Pro's value-add
|
||||||
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists && bl.SampleCount >= 10 {
|
formatBaselineComparison := func(metric string, currentVal float64, bl *baseline.MetricBaseline) string {
|
||||||
ratio := cpuVal / bl.Mean
|
if bl.SampleCount < 10 {
|
||||||
if ratio > 1.5 {
|
return fmt.Sprintf("- %s: %.1f%% (baseline learning: %d/10 samples)", metric, currentVal, bl.SampleCount)
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx higher** than baseline %.0f%% (σ=%.1f)", cpuVal, ratio, bl.Mean, bl.StdDev))
|
}
|
||||||
|
|
||||||
|
ratio := currentVal / bl.Mean
|
||||||
|
status := "✓ normal"
|
||||||
|
if ratio > 2.0 {
|
||||||
|
status = "🔴 **ANOMALY**"
|
||||||
|
} else if ratio > 1.5 {
|
||||||
|
status = "🟡 elevated"
|
||||||
|
} else if ratio < 0.3 {
|
||||||
|
status = "🔵 unusually low"
|
||||||
} else if ratio < 0.5 {
|
} else if ratio < 0.5 {
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx lower** than baseline %.0f%%", cpuVal, 1/ratio, bl.Mean))
|
status = "low"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf("- %s: %.1f%% vs baseline %.1f%% (σ=%.1f) → %s", metric, currentVal, bl.Mean, bl.StdDev, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check CPU baseline - always show if we have data
|
||||||
|
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
|
||||||
|
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists {
|
||||||
|
baselineInfo = append(baselineInfo, formatBaselineComparison("CPU", cpuVal, bl))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check memory baseline
|
// Check memory baseline - always show if we have data
|
||||||
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
|
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
|
||||||
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists && bl.SampleCount >= 10 {
|
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists {
|
||||||
ratio := memVal / bl.Mean
|
baselineInfo = append(baselineInfo, formatBaselineComparison("Memory", memVal, bl))
|
||||||
if ratio > 1.3 {
|
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("Memory %.0f%% is **%.1fx higher** than baseline %.0f%%", memVal, ratio, bl.Mean))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check disk baseline
|
// Check disk baseline - always show if we have data
|
||||||
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
|
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
|
||||||
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists && bl.SampleCount >= 10 {
|
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists {
|
||||||
ratio := diskVal / bl.Mean
|
baselineInfo = append(baselineInfo, formatBaselineComparison("Disk", diskVal, bl))
|
||||||
if ratio > 1.2 { // More sensitive for disk
|
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("Disk %.0f%% is **%.1fx higher** than baseline %.0f%%", diskVal, ratio, bl.Mean))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(baselineInfo) > 0 {
|
if len(baselineInfo) > 0 {
|
||||||
sections = append(sections, "### Baseline Comparisons\n"+strings.Join(baselineInfo, "\n"))
|
sections = append(sections, "### Learned Baselines (7-day patterns)\n"+strings.Join(baselineInfo, "\n"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -490,6 +490,8 @@ type Manager struct {
|
||||||
historyManager *HistoryManager
|
historyManager *HistoryManager
|
||||||
onAlert func(alert *Alert)
|
onAlert func(alert *Alert)
|
||||||
onResolved func(alertID string)
|
onResolved func(alertID string)
|
||||||
|
onAcknowledged func(alert *Alert, user string)
|
||||||
|
onUnacknowledged func(alert *Alert, user string)
|
||||||
onEscalate func(alert *Alert, level int)
|
onEscalate func(alert *Alert, level int)
|
||||||
escalationStop chan struct{}
|
escalationStop chan struct{}
|
||||||
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
|
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
|
||||||
|
|
@ -736,6 +738,20 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
|
||||||
m.onResolved = cb
|
m.onResolved = cb
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAcknowledgedCallback sets the callback for acknowledged alerts.
|
||||||
|
func (m *Manager) SetAcknowledgedCallback(cb func(alert *Alert, user string)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.onAcknowledged = cb
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetUnacknowledgedCallback sets the callback for unacknowledged alerts.
|
||||||
|
func (m *Manager) SetUnacknowledgedCallback(cb func(alert *Alert, user string)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.onUnacknowledged = cb
|
||||||
|
}
|
||||||
|
|
||||||
// SetEscalateCallback sets the callback for escalated alerts
|
// SetEscalateCallback sets the callback for escalated alerts
|
||||||
func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) {
|
func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
@ -768,6 +784,46 @@ func (m *Manager) safeCallResolvedCallback(alertID string, async bool) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// safeCallAcknowledgedCallback invokes onAcknowledged with panic recovery and alert cloning.
|
||||||
|
func (m *Manager) safeCallAcknowledgedCallback(alert *Alert, user string) {
|
||||||
|
if m.onAcknowledged == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
go func(a *Alert, u string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Str("alertID", a.ID).
|
||||||
|
Msg("Panic in onAcknowledged callback")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
m.onAcknowledged(a, u)
|
||||||
|
}(alertCopy, user)
|
||||||
|
}
|
||||||
|
|
||||||
|
// safeCallUnacknowledgedCallback invokes onUnacknowledged with panic recovery and alert cloning.
|
||||||
|
func (m *Manager) safeCallUnacknowledgedCallback(alert *Alert, user string) {
|
||||||
|
if m.onUnacknowledged == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
go func(a *Alert, u string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Str("alertID", a.ID).
|
||||||
|
Msg("Panic in onUnacknowledged callback")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
m.onUnacknowledged(a, u)
|
||||||
|
}(alertCopy, user)
|
||||||
|
}
|
||||||
|
|
||||||
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning
|
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning
|
||||||
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
|
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
|
||||||
if m.onEscalate == nil {
|
if m.onEscalate == nil {
|
||||||
|
|
@ -2457,7 +2513,6 @@ func (m *Manager) hasHostAgentForNode(nodeName string) bool {
|
||||||
return exists
|
return exists
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func hostResourceID(hostID string) string {
|
func hostResourceID(hostID string) string {
|
||||||
trimmed := strings.TrimSpace(hostID)
|
trimmed := strings.TrimSpace(hostID)
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
|
|
@ -5767,10 +5822,10 @@ func abs(x float64) float64 {
|
||||||
// AcknowledgeAlert acknowledges an alert
|
// AcknowledgeAlert acknowledges an alert
|
||||||
func (m *Manager) AcknowledgeAlert(alertID, user string) error {
|
func (m *Manager) AcknowledgeAlert(alertID, user string) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
alert, exists := m.activeAlerts[alertID]
|
alert, exists := m.activeAlerts[alertID]
|
||||||
if !exists {
|
if !exists {
|
||||||
|
m.mu.Unlock()
|
||||||
return fmt.Errorf("alert not found: %s", alertID)
|
return fmt.Errorf("alert not found: %s", alertID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -5787,22 +5842,26 @@ func (m *Manager) AcknowledgeAlert(alertID, user string) error {
|
||||||
time: now,
|
time: now,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("alertID", alertID).
|
Str("alertID", alertID).
|
||||||
Str("user", user).
|
Str("user", user).
|
||||||
Time("ackTime", now).
|
Time("ackTime", now).
|
||||||
Msg("Alert acknowledgment recorded")
|
Msg("Alert acknowledgment recorded")
|
||||||
|
|
||||||
|
m.safeCallAcknowledgedCallback(alertCopy, user)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnacknowledgeAlert removes the acknowledged status from an alert
|
// UnacknowledgeAlert removes the acknowledged status from an alert
|
||||||
func (m *Manager) UnacknowledgeAlert(alertID string) error {
|
func (m *Manager) UnacknowledgeAlert(alertID string) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
alert, exists := m.activeAlerts[alertID]
|
alert, exists := m.activeAlerts[alertID]
|
||||||
if !exists {
|
if !exists {
|
||||||
|
m.mu.Unlock()
|
||||||
return fmt.Errorf("alert not found: %s", alertID)
|
return fmt.Errorf("alert not found: %s", alertID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -5814,10 +5873,14 @@ func (m *Manager) UnacknowledgeAlert(alertID string) error {
|
||||||
m.activeAlerts[alertID] = alert
|
m.activeAlerts[alertID] = alert
|
||||||
delete(m.ackState, alertID)
|
delete(m.ackState, alertID)
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alertID", alertID).
|
Str("alertID", alertID).
|
||||||
Msg("Alert unacknowledged")
|
Msg("Alert unacknowledged")
|
||||||
|
|
||||||
|
m.safeCallUnacknowledgedCallback(alertCopy, "")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -7593,9 +7656,8 @@ func (m *Manager) ClearAlert(alertID string) bool {
|
||||||
// Cleanup removes old acknowledged alerts and cleans up tracking maps
|
// Cleanup removes old acknowledged alerts and cleans up tracking maps
|
||||||
func (m *Manager) Cleanup(maxAge time.Duration) {
|
func (m *Manager) Cleanup(maxAge time.Duration) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
var autoAcked []*Alert
|
||||||
|
|
||||||
// Auto-acknowledge old alerts if configured
|
// Auto-acknowledge old alerts if configured
|
||||||
if m.config.AutoAcknowledgeAfterHours > 0 {
|
if m.config.AutoAcknowledgeAfterHours > 0 {
|
||||||
|
|
@ -7610,6 +7672,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
|
||||||
ackTime := now
|
ackTime := now
|
||||||
alert.AckTime = &ackTime
|
alert.AckTime = &ackTime
|
||||||
alert.AckUser = "system-auto"
|
alert.AckUser = "system-auto"
|
||||||
|
autoAcked = append(autoAcked, alert.Clone())
|
||||||
|
|
||||||
if recordAlertAcknowledged != nil {
|
if recordAlertAcknowledged != nil {
|
||||||
recordAlertAcknowledged()
|
recordAlertAcknowledged()
|
||||||
|
|
@ -7776,6 +7839,12 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
|
||||||
Msg("Cleaned up stale PMG quarantine history")
|
Msg("Cleaned up stale PMG quarantine history")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
for _, alert := range autoAcked {
|
||||||
|
m.safeCallAcknowledgedCallback(alert, "system-auto")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold
|
// convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/license"
|
"github.com/rcourtman/pulse-go-rewrite/internal/license"
|
||||||
|
|
@ -118,6 +119,11 @@ func (h *AISettingsHandler) SetRemediationLog(remLog *ai.RemediationLog) {
|
||||||
h.aiService.SetRemediationLog(remLog)
|
h.aiService.SetRemediationLog(remLog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetIncidentStore sets the incident store for alert timelines.
|
||||||
|
func (h *AISettingsHandler) SetIncidentStore(store *memory.IncidentStore) {
|
||||||
|
h.aiService.SetIncidentStore(store)
|
||||||
|
}
|
||||||
|
|
||||||
// SetPatternDetector sets the pattern detector for failure prediction
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
|
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
|
||||||
h.aiService.SetPatternDetector(detector)
|
h.aiService.SetPatternDetector(detector)
|
||||||
|
|
@ -1767,11 +1773,29 @@ func (h *AISettingsHandler) HandleInvestigateAlert(w http.ResponseWriter, r *htt
|
||||||
data, _ := json.Marshal(finalEvent)
|
data, _ := json.Marshal(finalEvent)
|
||||||
safeWrite([]byte("data: " + string(data) + "\n\n"))
|
safeWrite([]byte("data: " + string(data) + "\n\n"))
|
||||||
|
|
||||||
|
if req.AlertID != "" {
|
||||||
|
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI alert investigation completed", map[string]interface{}{
|
||||||
|
"model": resp.Model,
|
||||||
|
"tool_calls": len(resp.ToolCalls),
|
||||||
|
"input_tokens": resp.InputTokens,
|
||||||
|
"output_tokens": resp.OutputTokens,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alert_id", req.AlertID).
|
Str("alert_id", req.AlertID).
|
||||||
Str("model", resp.Model).
|
Str("model", resp.Model).
|
||||||
Int("tool_calls", len(resp.ToolCalls)).
|
Int("tool_calls", len(resp.ToolCalls)).
|
||||||
Msg("AI alert investigation completed")
|
Msg("AI alert investigation completed")
|
||||||
|
|
||||||
|
if req.AlertID != "" {
|
||||||
|
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI investigation completed", map[string]interface{}{
|
||||||
|
"model": resp.Model,
|
||||||
|
"input_tokens": resp.InputTokens,
|
||||||
|
"output_tokens": resp.OutputTokens,
|
||||||
|
"tool_calls": len(resp.ToolCalls),
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetAlertProvider sets the alert provider for AI context
|
// SetAlertProvider sets the alert provider for AI context
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
||||||
|
|
@ -361,6 +362,130 @@ func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetAlertIncidentTimeline returns the incident timeline for an alert or resource.
|
||||||
|
func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
store := h.monitor.GetIncidentStore()
|
||||||
|
if store == nil {
|
||||||
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
query := r.URL.Query()
|
||||||
|
alertID := strings.TrimSpace(query.Get("alert_id"))
|
||||||
|
resourceID := strings.TrimSpace(query.Get("resource_id"))
|
||||||
|
startedAtRaw := strings.TrimSpace(query.Get("started_at"))
|
||||||
|
if startedAtRaw == "" {
|
||||||
|
startedAtRaw = strings.TrimSpace(query.Get("start_time"))
|
||||||
|
}
|
||||||
|
limit := 0
|
||||||
|
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
|
||||||
|
if parsed, err := strconv.Atoi(rawLimit); err == nil && parsed > 0 {
|
||||||
|
limit = parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if alertID != "" {
|
||||||
|
if !validateAlertID(alertID) {
|
||||||
|
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var startedAt time.Time
|
||||||
|
if startedAtRaw != "" {
|
||||||
|
parsed, err := time.Parse(time.RFC3339, startedAtRaw)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Invalid started_at time", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
startedAt = parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
var incident *memory.Incident
|
||||||
|
if !startedAt.IsZero() {
|
||||||
|
incident = store.GetTimelineByAlertAt(alertID, startedAt)
|
||||||
|
} else {
|
||||||
|
incident = store.GetTimelineByAlertID(alertID)
|
||||||
|
}
|
||||||
|
if err := utils.WriteJSONResponse(w, incident); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to write incident timeline response")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if resourceID != "" {
|
||||||
|
if len(resourceID) > 500 {
|
||||||
|
http.Error(w, "Invalid resource ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
incidents := store.ListIncidentsByResource(resourceID, limit)
|
||||||
|
if err := utils.WriteJSONResponse(w, incidents); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to write incident list response")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
http.Error(w, "Missing alert_id or resource_id", http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveAlertIncidentNote stores a user note in the incident timeline.
|
||||||
|
func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
store := h.monitor.GetIncidentStore()
|
||||||
|
if store == nil {
|
||||||
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, 8*1024)
|
||||||
|
var req struct {
|
||||||
|
AlertID string `json:"alert_id"`
|
||||||
|
IncidentID string `json:"incident_id"`
|
||||||
|
Note string `json:"note"`
|
||||||
|
User string `json:"user,omitempty"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req.AlertID = strings.TrimSpace(req.AlertID)
|
||||||
|
req.IncidentID = strings.TrimSpace(req.IncidentID)
|
||||||
|
req.Note = strings.TrimSpace(req.Note)
|
||||||
|
req.User = strings.TrimSpace(req.User)
|
||||||
|
|
||||||
|
if req.AlertID == "" && req.IncidentID == "" {
|
||||||
|
http.Error(w, "alert_id or incident_id is required", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.AlertID != "" && !validateAlertID(req.AlertID) {
|
||||||
|
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Note == "" {
|
||||||
|
http.Error(w, "note is required", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok := store.RecordNote(req.AlertID, req.IncidentID, req.Note, req.User); !ok {
|
||||||
|
http.Error(w, "Failed to save note", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
||||||
|
"success": true,
|
||||||
|
}); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to write incident note response")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ClearAlertHistory clears all alert history
|
// ClearAlertHistory clears all alert history
|
||||||
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
|
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
|
||||||
if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil {
|
if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil {
|
||||||
|
|
@ -737,6 +862,16 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
h.GetAlertHistory(w, r)
|
h.GetAlertHistory(w, r)
|
||||||
|
case path == "incidents" && r.Method == http.MethodGet:
|
||||||
|
if !ensureScope(w, r, config.ScopeMonitoringRead) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.GetAlertIncidentTimeline(w, r)
|
||||||
|
case path == "incidents/note" && r.Method == http.MethodPost:
|
||||||
|
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.SaveAlertIncidentNote(w, r)
|
||||||
case path == "history" && r.Method == http.MethodDelete:
|
case path == "history" && r.Method == http.MethodDelete:
|
||||||
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
||||||
return
|
return
|
||||||
|
|
|
||||||
|
|
@ -1109,6 +1109,9 @@ func (r *Router) setupRoutes() {
|
||||||
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
|
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
|
||||||
r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager))
|
r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager))
|
||||||
}
|
}
|
||||||
|
if incidentStore := r.monitor.GetIncidentStore(); incidentStore != nil {
|
||||||
|
r.aiSettingsHandler.SetIncidentStore(incidentStore)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view)
|
// Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view)
|
||||||
if r.resourceHandlers != nil {
|
if r.resourceHandlers != nil {
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
|
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
|
||||||
|
|
@ -604,6 +605,7 @@ type Monitor struct {
|
||||||
metricsHistory *MetricsHistory
|
metricsHistory *MetricsHistory
|
||||||
metricsStore *metrics.Store // Persistent SQLite metrics storage
|
metricsStore *metrics.Store // Persistent SQLite metrics storage
|
||||||
alertManager *alerts.Manager
|
alertManager *alerts.Manager
|
||||||
|
incidentStore *memory.IncidentStore
|
||||||
notificationMgr *notifications.NotificationManager
|
notificationMgr *notifications.NotificationManager
|
||||||
configPersist *config.ConfigPersistence
|
configPersist *config.ConfigPersistence
|
||||||
discoveryService *discovery.Service // Background discovery service
|
discoveryService *discovery.Service // Background discovery service
|
||||||
|
|
@ -3050,6 +3052,10 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||||
Msg("Persistent metrics store initialized with configurable retention")
|
Msg("Persistent metrics store initialized with configurable retention")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{
|
||||||
|
DataDir: cfg.DataPath,
|
||||||
|
})
|
||||||
|
|
||||||
m := &Monitor{
|
m := &Monitor{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
state: models.NewState(),
|
state: models.NewState(),
|
||||||
|
|
@ -3076,6 +3082,7 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||||
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
|
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
|
||||||
metricsStore: metricsStore, // Persistent SQLite storage
|
metricsStore: metricsStore, // Persistent SQLite storage
|
||||||
alertManager: alerts.NewManager(),
|
alertManager: alerts.NewManager(),
|
||||||
|
incidentStore: incidentStore,
|
||||||
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
|
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
|
||||||
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
||||||
discoveryService: nil, // Will be initialized in Start()
|
discoveryService: nil, // Will be initialized in Start()
|
||||||
|
|
@ -3756,25 +3763,19 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||||
|
|
||||||
// Set up alert callbacks
|
// Set up alert callbacks
|
||||||
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||||
wsHub.BroadcastAlert(alert)
|
m.handleAlertFired(alert)
|
||||||
// Send notifications
|
|
||||||
log.Debug().
|
|
||||||
Str("alertID", alert.ID).
|
|
||||||
Str("level", string(alert.Level)).
|
|
||||||
Msg("Alert raised, sending to notification manager")
|
|
||||||
go m.notificationMgr.SendAlert(alert)
|
|
||||||
})
|
})
|
||||||
m.alertManager.SetResolvedCallback(func(alertID string) {
|
m.alertManager.SetResolvedCallback(func(alertID string) {
|
||||||
wsHub.BroadcastAlertResolved(alertID)
|
m.handleAlertResolved(alertID)
|
||||||
m.notificationMgr.CancelAlert(alertID)
|
|
||||||
if m.notificationMgr.GetNotifyOnResolve() {
|
|
||||||
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
|
|
||||||
go m.notificationMgr.SendResolvedAlert(resolved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Don't broadcast full state here - it causes a cascade with many guests.
|
// Don't broadcast full state here - it causes a cascade with many guests.
|
||||||
// The frontend will get the updated alerts through the regular broadcast ticker.
|
// The frontend will get the updated alerts through the regular broadcast ticker.
|
||||||
})
|
})
|
||||||
|
m.alertManager.SetAcknowledgedCallback(func(alert *alerts.Alert, user string) {
|
||||||
|
m.handleAlertAcknowledged(alert, user)
|
||||||
|
})
|
||||||
|
m.alertManager.SetUnacknowledgedCallback(func(alert *alerts.Alert, user string) {
|
||||||
|
m.handleAlertUnacknowledged(alert, user)
|
||||||
|
})
|
||||||
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alertID", alert.ID).
|
Str("alertID", alert.ID).
|
||||||
|
|
@ -7731,6 +7732,11 @@ func (m *Monitor) GetAlertManager() *alerts.Manager {
|
||||||
return m.alertManager
|
return m.alertManager
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetIncidentStore returns the incident timeline store.
|
||||||
|
func (m *Monitor) GetIncidentStore() *memory.IncidentStore {
|
||||||
|
return m.incidentStore
|
||||||
|
}
|
||||||
|
|
||||||
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
|
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
|
||||||
// This enables token-efficient, real-time AI insights on specific resources
|
// This enables token-efficient, real-time AI insights on specific resources
|
||||||
func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
|
func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
|
||||||
|
|
@ -7738,31 +7744,67 @@ func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the current callback
|
|
||||||
originalCallback := m.alertManager
|
|
||||||
|
|
||||||
// Wrap the existing callback to also call the AI callback
|
// Wrap the existing callback to also call the AI callback
|
||||||
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||||
// Broadcast to WebSocket (this happens via the callback set in Start())
|
m.handleAlertFired(alert)
|
||||||
|
|
||||||
|
// Trigger AI analysis
|
||||||
|
go callback(alert)
|
||||||
|
})
|
||||||
|
|
||||||
|
log.Info().Msg("Alert-triggered AI callback registered")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertFired(alert *alerts.Alert) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if m.wsHub != nil {
|
if m.wsHub != nil {
|
||||||
m.wsHub.BroadcastAlert(alert)
|
m.wsHub.BroadcastAlert(alert)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send notifications
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("alertID", alert.ID).
|
Str("alertID", alert.ID).
|
||||||
Str("level", string(alert.Level)).
|
Str("level", string(alert.Level)).
|
||||||
Msg("Alert raised, sending to notification manager")
|
Msg("Alert raised, sending to notification manager")
|
||||||
go m.notificationMgr.SendAlert(alert)
|
go m.notificationMgr.SendAlert(alert)
|
||||||
|
|
||||||
// Trigger AI analysis
|
if m.incidentStore != nil {
|
||||||
go callback(alert)
|
m.incidentStore.RecordAlertFired(alert)
|
||||||
})
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Avoid unused variable warning
|
func (m *Monitor) handleAlertResolved(alertID string) {
|
||||||
_ = originalCallback
|
if m.wsHub != nil {
|
||||||
|
m.wsHub.BroadcastAlertResolved(alertID)
|
||||||
|
}
|
||||||
|
m.notificationMgr.CancelAlert(alertID)
|
||||||
|
if m.notificationMgr.GetNotifyOnResolve() {
|
||||||
|
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
|
||||||
|
go m.notificationMgr.SendResolvedAlert(resolved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.Info().Msg("Alert-triggered AI callback registered")
|
if m.incidentStore != nil {
|
||||||
|
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil && resolved.Alert != nil {
|
||||||
|
m.incidentStore.RecordAlertResolved(resolved.Alert, resolved.ResolvedTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertAcknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if m.incidentStore == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.incidentStore.RecordAlertAcknowledged(alert, user)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertUnacknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if m.incidentStore == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.incidentStore.RecordAlertUnacknowledged(alert, user)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetResourceStore sets the resource store for polling optimization.
|
// SetResourceStore sets the resource store for polling optimization.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue