feat: enhance AI baseline context visibility and incident timeline improvements
Backend: - Enhanced buildEnrichedResourceContext to ALWAYS show learned baselines with status indicators (normal/elevated/anomaly) instead of only when anomalous - This makes Pulse Pro's 'moat' visible - users can see the AI understands their infrastructure's normal behavior patterns - Added baseline import to service.go Frontend (user changes): - Added incident event type filtering with toggle buttons - Added resource incident panel to view all incidents for a resource - Added timeline expand/collapse functionality in alert history - Added incident note saving with proper incidentId tracking - Added startedAt parameter for proper incident timeline loading
This commit is contained in:
parent
f0b983667c
commit
82e5b28840
14 changed files with 2397 additions and 226 deletions
|
|
@ -1,4 +1,4 @@
|
||||||
import type { Alert } from '@/types/api';
|
import type { Alert, Incident } from '@/types/api';
|
||||||
import type { AlertConfig } from '@/types/alerts';
|
import type { AlertConfig } from '@/types/alerts';
|
||||||
import { apiFetchJSON } from '@/utils/apiClient';
|
import { apiFetchJSON } from '@/utils/apiClient';
|
||||||
|
|
||||||
|
|
@ -29,6 +29,37 @@ export class AlertsAPI {
|
||||||
return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`);
|
return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async getIncidentTimeline(alertId: string, startedAt?: string): Promise<Incident | null> {
|
||||||
|
const query = new URLSearchParams({ alert_id: alertId });
|
||||||
|
if (startedAt) {
|
||||||
|
query.set('started_at', startedAt);
|
||||||
|
}
|
||||||
|
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident | null>;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async getIncidentsForResource(resourceId: string, limit?: number): Promise<Incident[]> {
|
||||||
|
const query = new URLSearchParams({ resource_id: resourceId });
|
||||||
|
if (limit) query.set('limit', String(limit));
|
||||||
|
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
static async addIncidentNote(params: {
|
||||||
|
alertId?: string;
|
||||||
|
incidentId?: string;
|
||||||
|
note: string;
|
||||||
|
user?: string;
|
||||||
|
}): Promise<{ success: boolean }> {
|
||||||
|
return apiFetchJSON(`${this.baseUrl}/incidents/note`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify({
|
||||||
|
alert_id: params.alertId,
|
||||||
|
incident_id: params.incidentId,
|
||||||
|
note: params.note,
|
||||||
|
user: params.user,
|
||||||
|
}),
|
||||||
|
}) as Promise<{ success: boolean }>;
|
||||||
|
}
|
||||||
|
|
||||||
static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> {
|
static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> {
|
||||||
return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, {
|
return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1019,6 +1019,34 @@ export interface ResolvedAlert extends Alert {
|
||||||
resolvedTime: string;
|
resolvedTime: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface IncidentEvent {
|
||||||
|
id: string;
|
||||||
|
type: string;
|
||||||
|
timestamp: string;
|
||||||
|
summary: string;
|
||||||
|
details?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Incident {
|
||||||
|
id: string;
|
||||||
|
alertId: string;
|
||||||
|
alertType: string;
|
||||||
|
level: string;
|
||||||
|
resourceId: string;
|
||||||
|
resourceName: string;
|
||||||
|
resourceType?: string;
|
||||||
|
node?: string;
|
||||||
|
instance?: string;
|
||||||
|
message?: string;
|
||||||
|
status: string;
|
||||||
|
openedAt: string;
|
||||||
|
closedAt?: string;
|
||||||
|
acknowledged: boolean;
|
||||||
|
ackUser?: string;
|
||||||
|
ackTime?: string;
|
||||||
|
events?: IncidentEvent[];
|
||||||
|
}
|
||||||
|
|
||||||
// WebSocket message types
|
// WebSocket message types
|
||||||
export type WSMessage =
|
export type WSMessage =
|
||||||
| { type: 'initialState'; data: State }
|
| { type: 'initialState'; data: State }
|
||||||
|
|
|
||||||
|
|
@ -161,6 +161,17 @@ func (a *AlertTriggeredAnalyzer) analyzeResource(alert *alerts.Alert, resourceKe
|
||||||
Dur("duration", duration).
|
Dur("duration", duration).
|
||||||
Msg("Alert-triggered AI analysis completed with no additional findings")
|
Msg("Alert-triggered AI analysis completed with no additional findings")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if a.patrolService != nil && a.patrolService.aiService != nil {
|
||||||
|
summary := "Alert-triggered AI analysis completed"
|
||||||
|
if len(findings) > 0 {
|
||||||
|
summary = fmt.Sprintf("Alert-triggered AI analysis found %d findings", len(findings))
|
||||||
|
}
|
||||||
|
a.patrolService.aiService.RecordIncidentAnalysis(alert.ID, summary, map[string]interface{}{
|
||||||
|
"findings": len(findings),
|
||||||
|
"duration": duration.String(),
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// analyzeResourceByAlert determines the resource type from the alert and analyzes it
|
// analyzeResourceByAlert determines the resource type from the alert and analyzes it
|
||||||
|
|
|
||||||
848
internal/ai/memory/incidents.go
Normal file
848
internal/ai/memory/incidents.go
Normal file
|
|
@ -0,0 +1,848 @@
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncidentStatus represents the current state of an incident.
|
||||||
|
type IncidentStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
IncidentStatusOpen IncidentStatus = "open"
|
||||||
|
IncidentStatusResolved IncidentStatus = "resolved"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncidentEventType describes a timeline event type.
|
||||||
|
type IncidentEventType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
IncidentEventAlertFired IncidentEventType = "alert_fired"
|
||||||
|
IncidentEventAlertAcknowledged IncidentEventType = "alert_acknowledged"
|
||||||
|
IncidentEventAlertUnacknowledged IncidentEventType = "alert_unacknowledged"
|
||||||
|
IncidentEventAlertResolved IncidentEventType = "alert_resolved"
|
||||||
|
IncidentEventAnalysis IncidentEventType = "ai_analysis"
|
||||||
|
IncidentEventCommand IncidentEventType = "command"
|
||||||
|
IncidentEventRunbook IncidentEventType = "runbook"
|
||||||
|
IncidentEventNote IncidentEventType = "note"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncidentEvent represents a single timeline entry for an incident.
|
||||||
|
type IncidentEvent struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Type IncidentEventType `json:"type"`
|
||||||
|
Timestamp time.Time `json:"timestamp"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
Details map[string]interface{} `json:"details,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Incident captures an alert occurrence and its timeline.
|
||||||
|
type Incident struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
AlertID string `json:"alertId"`
|
||||||
|
AlertType string `json:"alertType"`
|
||||||
|
Level string `json:"level"`
|
||||||
|
ResourceID string `json:"resourceId"`
|
||||||
|
ResourceName string `json:"resourceName"`
|
||||||
|
ResourceType string `json:"resourceType,omitempty"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
|
Instance string `json:"instance,omitempty"`
|
||||||
|
Message string `json:"message,omitempty"`
|
||||||
|
Status IncidentStatus `json:"status"`
|
||||||
|
OpenedAt time.Time `json:"openedAt"`
|
||||||
|
ClosedAt *time.Time `json:"closedAt,omitempty"`
|
||||||
|
Acknowledged bool `json:"acknowledged"`
|
||||||
|
AckUser string `json:"ackUser,omitempty"`
|
||||||
|
AckTime *time.Time `json:"ackTime,omitempty"`
|
||||||
|
Events []IncidentEvent `json:"events,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncidentStoreConfig configures incident retention and persistence.
|
||||||
|
type IncidentStoreConfig struct {
|
||||||
|
DataDir string
|
||||||
|
MaxIncidents int
|
||||||
|
MaxEventsPerIncident int
|
||||||
|
MaxAgeDays int
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncidentStore maintains incident timelines and persistence.
|
||||||
|
type IncidentStore struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
saveMu sync.Mutex
|
||||||
|
incidents []*Incident
|
||||||
|
maxIncidents int
|
||||||
|
maxEvents int
|
||||||
|
maxAge time.Duration
|
||||||
|
dataDir string
|
||||||
|
filePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultIncidentMaxIncidents = 500
|
||||||
|
defaultIncidentMaxEvents = 120
|
||||||
|
defaultIncidentMaxAgeDays = 90
|
||||||
|
incidentFileName = "ai_incidents.json"
|
||||||
|
maxIncidentFileSize = 20 * 1024 * 1024 // 20MB
|
||||||
|
incidentStartMatchTolerance = 10 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewIncidentStore creates a new incident store with persistence.
|
||||||
|
func NewIncidentStore(cfg IncidentStoreConfig) *IncidentStore {
|
||||||
|
maxIncidents := cfg.MaxIncidents
|
||||||
|
if maxIncidents <= 0 {
|
||||||
|
maxIncidents = defaultIncidentMaxIncidents
|
||||||
|
}
|
||||||
|
maxEvents := cfg.MaxEventsPerIncident
|
||||||
|
if maxEvents <= 0 {
|
||||||
|
maxEvents = defaultIncidentMaxEvents
|
||||||
|
}
|
||||||
|
maxAgeDays := cfg.MaxAgeDays
|
||||||
|
if maxAgeDays <= 0 {
|
||||||
|
maxAgeDays = defaultIncidentMaxAgeDays
|
||||||
|
}
|
||||||
|
|
||||||
|
store := &IncidentStore{
|
||||||
|
incidents: make([]*Incident, 0),
|
||||||
|
maxIncidents: maxIncidents,
|
||||||
|
maxEvents: maxEvents,
|
||||||
|
maxAge: time.Duration(maxAgeDays) * 24 * time.Hour,
|
||||||
|
dataDir: cfg.DataDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
if store.dataDir != "" {
|
||||||
|
store.filePath = filepath.Join(store.dataDir, incidentFileName)
|
||||||
|
if err := store.loadFromDisk(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to load incident history from disk")
|
||||||
|
} else if len(store.incidents) > 0 {
|
||||||
|
log.Info().Int("count", len(store.incidents)).Msg("Loaded incident history from disk")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return store
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertFired opens or updates an incident for a fired alert.
|
||||||
|
func (s *IncidentStore) RecordAlertFired(alert *alerts.Alert) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = newIncidentFromAlert(alert)
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertFired, formatAlertSummary(alert), map[string]interface{}{
|
||||||
|
"type": alert.Type,
|
||||||
|
"level": string(alert.Level),
|
||||||
|
"value": alert.Value,
|
||||||
|
"threshold": alert.Threshold,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
updateIncidentFromAlert(incident, alert)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertAcknowledged records an acknowledgement event for an alert.
|
||||||
|
func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.ensureIncidentForAlertLocked(alert)
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
incident.Acknowledged = true
|
||||||
|
if alert.AckTime != nil {
|
||||||
|
incident.AckTime = alert.AckTime
|
||||||
|
} else {
|
||||||
|
now := time.Now()
|
||||||
|
incident.AckTime = &now
|
||||||
|
}
|
||||||
|
incident.AckUser = user
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{
|
||||||
|
"user": user,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertUnacknowledged records an unacknowledge event for an alert.
|
||||||
|
func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.ensureIncidentForAlertLocked(alert)
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
incident.Acknowledged = false
|
||||||
|
incident.AckTime = nil
|
||||||
|
incident.AckUser = ""
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{
|
||||||
|
"user": user,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAlertResolved records a resolved event and closes the incident.
|
||||||
|
func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time.Time) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = newIncidentFromAlert(alert)
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
incident.Status = IncidentStatusResolved
|
||||||
|
if resolvedAt.IsZero() {
|
||||||
|
now := time.Now()
|
||||||
|
resolvedAt = now
|
||||||
|
}
|
||||||
|
incident.ClosedAt = &resolvedAt
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{
|
||||||
|
"resolved_at": resolvedAt.Format(time.RFC3339),
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAnalysis adds an AI analysis event to the incident for an alert.
|
||||||
|
func (s *IncidentStore) RecordAnalysis(alertID, summary string, details map[string]interface{}) {
|
||||||
|
if alertID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alertID,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
if summary == "" {
|
||||||
|
summary = "AI analysis completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventAnalysis, summary, details)
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordCommand adds a command execution event to the incident for an alert.
|
||||||
|
func (s *IncidentStore) RecordCommand(alertID, command string, success bool, output string, details map[string]interface{}) {
|
||||||
|
if alertID == "" || command == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alertID,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
if details == nil {
|
||||||
|
details = make(map[string]interface{})
|
||||||
|
}
|
||||||
|
details["command"] = command
|
||||||
|
details["success"] = success
|
||||||
|
if output != "" {
|
||||||
|
details["output_excerpt"] = truncateOutput(output, 500)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := "failed"
|
||||||
|
if success {
|
||||||
|
status = "succeeded"
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("Command %s: %s", status, command)
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventCommand, summary, details)
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordRunbook adds a runbook execution event to the incident for an alert.
|
||||||
|
func (s *IncidentStore) RecordRunbook(alertID, runbookID, title string, outcome string, automatic bool, message string) {
|
||||||
|
if alertID == "" || runbookID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alertID,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := fmt.Sprintf("Runbook %s (%s)", title, outcome)
|
||||||
|
details := map[string]interface{}{
|
||||||
|
"runbook_id": runbookID,
|
||||||
|
"outcome": outcome,
|
||||||
|
"automatic": automatic,
|
||||||
|
}
|
||||||
|
if message != "" {
|
||||||
|
details["message"] = message
|
||||||
|
}
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventRunbook, summary, details)
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordNote appends a user note to an incident identified by alert ID or incident ID.
|
||||||
|
func (s *IncidentStore) RecordNote(alertID, incidentID, note, user string) bool {
|
||||||
|
note = strings.TrimSpace(note)
|
||||||
|
if note == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
var incident *Incident
|
||||||
|
if incidentID != "" {
|
||||||
|
incident = s.findIncidentByIDLocked(incidentID)
|
||||||
|
} else if alertID != "" {
|
||||||
|
incident = s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
}
|
||||||
|
if incident == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := "Note added"
|
||||||
|
if user != "" {
|
||||||
|
summary = fmt.Sprintf("Note added by %s", user)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.addEventLocked(incident, IncidentEventNote, summary, map[string]interface{}{
|
||||||
|
"note": note,
|
||||||
|
"user": user,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.trimLocked()
|
||||||
|
s.saveAsync()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTimelineByAlertID returns the most recent incident for the alert.
|
||||||
|
func (s *IncidentStore) GetTimelineByAlertID(alertID string) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return cloneIncident(incident)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTimelineByAlertAt returns the incident closest to the provided start time for an alert.
|
||||||
|
func (s *IncidentStore) GetTimelineByAlertAt(alertID string, startedAt time.Time) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if startedAt.IsZero() {
|
||||||
|
return s.GetTimelineByAlertID(alertID)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
var best *Incident
|
||||||
|
var bestDelta time.Duration
|
||||||
|
for _, incident := range s.incidents {
|
||||||
|
if incident == nil || incident.AlertID != alertID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delta := incident.OpenedAt.Sub(startedAt)
|
||||||
|
if delta < 0 {
|
||||||
|
delta = -delta
|
||||||
|
}
|
||||||
|
if best == nil || delta < bestDelta {
|
||||||
|
best = incident
|
||||||
|
bestDelta = delta
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if best == nil || bestDelta > incidentStartMatchTolerance {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return cloneIncident(best)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListIncidentsByResource returns recent incidents for a resource.
|
||||||
|
func (s *IncidentStore) ListIncidentsByResource(resourceID string, limit int) []*Incident {
|
||||||
|
if resourceID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
var matches []*Incident
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.ResourceID == resourceID {
|
||||||
|
matches = append(matches, cloneIncident(incident))
|
||||||
|
if limit > 0 && len(matches) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return matches
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForAlert returns a condensed incident timeline for prompt injection.
|
||||||
|
func (s *IncidentStore) FormatForAlert(alertID string, maxEvents int) string {
|
||||||
|
incident := s.GetTimelineByAlertID(alertID)
|
||||||
|
if incident == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("\n\n## Incident Memory\n")
|
||||||
|
b.WriteString(fmt.Sprintf("Alert incident for %s (%s, %s)\n",
|
||||||
|
incident.ResourceName, incident.AlertType, incident.Level))
|
||||||
|
b.WriteString(fmt.Sprintf("Status: %s\n", incident.Status))
|
||||||
|
|
||||||
|
events := incident.Events
|
||||||
|
if maxEvents > 0 && len(events) > maxEvents {
|
||||||
|
events = events[len(events)-maxEvents:]
|
||||||
|
}
|
||||||
|
for _, evt := range events {
|
||||||
|
b.WriteString("- ")
|
||||||
|
b.WriteString(evt.Timestamp.Format(time.RFC3339))
|
||||||
|
b.WriteString(": ")
|
||||||
|
b.WriteString(evt.Summary)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForResource returns a condensed incident summary for a resource.
|
||||||
|
func (s *IncidentStore) FormatForResource(resourceID string, limit int) string {
|
||||||
|
incidents := s.ListIncidentsByResource(resourceID, limit)
|
||||||
|
if len(incidents) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("\n\n## Incident Memory\n")
|
||||||
|
b.WriteString("Recent incidents for this resource:\n")
|
||||||
|
for _, incident := range incidents {
|
||||||
|
status := string(incident.Status)
|
||||||
|
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
|
||||||
|
status = "acknowledged"
|
||||||
|
}
|
||||||
|
b.WriteString("- ")
|
||||||
|
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
|
||||||
|
b.WriteString(": ")
|
||||||
|
b.WriteString(incident.AlertType)
|
||||||
|
if incident.Level != "" {
|
||||||
|
b.WriteString(" (")
|
||||||
|
b.WriteString(incident.Level)
|
||||||
|
b.WriteString(")")
|
||||||
|
}
|
||||||
|
b.WriteString(" - ")
|
||||||
|
b.WriteString(status)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatForPatrol returns a condensed incident summary for infrastructure-wide patrol analysis.
|
||||||
|
func (s *IncidentStore) FormatForPatrol(limit int) string {
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 8
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
if len(s.incidents) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("\n\n## Incident Memory\n")
|
||||||
|
b.WriteString("Recent incidents across infrastructure:\n")
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
for i := len(s.incidents) - 1; i >= 0 && count < limit; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
status := string(incident.Status)
|
||||||
|
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
|
||||||
|
status = "acknowledged"
|
||||||
|
}
|
||||||
|
|
||||||
|
lastSummary := ""
|
||||||
|
if len(incident.Events) > 0 {
|
||||||
|
lastSummary = incident.Events[len(incident.Events)-1].Summary
|
||||||
|
}
|
||||||
|
|
||||||
|
b.WriteString("- ")
|
||||||
|
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
|
||||||
|
b.WriteString(": ")
|
||||||
|
if incident.ResourceName != "" {
|
||||||
|
b.WriteString(incident.ResourceName)
|
||||||
|
b.WriteString(" - ")
|
||||||
|
}
|
||||||
|
if incident.AlertType != "" {
|
||||||
|
b.WriteString(incident.AlertType)
|
||||||
|
}
|
||||||
|
if incident.Level != "" {
|
||||||
|
b.WriteString(" (")
|
||||||
|
b.WriteString(incident.Level)
|
||||||
|
b.WriteString(")")
|
||||||
|
}
|
||||||
|
b.WriteString(" - ")
|
||||||
|
b.WriteString(status)
|
||||||
|
if lastSummary != "" {
|
||||||
|
b.WriteString(" - last: ")
|
||||||
|
b.WriteString(truncateOutput(lastSummary, 80))
|
||||||
|
} else if incident.Message != "" {
|
||||||
|
b.WriteString(" - ")
|
||||||
|
b.WriteString(truncateOutput(incident.Message, 80))
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIncidentFromAlert(alert *alerts.Alert) *Incident {
|
||||||
|
openedAt := alert.StartTime
|
||||||
|
if openedAt.IsZero() {
|
||||||
|
openedAt = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Incident{
|
||||||
|
ID: generateIncidentID(),
|
||||||
|
AlertID: alert.ID,
|
||||||
|
AlertType: alert.Type,
|
||||||
|
Level: string(alert.Level),
|
||||||
|
ResourceID: alert.ResourceID,
|
||||||
|
ResourceName: alert.ResourceName,
|
||||||
|
Node: alert.Node,
|
||||||
|
Instance: alert.Instance,
|
||||||
|
Message: alert.Message,
|
||||||
|
Status: IncidentStatusOpen,
|
||||||
|
OpenedAt: openedAt,
|
||||||
|
Acknowledged: alert.Acknowledged,
|
||||||
|
AckUser: alert.AckUser,
|
||||||
|
AckTime: alert.AckTime,
|
||||||
|
Events: make([]IncidentEvent, 0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateIncidentFromAlert(incident *Incident, alert *alerts.Alert) {
|
||||||
|
if incident == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
incident.AlertType = alert.Type
|
||||||
|
incident.Level = string(alert.Level)
|
||||||
|
incident.ResourceID = alert.ResourceID
|
||||||
|
incident.ResourceName = alert.ResourceName
|
||||||
|
incident.Node = alert.Node
|
||||||
|
incident.Instance = alert.Instance
|
||||||
|
incident.Message = alert.Message
|
||||||
|
incident.Acknowledged = alert.Acknowledged
|
||||||
|
incident.AckUser = alert.AckUser
|
||||||
|
incident.AckTime = alert.AckTime
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident {
|
||||||
|
incident := s.findLatestIncidentByAlertIDLocked(alert.ID)
|
||||||
|
if incident == nil {
|
||||||
|
incident = newIncidentFromAlert(alert)
|
||||||
|
s.incidents = append(s.incidents, incident)
|
||||||
|
}
|
||||||
|
updateIncidentFromAlert(incident, alert)
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) addEventLocked(incident *Incident, eventType IncidentEventType, summary string, details map[string]interface{}) {
|
||||||
|
if incident == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if summary == "" {
|
||||||
|
summary = string(eventType)
|
||||||
|
}
|
||||||
|
|
||||||
|
event := IncidentEvent{
|
||||||
|
ID: generateIncidentEventID(),
|
||||||
|
Type: eventType,
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
Summary: summary,
|
||||||
|
Details: details,
|
||||||
|
}
|
||||||
|
incident.Events = append(incident.Events, event)
|
||||||
|
if s.maxEvents > 0 && len(incident.Events) > s.maxEvents {
|
||||||
|
incident.Events = incident.Events[len(incident.Events)-s.maxEvents:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) findOpenIncidentByAlertIDLocked(alertID string) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.AlertID == alertID && incident.Status == IncidentStatusOpen {
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) findLatestIncidentByAlertIDLocked(alertID string) *Incident {
|
||||||
|
if alertID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.AlertID == alertID {
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) findIncidentByIDLocked(incidentID string) *Incident {
|
||||||
|
if incidentID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := len(s.incidents) - 1; i >= 0; i-- {
|
||||||
|
incident := s.incidents[i]
|
||||||
|
if incident != nil && incident.ID == incidentID {
|
||||||
|
return incident
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) trimLocked() {
|
||||||
|
if s.maxAge > 0 {
|
||||||
|
cutoff := time.Now().Add(-s.maxAge)
|
||||||
|
filtered := make([]*Incident, 0, len(s.incidents))
|
||||||
|
for _, incident := range s.incidents {
|
||||||
|
if incident == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
compareTime := incident.OpenedAt
|
||||||
|
if incident.ClosedAt != nil {
|
||||||
|
compareTime = *incident.ClosedAt
|
||||||
|
}
|
||||||
|
if compareTime.After(cutoff) {
|
||||||
|
filtered = append(filtered, incident)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.incidents = filtered
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.maxIncidents > 0 && len(s.incidents) > s.maxIncidents {
|
||||||
|
sort.Slice(s.incidents, func(i, j int) bool {
|
||||||
|
return s.incidents[i].OpenedAt.Before(s.incidents[j].OpenedAt)
|
||||||
|
})
|
||||||
|
if len(s.incidents) > s.maxIncidents {
|
||||||
|
s.incidents = s.incidents[len(s.incidents)-s.maxIncidents:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) saveAsync() {
|
||||||
|
if s.dataDir == "" || s.filePath == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
if err := s.saveToDisk(); err != nil {
|
||||||
|
log.Warn().Err(err).Msg("Failed to save incident history")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) saveToDisk() error {
|
||||||
|
s.saveMu.Lock()
|
||||||
|
defer s.saveMu.Unlock()
|
||||||
|
|
||||||
|
if s.dataDir == "" || s.filePath == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(s.dataDir, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.RLock()
|
||||||
|
snapshot := make([]*Incident, 0, len(s.incidents))
|
||||||
|
for _, incident := range s.incidents {
|
||||||
|
snapshot = append(snapshot, cloneIncident(incident))
|
||||||
|
}
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
data, err := json.Marshal(snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpFile := s.filePath + ".tmp"
|
||||||
|
if err := os.WriteFile(tmpFile, data, 0644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpFile, s.filePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *IncidentStore) loadFromDisk() error {
|
||||||
|
if s.filePath == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Stat(s.filePath)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Size() > maxIncidentFileSize {
|
||||||
|
return fmt.Errorf("incident history file too large (%d bytes)", info.Size())
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(s.filePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var incidents []*Incident
|
||||||
|
if err := json.Unmarshal(data, &incidents); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.incidents = incidents
|
||||||
|
s.trimLocked()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneIncident(src *Incident) *Incident {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
clone := *src
|
||||||
|
if src.AckTime != nil {
|
||||||
|
t := *src.AckTime
|
||||||
|
clone.AckTime = &t
|
||||||
|
}
|
||||||
|
if src.ClosedAt != nil {
|
||||||
|
t := *src.ClosedAt
|
||||||
|
clone.ClosedAt = &t
|
||||||
|
}
|
||||||
|
if len(src.Events) > 0 {
|
||||||
|
clone.Events = make([]IncidentEvent, len(src.Events))
|
||||||
|
for i, event := range src.Events {
|
||||||
|
cloneEvent := event
|
||||||
|
if event.Details != nil {
|
||||||
|
detailsCopy := make(map[string]interface{}, len(event.Details))
|
||||||
|
for key, value := range event.Details {
|
||||||
|
detailsCopy[key] = value
|
||||||
|
}
|
||||||
|
cloneEvent.Details = detailsCopy
|
||||||
|
}
|
||||||
|
clone.Events[i] = cloneEvent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &clone
|
||||||
|
}
|
||||||
|
|
||||||
|
var incidentCounter int64
|
||||||
|
|
||||||
|
func generateIncidentID() string {
|
||||||
|
incidentCounter++
|
||||||
|
return "inc-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentCounter%1000))
|
||||||
|
}
|
||||||
|
|
||||||
|
var incidentEventCounter int64
|
||||||
|
|
||||||
|
func generateIncidentEventID() string {
|
||||||
|
incidentEventCounter++
|
||||||
|
return "inc-evt-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentEventCounter%1000))
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatAlertSummary(alert *alerts.Alert) string {
|
||||||
|
if alert == nil {
|
||||||
|
return "Alert triggered"
|
||||||
|
}
|
||||||
|
summary := fmt.Sprintf("Alert triggered: %s (%s)", alert.Type, alert.Level)
|
||||||
|
if alert.Value > 0 || alert.Threshold > 0 {
|
||||||
|
summary = fmt.Sprintf("Alert triggered: %s (%s %.1f >= %.1f)", alert.Type, alert.Level, alert.Value, alert.Threshold)
|
||||||
|
}
|
||||||
|
return summary
|
||||||
|
}
|
||||||
108
internal/ai/memory/incidents_test.go
Normal file
108
internal/ai/memory/incidents_test.go
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIncidentStore_RecordTimeline(t *testing.T) {
|
||||||
|
store := NewIncidentStore(IncidentStoreConfig{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
MaxIncidents: 10,
|
||||||
|
MaxEventsPerIncident: 10,
|
||||||
|
MaxAgeDays: 30,
|
||||||
|
})
|
||||||
|
|
||||||
|
alert := &alerts.Alert{
|
||||||
|
ID: "alert-1",
|
||||||
|
Type: "cpu",
|
||||||
|
Level: alerts.AlertLevelWarning,
|
||||||
|
ResourceID: "res-1",
|
||||||
|
ResourceName: "vm-1",
|
||||||
|
StartTime: time.Now().Add(-5 * time.Minute),
|
||||||
|
Value: 92,
|
||||||
|
Threshold: 85,
|
||||||
|
}
|
||||||
|
|
||||||
|
store.RecordAlertFired(alert)
|
||||||
|
store.RecordAlertAcknowledged(alert, "admin")
|
||||||
|
store.RecordAnalysis(alert.ID, "analysis complete", map[string]interface{}{
|
||||||
|
"findings": 1,
|
||||||
|
})
|
||||||
|
store.RecordCommand(alert.ID, "systemctl restart nginx", true, "ok", nil)
|
||||||
|
store.RecordAlertResolved(alert, time.Now())
|
||||||
|
|
||||||
|
timeline := store.GetTimelineByAlertID(alert.ID)
|
||||||
|
if timeline == nil {
|
||||||
|
t.Fatalf("expected timeline, got nil")
|
||||||
|
}
|
||||||
|
if timeline.Status != IncidentStatusResolved {
|
||||||
|
t.Fatalf("expected status %q, got %q", IncidentStatusResolved, timeline.Status)
|
||||||
|
}
|
||||||
|
if timeline.AckUser != "admin" {
|
||||||
|
t.Fatalf("expected ack user admin, got %q", timeline.AckUser)
|
||||||
|
}
|
||||||
|
if len(timeline.Events) < 4 {
|
||||||
|
t.Fatalf("expected events recorded, got %d", len(timeline.Events))
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok := store.RecordNote(alert.ID, "", "note text", ""); !ok {
|
||||||
|
t.Fatalf("expected note to be saved")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) {
|
||||||
|
store := NewIncidentStore(IncidentStoreConfig{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
MaxIncidents: 10,
|
||||||
|
MaxEventsPerIncident: 10,
|
||||||
|
MaxAgeDays: 30,
|
||||||
|
})
|
||||||
|
|
||||||
|
base := time.Now().UTC()
|
||||||
|
|
||||||
|
first := &alerts.Alert{
|
||||||
|
ID: "alert-2",
|
||||||
|
Type: "cpu",
|
||||||
|
Level: alerts.AlertLevelWarning,
|
||||||
|
ResourceID: "res-2",
|
||||||
|
ResourceName: "vm-2",
|
||||||
|
StartTime: base.Add(-2 * time.Hour),
|
||||||
|
}
|
||||||
|
|
||||||
|
second := &alerts.Alert{
|
||||||
|
ID: "alert-2",
|
||||||
|
Type: "cpu",
|
||||||
|
Level: alerts.AlertLevelWarning,
|
||||||
|
ResourceID: "res-2",
|
||||||
|
ResourceName: "vm-2",
|
||||||
|
StartTime: base.Add(-10 * time.Minute),
|
||||||
|
}
|
||||||
|
|
||||||
|
store.RecordAlertFired(first)
|
||||||
|
store.RecordAlertResolved(first, base.Add(-90*time.Minute))
|
||||||
|
store.RecordAlertFired(second)
|
||||||
|
|
||||||
|
timeline := store.GetTimelineByAlertAt(first.ID, first.StartTime)
|
||||||
|
if timeline == nil {
|
||||||
|
t.Fatalf("expected timeline for first incident, got nil")
|
||||||
|
}
|
||||||
|
if !timeline.OpenedAt.Equal(first.StartTime) {
|
||||||
|
t.Fatalf("expected openedAt %s, got %s", first.StartTime, timeline.OpenedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeline = store.GetTimelineByAlertAt(second.ID, second.StartTime)
|
||||||
|
if timeline == nil {
|
||||||
|
t.Fatalf("expected timeline for second incident, got nil")
|
||||||
|
}
|
||||||
|
if !timeline.OpenedAt.Equal(second.StartTime) {
|
||||||
|
t.Fatalf("expected openedAt %s, got %s", second.StartTime, timeline.OpenedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeline = store.GetTimelineByAlertAt(second.ID, base.Add(-45*time.Minute))
|
||||||
|
if timeline != nil {
|
||||||
|
t.Fatalf("expected no timeline for mismatched start time")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||||
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
|
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
@ -217,6 +218,7 @@ type PatrolService struct {
|
||||||
remediationLog *RemediationLog // For tracking remediation actions
|
remediationLog *RemediationLog // For tracking remediation actions
|
||||||
patternDetector *PatternDetector // For failure prediction from historical patterns
|
patternDetector *PatternDetector // For failure prediction from historical patterns
|
||||||
correlationDetector *CorrelationDetector // For multi-resource correlation
|
correlationDetector *CorrelationDetector // For multi-resource correlation
|
||||||
|
incidentStore *memory.IncidentStore // For incident timeline capture
|
||||||
|
|
||||||
// Cached thresholds (recalculated when thresholdProvider changes)
|
// Cached thresholds (recalculated when thresholdProvider changes)
|
||||||
thresholds PatrolThresholds
|
thresholds PatrolThresholds
|
||||||
|
|
@ -263,6 +265,20 @@ func NewPatrolService(aiService *Service, stateProvider StateProvider) *PatrolSe
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetIncidentStore attaches an incident store for alert timeline capture.
|
||||||
|
func (p *PatrolService) SetIncidentStore(store *memory.IncidentStore) {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
p.incidentStore = store
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIncidentStore returns the incident store if configured.
|
||||||
|
func (p *PatrolService) GetIncidentStore() *memory.IncidentStore {
|
||||||
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
return p.incidentStore
|
||||||
|
}
|
||||||
|
|
||||||
// SetConfig updates the patrol configuration
|
// SetConfig updates the patrol configuration
|
||||||
func (p *PatrolService) SetConfig(cfg PatrolConfig) {
|
func (p *PatrolService) SetConfig(cfg PatrolConfig) {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
|
|
@ -2185,12 +2201,17 @@ func (p *PatrolService) buildPatrolPrompt(summary string) string {
|
||||||
|
|
||||||
// Get resource notes from knowledge store (per-resource user notes)
|
// Get resource notes from knowledge store (per-resource user notes)
|
||||||
var knowledgeContext string
|
var knowledgeContext string
|
||||||
|
var incidentContext string
|
||||||
p.mu.RLock()
|
p.mu.RLock()
|
||||||
knowledgeStore := p.knowledgeStore
|
knowledgeStore := p.knowledgeStore
|
||||||
|
incidentStore := p.incidentStore
|
||||||
p.mu.RUnlock()
|
p.mu.RUnlock()
|
||||||
if knowledgeStore != nil {
|
if knowledgeStore != nil {
|
||||||
knowledgeContext = knowledgeStore.FormatAllForContext()
|
knowledgeContext = knowledgeStore.FormatAllForContext()
|
||||||
}
|
}
|
||||||
|
if incidentStore != nil {
|
||||||
|
incidentContext = incidentStore.FormatForPatrol(8)
|
||||||
|
}
|
||||||
|
|
||||||
basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities.
|
basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities.
|
||||||
|
|
||||||
|
|
@ -2235,6 +2256,12 @@ IMPORTANT: Respect the user's feedback above. Do NOT re-raise findings that are:
|
||||||
Only report NEW issues or issues where the severity has clearly escalated.`)
|
Only report NEW issues or issues where the severity has clearly escalated.`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if incidentContext != "" {
|
||||||
|
contextAdditions.WriteString("\n\n")
|
||||||
|
contextAdditions.WriteString(incidentContext)
|
||||||
|
contextAdditions.WriteString("\nIMPORTANT: Use incident memory to avoid repeating known issues and to build on successful past investigations.")
|
||||||
|
}
|
||||||
|
|
||||||
if contextAdditions.Len() > 0 {
|
if contextAdditions.Len() > 0 {
|
||||||
return basePrompt + contextAdditions.String()
|
return basePrompt + contextAdditions.String()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -323,6 +323,10 @@ func (p *PatrolService) logRunbookExecution(finding *Finding, runbook Runbook, s
|
||||||
if err := p.remediationLog.Log(record); err != nil {
|
if err := p.remediationLog.Log(record); err != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to log runbook execution")
|
log.Warn().Err(err).Msg("Failed to log runbook execution")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if p.aiService != nil && finding != nil && finding.AlertID != "" {
|
||||||
|
p.aiService.RecordIncidentRunbook(finding.AlertID, runbook.ID, runbook.Title, outcome, automatic, message)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type runbookContext struct {
|
type runbookContext struct {
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,10 @@ import (
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
|
|
@ -73,9 +75,10 @@ type Service struct {
|
||||||
alertProvider AlertProvider
|
alertProvider AlertProvider
|
||||||
knowledgeStore *knowledge.Store
|
knowledgeStore *knowledge.Store
|
||||||
costStore *cost.Store
|
costStore *cost.Store
|
||||||
resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
|
resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
|
||||||
patrolService *PatrolService // Background AI monitoring service
|
patrolService *PatrolService // Background AI monitoring service
|
||||||
metadataProvider MetadataProvider // Enables AI to update resource URLs
|
metadataProvider MetadataProvider // Enables AI to update resource URLs
|
||||||
|
incidentStore *memory.IncidentStore // Incident timelines for alert memory
|
||||||
|
|
||||||
// Alert-triggered analysis - token-efficient real-time AI insights
|
// Alert-triggered analysis - token-efficient real-time AI insights
|
||||||
alertTriggeredAnalyzer *AlertTriggeredAnalyzer
|
alertTriggeredAnalyzer *AlertTriggeredAnalyzer
|
||||||
|
|
@ -197,6 +200,9 @@ func (s *Service) SetStateProvider(sp StateProvider) {
|
||||||
if s.knowledgeStore != nil {
|
if s.knowledgeStore != nil {
|
||||||
s.patrolService.SetKnowledgeStore(s.knowledgeStore)
|
s.patrolService.SetKnowledgeStore(s.knowledgeStore)
|
||||||
}
|
}
|
||||||
|
if s.incidentStore != nil {
|
||||||
|
s.patrolService.SetIncidentStore(s.incidentStore)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize alert-triggered analyzer if not already done
|
// Initialize alert-triggered analyzer if not already done
|
||||||
|
|
@ -350,6 +356,17 @@ func (s *Service) SetRemediationLog(remLog *RemediationLog) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetIncidentStore sets the incident store for alert timeline memory.
|
||||||
|
func (s *Service) SetIncidentStore(store *memory.IncidentStore) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.incidentStore = store
|
||||||
|
|
||||||
|
if s.patrolService != nil {
|
||||||
|
s.patrolService.SetIncidentStore(store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// SetPatternDetector sets the pattern detector for failure prediction
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
|
|
@ -1740,11 +1757,11 @@ func isActionableCommand(cmd string) bool {
|
||||||
"service restart", "service start", "service stop",
|
"service restart", "service start", "service stop",
|
||||||
"pct resize", "pct start", "pct stop", "pct shutdown", "pct reboot",
|
"pct resize", "pct start", "pct stop", "pct shutdown", "pct reboot",
|
||||||
"qm resize", "qm start", "qm stop", "qm shutdown", "qm reboot",
|
"qm resize", "qm start", "qm stop", "qm shutdown", "qm reboot",
|
||||||
"rm -", "rm /", // File deletion/cleanup
|
"rm -", "rm /", // File deletion/cleanup
|
||||||
"chmod", "chown", // Permission fixes
|
"chmod", "chown", // Permission fixes
|
||||||
"mkdir", // Creating directories
|
"mkdir", // Creating directories
|
||||||
"mv ", "cp ", // File operations
|
"mv ", "cp ", // File operations
|
||||||
"echo >", "tee ", // Writing to files
|
"echo >", "tee ", // Writing to files
|
||||||
"apt install", "apt upgrade", "apt remove",
|
"apt install", "apt upgrade", "apt remove",
|
||||||
"yum install", "dnf install",
|
"yum install", "dnf install",
|
||||||
"pip install", "npm install",
|
"pip install", "npm install",
|
||||||
|
|
@ -2183,11 +2200,34 @@ func (s *Service) executeTool(ctx context.Context, req ExecuteRequest, tc provid
|
||||||
|
|
||||||
// Execute via agent
|
// Execute via agent
|
||||||
result, err := s.executeOnAgent(ctx, execReq, command)
|
result, err := s.executeOnAgent(ctx, execReq, command)
|
||||||
|
recordIncident := func(success bool, output string) {
|
||||||
|
alertID := extractAlertID(req.Context)
|
||||||
|
if alertID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
incidentStore := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
if incidentStore == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
details := map[string]interface{}{
|
||||||
|
"resource_id": req.TargetID,
|
||||||
|
"resource_type": req.TargetType,
|
||||||
|
"run_on_host": runOnHost,
|
||||||
|
}
|
||||||
|
if targetHost != "" {
|
||||||
|
details["target_host"] = targetHost
|
||||||
|
}
|
||||||
|
incidentStore.RecordCommand(alertID, command, success, output, details)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
recordIncident(false, result)
|
||||||
execution.Output = fmt.Sprintf("Error executing command: %s", err)
|
execution.Output = fmt.Sprintf("Error executing command: %s", err)
|
||||||
return execution.Output, execution
|
return execution.Output, execution
|
||||||
}
|
}
|
||||||
|
|
||||||
|
recordIncident(true, result)
|
||||||
execution.Output = result
|
execution.Output = result
|
||||||
execution.Success = true
|
execution.Success = true
|
||||||
return result, execution
|
return result, execution
|
||||||
|
|
@ -2842,6 +2882,31 @@ This is a 3-command job. Don't over-investigate.`
|
||||||
// Add current alert status - this gives AI awareness of active issues
|
// Add current alert status - this gives AI awareness of active issues
|
||||||
prompt += s.buildAlertContext()
|
prompt += s.buildAlertContext()
|
||||||
|
|
||||||
|
// Add incident memory for alert or resource context
|
||||||
|
alertID := ""
|
||||||
|
resourceID := req.TargetID
|
||||||
|
if req.Context != nil {
|
||||||
|
if val, ok := req.Context["alertId"].(string); ok && val != "" {
|
||||||
|
alertID = val
|
||||||
|
} else if val, ok := req.Context["alert_id"].(string); ok && val != "" {
|
||||||
|
alertID = val
|
||||||
|
}
|
||||||
|
|
||||||
|
if resourceID == "" {
|
||||||
|
if val, ok := req.Context["resourceId"].(string); ok && val != "" {
|
||||||
|
resourceID = val
|
||||||
|
} else if val, ok := req.Context["resource_id"].(string); ok && val != "" {
|
||||||
|
resourceID = val
|
||||||
|
} else if val, ok := req.Context["guest_id"].(string); ok && val != "" {
|
||||||
|
resourceID = val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if incidentContext := s.buildIncidentContext(resourceID, alertID); incidentContext != "" {
|
||||||
|
prompt += incidentContext
|
||||||
|
}
|
||||||
|
|
||||||
// Add all saved knowledge when no specific target is selected
|
// Add all saved knowledge when no specific target is selected
|
||||||
// This gives the AI context about everything learned from previous sessions
|
// This gives the AI context about everything learned from previous sessions
|
||||||
if req.TargetType == "" && s.knowledgeStore != nil {
|
if req.TargetType == "" && s.knowledgeStore != nil {
|
||||||
|
|
@ -2877,6 +2942,7 @@ This is a 3-command job. Don't over-investigate.`
|
||||||
|
|
||||||
// Add past remediation history for this resource
|
// Add past remediation history for this resource
|
||||||
prompt += s.buildRemediationContext(req.TargetID, req.Prompt)
|
prompt += s.buildRemediationContext(req.TargetID, req.Prompt)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add any provided context in a structured way
|
// Add any provided context in a structured way
|
||||||
|
|
@ -3258,6 +3324,66 @@ func (s *Service) buildRemediationContext(resourceID, currentProblem string) str
|
||||||
return context
|
return context
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildIncidentContext adds incident timeline context for alerts/resources.
|
||||||
|
func (s *Service) buildIncidentContext(resourceID, alertID string) string {
|
||||||
|
s.mu.RLock()
|
||||||
|
store := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
if store == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if alertID != "" {
|
||||||
|
return store.FormatForAlert(alertID, 8)
|
||||||
|
}
|
||||||
|
if resourceID != "" {
|
||||||
|
return store.FormatForResource(resourceID, 4)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordIncidentAnalysis stores an AI analysis event for an alert.
|
||||||
|
func (s *Service) RecordIncidentAnalysis(alertID, summary string, details map[string]interface{}) {
|
||||||
|
if alertID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
store := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
if store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
store.RecordAnalysis(alertID, summary, details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordIncidentRunbook stores a runbook execution event for an alert.
|
||||||
|
func (s *Service) RecordIncidentRunbook(alertID, runbookID, title string, outcome memory.Outcome, automatic bool, message string) {
|
||||||
|
if alertID == "" || runbookID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
store := s.incidentStore
|
||||||
|
s.mu.RUnlock()
|
||||||
|
if store == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
store.RecordRunbook(alertID, runbookID, title, string(outcome), automatic, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractAlertID(ctx map[string]interface{}) string {
|
||||||
|
if ctx == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if alertID, ok := ctx["alertId"].(string); ok && alertID != "" {
|
||||||
|
return alertID
|
||||||
|
}
|
||||||
|
if alertID, ok := ctx["alert_id"].(string); ok && alertID != "" {
|
||||||
|
return alertID
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// truncateString truncates a string to maxLen characters
|
// truncateString truncates a string to maxLen characters
|
||||||
func truncateString(s string, maxLen int) string {
|
func truncateString(s string, maxLen int) string {
|
||||||
if len(s) <= maxLen {
|
if len(s) <= maxLen {
|
||||||
|
|
@ -3302,40 +3428,51 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check CPU baseline
|
// Helper to format baseline comparison with status
|
||||||
|
// ALWAYS shows baseline info when available - this is Pulse Pro's value-add
|
||||||
|
formatBaselineComparison := func(metric string, currentVal float64, bl *baseline.MetricBaseline) string {
|
||||||
|
if bl.SampleCount < 10 {
|
||||||
|
return fmt.Sprintf("- %s: %.1f%% (baseline learning: %d/10 samples)", metric, currentVal, bl.SampleCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
ratio := currentVal / bl.Mean
|
||||||
|
status := "✓ normal"
|
||||||
|
if ratio > 2.0 {
|
||||||
|
status = "🔴 **ANOMALY**"
|
||||||
|
} else if ratio > 1.5 {
|
||||||
|
status = "🟡 elevated"
|
||||||
|
} else if ratio < 0.3 {
|
||||||
|
status = "🔵 unusually low"
|
||||||
|
} else if ratio < 0.5 {
|
||||||
|
status = "low"
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf("- %s: %.1f%% vs baseline %.1f%% (σ=%.1f) → %s", metric, currentVal, bl.Mean, bl.StdDev, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check CPU baseline - always show if we have data
|
||||||
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
|
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
|
||||||
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists && bl.SampleCount >= 10 {
|
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists {
|
||||||
ratio := cpuVal / bl.Mean
|
baselineInfo = append(baselineInfo, formatBaselineComparison("CPU", cpuVal, bl))
|
||||||
if ratio > 1.5 {
|
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx higher** than baseline %.0f%% (σ=%.1f)", cpuVal, ratio, bl.Mean, bl.StdDev))
|
|
||||||
} else if ratio < 0.5 {
|
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx lower** than baseline %.0f%%", cpuVal, 1/ratio, bl.Mean))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check memory baseline
|
// Check memory baseline - always show if we have data
|
||||||
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
|
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
|
||||||
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists && bl.SampleCount >= 10 {
|
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists {
|
||||||
ratio := memVal / bl.Mean
|
baselineInfo = append(baselineInfo, formatBaselineComparison("Memory", memVal, bl))
|
||||||
if ratio > 1.3 {
|
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("Memory %.0f%% is **%.1fx higher** than baseline %.0f%%", memVal, ratio, bl.Mean))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check disk baseline
|
// Check disk baseline - always show if we have data
|
||||||
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
|
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
|
||||||
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists && bl.SampleCount >= 10 {
|
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists {
|
||||||
ratio := diskVal / bl.Mean
|
baselineInfo = append(baselineInfo, formatBaselineComparison("Disk", diskVal, bl))
|
||||||
if ratio > 1.2 { // More sensitive for disk
|
|
||||||
baselineInfo = append(baselineInfo, fmt.Sprintf("Disk %.0f%% is **%.1fx higher** than baseline %.0f%%", diskVal, ratio, bl.Mean))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(baselineInfo) > 0 {
|
if len(baselineInfo) > 0 {
|
||||||
sections = append(sections, "### Baseline Comparisons\n"+strings.Join(baselineInfo, "\n"))
|
sections = append(sections, "### Learned Baselines (7-day patterns)\n"+strings.Join(baselineInfo, "\n"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ const (
|
||||||
DefaultObservationWindow = 24 // hours
|
DefaultObservationWindow = 24 // hours
|
||||||
|
|
||||||
// Flapping detection
|
// Flapping detection
|
||||||
DefaultFlappingWindow = 300 // seconds (5 minutes)
|
DefaultFlappingWindow = 300 // seconds (5 minutes)
|
||||||
DefaultFlappingThreshold = 5 // state changes to trigger flapping
|
DefaultFlappingThreshold = 5 // state changes to trigger flapping
|
||||||
DefaultFlappingCooldown = 15 // minutes
|
DefaultFlappingCooldown = 15 // minutes
|
||||||
|
|
||||||
|
|
@ -256,10 +256,10 @@ type GroupingConfig struct {
|
||||||
// ScheduleConfig represents alerting schedule configuration
|
// ScheduleConfig represents alerting schedule configuration
|
||||||
type ScheduleConfig struct {
|
type ScheduleConfig struct {
|
||||||
QuietHours QuietHours `json:"quietHours"`
|
QuietHours QuietHours `json:"quietHours"`
|
||||||
Cooldown int `json:"cooldown"` // minutes
|
Cooldown int `json:"cooldown"` // minutes
|
||||||
GroupingWindow int `json:"groupingWindow,omitempty"` // Deprecated: use Grouping.Window instead. Will be auto-migrated on config update.
|
GroupingWindow int `json:"groupingWindow,omitempty"` // Deprecated: use Grouping.Window instead. Will be auto-migrated on config update.
|
||||||
MaxAlertsHour int `json:"maxAlertsHour"` // max alerts per hour per resource
|
MaxAlertsHour int `json:"maxAlertsHour"` // max alerts per hour per resource
|
||||||
NotifyOnResolve bool `json:"notifyOnResolve"` // Send notification when alert clears
|
NotifyOnResolve bool `json:"notifyOnResolve"` // Send notification when alert clears
|
||||||
Escalation EscalationConfig `json:"escalation"`
|
Escalation EscalationConfig `json:"escalation"`
|
||||||
Grouping GroupingConfig `json:"grouping"`
|
Grouping GroupingConfig `json:"grouping"`
|
||||||
}
|
}
|
||||||
|
|
@ -484,15 +484,17 @@ func SetMetricHooks(fired func(*Alert), resolved func(*Alert), suppressed func(s
|
||||||
}
|
}
|
||||||
|
|
||||||
type Manager struct {
|
type Manager struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
config AlertConfig
|
config AlertConfig
|
||||||
activeAlerts map[string]*Alert
|
activeAlerts map[string]*Alert
|
||||||
historyManager *HistoryManager
|
historyManager *HistoryManager
|
||||||
onAlert func(alert *Alert)
|
onAlert func(alert *Alert)
|
||||||
onResolved func(alertID string)
|
onResolved func(alertID string)
|
||||||
onEscalate func(alert *Alert, level int)
|
onAcknowledged func(alert *Alert, user string)
|
||||||
escalationStop chan struct{}
|
onUnacknowledged func(alert *Alert, user string)
|
||||||
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
|
onEscalate func(alert *Alert, level int)
|
||||||
|
escalationStop chan struct{}
|
||||||
|
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
|
||||||
// New fields for deduplication and suppression
|
// New fields for deduplication and suppression
|
||||||
recentAlerts map[string]*Alert // Track recent alerts for deduplication
|
recentAlerts map[string]*Alert // Track recent alerts for deduplication
|
||||||
suppressedUntil map[string]time.Time // Track suppression windows
|
suppressedUntil map[string]time.Time // Track suppression windows
|
||||||
|
|
@ -736,6 +738,20 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
|
||||||
m.onResolved = cb
|
m.onResolved = cb
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAcknowledgedCallback sets the callback for acknowledged alerts.
|
||||||
|
func (m *Manager) SetAcknowledgedCallback(cb func(alert *Alert, user string)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.onAcknowledged = cb
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetUnacknowledgedCallback sets the callback for unacknowledged alerts.
|
||||||
|
func (m *Manager) SetUnacknowledgedCallback(cb func(alert *Alert, user string)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.onUnacknowledged = cb
|
||||||
|
}
|
||||||
|
|
||||||
// SetEscalateCallback sets the callback for escalated alerts
|
// SetEscalateCallback sets the callback for escalated alerts
|
||||||
func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) {
|
func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
@ -768,6 +784,46 @@ func (m *Manager) safeCallResolvedCallback(alertID string, async bool) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// safeCallAcknowledgedCallback invokes onAcknowledged with panic recovery and alert cloning.
|
||||||
|
func (m *Manager) safeCallAcknowledgedCallback(alert *Alert, user string) {
|
||||||
|
if m.onAcknowledged == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
go func(a *Alert, u string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Str("alertID", a.ID).
|
||||||
|
Msg("Panic in onAcknowledged callback")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
m.onAcknowledged(a, u)
|
||||||
|
}(alertCopy, user)
|
||||||
|
}
|
||||||
|
|
||||||
|
// safeCallUnacknowledgedCallback invokes onUnacknowledged with panic recovery and alert cloning.
|
||||||
|
func (m *Manager) safeCallUnacknowledgedCallback(alert *Alert, user string) {
|
||||||
|
if m.onUnacknowledged == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
go func(a *Alert, u string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().
|
||||||
|
Interface("panic", r).
|
||||||
|
Str("alertID", a.ID).
|
||||||
|
Msg("Panic in onUnacknowledged callback")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
m.onUnacknowledged(a, u)
|
||||||
|
}(alertCopy, user)
|
||||||
|
}
|
||||||
|
|
||||||
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning
|
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning
|
||||||
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
|
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
|
||||||
if m.onEscalate == nil {
|
if m.onEscalate == nil {
|
||||||
|
|
@ -2457,7 +2513,6 @@ func (m *Manager) hasHostAgentForNode(nodeName string) bool {
|
||||||
return exists
|
return exists
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func hostResourceID(hostID string) string {
|
func hostResourceID(hostID string) string {
|
||||||
trimmed := strings.TrimSpace(hostID)
|
trimmed := strings.TrimSpace(hostID)
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
|
|
@ -5767,10 +5822,10 @@ func abs(x float64) float64 {
|
||||||
// AcknowledgeAlert acknowledges an alert
|
// AcknowledgeAlert acknowledges an alert
|
||||||
func (m *Manager) AcknowledgeAlert(alertID, user string) error {
|
func (m *Manager) AcknowledgeAlert(alertID, user string) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
alert, exists := m.activeAlerts[alertID]
|
alert, exists := m.activeAlerts[alertID]
|
||||||
if !exists {
|
if !exists {
|
||||||
|
m.mu.Unlock()
|
||||||
return fmt.Errorf("alert not found: %s", alertID)
|
return fmt.Errorf("alert not found: %s", alertID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -5787,22 +5842,26 @@ func (m *Manager) AcknowledgeAlert(alertID, user string) error {
|
||||||
time: now,
|
time: now,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("alertID", alertID).
|
Str("alertID", alertID).
|
||||||
Str("user", user).
|
Str("user", user).
|
||||||
Time("ackTime", now).
|
Time("ackTime", now).
|
||||||
Msg("Alert acknowledgment recorded")
|
Msg("Alert acknowledgment recorded")
|
||||||
|
|
||||||
|
m.safeCallAcknowledgedCallback(alertCopy, user)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnacknowledgeAlert removes the acknowledged status from an alert
|
// UnacknowledgeAlert removes the acknowledged status from an alert
|
||||||
func (m *Manager) UnacknowledgeAlert(alertID string) error {
|
func (m *Manager) UnacknowledgeAlert(alertID string) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
alert, exists := m.activeAlerts[alertID]
|
alert, exists := m.activeAlerts[alertID]
|
||||||
if !exists {
|
if !exists {
|
||||||
|
m.mu.Unlock()
|
||||||
return fmt.Errorf("alert not found: %s", alertID)
|
return fmt.Errorf("alert not found: %s", alertID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -5814,10 +5873,14 @@ func (m *Manager) UnacknowledgeAlert(alertID string) error {
|
||||||
m.activeAlerts[alertID] = alert
|
m.activeAlerts[alertID] = alert
|
||||||
delete(m.ackState, alertID)
|
delete(m.ackState, alertID)
|
||||||
|
|
||||||
|
alertCopy := alert.Clone()
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alertID", alertID).
|
Str("alertID", alertID).
|
||||||
Msg("Alert unacknowledged")
|
Msg("Alert unacknowledged")
|
||||||
|
|
||||||
|
m.safeCallUnacknowledgedCallback(alertCopy, "")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -7593,9 +7656,8 @@ func (m *Manager) ClearAlert(alertID string) bool {
|
||||||
// Cleanup removes old acknowledged alerts and cleans up tracking maps
|
// Cleanup removes old acknowledged alerts and cleans up tracking maps
|
||||||
func (m *Manager) Cleanup(maxAge time.Duration) {
|
func (m *Manager) Cleanup(maxAge time.Duration) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
var autoAcked []*Alert
|
||||||
|
|
||||||
// Auto-acknowledge old alerts if configured
|
// Auto-acknowledge old alerts if configured
|
||||||
if m.config.AutoAcknowledgeAfterHours > 0 {
|
if m.config.AutoAcknowledgeAfterHours > 0 {
|
||||||
|
|
@ -7610,6 +7672,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
|
||||||
ackTime := now
|
ackTime := now
|
||||||
alert.AckTime = &ackTime
|
alert.AckTime = &ackTime
|
||||||
alert.AckUser = "system-auto"
|
alert.AckUser = "system-auto"
|
||||||
|
autoAcked = append(autoAcked, alert.Clone())
|
||||||
|
|
||||||
if recordAlertAcknowledged != nil {
|
if recordAlertAcknowledged != nil {
|
||||||
recordAlertAcknowledged()
|
recordAlertAcknowledged()
|
||||||
|
|
@ -7776,6 +7839,12 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
|
||||||
Msg("Cleaned up stale PMG quarantine history")
|
Msg("Cleaned up stale PMG quarantine history")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
for _, alert := range autoAcked {
|
||||||
|
m.safeCallAcknowledgedCallback(alert, "system-auto")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold
|
// convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/license"
|
"github.com/rcourtman/pulse-go-rewrite/internal/license"
|
||||||
|
|
@ -118,6 +119,11 @@ func (h *AISettingsHandler) SetRemediationLog(remLog *ai.RemediationLog) {
|
||||||
h.aiService.SetRemediationLog(remLog)
|
h.aiService.SetRemediationLog(remLog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetIncidentStore sets the incident store for alert timelines.
|
||||||
|
func (h *AISettingsHandler) SetIncidentStore(store *memory.IncidentStore) {
|
||||||
|
h.aiService.SetIncidentStore(store)
|
||||||
|
}
|
||||||
|
|
||||||
// SetPatternDetector sets the pattern detector for failure prediction
|
// SetPatternDetector sets the pattern detector for failure prediction
|
||||||
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
|
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
|
||||||
h.aiService.SetPatternDetector(detector)
|
h.aiService.SetPatternDetector(detector)
|
||||||
|
|
@ -1767,11 +1773,29 @@ func (h *AISettingsHandler) HandleInvestigateAlert(w http.ResponseWriter, r *htt
|
||||||
data, _ := json.Marshal(finalEvent)
|
data, _ := json.Marshal(finalEvent)
|
||||||
safeWrite([]byte("data: " + string(data) + "\n\n"))
|
safeWrite([]byte("data: " + string(data) + "\n\n"))
|
||||||
|
|
||||||
|
if req.AlertID != "" {
|
||||||
|
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI alert investigation completed", map[string]interface{}{
|
||||||
|
"model": resp.Model,
|
||||||
|
"tool_calls": len(resp.ToolCalls),
|
||||||
|
"input_tokens": resp.InputTokens,
|
||||||
|
"output_tokens": resp.OutputTokens,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alert_id", req.AlertID).
|
Str("alert_id", req.AlertID).
|
||||||
Str("model", resp.Model).
|
Str("model", resp.Model).
|
||||||
Int("tool_calls", len(resp.ToolCalls)).
|
Int("tool_calls", len(resp.ToolCalls)).
|
||||||
Msg("AI alert investigation completed")
|
Msg("AI alert investigation completed")
|
||||||
|
|
||||||
|
if req.AlertID != "" {
|
||||||
|
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI investigation completed", map[string]interface{}{
|
||||||
|
"model": resp.Model,
|
||||||
|
"input_tokens": resp.InputTokens,
|
||||||
|
"output_tokens": resp.OutputTokens,
|
||||||
|
"tool_calls": len(resp.ToolCalls),
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetAlertProvider sets the alert provider for AI context
|
// SetAlertProvider sets the alert provider for AI context
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
||||||
|
|
@ -361,6 +362,130 @@ func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetAlertIncidentTimeline returns the incident timeline for an alert or resource.
|
||||||
|
func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
store := h.monitor.GetIncidentStore()
|
||||||
|
if store == nil {
|
||||||
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
query := r.URL.Query()
|
||||||
|
alertID := strings.TrimSpace(query.Get("alert_id"))
|
||||||
|
resourceID := strings.TrimSpace(query.Get("resource_id"))
|
||||||
|
startedAtRaw := strings.TrimSpace(query.Get("started_at"))
|
||||||
|
if startedAtRaw == "" {
|
||||||
|
startedAtRaw = strings.TrimSpace(query.Get("start_time"))
|
||||||
|
}
|
||||||
|
limit := 0
|
||||||
|
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
|
||||||
|
if parsed, err := strconv.Atoi(rawLimit); err == nil && parsed > 0 {
|
||||||
|
limit = parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if alertID != "" {
|
||||||
|
if !validateAlertID(alertID) {
|
||||||
|
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var startedAt time.Time
|
||||||
|
if startedAtRaw != "" {
|
||||||
|
parsed, err := time.Parse(time.RFC3339, startedAtRaw)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Invalid started_at time", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
startedAt = parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
var incident *memory.Incident
|
||||||
|
if !startedAt.IsZero() {
|
||||||
|
incident = store.GetTimelineByAlertAt(alertID, startedAt)
|
||||||
|
} else {
|
||||||
|
incident = store.GetTimelineByAlertID(alertID)
|
||||||
|
}
|
||||||
|
if err := utils.WriteJSONResponse(w, incident); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to write incident timeline response")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if resourceID != "" {
|
||||||
|
if len(resourceID) > 500 {
|
||||||
|
http.Error(w, "Invalid resource ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
incidents := store.ListIncidentsByResource(resourceID, limit)
|
||||||
|
if err := utils.WriteJSONResponse(w, incidents); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to write incident list response")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
http.Error(w, "Missing alert_id or resource_id", http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveAlertIncidentNote stores a user note in the incident timeline.
|
||||||
|
func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
store := h.monitor.GetIncidentStore()
|
||||||
|
if store == nil {
|
||||||
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, 8*1024)
|
||||||
|
var req struct {
|
||||||
|
AlertID string `json:"alert_id"`
|
||||||
|
IncidentID string `json:"incident_id"`
|
||||||
|
Note string `json:"note"`
|
||||||
|
User string `json:"user,omitempty"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req.AlertID = strings.TrimSpace(req.AlertID)
|
||||||
|
req.IncidentID = strings.TrimSpace(req.IncidentID)
|
||||||
|
req.Note = strings.TrimSpace(req.Note)
|
||||||
|
req.User = strings.TrimSpace(req.User)
|
||||||
|
|
||||||
|
if req.AlertID == "" && req.IncidentID == "" {
|
||||||
|
http.Error(w, "alert_id or incident_id is required", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.AlertID != "" && !validateAlertID(req.AlertID) {
|
||||||
|
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Note == "" {
|
||||||
|
http.Error(w, "note is required", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if ok := store.RecordNote(req.AlertID, req.IncidentID, req.Note, req.User); !ok {
|
||||||
|
http.Error(w, "Failed to save note", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
||||||
|
"success": true,
|
||||||
|
}); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to write incident note response")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ClearAlertHistory clears all alert history
|
// ClearAlertHistory clears all alert history
|
||||||
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
|
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
|
||||||
if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil {
|
if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil {
|
||||||
|
|
@ -737,6 +862,16 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
h.GetAlertHistory(w, r)
|
h.GetAlertHistory(w, r)
|
||||||
|
case path == "incidents" && r.Method == http.MethodGet:
|
||||||
|
if !ensureScope(w, r, config.ScopeMonitoringRead) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.GetAlertIncidentTimeline(w, r)
|
||||||
|
case path == "incidents/note" && r.Method == http.MethodPost:
|
||||||
|
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.SaveAlertIncidentNote(w, r)
|
||||||
case path == "history" && r.Method == http.MethodDelete:
|
case path == "history" && r.Method == http.MethodDelete:
|
||||||
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
||||||
return
|
return
|
||||||
|
|
|
||||||
|
|
@ -1109,6 +1109,9 @@ func (r *Router) setupRoutes() {
|
||||||
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
|
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
|
||||||
r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager))
|
r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager))
|
||||||
}
|
}
|
||||||
|
if incidentStore := r.monitor.GetIncidentStore(); incidentStore != nil {
|
||||||
|
r.aiSettingsHandler.SetIncidentStore(incidentStore)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view)
|
// Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view)
|
||||||
if r.resourceHandlers != nil {
|
if r.resourceHandlers != nil {
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
|
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
|
||||||
|
|
@ -604,6 +605,7 @@ type Monitor struct {
|
||||||
metricsHistory *MetricsHistory
|
metricsHistory *MetricsHistory
|
||||||
metricsStore *metrics.Store // Persistent SQLite metrics storage
|
metricsStore *metrics.Store // Persistent SQLite metrics storage
|
||||||
alertManager *alerts.Manager
|
alertManager *alerts.Manager
|
||||||
|
incidentStore *memory.IncidentStore
|
||||||
notificationMgr *notifications.NotificationManager
|
notificationMgr *notifications.NotificationManager
|
||||||
configPersist *config.ConfigPersistence
|
configPersist *config.ConfigPersistence
|
||||||
discoveryService *discovery.Service // Background discovery service
|
discoveryService *discovery.Service // Background discovery service
|
||||||
|
|
@ -3050,6 +3052,10 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||||
Msg("Persistent metrics store initialized with configurable retention")
|
Msg("Persistent metrics store initialized with configurable retention")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{
|
||||||
|
DataDir: cfg.DataPath,
|
||||||
|
})
|
||||||
|
|
||||||
m := &Monitor{
|
m := &Monitor{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
state: models.NewState(),
|
state: models.NewState(),
|
||||||
|
|
@ -3076,6 +3082,7 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||||
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
|
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
|
||||||
metricsStore: metricsStore, // Persistent SQLite storage
|
metricsStore: metricsStore, // Persistent SQLite storage
|
||||||
alertManager: alerts.NewManager(),
|
alertManager: alerts.NewManager(),
|
||||||
|
incidentStore: incidentStore,
|
||||||
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
|
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
|
||||||
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
||||||
discoveryService: nil, // Will be initialized in Start()
|
discoveryService: nil, // Will be initialized in Start()
|
||||||
|
|
@ -3756,25 +3763,19 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||||
|
|
||||||
// Set up alert callbacks
|
// Set up alert callbacks
|
||||||
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||||
wsHub.BroadcastAlert(alert)
|
m.handleAlertFired(alert)
|
||||||
// Send notifications
|
|
||||||
log.Debug().
|
|
||||||
Str("alertID", alert.ID).
|
|
||||||
Str("level", string(alert.Level)).
|
|
||||||
Msg("Alert raised, sending to notification manager")
|
|
||||||
go m.notificationMgr.SendAlert(alert)
|
|
||||||
})
|
})
|
||||||
m.alertManager.SetResolvedCallback(func(alertID string) {
|
m.alertManager.SetResolvedCallback(func(alertID string) {
|
||||||
wsHub.BroadcastAlertResolved(alertID)
|
m.handleAlertResolved(alertID)
|
||||||
m.notificationMgr.CancelAlert(alertID)
|
|
||||||
if m.notificationMgr.GetNotifyOnResolve() {
|
|
||||||
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
|
|
||||||
go m.notificationMgr.SendResolvedAlert(resolved)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Don't broadcast full state here - it causes a cascade with many guests.
|
// Don't broadcast full state here - it causes a cascade with many guests.
|
||||||
// The frontend will get the updated alerts through the regular broadcast ticker.
|
// The frontend will get the updated alerts through the regular broadcast ticker.
|
||||||
})
|
})
|
||||||
|
m.alertManager.SetAcknowledgedCallback(func(alert *alerts.Alert, user string) {
|
||||||
|
m.handleAlertAcknowledged(alert, user)
|
||||||
|
})
|
||||||
|
m.alertManager.SetUnacknowledgedCallback(func(alert *alerts.Alert, user string) {
|
||||||
|
m.handleAlertUnacknowledged(alert, user)
|
||||||
|
})
|
||||||
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("alertID", alert.ID).
|
Str("alertID", alert.ID).
|
||||||
|
|
@ -7731,6 +7732,11 @@ func (m *Monitor) GetAlertManager() *alerts.Manager {
|
||||||
return m.alertManager
|
return m.alertManager
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetIncidentStore returns the incident timeline store.
|
||||||
|
func (m *Monitor) GetIncidentStore() *memory.IncidentStore {
|
||||||
|
return m.incidentStore
|
||||||
|
}
|
||||||
|
|
||||||
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
|
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
|
||||||
// This enables token-efficient, real-time AI insights on specific resources
|
// This enables token-efficient, real-time AI insights on specific resources
|
||||||
func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
|
func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
|
||||||
|
|
@ -7738,33 +7744,69 @@ func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the current callback
|
|
||||||
originalCallback := m.alertManager
|
|
||||||
|
|
||||||
// Wrap the existing callback to also call the AI callback
|
// Wrap the existing callback to also call the AI callback
|
||||||
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||||
// Broadcast to WebSocket (this happens via the callback set in Start())
|
m.handleAlertFired(alert)
|
||||||
if m.wsHub != nil {
|
|
||||||
m.wsHub.BroadcastAlert(alert)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send notifications
|
|
||||||
log.Debug().
|
|
||||||
Str("alertID", alert.ID).
|
|
||||||
Str("level", string(alert.Level)).
|
|
||||||
Msg("Alert raised, sending to notification manager")
|
|
||||||
go m.notificationMgr.SendAlert(alert)
|
|
||||||
|
|
||||||
// Trigger AI analysis
|
// Trigger AI analysis
|
||||||
go callback(alert)
|
go callback(alert)
|
||||||
})
|
})
|
||||||
|
|
||||||
// Avoid unused variable warning
|
|
||||||
_ = originalCallback
|
|
||||||
|
|
||||||
log.Info().Msg("Alert-triggered AI callback registered")
|
log.Info().Msg("Alert-triggered AI callback registered")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertFired(alert *alerts.Alert) {
|
||||||
|
if alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.wsHub != nil {
|
||||||
|
m.wsHub.BroadcastAlert(alert)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug().
|
||||||
|
Str("alertID", alert.ID).
|
||||||
|
Str("level", string(alert.Level)).
|
||||||
|
Msg("Alert raised, sending to notification manager")
|
||||||
|
go m.notificationMgr.SendAlert(alert)
|
||||||
|
|
||||||
|
if m.incidentStore != nil {
|
||||||
|
m.incidentStore.RecordAlertFired(alert)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertResolved(alertID string) {
|
||||||
|
if m.wsHub != nil {
|
||||||
|
m.wsHub.BroadcastAlertResolved(alertID)
|
||||||
|
}
|
||||||
|
m.notificationMgr.CancelAlert(alertID)
|
||||||
|
if m.notificationMgr.GetNotifyOnResolve() {
|
||||||
|
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
|
||||||
|
go m.notificationMgr.SendResolvedAlert(resolved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.incidentStore != nil {
|
||||||
|
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil && resolved.Alert != nil {
|
||||||
|
m.incidentStore.RecordAlertResolved(resolved.Alert, resolved.ResolvedTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertAcknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if m.incidentStore == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.incidentStore.RecordAlertAcknowledged(alert, user)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) handleAlertUnacknowledged(alert *alerts.Alert, user string) {
|
||||||
|
if m.incidentStore == nil || alert == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.incidentStore.RecordAlertUnacknowledged(alert, user)
|
||||||
|
}
|
||||||
|
|
||||||
// SetResourceStore sets the resource store for polling optimization.
|
// SetResourceStore sets the resource store for polling optimization.
|
||||||
// When set, the monitor will check if it should reduce polling frequency
|
// When set, the monitor will check if it should reduce polling frequency
|
||||||
// for nodes that have host agents providing data.
|
// for nodes that have host agents providing data.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue