feat: enhance AI baseline context visibility and incident timeline improvements

Backend:
- Enhanced buildEnrichedResourceContext to ALWAYS show learned baselines with
  status indicators (normal/elevated/anomaly) instead of only when anomalous
- This makes Pulse Pro's 'moat' visible - users can see the AI understands
  their infrastructure's normal behavior patterns
- Added baseline import to service.go

Frontend (user changes):
- Added incident event type filtering with toggle buttons
- Added resource incident panel to view all incidents for a resource
- Added timeline expand/collapse functionality in alert history
- Added incident note saving with proper incidentId tracking
- Added startedAt parameter for proper incident timeline loading
This commit is contained in:
rcourtman 2025-12-21 00:14:20 +00:00
parent f0b983667c
commit 82e5b28840
14 changed files with 2397 additions and 226 deletions

View file

@ -1,4 +1,4 @@
import type { Alert } from '@/types/api'; import type { Alert, Incident } from '@/types/api';
import type { AlertConfig } from '@/types/alerts'; import type { AlertConfig } from '@/types/alerts';
import { apiFetchJSON } from '@/utils/apiClient'; import { apiFetchJSON } from '@/utils/apiClient';
@ -29,6 +29,37 @@ export class AlertsAPI {
return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`); return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`);
} }
static async getIncidentTimeline(alertId: string, startedAt?: string): Promise<Incident | null> {
const query = new URLSearchParams({ alert_id: alertId });
if (startedAt) {
query.set('started_at', startedAt);
}
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident | null>;
}
static async getIncidentsForResource(resourceId: string, limit?: number): Promise<Incident[]> {
const query = new URLSearchParams({ resource_id: resourceId });
if (limit) query.set('limit', String(limit));
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident[]>;
}
static async addIncidentNote(params: {
alertId?: string;
incidentId?: string;
note: string;
user?: string;
}): Promise<{ success: boolean }> {
return apiFetchJSON(`${this.baseUrl}/incidents/note`, {
method: 'POST',
body: JSON.stringify({
alert_id: params.alertId,
incident_id: params.incidentId,
note: params.note,
user: params.user,
}),
}) as Promise<{ success: boolean }>;
}
static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> { static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> {
return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, { return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, {
method: 'POST', method: 'POST',

File diff suppressed because it is too large Load diff

View file

@ -1019,6 +1019,34 @@ export interface ResolvedAlert extends Alert {
resolvedTime: string; resolvedTime: string;
} }
export interface IncidentEvent {
id: string;
type: string;
timestamp: string;
summary: string;
details?: Record<string, unknown>;
}
export interface Incident {
id: string;
alertId: string;
alertType: string;
level: string;
resourceId: string;
resourceName: string;
resourceType?: string;
node?: string;
instance?: string;
message?: string;
status: string;
openedAt: string;
closedAt?: string;
acknowledged: boolean;
ackUser?: string;
ackTime?: string;
events?: IncidentEvent[];
}
// WebSocket message types // WebSocket message types
export type WSMessage = export type WSMessage =
| { type: 'initialState'; data: State } | { type: 'initialState'; data: State }

View file

@ -161,6 +161,17 @@ func (a *AlertTriggeredAnalyzer) analyzeResource(alert *alerts.Alert, resourceKe
Dur("duration", duration). Dur("duration", duration).
Msg("Alert-triggered AI analysis completed with no additional findings") Msg("Alert-triggered AI analysis completed with no additional findings")
} }
if a.patrolService != nil && a.patrolService.aiService != nil {
summary := "Alert-triggered AI analysis completed"
if len(findings) > 0 {
summary = fmt.Sprintf("Alert-triggered AI analysis found %d findings", len(findings))
}
a.patrolService.aiService.RecordIncidentAnalysis(alert.ID, summary, map[string]interface{}{
"findings": len(findings),
"duration": duration.String(),
})
}
} }
// analyzeResourceByAlert determines the resource type from the alert and analyzes it // analyzeResourceByAlert determines the resource type from the alert and analyzes it

View file

@ -0,0 +1,848 @@
package memory
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rs/zerolog/log"
)
// IncidentStatus represents the current state of an incident.
type IncidentStatus string
const (
IncidentStatusOpen IncidentStatus = "open"
IncidentStatusResolved IncidentStatus = "resolved"
)
// IncidentEventType describes a timeline event type.
type IncidentEventType string
const (
IncidentEventAlertFired IncidentEventType = "alert_fired"
IncidentEventAlertAcknowledged IncidentEventType = "alert_acknowledged"
IncidentEventAlertUnacknowledged IncidentEventType = "alert_unacknowledged"
IncidentEventAlertResolved IncidentEventType = "alert_resolved"
IncidentEventAnalysis IncidentEventType = "ai_analysis"
IncidentEventCommand IncidentEventType = "command"
IncidentEventRunbook IncidentEventType = "runbook"
IncidentEventNote IncidentEventType = "note"
)
// IncidentEvent represents a single timeline entry for an incident.
type IncidentEvent struct {
ID string `json:"id"`
Type IncidentEventType `json:"type"`
Timestamp time.Time `json:"timestamp"`
Summary string `json:"summary"`
Details map[string]interface{} `json:"details,omitempty"`
}
// Incident captures an alert occurrence and its timeline.
type Incident struct {
ID string `json:"id"`
AlertID string `json:"alertId"`
AlertType string `json:"alertType"`
Level string `json:"level"`
ResourceID string `json:"resourceId"`
ResourceName string `json:"resourceName"`
ResourceType string `json:"resourceType,omitempty"`
Node string `json:"node,omitempty"`
Instance string `json:"instance,omitempty"`
Message string `json:"message,omitempty"`
Status IncidentStatus `json:"status"`
OpenedAt time.Time `json:"openedAt"`
ClosedAt *time.Time `json:"closedAt,omitempty"`
Acknowledged bool `json:"acknowledged"`
AckUser string `json:"ackUser,omitempty"`
AckTime *time.Time `json:"ackTime,omitempty"`
Events []IncidentEvent `json:"events,omitempty"`
}
// IncidentStoreConfig configures incident retention and persistence.
type IncidentStoreConfig struct {
DataDir string
MaxIncidents int
MaxEventsPerIncident int
MaxAgeDays int
}
// IncidentStore maintains incident timelines and persistence.
type IncidentStore struct {
mu sync.RWMutex
saveMu sync.Mutex
incidents []*Incident
maxIncidents int
maxEvents int
maxAge time.Duration
dataDir string
filePath string
}
const (
defaultIncidentMaxIncidents = 500
defaultIncidentMaxEvents = 120
defaultIncidentMaxAgeDays = 90
incidentFileName = "ai_incidents.json"
maxIncidentFileSize = 20 * 1024 * 1024 // 20MB
incidentStartMatchTolerance = 10 * time.Minute
)
// NewIncidentStore creates a new incident store with persistence.
func NewIncidentStore(cfg IncidentStoreConfig) *IncidentStore {
maxIncidents := cfg.MaxIncidents
if maxIncidents <= 0 {
maxIncidents = defaultIncidentMaxIncidents
}
maxEvents := cfg.MaxEventsPerIncident
if maxEvents <= 0 {
maxEvents = defaultIncidentMaxEvents
}
maxAgeDays := cfg.MaxAgeDays
if maxAgeDays <= 0 {
maxAgeDays = defaultIncidentMaxAgeDays
}
store := &IncidentStore{
incidents: make([]*Incident, 0),
maxIncidents: maxIncidents,
maxEvents: maxEvents,
maxAge: time.Duration(maxAgeDays) * 24 * time.Hour,
dataDir: cfg.DataDir,
}
if store.dataDir != "" {
store.filePath = filepath.Join(store.dataDir, incidentFileName)
if err := store.loadFromDisk(); err != nil {
log.Warn().Err(err).Msg("Failed to load incident history from disk")
} else if len(store.incidents) > 0 {
log.Info().Int("count", len(store.incidents)).Msg("Loaded incident history from disk")
}
}
return store
}
// RecordAlertFired opens or updates an incident for a fired alert.
func (s *IncidentStore) RecordAlertFired(alert *alerts.Alert) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
if incident == nil {
incident = newIncidentFromAlert(alert)
s.incidents = append(s.incidents, incident)
s.addEventLocked(incident, IncidentEventAlertFired, formatAlertSummary(alert), map[string]interface{}{
"type": alert.Type,
"level": string(alert.Level),
"value": alert.Value,
"threshold": alert.Threshold,
})
} else {
updateIncidentFromAlert(incident, alert)
}
s.trimLocked()
s.saveAsync()
}
// RecordAlertAcknowledged records an acknowledgement event for an alert.
func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.ensureIncidentForAlertLocked(alert)
if incident == nil {
return
}
incident.Acknowledged = true
if alert.AckTime != nil {
incident.AckTime = alert.AckTime
} else {
now := time.Now()
incident.AckTime = &now
}
incident.AckUser = user
s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{
"user": user,
})
s.trimLocked()
s.saveAsync()
}
// RecordAlertUnacknowledged records an unacknowledge event for an alert.
func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user string) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.ensureIncidentForAlertLocked(alert)
if incident == nil {
return
}
incident.Acknowledged = false
incident.AckTime = nil
incident.AckUser = ""
s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{
"user": user,
})
s.trimLocked()
s.saveAsync()
}
// RecordAlertResolved records a resolved event and closes the incident.
func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time.Time) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
if incident == nil {
incident = newIncidentFromAlert(alert)
s.incidents = append(s.incidents, incident)
}
incident.Status = IncidentStatusResolved
if resolvedAt.IsZero() {
now := time.Now()
resolvedAt = now
}
incident.ClosedAt = &resolvedAt
s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{
"resolved_at": resolvedAt.Format(time.RFC3339),
})
s.trimLocked()
s.saveAsync()
}
// RecordAnalysis adds an AI analysis event to the incident for an alert.
func (s *IncidentStore) RecordAnalysis(alertID, summary string, details map[string]interface{}) {
if alertID == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
incident = &Incident{
ID: generateIncidentID(),
AlertID: alertID,
Status: IncidentStatusOpen,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, incident)
}
if summary == "" {
summary = "AI analysis completed"
}
s.addEventLocked(incident, IncidentEventAnalysis, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordCommand adds a command execution event to the incident for an alert.
func (s *IncidentStore) RecordCommand(alertID, command string, success bool, output string, details map[string]interface{}) {
if alertID == "" || command == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
incident = &Incident{
ID: generateIncidentID(),
AlertID: alertID,
Status: IncidentStatusOpen,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, incident)
}
if details == nil {
details = make(map[string]interface{})
}
details["command"] = command
details["success"] = success
if output != "" {
details["output_excerpt"] = truncateOutput(output, 500)
}
status := "failed"
if success {
status = "succeeded"
}
summary := fmt.Sprintf("Command %s: %s", status, command)
s.addEventLocked(incident, IncidentEventCommand, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordRunbook adds a runbook execution event to the incident for an alert.
func (s *IncidentStore) RecordRunbook(alertID, runbookID, title string, outcome string, automatic bool, message string) {
if alertID == "" || runbookID == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
incident = &Incident{
ID: generateIncidentID(),
AlertID: alertID,
Status: IncidentStatusOpen,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, incident)
}
summary := fmt.Sprintf("Runbook %s (%s)", title, outcome)
details := map[string]interface{}{
"runbook_id": runbookID,
"outcome": outcome,
"automatic": automatic,
}
if message != "" {
details["message"] = message
}
s.addEventLocked(incident, IncidentEventRunbook, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordNote appends a user note to an incident identified by alert ID or incident ID.
func (s *IncidentStore) RecordNote(alertID, incidentID, note, user string) bool {
note = strings.TrimSpace(note)
if note == "" {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
var incident *Incident
if incidentID != "" {
incident = s.findIncidentByIDLocked(incidentID)
} else if alertID != "" {
incident = s.findLatestIncidentByAlertIDLocked(alertID)
}
if incident == nil {
return false
}
summary := "Note added"
if user != "" {
summary = fmt.Sprintf("Note added by %s", user)
}
s.addEventLocked(incident, IncidentEventNote, summary, map[string]interface{}{
"note": note,
"user": user,
})
s.trimLocked()
s.saveAsync()
return true
}
// GetTimelineByAlertID returns the most recent incident for the alert.
func (s *IncidentStore) GetTimelineByAlertID(alertID string) *Incident {
if alertID == "" {
return nil
}
s.mu.RLock()
defer s.mu.RUnlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
return nil
}
return cloneIncident(incident)
}
// GetTimelineByAlertAt returns the incident closest to the provided start time for an alert.
func (s *IncidentStore) GetTimelineByAlertAt(alertID string, startedAt time.Time) *Incident {
if alertID == "" {
return nil
}
if startedAt.IsZero() {
return s.GetTimelineByAlertID(alertID)
}
s.mu.RLock()
defer s.mu.RUnlock()
var best *Incident
var bestDelta time.Duration
for _, incident := range s.incidents {
if incident == nil || incident.AlertID != alertID {
continue
}
delta := incident.OpenedAt.Sub(startedAt)
if delta < 0 {
delta = -delta
}
if best == nil || delta < bestDelta {
best = incident
bestDelta = delta
}
}
if best == nil || bestDelta > incidentStartMatchTolerance {
return nil
}
return cloneIncident(best)
}
// ListIncidentsByResource returns recent incidents for a resource.
func (s *IncidentStore) ListIncidentsByResource(resourceID string, limit int) []*Incident {
if resourceID == "" {
return nil
}
s.mu.RLock()
defer s.mu.RUnlock()
var matches []*Incident
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.ResourceID == resourceID {
matches = append(matches, cloneIncident(incident))
if limit > 0 && len(matches) >= limit {
break
}
}
}
return matches
}
// FormatForAlert returns a condensed incident timeline for prompt injection.
func (s *IncidentStore) FormatForAlert(alertID string, maxEvents int) string {
incident := s.GetTimelineByAlertID(alertID)
if incident == nil {
return ""
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString(fmt.Sprintf("Alert incident for %s (%s, %s)\n",
incident.ResourceName, incident.AlertType, incident.Level))
b.WriteString(fmt.Sprintf("Status: %s\n", incident.Status))
events := incident.Events
if maxEvents > 0 && len(events) > maxEvents {
events = events[len(events)-maxEvents:]
}
for _, evt := range events {
b.WriteString("- ")
b.WriteString(evt.Timestamp.Format(time.RFC3339))
b.WriteString(": ")
b.WriteString(evt.Summary)
b.WriteString("\n")
}
return b.String()
}
// FormatForResource returns a condensed incident summary for a resource.
func (s *IncidentStore) FormatForResource(resourceID string, limit int) string {
incidents := s.ListIncidentsByResource(resourceID, limit)
if len(incidents) == 0 {
return ""
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString("Recent incidents for this resource:\n")
for _, incident := range incidents {
status := string(incident.Status)
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
status = "acknowledged"
}
b.WriteString("- ")
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
b.WriteString(": ")
b.WriteString(incident.AlertType)
if incident.Level != "" {
b.WriteString(" (")
b.WriteString(incident.Level)
b.WriteString(")")
}
b.WriteString(" - ")
b.WriteString(status)
b.WriteString("\n")
}
return b.String()
}
// FormatForPatrol returns a condensed incident summary for infrastructure-wide patrol analysis.
func (s *IncidentStore) FormatForPatrol(limit int) string {
if limit <= 0 {
limit = 8
}
s.mu.RLock()
defer s.mu.RUnlock()
if len(s.incidents) == 0 {
return ""
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString("Recent incidents across infrastructure:\n")
count := 0
for i := len(s.incidents) - 1; i >= 0 && count < limit; i-- {
incident := s.incidents[i]
if incident == nil {
continue
}
status := string(incident.Status)
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
status = "acknowledged"
}
lastSummary := ""
if len(incident.Events) > 0 {
lastSummary = incident.Events[len(incident.Events)-1].Summary
}
b.WriteString("- ")
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
b.WriteString(": ")
if incident.ResourceName != "" {
b.WriteString(incident.ResourceName)
b.WriteString(" - ")
}
if incident.AlertType != "" {
b.WriteString(incident.AlertType)
}
if incident.Level != "" {
b.WriteString(" (")
b.WriteString(incident.Level)
b.WriteString(")")
}
b.WriteString(" - ")
b.WriteString(status)
if lastSummary != "" {
b.WriteString(" - last: ")
b.WriteString(truncateOutput(lastSummary, 80))
} else if incident.Message != "" {
b.WriteString(" - ")
b.WriteString(truncateOutput(incident.Message, 80))
}
b.WriteString("\n")
count++
}
return b.String()
}
func newIncidentFromAlert(alert *alerts.Alert) *Incident {
openedAt := alert.StartTime
if openedAt.IsZero() {
openedAt = time.Now()
}
return &Incident{
ID: generateIncidentID(),
AlertID: alert.ID,
AlertType: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
Instance: alert.Instance,
Message: alert.Message,
Status: IncidentStatusOpen,
OpenedAt: openedAt,
Acknowledged: alert.Acknowledged,
AckUser: alert.AckUser,
AckTime: alert.AckTime,
Events: make([]IncidentEvent, 0),
}
}
func updateIncidentFromAlert(incident *Incident, alert *alerts.Alert) {
if incident == nil || alert == nil {
return
}
incident.AlertType = alert.Type
incident.Level = string(alert.Level)
incident.ResourceID = alert.ResourceID
incident.ResourceName = alert.ResourceName
incident.Node = alert.Node
incident.Instance = alert.Instance
incident.Message = alert.Message
incident.Acknowledged = alert.Acknowledged
incident.AckUser = alert.AckUser
incident.AckTime = alert.AckTime
}
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident {
incident := s.findLatestIncidentByAlertIDLocked(alert.ID)
if incident == nil {
incident = newIncidentFromAlert(alert)
s.incidents = append(s.incidents, incident)
}
updateIncidentFromAlert(incident, alert)
return incident
}
func (s *IncidentStore) addEventLocked(incident *Incident, eventType IncidentEventType, summary string, details map[string]interface{}) {
if incident == nil {
return
}
if summary == "" {
summary = string(eventType)
}
event := IncidentEvent{
ID: generateIncidentEventID(),
Type: eventType,
Timestamp: time.Now(),
Summary: summary,
Details: details,
}
incident.Events = append(incident.Events, event)
if s.maxEvents > 0 && len(incident.Events) > s.maxEvents {
incident.Events = incident.Events[len(incident.Events)-s.maxEvents:]
}
}
func (s *IncidentStore) findOpenIncidentByAlertIDLocked(alertID string) *Incident {
if alertID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.AlertID == alertID && incident.Status == IncidentStatusOpen {
return incident
}
}
return nil
}
func (s *IncidentStore) findLatestIncidentByAlertIDLocked(alertID string) *Incident {
if alertID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.AlertID == alertID {
return incident
}
}
return nil
}
func (s *IncidentStore) findIncidentByIDLocked(incidentID string) *Incident {
if incidentID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.ID == incidentID {
return incident
}
}
return nil
}
func (s *IncidentStore) trimLocked() {
if s.maxAge > 0 {
cutoff := time.Now().Add(-s.maxAge)
filtered := make([]*Incident, 0, len(s.incidents))
for _, incident := range s.incidents {
if incident == nil {
continue
}
compareTime := incident.OpenedAt
if incident.ClosedAt != nil {
compareTime = *incident.ClosedAt
}
if compareTime.After(cutoff) {
filtered = append(filtered, incident)
}
}
s.incidents = filtered
}
if s.maxIncidents > 0 && len(s.incidents) > s.maxIncidents {
sort.Slice(s.incidents, func(i, j int) bool {
return s.incidents[i].OpenedAt.Before(s.incidents[j].OpenedAt)
})
if len(s.incidents) > s.maxIncidents {
s.incidents = s.incidents[len(s.incidents)-s.maxIncidents:]
}
}
}
func (s *IncidentStore) saveAsync() {
if s.dataDir == "" || s.filePath == "" {
return
}
go func() {
if err := s.saveToDisk(); err != nil {
log.Warn().Err(err).Msg("Failed to save incident history")
}
}()
}
func (s *IncidentStore) saveToDisk() error {
s.saveMu.Lock()
defer s.saveMu.Unlock()
if s.dataDir == "" || s.filePath == "" {
return nil
}
if err := os.MkdirAll(s.dataDir, 0755); err != nil {
return err
}
s.mu.RLock()
snapshot := make([]*Incident, 0, len(s.incidents))
for _, incident := range s.incidents {
snapshot = append(snapshot, cloneIncident(incident))
}
s.mu.RUnlock()
data, err := json.Marshal(snapshot)
if err != nil {
return err
}
tmpFile := s.filePath + ".tmp"
if err := os.WriteFile(tmpFile, data, 0644); err != nil {
return err
}
if err := os.Rename(tmpFile, s.filePath); err != nil {
return err
}
return nil
}
func (s *IncidentStore) loadFromDisk() error {
if s.filePath == "" {
return nil
}
info, err := os.Stat(s.filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if info.Size() > maxIncidentFileSize {
return fmt.Errorf("incident history file too large (%d bytes)", info.Size())
}
data, err := os.ReadFile(s.filePath)
if err != nil {
return err
}
var incidents []*Incident
if err := json.Unmarshal(data, &incidents); err != nil {
return err
}
s.incidents = incidents
s.trimLocked()
return nil
}
func cloneIncident(src *Incident) *Incident {
if src == nil {
return nil
}
clone := *src
if src.AckTime != nil {
t := *src.AckTime
clone.AckTime = &t
}
if src.ClosedAt != nil {
t := *src.ClosedAt
clone.ClosedAt = &t
}
if len(src.Events) > 0 {
clone.Events = make([]IncidentEvent, len(src.Events))
for i, event := range src.Events {
cloneEvent := event
if event.Details != nil {
detailsCopy := make(map[string]interface{}, len(event.Details))
for key, value := range event.Details {
detailsCopy[key] = value
}
cloneEvent.Details = detailsCopy
}
clone.Events[i] = cloneEvent
}
}
return &clone
}
var incidentCounter int64
func generateIncidentID() string {
incidentCounter++
return "inc-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentCounter%1000))
}
var incidentEventCounter int64
func generateIncidentEventID() string {
incidentEventCounter++
return "inc-evt-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentEventCounter%1000))
}
func formatAlertSummary(alert *alerts.Alert) string {
if alert == nil {
return "Alert triggered"
}
summary := fmt.Sprintf("Alert triggered: %s (%s)", alert.Type, alert.Level)
if alert.Value > 0 || alert.Threshold > 0 {
summary = fmt.Sprintf("Alert triggered: %s (%s %.1f >= %.1f)", alert.Type, alert.Level, alert.Value, alert.Threshold)
}
return summary
}

View file

@ -0,0 +1,108 @@
package memory
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
)
func TestIncidentStore_RecordTimeline(t *testing.T) {
store := NewIncidentStore(IncidentStoreConfig{
DataDir: t.TempDir(),
MaxIncidents: 10,
MaxEventsPerIncident: 10,
MaxAgeDays: 30,
})
alert := &alerts.Alert{
ID: "alert-1",
Type: "cpu",
Level: alerts.AlertLevelWarning,
ResourceID: "res-1",
ResourceName: "vm-1",
StartTime: time.Now().Add(-5 * time.Minute),
Value: 92,
Threshold: 85,
}
store.RecordAlertFired(alert)
store.RecordAlertAcknowledged(alert, "admin")
store.RecordAnalysis(alert.ID, "analysis complete", map[string]interface{}{
"findings": 1,
})
store.RecordCommand(alert.ID, "systemctl restart nginx", true, "ok", nil)
store.RecordAlertResolved(alert, time.Now())
timeline := store.GetTimelineByAlertID(alert.ID)
if timeline == nil {
t.Fatalf("expected timeline, got nil")
}
if timeline.Status != IncidentStatusResolved {
t.Fatalf("expected status %q, got %q", IncidentStatusResolved, timeline.Status)
}
if timeline.AckUser != "admin" {
t.Fatalf("expected ack user admin, got %q", timeline.AckUser)
}
if len(timeline.Events) < 4 {
t.Fatalf("expected events recorded, got %d", len(timeline.Events))
}
if ok := store.RecordNote(alert.ID, "", "note text", ""); !ok {
t.Fatalf("expected note to be saved")
}
}
func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) {
store := NewIncidentStore(IncidentStoreConfig{
DataDir: t.TempDir(),
MaxIncidents: 10,
MaxEventsPerIncident: 10,
MaxAgeDays: 30,
})
base := time.Now().UTC()
first := &alerts.Alert{
ID: "alert-2",
Type: "cpu",
Level: alerts.AlertLevelWarning,
ResourceID: "res-2",
ResourceName: "vm-2",
StartTime: base.Add(-2 * time.Hour),
}
second := &alerts.Alert{
ID: "alert-2",
Type: "cpu",
Level: alerts.AlertLevelWarning,
ResourceID: "res-2",
ResourceName: "vm-2",
StartTime: base.Add(-10 * time.Minute),
}
store.RecordAlertFired(first)
store.RecordAlertResolved(first, base.Add(-90*time.Minute))
store.RecordAlertFired(second)
timeline := store.GetTimelineByAlertAt(first.ID, first.StartTime)
if timeline == nil {
t.Fatalf("expected timeline for first incident, got nil")
}
if !timeline.OpenedAt.Equal(first.StartTime) {
t.Fatalf("expected openedAt %s, got %s", first.StartTime, timeline.OpenedAt)
}
timeline = store.GetTimelineByAlertAt(second.ID, second.StartTime)
if timeline == nil {
t.Fatalf("expected timeline for second incident, got nil")
}
if !timeline.OpenedAt.Equal(second.StartTime) {
t.Fatalf("expected openedAt %s, got %s", second.StartTime, timeline.OpenedAt)
}
timeline = store.GetTimelineByAlertAt(second.ID, base.Add(-45*time.Minute))
if timeline != nil {
t.Fatalf("expected no timeline for mismatched start time")
}
}

View file

@ -13,6 +13,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline" "github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context" aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge" "github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
@ -217,6 +218,7 @@ type PatrolService struct {
remediationLog *RemediationLog // For tracking remediation actions remediationLog *RemediationLog // For tracking remediation actions
patternDetector *PatternDetector // For failure prediction from historical patterns patternDetector *PatternDetector // For failure prediction from historical patterns
correlationDetector *CorrelationDetector // For multi-resource correlation correlationDetector *CorrelationDetector // For multi-resource correlation
incidentStore *memory.IncidentStore // For incident timeline capture
// Cached thresholds (recalculated when thresholdProvider changes) // Cached thresholds (recalculated when thresholdProvider changes)
thresholds PatrolThresholds thresholds PatrolThresholds
@ -263,6 +265,20 @@ func NewPatrolService(aiService *Service, stateProvider StateProvider) *PatrolSe
} }
} }
// SetIncidentStore attaches an incident store for alert timeline capture.
func (p *PatrolService) SetIncidentStore(store *memory.IncidentStore) {
p.mu.Lock()
defer p.mu.Unlock()
p.incidentStore = store
}
// GetIncidentStore returns the incident store if configured.
func (p *PatrolService) GetIncidentStore() *memory.IncidentStore {
p.mu.RLock()
defer p.mu.RUnlock()
return p.incidentStore
}
// SetConfig updates the patrol configuration // SetConfig updates the patrol configuration
func (p *PatrolService) SetConfig(cfg PatrolConfig) { func (p *PatrolService) SetConfig(cfg PatrolConfig) {
p.mu.Lock() p.mu.Lock()
@ -2185,12 +2201,17 @@ func (p *PatrolService) buildPatrolPrompt(summary string) string {
// Get resource notes from knowledge store (per-resource user notes) // Get resource notes from knowledge store (per-resource user notes)
var knowledgeContext string var knowledgeContext string
var incidentContext string
p.mu.RLock() p.mu.RLock()
knowledgeStore := p.knowledgeStore knowledgeStore := p.knowledgeStore
incidentStore := p.incidentStore
p.mu.RUnlock() p.mu.RUnlock()
if knowledgeStore != nil { if knowledgeStore != nil {
knowledgeContext = knowledgeStore.FormatAllForContext() knowledgeContext = knowledgeStore.FormatAllForContext()
} }
if incidentStore != nil {
incidentContext = incidentStore.FormatForPatrol(8)
}
basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities. basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities.
@ -2235,6 +2256,12 @@ IMPORTANT: Respect the user's feedback above. Do NOT re-raise findings that are:
Only report NEW issues or issues where the severity has clearly escalated.`) Only report NEW issues or issues where the severity has clearly escalated.`)
} }
if incidentContext != "" {
contextAdditions.WriteString("\n\n")
contextAdditions.WriteString(incidentContext)
contextAdditions.WriteString("\nIMPORTANT: Use incident memory to avoid repeating known issues and to build on successful past investigations.")
}
if contextAdditions.Len() > 0 { if contextAdditions.Len() > 0 {
return basePrompt + contextAdditions.String() return basePrompt + contextAdditions.String()
} }

View file

@ -323,6 +323,10 @@ func (p *PatrolService) logRunbookExecution(finding *Finding, runbook Runbook, s
if err := p.remediationLog.Log(record); err != nil { if err := p.remediationLog.Log(record); err != nil {
log.Warn().Err(err).Msg("Failed to log runbook execution") log.Warn().Err(err).Msg("Failed to log runbook execution")
} }
if p.aiService != nil && finding != nil && finding.AlertID != "" {
p.aiService.RecordIncidentRunbook(finding.AlertID, runbook.ID, runbook.Title, outcome, automatic, message)
}
} }
type runbookContext struct { type runbookContext struct {

View file

@ -18,8 +18,10 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost" "github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge" "github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers" "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
"github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/models"
@ -73,9 +75,10 @@ type Service struct {
alertProvider AlertProvider alertProvider AlertProvider
knowledgeStore *knowledge.Store knowledgeStore *knowledge.Store
costStore *cost.Store costStore *cost.Store
resourceProvider ResourceProvider // Unified resource model provider (Phase 2) resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
patrolService *PatrolService // Background AI monitoring service patrolService *PatrolService // Background AI monitoring service
metadataProvider MetadataProvider // Enables AI to update resource URLs metadataProvider MetadataProvider // Enables AI to update resource URLs
incidentStore *memory.IncidentStore // Incident timelines for alert memory
// Alert-triggered analysis - token-efficient real-time AI insights // Alert-triggered analysis - token-efficient real-time AI insights
alertTriggeredAnalyzer *AlertTriggeredAnalyzer alertTriggeredAnalyzer *AlertTriggeredAnalyzer
@ -197,6 +200,9 @@ func (s *Service) SetStateProvider(sp StateProvider) {
if s.knowledgeStore != nil { if s.knowledgeStore != nil {
s.patrolService.SetKnowledgeStore(s.knowledgeStore) s.patrolService.SetKnowledgeStore(s.knowledgeStore)
} }
if s.incidentStore != nil {
s.patrolService.SetIncidentStore(s.incidentStore)
}
} }
// Initialize alert-triggered analyzer if not already done // Initialize alert-triggered analyzer if not already done
@ -350,6 +356,17 @@ func (s *Service) SetRemediationLog(remLog *RemediationLog) {
} }
} }
// SetIncidentStore sets the incident store for alert timeline memory.
func (s *Service) SetIncidentStore(store *memory.IncidentStore) {
s.mu.Lock()
defer s.mu.Unlock()
s.incidentStore = store
if s.patrolService != nil {
s.patrolService.SetIncidentStore(store)
}
}
// SetPatternDetector sets the pattern detector for failure prediction // SetPatternDetector sets the pattern detector for failure prediction
func (s *Service) SetPatternDetector(detector *PatternDetector) { func (s *Service) SetPatternDetector(detector *PatternDetector) {
s.mu.RLock() s.mu.RLock()
@ -1740,11 +1757,11 @@ func isActionableCommand(cmd string) bool {
"service restart", "service start", "service stop", "service restart", "service start", "service stop",
"pct resize", "pct start", "pct stop", "pct shutdown", "pct reboot", "pct resize", "pct start", "pct stop", "pct shutdown", "pct reboot",
"qm resize", "qm start", "qm stop", "qm shutdown", "qm reboot", "qm resize", "qm start", "qm stop", "qm shutdown", "qm reboot",
"rm -", "rm /", // File deletion/cleanup "rm -", "rm /", // File deletion/cleanup
"chmod", "chown", // Permission fixes "chmod", "chown", // Permission fixes
"mkdir", // Creating directories "mkdir", // Creating directories
"mv ", "cp ", // File operations "mv ", "cp ", // File operations
"echo >", "tee ", // Writing to files "echo >", "tee ", // Writing to files
"apt install", "apt upgrade", "apt remove", "apt install", "apt upgrade", "apt remove",
"yum install", "dnf install", "yum install", "dnf install",
"pip install", "npm install", "pip install", "npm install",
@ -2183,11 +2200,34 @@ func (s *Service) executeTool(ctx context.Context, req ExecuteRequest, tc provid
// Execute via agent // Execute via agent
result, err := s.executeOnAgent(ctx, execReq, command) result, err := s.executeOnAgent(ctx, execReq, command)
recordIncident := func(success bool, output string) {
alertID := extractAlertID(req.Context)
if alertID == "" {
return
}
s.mu.RLock()
incidentStore := s.incidentStore
s.mu.RUnlock()
if incidentStore == nil {
return
}
details := map[string]interface{}{
"resource_id": req.TargetID,
"resource_type": req.TargetType,
"run_on_host": runOnHost,
}
if targetHost != "" {
details["target_host"] = targetHost
}
incidentStore.RecordCommand(alertID, command, success, output, details)
}
if err != nil { if err != nil {
recordIncident(false, result)
execution.Output = fmt.Sprintf("Error executing command: %s", err) execution.Output = fmt.Sprintf("Error executing command: %s", err)
return execution.Output, execution return execution.Output, execution
} }
recordIncident(true, result)
execution.Output = result execution.Output = result
execution.Success = true execution.Success = true
return result, execution return result, execution
@ -2842,6 +2882,31 @@ This is a 3-command job. Don't over-investigate.`
// Add current alert status - this gives AI awareness of active issues // Add current alert status - this gives AI awareness of active issues
prompt += s.buildAlertContext() prompt += s.buildAlertContext()
// Add incident memory for alert or resource context
alertID := ""
resourceID := req.TargetID
if req.Context != nil {
if val, ok := req.Context["alertId"].(string); ok && val != "" {
alertID = val
} else if val, ok := req.Context["alert_id"].(string); ok && val != "" {
alertID = val
}
if resourceID == "" {
if val, ok := req.Context["resourceId"].(string); ok && val != "" {
resourceID = val
} else if val, ok := req.Context["resource_id"].(string); ok && val != "" {
resourceID = val
} else if val, ok := req.Context["guest_id"].(string); ok && val != "" {
resourceID = val
}
}
}
if incidentContext := s.buildIncidentContext(resourceID, alertID); incidentContext != "" {
prompt += incidentContext
}
// Add all saved knowledge when no specific target is selected // Add all saved knowledge when no specific target is selected
// This gives the AI context about everything learned from previous sessions // This gives the AI context about everything learned from previous sessions
if req.TargetType == "" && s.knowledgeStore != nil { if req.TargetType == "" && s.knowledgeStore != nil {
@ -2877,6 +2942,7 @@ This is a 3-command job. Don't over-investigate.`
// Add past remediation history for this resource // Add past remediation history for this resource
prompt += s.buildRemediationContext(req.TargetID, req.Prompt) prompt += s.buildRemediationContext(req.TargetID, req.Prompt)
} }
// Add any provided context in a structured way // Add any provided context in a structured way
@ -3258,6 +3324,66 @@ func (s *Service) buildRemediationContext(resourceID, currentProblem string) str
return context return context
} }
// buildIncidentContext adds incident timeline context for alerts/resources.
func (s *Service) buildIncidentContext(resourceID, alertID string) string {
s.mu.RLock()
store := s.incidentStore
s.mu.RUnlock()
if store == nil {
return ""
}
if alertID != "" {
return store.FormatForAlert(alertID, 8)
}
if resourceID != "" {
return store.FormatForResource(resourceID, 4)
}
return ""
}
// RecordIncidentAnalysis stores an AI analysis event for an alert.
func (s *Service) RecordIncidentAnalysis(alertID, summary string, details map[string]interface{}) {
if alertID == "" {
return
}
s.mu.RLock()
store := s.incidentStore
s.mu.RUnlock()
if store == nil {
return
}
store.RecordAnalysis(alertID, summary, details)
}
// RecordIncidentRunbook stores a runbook execution event for an alert.
func (s *Service) RecordIncidentRunbook(alertID, runbookID, title string, outcome memory.Outcome, automatic bool, message string) {
if alertID == "" || runbookID == "" {
return
}
s.mu.RLock()
store := s.incidentStore
s.mu.RUnlock()
if store == nil {
return
}
store.RecordRunbook(alertID, runbookID, title, string(outcome), automatic, message)
}
func extractAlertID(ctx map[string]interface{}) string {
if ctx == nil {
return ""
}
if alertID, ok := ctx["alertId"].(string); ok && alertID != "" {
return alertID
}
if alertID, ok := ctx["alert_id"].(string); ok && alertID != "" {
return alertID
}
return ""
}
// truncateString truncates a string to maxLen characters // truncateString truncates a string to maxLen characters
func truncateString(s string, maxLen int) string { func truncateString(s string, maxLen int) string {
if len(s) <= maxLen { if len(s) <= maxLen {
@ -3302,40 +3428,51 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
return 0, false return 0, false
} }
// Check CPU baseline // Helper to format baseline comparison with status
// ALWAYS shows baseline info when available - this is Pulse Pro's value-add
formatBaselineComparison := func(metric string, currentVal float64, bl *baseline.MetricBaseline) string {
if bl.SampleCount < 10 {
return fmt.Sprintf("- %s: %.1f%% (baseline learning: %d/10 samples)", metric, currentVal, bl.SampleCount)
}
ratio := currentVal / bl.Mean
status := "✓ normal"
if ratio > 2.0 {
status = "🔴 **ANOMALY**"
} else if ratio > 1.5 {
status = "🟡 elevated"
} else if ratio < 0.3 {
status = "🔵 unusually low"
} else if ratio < 0.5 {
status = "low"
}
return fmt.Sprintf("- %s: %.1f%% vs baseline %.1f%% (σ=%.1f) → %s", metric, currentVal, bl.Mean, bl.StdDev, status)
}
// Check CPU baseline - always show if we have data
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok { if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists && bl.SampleCount >= 10 { if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists {
ratio := cpuVal / bl.Mean baselineInfo = append(baselineInfo, formatBaselineComparison("CPU", cpuVal, bl))
if ratio > 1.5 {
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx higher** than baseline %.0f%% (σ=%.1f)", cpuVal, ratio, bl.Mean, bl.StdDev))
} else if ratio < 0.5 {
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx lower** than baseline %.0f%%", cpuVal, 1/ratio, bl.Mean))
}
} }
} }
// Check memory baseline // Check memory baseline - always show if we have data
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok { if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists && bl.SampleCount >= 10 { if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists {
ratio := memVal / bl.Mean baselineInfo = append(baselineInfo, formatBaselineComparison("Memory", memVal, bl))
if ratio > 1.3 {
baselineInfo = append(baselineInfo, fmt.Sprintf("Memory %.0f%% is **%.1fx higher** than baseline %.0f%%", memVal, ratio, bl.Mean))
}
} }
} }
// Check disk baseline // Check disk baseline - always show if we have data
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok { if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists && bl.SampleCount >= 10 { if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists {
ratio := diskVal / bl.Mean baselineInfo = append(baselineInfo, formatBaselineComparison("Disk", diskVal, bl))
if ratio > 1.2 { // More sensitive for disk
baselineInfo = append(baselineInfo, fmt.Sprintf("Disk %.0f%% is **%.1fx higher** than baseline %.0f%%", diskVal, ratio, bl.Mean))
}
} }
} }
if len(baselineInfo) > 0 { if len(baselineInfo) > 0 {
sections = append(sections, "### Baseline Comparisons\n"+strings.Join(baselineInfo, "\n")) sections = append(sections, "### Learned Baselines (7-day patterns)\n"+strings.Join(baselineInfo, "\n"))
} }
} }

View file

@ -58,7 +58,7 @@ const (
DefaultObservationWindow = 24 // hours DefaultObservationWindow = 24 // hours
// Flapping detection // Flapping detection
DefaultFlappingWindow = 300 // seconds (5 minutes) DefaultFlappingWindow = 300 // seconds (5 minutes)
DefaultFlappingThreshold = 5 // state changes to trigger flapping DefaultFlappingThreshold = 5 // state changes to trigger flapping
DefaultFlappingCooldown = 15 // minutes DefaultFlappingCooldown = 15 // minutes
@ -256,10 +256,10 @@ type GroupingConfig struct {
// ScheduleConfig represents alerting schedule configuration // ScheduleConfig represents alerting schedule configuration
type ScheduleConfig struct { type ScheduleConfig struct {
QuietHours QuietHours `json:"quietHours"` QuietHours QuietHours `json:"quietHours"`
Cooldown int `json:"cooldown"` // minutes Cooldown int `json:"cooldown"` // minutes
GroupingWindow int `json:"groupingWindow,omitempty"` // Deprecated: use Grouping.Window instead. Will be auto-migrated on config update. GroupingWindow int `json:"groupingWindow,omitempty"` // Deprecated: use Grouping.Window instead. Will be auto-migrated on config update.
MaxAlertsHour int `json:"maxAlertsHour"` // max alerts per hour per resource MaxAlertsHour int `json:"maxAlertsHour"` // max alerts per hour per resource
NotifyOnResolve bool `json:"notifyOnResolve"` // Send notification when alert clears NotifyOnResolve bool `json:"notifyOnResolve"` // Send notification when alert clears
Escalation EscalationConfig `json:"escalation"` Escalation EscalationConfig `json:"escalation"`
Grouping GroupingConfig `json:"grouping"` Grouping GroupingConfig `json:"grouping"`
} }
@ -484,15 +484,17 @@ func SetMetricHooks(fired func(*Alert), resolved func(*Alert), suppressed func(s
} }
type Manager struct { type Manager struct {
mu sync.RWMutex mu sync.RWMutex
config AlertConfig config AlertConfig
activeAlerts map[string]*Alert activeAlerts map[string]*Alert
historyManager *HistoryManager historyManager *HistoryManager
onAlert func(alert *Alert) onAlert func(alert *Alert)
onResolved func(alertID string) onResolved func(alertID string)
onEscalate func(alert *Alert, level int) onAcknowledged func(alert *Alert, user string)
escalationStop chan struct{} onUnacknowledged func(alert *Alert, user string)
alertRateLimit map[string][]time.Time // Track alert times for rate limiting onEscalate func(alert *Alert, level int)
escalationStop chan struct{}
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
// New fields for deduplication and suppression // New fields for deduplication and suppression
recentAlerts map[string]*Alert // Track recent alerts for deduplication recentAlerts map[string]*Alert // Track recent alerts for deduplication
suppressedUntil map[string]time.Time // Track suppression windows suppressedUntil map[string]time.Time // Track suppression windows
@ -736,6 +738,20 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
m.onResolved = cb m.onResolved = cb
} }
// SetAcknowledgedCallback sets the callback for acknowledged alerts.
func (m *Manager) SetAcknowledgedCallback(cb func(alert *Alert, user string)) {
m.mu.Lock()
defer m.mu.Unlock()
m.onAcknowledged = cb
}
// SetUnacknowledgedCallback sets the callback for unacknowledged alerts.
func (m *Manager) SetUnacknowledgedCallback(cb func(alert *Alert, user string)) {
m.mu.Lock()
defer m.mu.Unlock()
m.onUnacknowledged = cb
}
// SetEscalateCallback sets the callback for escalated alerts // SetEscalateCallback sets the callback for escalated alerts
func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) { func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) {
m.mu.Lock() m.mu.Lock()
@ -768,6 +784,46 @@ func (m *Manager) safeCallResolvedCallback(alertID string, async bool) {
} }
} }
// safeCallAcknowledgedCallback invokes onAcknowledged with panic recovery and alert cloning.
func (m *Manager) safeCallAcknowledgedCallback(alert *Alert, user string) {
if m.onAcknowledged == nil || alert == nil {
return
}
alertCopy := alert.Clone()
go func(a *Alert, u string) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", a.ID).
Msg("Panic in onAcknowledged callback")
}
}()
m.onAcknowledged(a, u)
}(alertCopy, user)
}
// safeCallUnacknowledgedCallback invokes onUnacknowledged with panic recovery and alert cloning.
func (m *Manager) safeCallUnacknowledgedCallback(alert *Alert, user string) {
if m.onUnacknowledged == nil || alert == nil {
return
}
alertCopy := alert.Clone()
go func(a *Alert, u string) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", a.ID).
Msg("Panic in onUnacknowledged callback")
}
}()
m.onUnacknowledged(a, u)
}(alertCopy, user)
}
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning // safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) { func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
if m.onEscalate == nil { if m.onEscalate == nil {
@ -2457,7 +2513,6 @@ func (m *Manager) hasHostAgentForNode(nodeName string) bool {
return exists return exists
} }
func hostResourceID(hostID string) string { func hostResourceID(hostID string) string {
trimmed := strings.TrimSpace(hostID) trimmed := strings.TrimSpace(hostID)
if trimmed == "" { if trimmed == "" {
@ -5767,10 +5822,10 @@ func abs(x float64) float64 {
// AcknowledgeAlert acknowledges an alert // AcknowledgeAlert acknowledges an alert
func (m *Manager) AcknowledgeAlert(alertID, user string) error { func (m *Manager) AcknowledgeAlert(alertID, user string) error {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.activeAlerts[alertID] alert, exists := m.activeAlerts[alertID]
if !exists { if !exists {
m.mu.Unlock()
return fmt.Errorf("alert not found: %s", alertID) return fmt.Errorf("alert not found: %s", alertID)
} }
@ -5787,22 +5842,26 @@ func (m *Manager) AcknowledgeAlert(alertID, user string) error {
time: now, time: now,
} }
alertCopy := alert.Clone()
m.mu.Unlock()
log.Debug(). log.Debug().
Str("alertID", alertID). Str("alertID", alertID).
Str("user", user). Str("user", user).
Time("ackTime", now). Time("ackTime", now).
Msg("Alert acknowledgment recorded") Msg("Alert acknowledgment recorded")
m.safeCallAcknowledgedCallback(alertCopy, user)
return nil return nil
} }
// UnacknowledgeAlert removes the acknowledged status from an alert // UnacknowledgeAlert removes the acknowledged status from an alert
func (m *Manager) UnacknowledgeAlert(alertID string) error { func (m *Manager) UnacknowledgeAlert(alertID string) error {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.activeAlerts[alertID] alert, exists := m.activeAlerts[alertID]
if !exists { if !exists {
m.mu.Unlock()
return fmt.Errorf("alert not found: %s", alertID) return fmt.Errorf("alert not found: %s", alertID)
} }
@ -5814,10 +5873,14 @@ func (m *Manager) UnacknowledgeAlert(alertID string) error {
m.activeAlerts[alertID] = alert m.activeAlerts[alertID] = alert
delete(m.ackState, alertID) delete(m.ackState, alertID)
alertCopy := alert.Clone()
m.mu.Unlock()
log.Info(). log.Info().
Str("alertID", alertID). Str("alertID", alertID).
Msg("Alert unacknowledged") Msg("Alert unacknowledged")
m.safeCallUnacknowledgedCallback(alertCopy, "")
return nil return nil
} }
@ -7593,9 +7656,8 @@ func (m *Manager) ClearAlert(alertID string) bool {
// Cleanup removes old acknowledged alerts and cleans up tracking maps // Cleanup removes old acknowledged alerts and cleans up tracking maps
func (m *Manager) Cleanup(maxAge time.Duration) { func (m *Manager) Cleanup(maxAge time.Duration) {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock()
now := time.Now() now := time.Now()
var autoAcked []*Alert
// Auto-acknowledge old alerts if configured // Auto-acknowledge old alerts if configured
if m.config.AutoAcknowledgeAfterHours > 0 { if m.config.AutoAcknowledgeAfterHours > 0 {
@ -7610,6 +7672,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
ackTime := now ackTime := now
alert.AckTime = &ackTime alert.AckTime = &ackTime
alert.AckUser = "system-auto" alert.AckUser = "system-auto"
autoAcked = append(autoAcked, alert.Clone())
if recordAlertAcknowledged != nil { if recordAlertAcknowledged != nil {
recordAlertAcknowledged() recordAlertAcknowledged()
@ -7776,6 +7839,12 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
Msg("Cleaned up stale PMG quarantine history") Msg("Cleaned up stale PMG quarantine history")
} }
} }
m.mu.Unlock()
for _, alert := range autoAcked {
m.safeCallAcknowledgedCallback(alert, "system-auto")
}
} }
// convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold // convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold

View file

@ -19,6 +19,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai" "github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost" "github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers" "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
"github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/license" "github.com/rcourtman/pulse-go-rewrite/internal/license"
@ -118,6 +119,11 @@ func (h *AISettingsHandler) SetRemediationLog(remLog *ai.RemediationLog) {
h.aiService.SetRemediationLog(remLog) h.aiService.SetRemediationLog(remLog)
} }
// SetIncidentStore sets the incident store for alert timelines.
func (h *AISettingsHandler) SetIncidentStore(store *memory.IncidentStore) {
h.aiService.SetIncidentStore(store)
}
// SetPatternDetector sets the pattern detector for failure prediction // SetPatternDetector sets the pattern detector for failure prediction
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) { func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
h.aiService.SetPatternDetector(detector) h.aiService.SetPatternDetector(detector)
@ -1767,11 +1773,29 @@ func (h *AISettingsHandler) HandleInvestigateAlert(w http.ResponseWriter, r *htt
data, _ := json.Marshal(finalEvent) data, _ := json.Marshal(finalEvent)
safeWrite([]byte("data: " + string(data) + "\n\n")) safeWrite([]byte("data: " + string(data) + "\n\n"))
if req.AlertID != "" {
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI alert investigation completed", map[string]interface{}{
"model": resp.Model,
"tool_calls": len(resp.ToolCalls),
"input_tokens": resp.InputTokens,
"output_tokens": resp.OutputTokens,
})
}
log.Info(). log.Info().
Str("alert_id", req.AlertID). Str("alert_id", req.AlertID).
Str("model", resp.Model). Str("model", resp.Model).
Int("tool_calls", len(resp.ToolCalls)). Int("tool_calls", len(resp.ToolCalls)).
Msg("AI alert investigation completed") Msg("AI alert investigation completed")
if req.AlertID != "" {
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI investigation completed", map[string]interface{}{
"model": resp.Model,
"input_tokens": resp.InputTokens,
"output_tokens": resp.OutputTokens,
"tool_calls": len(resp.ToolCalls),
})
}
} }
// SetAlertProvider sets the alert provider for AI context // SetAlertProvider sets the alert provider for AI context

View file

@ -8,6 +8,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/mock"
@ -361,6 +362,130 @@ func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request)
} }
} }
// GetAlertIncidentTimeline returns the incident timeline for an alert or resource.
func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
store := h.monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
}
query := r.URL.Query()
alertID := strings.TrimSpace(query.Get("alert_id"))
resourceID := strings.TrimSpace(query.Get("resource_id"))
startedAtRaw := strings.TrimSpace(query.Get("started_at"))
if startedAtRaw == "" {
startedAtRaw = strings.TrimSpace(query.Get("start_time"))
}
limit := 0
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
if parsed, err := strconv.Atoi(rawLimit); err == nil && parsed > 0 {
limit = parsed
}
}
if alertID != "" {
if !validateAlertID(alertID) {
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
return
}
var startedAt time.Time
if startedAtRaw != "" {
parsed, err := time.Parse(time.RFC3339, startedAtRaw)
if err != nil {
http.Error(w, "Invalid started_at time", http.StatusBadRequest)
return
}
startedAt = parsed
}
var incident *memory.Incident
if !startedAt.IsZero() {
incident = store.GetTimelineByAlertAt(alertID, startedAt)
} else {
incident = store.GetTimelineByAlertID(alertID)
}
if err := utils.WriteJSONResponse(w, incident); err != nil {
log.Error().Err(err).Msg("Failed to write incident timeline response")
}
return
}
if resourceID != "" {
if len(resourceID) > 500 {
http.Error(w, "Invalid resource ID", http.StatusBadRequest)
return
}
incidents := store.ListIncidentsByResource(resourceID, limit)
if err := utils.WriteJSONResponse(w, incidents); err != nil {
log.Error().Err(err).Msg("Failed to write incident list response")
}
return
}
http.Error(w, "Missing alert_id or resource_id", http.StatusBadRequest)
}
// SaveAlertIncidentNote stores a user note in the incident timeline.
func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
store := h.monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 8*1024)
var req struct {
AlertID string `json:"alert_id"`
IncidentID string `json:"incident_id"`
Note string `json:"note"`
User string `json:"user,omitempty"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid request body", http.StatusBadRequest)
return
}
req.AlertID = strings.TrimSpace(req.AlertID)
req.IncidentID = strings.TrimSpace(req.IncidentID)
req.Note = strings.TrimSpace(req.Note)
req.User = strings.TrimSpace(req.User)
if req.AlertID == "" && req.IncidentID == "" {
http.Error(w, "alert_id or incident_id is required", http.StatusBadRequest)
return
}
if req.AlertID != "" && !validateAlertID(req.AlertID) {
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
return
}
if req.Note == "" {
http.Error(w, "note is required", http.StatusBadRequest)
return
}
if ok := store.RecordNote(req.AlertID, req.IncidentID, req.Note, req.User); !ok {
http.Error(w, "Failed to save note", http.StatusBadRequest)
return
}
if err := utils.WriteJSONResponse(w, map[string]interface{}{
"success": true,
}); err != nil {
log.Error().Err(err).Msg("Failed to write incident note response")
}
}
// ClearAlertHistory clears all alert history // ClearAlertHistory clears all alert history
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) { func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil { if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil {
@ -737,6 +862,16 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
return return
} }
h.GetAlertHistory(w, r) h.GetAlertHistory(w, r)
case path == "incidents" && r.Method == http.MethodGet:
if !ensureScope(w, r, config.ScopeMonitoringRead) {
return
}
h.GetAlertIncidentTimeline(w, r)
case path == "incidents/note" && r.Method == http.MethodPost:
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
return
}
h.SaveAlertIncidentNote(w, r)
case path == "history" && r.Method == http.MethodDelete: case path == "history" && r.Method == http.MethodDelete:
if !ensureScope(w, r, config.ScopeMonitoringWrite) { if !ensureScope(w, r, config.ScopeMonitoringWrite) {
return return

View file

@ -1109,6 +1109,9 @@ func (r *Router) setupRoutes() {
if alertManager := r.monitor.GetAlertManager(); alertManager != nil { if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager)) r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager))
} }
if incidentStore := r.monitor.GetIncidentStore(); incidentStore != nil {
r.aiSettingsHandler.SetIncidentStore(incidentStore)
}
} }
// Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view) // Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view)
if r.resourceHandlers != nil { if r.resourceHandlers != nil {

View file

@ -20,6 +20,7 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/discovery" "github.com/rcourtman/pulse-go-rewrite/internal/discovery"
@ -604,6 +605,7 @@ type Monitor struct {
metricsHistory *MetricsHistory metricsHistory *MetricsHistory
metricsStore *metrics.Store // Persistent SQLite metrics storage metricsStore *metrics.Store // Persistent SQLite metrics storage
alertManager *alerts.Manager alertManager *alerts.Manager
incidentStore *memory.IncidentStore
notificationMgr *notifications.NotificationManager notificationMgr *notifications.NotificationManager
configPersist *config.ConfigPersistence configPersist *config.ConfigPersistence
discoveryService *discovery.Service // Background discovery service discoveryService *discovery.Service // Background discovery service
@ -3050,6 +3052,10 @@ func New(cfg *config.Config) (*Monitor, error) {
Msg("Persistent metrics store initialized with configurable retention") Msg("Persistent metrics store initialized with configurable retention")
} }
incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{
DataDir: cfg.DataPath,
})
m := &Monitor{ m := &Monitor{
config: cfg, config: cfg,
state: models.NewState(), state: models.NewState(),
@ -3076,6 +3082,7 @@ func New(cfg *config.Config) (*Monitor, error) {
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
metricsStore: metricsStore, // Persistent SQLite storage metricsStore: metricsStore, // Persistent SQLite storage
alertManager: alerts.NewManager(), alertManager: alerts.NewManager(),
incidentStore: incidentStore,
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL), notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
configPersist: config.NewConfigPersistence(cfg.DataPath), configPersist: config.NewConfigPersistence(cfg.DataPath),
discoveryService: nil, // Will be initialized in Start() discoveryService: nil, // Will be initialized in Start()
@ -3756,25 +3763,19 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
// Set up alert callbacks // Set up alert callbacks
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) { m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
wsHub.BroadcastAlert(alert) m.handleAlertFired(alert)
// Send notifications
log.Debug().
Str("alertID", alert.ID).
Str("level", string(alert.Level)).
Msg("Alert raised, sending to notification manager")
go m.notificationMgr.SendAlert(alert)
}) })
m.alertManager.SetResolvedCallback(func(alertID string) { m.alertManager.SetResolvedCallback(func(alertID string) {
wsHub.BroadcastAlertResolved(alertID) m.handleAlertResolved(alertID)
m.notificationMgr.CancelAlert(alertID)
if m.notificationMgr.GetNotifyOnResolve() {
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
go m.notificationMgr.SendResolvedAlert(resolved)
}
}
// Don't broadcast full state here - it causes a cascade with many guests. // Don't broadcast full state here - it causes a cascade with many guests.
// The frontend will get the updated alerts through the regular broadcast ticker. // The frontend will get the updated alerts through the regular broadcast ticker.
}) })
m.alertManager.SetAcknowledgedCallback(func(alert *alerts.Alert, user string) {
m.handleAlertAcknowledged(alert, user)
})
m.alertManager.SetUnacknowledgedCallback(func(alert *alerts.Alert, user string) {
m.handleAlertUnacknowledged(alert, user)
})
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) { m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
log.Info(). log.Info().
Str("alertID", alert.ID). Str("alertID", alert.ID).
@ -7731,6 +7732,11 @@ func (m *Monitor) GetAlertManager() *alerts.Manager {
return m.alertManager return m.alertManager
} }
// GetIncidentStore returns the incident timeline store.
func (m *Monitor) GetIncidentStore() *memory.IncidentStore {
return m.incidentStore
}
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire // SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
// This enables token-efficient, real-time AI insights on specific resources // This enables token-efficient, real-time AI insights on specific resources
func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) { func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
@ -7738,33 +7744,69 @@ func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
return return
} }
// Get the current callback
originalCallback := m.alertManager
// Wrap the existing callback to also call the AI callback // Wrap the existing callback to also call the AI callback
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) { m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
// Broadcast to WebSocket (this happens via the callback set in Start()) m.handleAlertFired(alert)
if m.wsHub != nil {
m.wsHub.BroadcastAlert(alert)
}
// Send notifications
log.Debug().
Str("alertID", alert.ID).
Str("level", string(alert.Level)).
Msg("Alert raised, sending to notification manager")
go m.notificationMgr.SendAlert(alert)
// Trigger AI analysis // Trigger AI analysis
go callback(alert) go callback(alert)
}) })
// Avoid unused variable warning
_ = originalCallback
log.Info().Msg("Alert-triggered AI callback registered") log.Info().Msg("Alert-triggered AI callback registered")
} }
func (m *Monitor) handleAlertFired(alert *alerts.Alert) {
if alert == nil {
return
}
if m.wsHub != nil {
m.wsHub.BroadcastAlert(alert)
}
log.Debug().
Str("alertID", alert.ID).
Str("level", string(alert.Level)).
Msg("Alert raised, sending to notification manager")
go m.notificationMgr.SendAlert(alert)
if m.incidentStore != nil {
m.incidentStore.RecordAlertFired(alert)
}
}
func (m *Monitor) handleAlertResolved(alertID string) {
if m.wsHub != nil {
m.wsHub.BroadcastAlertResolved(alertID)
}
m.notificationMgr.CancelAlert(alertID)
if m.notificationMgr.GetNotifyOnResolve() {
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
go m.notificationMgr.SendResolvedAlert(resolved)
}
}
if m.incidentStore != nil {
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil && resolved.Alert != nil {
m.incidentStore.RecordAlertResolved(resolved.Alert, resolved.ResolvedTime)
}
}
}
func (m *Monitor) handleAlertAcknowledged(alert *alerts.Alert, user string) {
if m.incidentStore == nil || alert == nil {
return
}
m.incidentStore.RecordAlertAcknowledged(alert, user)
}
func (m *Monitor) handleAlertUnacknowledged(alert *alerts.Alert, user string) {
if m.incidentStore == nil || alert == nil {
return
}
m.incidentStore.RecordAlertUnacknowledged(alert, user)
}
// SetResourceStore sets the resource store for polling optimization. // SetResourceStore sets the resource store for polling optimization.
// When set, the monitor will check if it should reduce polling frequency // When set, the monitor will check if it should reduce polling frequency
// for nodes that have host agents providing data. // for nodes that have host agents providing data.