danvics/Pulse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: enhance AI baseline context visibility and incident timeline improvements

Browse source 
Backend:
- Enhanced buildEnrichedResourceContext to ALWAYS show learned baselines with
  status indicators (normal/elevated/anomaly) instead of only when anomalous
- This makes Pulse Pro's 'moat' visible - users can see the AI understands
  their infrastructure's normal behavior patterns
- Added baseline import to service.go

Frontend (user changes):
- Added incident event type filtering with toggle buttons
- Added resource incident panel to view all incidents for a resource
- Added timeline expand/collapse functionality in alert history
- Added incident note saving with proper incidentId tracking
- Added startedAt parameter for proper incident timeline loading
This commit is contained in:
rcourtman 2025-12-21 00:14:20 +00:00
parent f0b983667c
commit 82e5b28840
14 changed files with 2397 additions and 226 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
frontend-modern/src/api/alerts.ts
View file

@ -1,4 +1,4 @@
import type { Alert } from '@/types/api';
import type { Alert, Incident } from '@/types/api';
import type { AlertConfig } from '@/types/alerts';
import { apiFetchJSON } from '@/utils/apiClient';
@ -29,6 +29,37 @@ export class AlertsAPI {
return apiFetchJSON(`${this.baseUrl}/history?${queryParams}`);
}
static async getIncidentTimeline(alertId: string, startedAt?: string): Promise<Incident | null> {
const query = new URLSearchParams({ alert_id: alertId });
if (startedAt) {
query.set('started_at', startedAt);
}
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident | null>;
}
static async getIncidentsForResource(resourceId: string, limit?: number): Promise<Incident[]> {
const query = new URLSearchParams({ resource_id: resourceId });
if (limit) query.set('limit', String(limit));
return apiFetchJSON(`${this.baseUrl}/incidents?${query.toString()}`) as Promise<Incident[]>;
}
static async addIncidentNote(params: {
alertId?: string;
incidentId?: string;
note: string;
user?: string;
}): Promise<{ success: boolean }> {
return apiFetchJSON(`${this.baseUrl}/incidents/note`, {
method: 'POST',
body: JSON.stringify({
alert_id: params.alertId,
incident_id: params.incidentId,
note: params.note,
user: params.user,
}),
}) as Promise<{ success: boolean }>;
}
static async acknowledge(alertId: string, user?: string): Promise<{ success: boolean }> {
return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(alertId)}/acknowledge`, {
method: 'POST',

932
frontend-modern/src/pages/Alerts.tsx
View file

File diff suppressed because it is too large Load diff

28
frontend-modern/src/types/api.ts
View file

@ -1019,6 +1019,34 @@ export interface ResolvedAlert extends Alert {
resolvedTime: string;
}
export interface IncidentEvent {
id: string;
type: string;
timestamp: string;
summary: string;
details?: Record<string, unknown>;
}
export interface Incident {
id: string;
alertId: string;
alertType: string;
level: string;
resourceId: string;
resourceName: string;
resourceType?: string;
node?: string;
instance?: string;
message?: string;
status: string;
openedAt: string;
closedAt?: string;
acknowledged: boolean;
ackUser?: string;
ackTime?: string;
events?: IncidentEvent[];
}
// WebSocket message types
export type WSMessage =
| { type: 'initialState'; data: State }

11
internal/ai/alert_triggered.go
View file

@ -161,6 +161,17 @@ func (a *AlertTriggeredAnalyzer) analyzeResource(alert *alerts.Alert, resourceKe
Dur("duration", duration).
Msg("Alert-triggered AI analysis completed with no additional findings")
}
if a.patrolService != nil && a.patrolService.aiService != nil {
summary := "Alert-triggered AI analysis completed"
if len(findings) > 0 {
summary = fmt.Sprintf("Alert-triggered AI analysis found %d findings", len(findings))
}
a.patrolService.aiService.RecordIncidentAnalysis(alert.ID, summary, map[string]interface{}{
"findings": len(findings),
"duration": duration.String(),
})
}
}
// analyzeResourceByAlert determines the resource type from the alert and analyzes it

848
internal/ai/memory/incidents.go Normal file
View file

@ -0,0 +1,848 @@
package memory
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rs/zerolog/log"
)
// IncidentStatus represents the current state of an incident.
type IncidentStatus string
const (
IncidentStatusOpen IncidentStatus = "open"
IncidentStatusResolved IncidentStatus = "resolved"
)
// IncidentEventType describes a timeline event type.
type IncidentEventType string
const (
IncidentEventAlertFired IncidentEventType = "alert_fired"
IncidentEventAlertAcknowledged IncidentEventType = "alert_acknowledged"
IncidentEventAlertUnacknowledged IncidentEventType = "alert_unacknowledged"
IncidentEventAlertResolved IncidentEventType = "alert_resolved"
IncidentEventAnalysis IncidentEventType = "ai_analysis"
IncidentEventCommand IncidentEventType = "command"
IncidentEventRunbook IncidentEventType = "runbook"
IncidentEventNote IncidentEventType = "note"
)
// IncidentEvent represents a single timeline entry for an incident.
type IncidentEvent struct {
ID string `json:"id"`
Type IncidentEventType `json:"type"`
Timestamp time.Time `json:"timestamp"`
Summary string `json:"summary"`
Details map[string]interface{} `json:"details,omitempty"`
}
// Incident captures an alert occurrence and its timeline.
type Incident struct {
ID string `json:"id"`
AlertID string `json:"alertId"`
AlertType string `json:"alertType"`
Level string `json:"level"`
ResourceID string `json:"resourceId"`
ResourceName string `json:"resourceName"`
ResourceType string `json:"resourceType,omitempty"`
Node string `json:"node,omitempty"`
Instance string `json:"instance,omitempty"`
Message string `json:"message,omitempty"`
Status IncidentStatus `json:"status"`
OpenedAt time.Time `json:"openedAt"`
ClosedAt *time.Time `json:"closedAt,omitempty"`
Acknowledged bool `json:"acknowledged"`
AckUser string `json:"ackUser,omitempty"`
AckTime *time.Time `json:"ackTime,omitempty"`
Events []IncidentEvent `json:"events,omitempty"`
}
// IncidentStoreConfig configures incident retention and persistence.
type IncidentStoreConfig struct {
DataDir string
MaxIncidents int
MaxEventsPerIncident int
MaxAgeDays int
}
// IncidentStore maintains incident timelines and persistence.
type IncidentStore struct {
mu sync.RWMutex
saveMu sync.Mutex
incidents []*Incident
maxIncidents int
maxEvents int
maxAge time.Duration
dataDir string
filePath string
}
const (
defaultIncidentMaxIncidents = 500
defaultIncidentMaxEvents = 120
defaultIncidentMaxAgeDays = 90
incidentFileName = "ai_incidents.json"
maxIncidentFileSize = 20 * 1024 * 1024 // 20MB
incidentStartMatchTolerance = 10 * time.Minute
)
// NewIncidentStore creates a new incident store with persistence.
func NewIncidentStore(cfg IncidentStoreConfig) *IncidentStore {
maxIncidents := cfg.MaxIncidents
if maxIncidents <= 0 {
maxIncidents = defaultIncidentMaxIncidents
}
maxEvents := cfg.MaxEventsPerIncident
if maxEvents <= 0 {
maxEvents = defaultIncidentMaxEvents
}
maxAgeDays := cfg.MaxAgeDays
if maxAgeDays <= 0 {
maxAgeDays = defaultIncidentMaxAgeDays
}
store := &IncidentStore{
incidents: make([]*Incident, 0),
maxIncidents: maxIncidents,
maxEvents: maxEvents,
maxAge: time.Duration(maxAgeDays) * 24 * time.Hour,
dataDir: cfg.DataDir,
}
if store.dataDir != "" {
store.filePath = filepath.Join(store.dataDir, incidentFileName)
if err := store.loadFromDisk(); err != nil {
log.Warn().Err(err).Msg("Failed to load incident history from disk")
} else if len(store.incidents) > 0 {
log.Info().Int("count", len(store.incidents)).Msg("Loaded incident history from disk")
}
}
return store
}
// RecordAlertFired opens or updates an incident for a fired alert.
func (s *IncidentStore) RecordAlertFired(alert *alerts.Alert) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
if incident == nil {
incident = newIncidentFromAlert(alert)
s.incidents = append(s.incidents, incident)
s.addEventLocked(incident, IncidentEventAlertFired, formatAlertSummary(alert), map[string]interface{}{
"type": alert.Type,
"level": string(alert.Level),
"value": alert.Value,
"threshold": alert.Threshold,
})
} else {
updateIncidentFromAlert(incident, alert)
}
s.trimLocked()
s.saveAsync()
}
// RecordAlertAcknowledged records an acknowledgement event for an alert.
func (s *IncidentStore) RecordAlertAcknowledged(alert *alerts.Alert, user string) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.ensureIncidentForAlertLocked(alert)
if incident == nil {
return
}
incident.Acknowledged = true
if alert.AckTime != nil {
incident.AckTime = alert.AckTime
} else {
now := time.Now()
incident.AckTime = &now
}
incident.AckUser = user
s.addEventLocked(incident, IncidentEventAlertAcknowledged, "Alert acknowledged", map[string]interface{}{
"user": user,
})
s.trimLocked()
s.saveAsync()
}
// RecordAlertUnacknowledged records an unacknowledge event for an alert.
func (s *IncidentStore) RecordAlertUnacknowledged(alert *alerts.Alert, user string) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.ensureIncidentForAlertLocked(alert)
if incident == nil {
return
}
incident.Acknowledged = false
incident.AckTime = nil
incident.AckUser = ""
s.addEventLocked(incident, IncidentEventAlertUnacknowledged, "Alert unacknowledged", map[string]interface{}{
"user": user,
})
s.trimLocked()
s.saveAsync()
}
// RecordAlertResolved records a resolved event and closes the incident.
func (s *IncidentStore) RecordAlertResolved(alert *alerts.Alert, resolvedAt time.Time) {
if alert == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findOpenIncidentByAlertIDLocked(alert.ID)
if incident == nil {
incident = newIncidentFromAlert(alert)
s.incidents = append(s.incidents, incident)
}
incident.Status = IncidentStatusResolved
if resolvedAt.IsZero() {
now := time.Now()
resolvedAt = now
}
incident.ClosedAt = &resolvedAt
s.addEventLocked(incident, IncidentEventAlertResolved, "Alert resolved", map[string]interface{}{
"resolved_at": resolvedAt.Format(time.RFC3339),
})
s.trimLocked()
s.saveAsync()
}
// RecordAnalysis adds an AI analysis event to the incident for an alert.
func (s *IncidentStore) RecordAnalysis(alertID, summary string, details map[string]interface{}) {
if alertID == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
incident = &Incident{
ID: generateIncidentID(),
AlertID: alertID,
Status: IncidentStatusOpen,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, incident)
}
if summary == "" {
summary = "AI analysis completed"
}
s.addEventLocked(incident, IncidentEventAnalysis, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordCommand adds a command execution event to the incident for an alert.
func (s *IncidentStore) RecordCommand(alertID, command string, success bool, output string, details map[string]interface{}) {
if alertID == "" || command == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
incident = &Incident{
ID: generateIncidentID(),
AlertID: alertID,
Status: IncidentStatusOpen,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, incident)
}
if details == nil {
details = make(map[string]interface{})
}
details["command"] = command
details["success"] = success
if output != "" {
details["output_excerpt"] = truncateOutput(output, 500)
}
status := "failed"
if success {
status = "succeeded"
}
summary := fmt.Sprintf("Command %s: %s", status, command)
s.addEventLocked(incident, IncidentEventCommand, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordRunbook adds a runbook execution event to the incident for an alert.
func (s *IncidentStore) RecordRunbook(alertID, runbookID, title string, outcome string, automatic bool, message string) {
if alertID == "" || runbookID == "" {
return
}
s.mu.Lock()
defer s.mu.Unlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
incident = &Incident{
ID: generateIncidentID(),
AlertID: alertID,
Status: IncidentStatusOpen,
OpenedAt: time.Now(),
}
s.incidents = append(s.incidents, incident)
}
summary := fmt.Sprintf("Runbook %s (%s)", title, outcome)
details := map[string]interface{}{
"runbook_id": runbookID,
"outcome": outcome,
"automatic": automatic,
}
if message != "" {
details["message"] = message
}
s.addEventLocked(incident, IncidentEventRunbook, summary, details)
s.trimLocked()
s.saveAsync()
}
// RecordNote appends a user note to an incident identified by alert ID or incident ID.
func (s *IncidentStore) RecordNote(alertID, incidentID, note, user string) bool {
note = strings.TrimSpace(note)
if note == "" {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
var incident *Incident
if incidentID != "" {
incident = s.findIncidentByIDLocked(incidentID)
} else if alertID != "" {
incident = s.findLatestIncidentByAlertIDLocked(alertID)
}
if incident == nil {
return false
}
summary := "Note added"
if user != "" {
summary = fmt.Sprintf("Note added by %s", user)
}
s.addEventLocked(incident, IncidentEventNote, summary, map[string]interface{}{
"note": note,
"user": user,
})
s.trimLocked()
s.saveAsync()
return true
}
// GetTimelineByAlertID returns the most recent incident for the alert.
func (s *IncidentStore) GetTimelineByAlertID(alertID string) *Incident {
if alertID == "" {
return nil
}
s.mu.RLock()
defer s.mu.RUnlock()
incident := s.findLatestIncidentByAlertIDLocked(alertID)
if incident == nil {
return nil
}
return cloneIncident(incident)
}
// GetTimelineByAlertAt returns the incident closest to the provided start time for an alert.
func (s *IncidentStore) GetTimelineByAlertAt(alertID string, startedAt time.Time) *Incident {
if alertID == "" {
return nil
}
if startedAt.IsZero() {
return s.GetTimelineByAlertID(alertID)
}
s.mu.RLock()
defer s.mu.RUnlock()
var best *Incident
var bestDelta time.Duration
for _, incident := range s.incidents {
if incident == nil || incident.AlertID != alertID {
continue
}
delta := incident.OpenedAt.Sub(startedAt)
if delta < 0 {
delta = -delta
}
if best == nil || delta < bestDelta {
best = incident
bestDelta = delta
}
}
if best == nil || bestDelta > incidentStartMatchTolerance {
return nil
}
return cloneIncident(best)
}
// ListIncidentsByResource returns recent incidents for a resource.
func (s *IncidentStore) ListIncidentsByResource(resourceID string, limit int) []*Incident {
if resourceID == "" {
return nil
}
s.mu.RLock()
defer s.mu.RUnlock()
var matches []*Incident
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.ResourceID == resourceID {
matches = append(matches, cloneIncident(incident))
if limit > 0 && len(matches) >= limit {
break
}
}
}
return matches
}
// FormatForAlert returns a condensed incident timeline for prompt injection.
func (s *IncidentStore) FormatForAlert(alertID string, maxEvents int) string {
incident := s.GetTimelineByAlertID(alertID)
if incident == nil {
return ""
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString(fmt.Sprintf("Alert incident for %s (%s, %s)\n",
incident.ResourceName, incident.AlertType, incident.Level))
b.WriteString(fmt.Sprintf("Status: %s\n", incident.Status))
events := incident.Events
if maxEvents > 0 && len(events) > maxEvents {
events = events[len(events)-maxEvents:]
}
for _, evt := range events {
b.WriteString("- ")
b.WriteString(evt.Timestamp.Format(time.RFC3339))
b.WriteString(": ")
b.WriteString(evt.Summary)
b.WriteString("\n")
}
return b.String()
}
// FormatForResource returns a condensed incident summary for a resource.
func (s *IncidentStore) FormatForResource(resourceID string, limit int) string {
incidents := s.ListIncidentsByResource(resourceID, limit)
if len(incidents) == 0 {
return ""
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString("Recent incidents for this resource:\n")
for _, incident := range incidents {
status := string(incident.Status)
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
status = "acknowledged"
}
b.WriteString("- ")
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
b.WriteString(": ")
b.WriteString(incident.AlertType)
if incident.Level != "" {
b.WriteString(" (")
b.WriteString(incident.Level)
b.WriteString(")")
}
b.WriteString(" - ")
b.WriteString(status)
b.WriteString("\n")
}
return b.String()
}
// FormatForPatrol returns a condensed incident summary for infrastructure-wide patrol analysis.
func (s *IncidentStore) FormatForPatrol(limit int) string {
if limit <= 0 {
limit = 8
}
s.mu.RLock()
defer s.mu.RUnlock()
if len(s.incidents) == 0 {
return ""
}
var b strings.Builder
b.WriteString("\n\n## Incident Memory\n")
b.WriteString("Recent incidents across infrastructure:\n")
count := 0
for i := len(s.incidents) - 1; i >= 0 && count < limit; i-- {
incident := s.incidents[i]
if incident == nil {
continue
}
status := string(incident.Status)
if incident.Acknowledged && incident.Status == IncidentStatusOpen {
status = "acknowledged"
}
lastSummary := ""
if len(incident.Events) > 0 {
lastSummary = incident.Events[len(incident.Events)-1].Summary
}
b.WriteString("- ")
b.WriteString(incident.OpenedAt.Format(time.RFC3339))
b.WriteString(": ")
if incident.ResourceName != "" {
b.WriteString(incident.ResourceName)
b.WriteString(" - ")
}
if incident.AlertType != "" {
b.WriteString(incident.AlertType)
}
if incident.Level != "" {
b.WriteString(" (")
b.WriteString(incident.Level)
b.WriteString(")")
}
b.WriteString(" - ")
b.WriteString(status)
if lastSummary != "" {
b.WriteString(" - last: ")
b.WriteString(truncateOutput(lastSummary, 80))
} else if incident.Message != "" {
b.WriteString(" - ")
b.WriteString(truncateOutput(incident.Message, 80))
}
b.WriteString("\n")
count++
}
return b.String()
}
func newIncidentFromAlert(alert *alerts.Alert) *Incident {
openedAt := alert.StartTime
if openedAt.IsZero() {
openedAt = time.Now()
}
return &Incident{
ID: generateIncidentID(),
AlertID: alert.ID,
AlertType: alert.Type,
Level: string(alert.Level),
ResourceID: alert.ResourceID,
ResourceName: alert.ResourceName,
Node: alert.Node,
Instance: alert.Instance,
Message: alert.Message,
Status: IncidentStatusOpen,
OpenedAt: openedAt,
Acknowledged: alert.Acknowledged,
AckUser: alert.AckUser,
AckTime: alert.AckTime,
Events: make([]IncidentEvent, 0),
}
}
func updateIncidentFromAlert(incident *Incident, alert *alerts.Alert) {
if incident == nil || alert == nil {
return
}
incident.AlertType = alert.Type
incident.Level = string(alert.Level)
incident.ResourceID = alert.ResourceID
incident.ResourceName = alert.ResourceName
incident.Node = alert.Node
incident.Instance = alert.Instance
incident.Message = alert.Message
incident.Acknowledged = alert.Acknowledged
incident.AckUser = alert.AckUser
incident.AckTime = alert.AckTime
}
func (s *IncidentStore) ensureIncidentForAlertLocked(alert *alerts.Alert) *Incident {
incident := s.findLatestIncidentByAlertIDLocked(alert.ID)
if incident == nil {
incident = newIncidentFromAlert(alert)
s.incidents = append(s.incidents, incident)
}
updateIncidentFromAlert(incident, alert)
return incident
}
func (s *IncidentStore) addEventLocked(incident *Incident, eventType IncidentEventType, summary string, details map[string]interface{}) {
if incident == nil {
return
}
if summary == "" {
summary = string(eventType)
}
event := IncidentEvent{
ID: generateIncidentEventID(),
Type: eventType,
Timestamp: time.Now(),
Summary: summary,
Details: details,
}
incident.Events = append(incident.Events, event)
if s.maxEvents > 0 && len(incident.Events) > s.maxEvents {
incident.Events = incident.Events[len(incident.Events)-s.maxEvents:]
}
}
func (s *IncidentStore) findOpenIncidentByAlertIDLocked(alertID string) *Incident {
if alertID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.AlertID == alertID && incident.Status == IncidentStatusOpen {
return incident
}
}
return nil
}
func (s *IncidentStore) findLatestIncidentByAlertIDLocked(alertID string) *Incident {
if alertID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.AlertID == alertID {
return incident
}
}
return nil
}
func (s *IncidentStore) findIncidentByIDLocked(incidentID string) *Incident {
if incidentID == "" {
return nil
}
for i := len(s.incidents) - 1; i >= 0; i-- {
incident := s.incidents[i]
if incident != nil && incident.ID == incidentID {
return incident
}
}
return nil
}
func (s *IncidentStore) trimLocked() {
if s.maxAge > 0 {
cutoff := time.Now().Add(-s.maxAge)
filtered := make([]*Incident, 0, len(s.incidents))
for _, incident := range s.incidents {
if incident == nil {
continue
}
compareTime := incident.OpenedAt
if incident.ClosedAt != nil {
compareTime = *incident.ClosedAt
}
if compareTime.After(cutoff) {
filtered = append(filtered, incident)
}
}
s.incidents = filtered
}
if s.maxIncidents > 0 && len(s.incidents) > s.maxIncidents {
sort.Slice(s.incidents, func(i, j int) bool {
return s.incidents[i].OpenedAt.Before(s.incidents[j].OpenedAt)
})
if len(s.incidents) > s.maxIncidents {
s.incidents = s.incidents[len(s.incidents)-s.maxIncidents:]
}
}
}
func (s *IncidentStore) saveAsync() {
if s.dataDir == "" || s.filePath == "" {
return
}
go func() {
if err := s.saveToDisk(); err != nil {
log.Warn().Err(err).Msg("Failed to save incident history")
}
}()
}
func (s *IncidentStore) saveToDisk() error {
s.saveMu.Lock()
defer s.saveMu.Unlock()
if s.dataDir == "" || s.filePath == "" {
return nil
}
if err := os.MkdirAll(s.dataDir, 0755); err != nil {
return err
}
s.mu.RLock()
snapshot := make([]*Incident, 0, len(s.incidents))
for _, incident := range s.incidents {
snapshot = append(snapshot, cloneIncident(incident))
}
s.mu.RUnlock()
data, err := json.Marshal(snapshot)
if err != nil {
return err
}
tmpFile := s.filePath + ".tmp"
if err := os.WriteFile(tmpFile, data, 0644); err != nil {
return err
}
if err := os.Rename(tmpFile, s.filePath); err != nil {
return err
}
return nil
}
func (s *IncidentStore) loadFromDisk() error {
if s.filePath == "" {
return nil
}
info, err := os.Stat(s.filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
if info.Size() > maxIncidentFileSize {
return fmt.Errorf("incident history file too large (%d bytes)", info.Size())
}
data, err := os.ReadFile(s.filePath)
if err != nil {
return err
}
var incidents []*Incident
if err := json.Unmarshal(data, &incidents); err != nil {
return err
}
s.incidents = incidents
s.trimLocked()
return nil
}
func cloneIncident(src *Incident) *Incident {
if src == nil {
return nil
}
clone := *src
if src.AckTime != nil {
t := *src.AckTime
clone.AckTime = &t
}
if src.ClosedAt != nil {
t := *src.ClosedAt
clone.ClosedAt = &t
}
if len(src.Events) > 0 {
clone.Events = make([]IncidentEvent, len(src.Events))
for i, event := range src.Events {
cloneEvent := event
if event.Details != nil {
detailsCopy := make(map[string]interface{}, len(event.Details))
for key, value := range event.Details {
detailsCopy[key] = value
}
cloneEvent.Details = detailsCopy
}
clone.Events[i] = cloneEvent
}
}
return &clone
}
var incidentCounter int64
func generateIncidentID() string {
incidentCounter++
return "inc-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentCounter%1000))
}
var incidentEventCounter int64
func generateIncidentEventID() string {
incidentEventCounter++
return "inc-evt-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentEventCounter%1000))
}
func formatAlertSummary(alert *alerts.Alert) string {
if alert == nil {
return "Alert triggered"
}
summary := fmt.Sprintf("Alert triggered: %s (%s)", alert.Type, alert.Level)
if alert.Value > 0 || alert.Threshold > 0 {
summary = fmt.Sprintf("Alert triggered: %s (%s %.1f >= %.1f)", alert.Type, alert.Level, alert.Value, alert.Threshold)
}
return summary
}

108
internal/ai/memory/incidents_test.go Normal file
View file

@ -0,0 +1,108 @@
package memory
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
)
func TestIncidentStore_RecordTimeline(t *testing.T) {
store := NewIncidentStore(IncidentStoreConfig{
DataDir: t.TempDir(),
MaxIncidents: 10,
MaxEventsPerIncident: 10,
MaxAgeDays: 30,
})
alert := &alerts.Alert{
ID: "alert-1",
Type: "cpu",
Level: alerts.AlertLevelWarning,
ResourceID: "res-1",
ResourceName: "vm-1",
StartTime: time.Now().Add(-5 * time.Minute),
Value: 92,
Threshold: 85,
}
store.RecordAlertFired(alert)
store.RecordAlertAcknowledged(alert, "admin")
store.RecordAnalysis(alert.ID, "analysis complete", map[string]interface{}{
"findings": 1,
})
store.RecordCommand(alert.ID, "systemctl restart nginx", true, "ok", nil)
store.RecordAlertResolved(alert, time.Now())
timeline := store.GetTimelineByAlertID(alert.ID)
if timeline == nil {
t.Fatalf("expected timeline, got nil")
}
if timeline.Status != IncidentStatusResolved {
t.Fatalf("expected status %q, got %q", IncidentStatusResolved, timeline.Status)
}
if timeline.AckUser != "admin" {
t.Fatalf("expected ack user admin, got %q", timeline.AckUser)
}
if len(timeline.Events) < 4 {
t.Fatalf("expected events recorded, got %d", len(timeline.Events))
}
if ok := store.RecordNote(alert.ID, "", "note text", ""); !ok {
t.Fatalf("expected note to be saved")
}
}
func TestIncidentStore_GetTimelineByAlertAt(t *testing.T) {
store := NewIncidentStore(IncidentStoreConfig{
DataDir: t.TempDir(),
MaxIncidents: 10,
MaxEventsPerIncident: 10,
MaxAgeDays: 30,
})
base := time.Now().UTC()
first := &alerts.Alert{
ID: "alert-2",
Type: "cpu",
Level: alerts.AlertLevelWarning,
ResourceID: "res-2",
ResourceName: "vm-2",
StartTime: base.Add(-2 * time.Hour),
}
second := &alerts.Alert{
ID: "alert-2",
Type: "cpu",
Level: alerts.AlertLevelWarning,
ResourceID: "res-2",
ResourceName: "vm-2",
StartTime: base.Add(-10 * time.Minute),
}
store.RecordAlertFired(first)
store.RecordAlertResolved(first, base.Add(-90*time.Minute))
store.RecordAlertFired(second)
timeline := store.GetTimelineByAlertAt(first.ID, first.StartTime)
if timeline == nil {
t.Fatalf("expected timeline for first incident, got nil")
}
if !timeline.OpenedAt.Equal(first.StartTime) {
t.Fatalf("expected openedAt %s, got %s", first.StartTime, timeline.OpenedAt)
}
timeline = store.GetTimelineByAlertAt(second.ID, second.StartTime)
if timeline == nil {
t.Fatalf("expected timeline for second incident, got nil")
}
if !timeline.OpenedAt.Equal(second.StartTime) {
t.Fatalf("expected openedAt %s, got %s", second.StartTime, timeline.OpenedAt)
}
timeline = store.GetTimelineByAlertAt(second.ID, base.Add(-45*time.Minute))
if timeline != nil {
t.Fatalf("expected no timeline for mismatched start time")
}
}

27
internal/ai/patrol.go
View file

@ -13,6 +13,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
aicontext "github.com/rcourtman/pulse-go-rewrite/internal/ai/context"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rs/zerolog/log"
)
@ -217,6 +218,7 @@ type PatrolService struct {
remediationLog *RemediationLog // For tracking remediation actions
patternDetector *PatternDetector // For failure prediction from historical patterns
correlationDetector *CorrelationDetector // For multi-resource correlation
incidentStore *memory.IncidentStore // For incident timeline capture
// Cached thresholds (recalculated when thresholdProvider changes)
thresholds PatrolThresholds
@ -263,6 +265,20 @@ func NewPatrolService(aiService *Service, stateProvider StateProvider) *PatrolSe
}
}
// SetIncidentStore attaches an incident store for alert timeline capture.
func (p *PatrolService) SetIncidentStore(store *memory.IncidentStore) {
p.mu.Lock()
defer p.mu.Unlock()
p.incidentStore = store
}
// GetIncidentStore returns the incident store if configured.
func (p *PatrolService) GetIncidentStore() *memory.IncidentStore {
p.mu.RLock()
defer p.mu.RUnlock()
return p.incidentStore
}
// SetConfig updates the patrol configuration
func (p *PatrolService) SetConfig(cfg PatrolConfig) {
p.mu.Lock()
@ -2185,12 +2201,17 @@ func (p *PatrolService) buildPatrolPrompt(summary string) string {
// Get resource notes from knowledge store (per-resource user notes)
var knowledgeContext string
var incidentContext string
p.mu.RLock()
knowledgeStore := p.knowledgeStore
incidentStore := p.incidentStore
p.mu.RUnlock()
if knowledgeStore != nil {
knowledgeContext = knowledgeStore.FormatAllForContext()
}
if incidentStore != nil {
incidentContext = incidentStore.FormatForPatrol(8)
}
basePrompt := fmt.Sprintf(`Please perform a comprehensive analysis of the following infrastructure and identify any issues, potential problems, or optimization opportunities.
@ -2235,6 +2256,12 @@ IMPORTANT: Respect the user's feedback above. Do NOT re-raise findings that are:
Only report NEW issues or issues where the severity has clearly escalated.`)
}
if incidentContext != "" {
contextAdditions.WriteString("\n\n")
contextAdditions.WriteString(incidentContext)
contextAdditions.WriteString("\nIMPORTANT: Use incident memory to avoid repeating known issues and to build on successful past investigations.")
}
if contextAdditions.Len() > 0 {
return basePrompt + contextAdditions.String()
}

4
internal/ai/runbooks.go
View file

@ -323,6 +323,10 @@ func (p *PatrolService) logRunbookExecution(finding *Finding, runbook Runbook, s
if err := p.remediationLog.Log(record); err != nil {
log.Warn().Err(err).Msg("Failed to log runbook execution")
}
if p.aiService != nil && finding != nil && finding.AlertID != "" {
p.aiService.RecordIncidentRunbook(finding.AlertID, runbook.ID, runbook.Title, outcome, automatic, message)
}
}
type runbookContext struct {

259
internal/ai/service.go
View file

@ -18,8 +18,10 @@ import (
"github.com/google/uuid"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
@ -73,9 +75,10 @@ type Service struct {
alertProvider AlertProvider
knowledgeStore *knowledge.Store
costStore *cost.Store
resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
patrolService *PatrolService // Background AI monitoring service
metadataProvider MetadataProvider // Enables AI to update resource URLs
resourceProvider ResourceProvider // Unified resource model provider (Phase 2)
patrolService *PatrolService // Background AI monitoring service
metadataProvider MetadataProvider // Enables AI to update resource URLs
incidentStore *memory.IncidentStore // Incident timelines for alert memory
// Alert-triggered analysis - token-efficient real-time AI insights
alertTriggeredAnalyzer *AlertTriggeredAnalyzer
@ -197,6 +200,9 @@ func (s *Service) SetStateProvider(sp StateProvider) {
if s.knowledgeStore != nil {
s.patrolService.SetKnowledgeStore(s.knowledgeStore)
}
if s.incidentStore != nil {
s.patrolService.SetIncidentStore(s.incidentStore)
}
}
// Initialize alert-triggered analyzer if not already done
@ -350,6 +356,17 @@ func (s *Service) SetRemediationLog(remLog *RemediationLog) {
}
}
// SetIncidentStore sets the incident store for alert timeline memory.
func (s *Service) SetIncidentStore(store *memory.IncidentStore) {
s.mu.Lock()
defer s.mu.Unlock()
s.incidentStore = store
if s.patrolService != nil {
s.patrolService.SetIncidentStore(store)
}
}
// SetPatternDetector sets the pattern detector for failure prediction
func (s *Service) SetPatternDetector(detector *PatternDetector) {
s.mu.RLock()
@ -1724,14 +1741,14 @@ func (s *Service) logRemediation(req ExecuteRequest, command, output string, suc
// that's worth logging as an achievement, false for read-only diagnostics
func isActionableCommand(cmd string) bool {
cmd = strings.TrimSpace(cmd)
// Strip [host] or [hostname] prefix if present
if strings.HasPrefix(cmd, "[") {
if idx := strings.Index(cmd, "]"); idx != -1 {
cmd = strings.TrimSpace(cmd[idx+1:])
}
}
// Commands that PERFORM ACTIONS (should be logged)
actionPatterns := []string{
"docker restart", "docker start", "docker stop", "docker rm",
@ -1740,24 +1757,24 @@ func isActionableCommand(cmd string) bool {
"service restart", "service start", "service stop",
"pct resize", "pct start", "pct stop", "pct shutdown", "pct reboot",
"qm resize", "qm start", "qm stop", "qm shutdown", "qm reboot",
"rm -", "rm /", // File deletion/cleanup
"chmod", "chown", // Permission fixes
"mkdir", // Creating directories
"mv ", "cp ", // File operations
"echo >", "tee ", // Writing to files
"rm -", "rm /", // File deletion/cleanup
"chmod", "chown", // Permission fixes
"mkdir", // Creating directories
"mv ", "cp ", // File operations
"echo >", "tee ", // Writing to files
"apt install", "apt upgrade", "apt remove",
"yum install", "dnf install",
"pip install", "npm install",
"kill ", "pkill ", "killall ",
"reboot", "shutdown",
}
for _, pattern := range actionPatterns {
if strings.Contains(cmd, pattern) {
return true
}
}
// Everything else is diagnostic (df, grep, cat, tail, ps, ls, etc.)
return false
}
@ -1766,7 +1783,7 @@ func isActionableCommand(cmd string) bool {
// This is used to display meaningful descriptions in the Pulse AI Impact section
func generateRemediationSummary(command string, targetType string, context map[string]interface{}) string {
cmd := strings.TrimSpace(command)
// Extract target name from context
targetName := ""
if context != nil {
@ -1778,7 +1795,7 @@ func generateRemediationSummary(command string, targetType string, context map[s
targetName = name
}
}
// Extract meaningful path or service from command
extractPath := func() string {
// Look for common path patterns
@ -1798,7 +1815,7 @@ func generateRemediationSummary(command string, targetType string, context map[s
}
return ""
}
// Extract container/service name from docker commands
extractDockerTarget := func() string {
// docker ps --filter name=XXX
@ -1811,10 +1828,10 @@ func generateRemediationSummary(command string, targetType string, context map[s
}
return ""
}
path := extractPath()
dockerTarget := extractDockerTarget()
// Generate summary based on command type
switch {
case strings.Contains(cmd, "docker restart") || strings.Contains(cmd, "docker start"):
@ -1822,37 +1839,37 @@ func generateRemediationSummary(command string, targetType string, context map[s
return fmt.Sprintf("Restarted %s container", dockerTarget)
}
return "Restarted container"
case strings.Contains(cmd, "docker stop"):
if dockerTarget != "" {
return fmt.Sprintf("Stopped %s container", dockerTarget)
}
return "Stopped container"
case strings.Contains(cmd, "docker ps"):
if dockerTarget != "" {
return fmt.Sprintf("Verified %s container is running", dockerTarget)
}
return "Checked container status"
case strings.Contains(cmd, "docker logs"):
if dockerTarget != "" {
return fmt.Sprintf("Retrieved %s logs", dockerTarget)
}
return "Retrieved container logs"
case strings.Contains(cmd, "systemctl restart"):
if match := regexp.MustCompile(`systemctl\s+restart\s+(\S+)`).FindStringSubmatch(cmd); len(match) > 1 {
return fmt.Sprintf("Restarted %s service", match[1])
}
return "Restarted system service"
case strings.Contains(cmd, "systemctl status"):
if match := regexp.MustCompile(`systemctl\s+status\s+(\S+)`).FindStringSubmatch(cmd); len(match) > 1 {
return fmt.Sprintf("Checked %s service status", match[1])
}
return "Checked service status"
case strings.Contains(cmd, "df ") || strings.Contains(cmd, "du "):
if path != "" {
// Check for known services in path
@ -1869,7 +1886,7 @@ func generateRemediationSummary(command string, targetType string, context map[s
return fmt.Sprintf("Analyzed %s storage", path)
}
return "Analyzed disk usage"
case strings.Contains(cmd, "grep") && (strings.Contains(cmd, "config") || strings.Contains(cmd, ".yml") || strings.Contains(cmd, ".yaml")):
if path != "" && strings.Contains(strings.ToLower(path), "frigate") {
return "Inspected Frigate configuration"
@ -1878,40 +1895,40 @@ func generateRemediationSummary(command string, targetType string, context map[s
return fmt.Sprintf("Inspected %s configuration", path)
}
return "Inspected configuration"
case strings.Contains(cmd, "tail") || strings.Contains(cmd, "journalctl"):
if targetName != "" {
return fmt.Sprintf("Reviewed %s logs", targetName)
}
return "Reviewed system logs"
case strings.Contains(cmd, "pct resize"):
if match := regexp.MustCompile(`pct\s+resize\s+(\d+)`).FindStringSubmatch(cmd); len(match) > 1 {
return fmt.Sprintf("Resized container %s disk", match[1])
}
return "Resized container disk"
case strings.Contains(cmd, "qm resize"):
if match := regexp.MustCompile(`qm\s+resize\s+(\d+)`).FindStringSubmatch(cmd); len(match) > 1 {
return fmt.Sprintf("Resized VM %s disk", match[1])
}
return "Resized VM disk"
case strings.Contains(cmd, "ping") || strings.Contains(cmd, "curl"):
return "Tested network connectivity"
case strings.Contains(cmd, "free") || strings.Contains(cmd, "meminfo"):
return "Checked memory usage"
case strings.Contains(cmd, "ps aux") || strings.Contains(cmd, "top"):
return "Analyzed running processes"
case strings.Contains(cmd, "rm "):
return "Cleaned up files"
case strings.Contains(cmd, "chmod") || strings.Contains(cmd, "chown"):
return "Fixed file permissions"
default:
// Generic fallback - try to use target name if available
if targetName != "" {
@ -2183,11 +2200,34 @@ func (s *Service) executeTool(ctx context.Context, req ExecuteRequest, tc provid
// Execute via agent
result, err := s.executeOnAgent(ctx, execReq, command)
recordIncident := func(success bool, output string) {
alertID := extractAlertID(req.Context)
if alertID == "" {
return
}
s.mu.RLock()
incidentStore := s.incidentStore
s.mu.RUnlock()
if incidentStore == nil {
return
}
details := map[string]interface{}{
"resource_id": req.TargetID,
"resource_type": req.TargetType,
"run_on_host": runOnHost,
}
if targetHost != "" {
details["target_host"] = targetHost
}
incidentStore.RecordCommand(alertID, command, success, output, details)
}
if err != nil {
recordIncident(false, result)
execution.Output = fmt.Sprintf("Error executing command: %s", err)
return execution.Output, execution
}
recordIncident(true, result)
execution.Output = result
execution.Success = true
return result, execution
@ -2842,6 +2882,31 @@ This is a 3-command job. Don't over-investigate.`
// Add current alert status - this gives AI awareness of active issues
prompt += s.buildAlertContext()
// Add incident memory for alert or resource context
alertID := ""
resourceID := req.TargetID
if req.Context != nil {
if val, ok := req.Context["alertId"].(string); ok && val != "" {
alertID = val
} else if val, ok := req.Context["alert_id"].(string); ok && val != "" {
alertID = val
}
if resourceID == "" {
if val, ok := req.Context["resourceId"].(string); ok && val != "" {
resourceID = val
} else if val, ok := req.Context["resource_id"].(string); ok && val != "" {
resourceID = val
} else if val, ok := req.Context["guest_id"].(string); ok && val != "" {
resourceID = val
}
}
}
if incidentContext := s.buildIncidentContext(resourceID, alertID); incidentContext != "" {
prompt += incidentContext
}
// Add all saved knowledge when no specific target is selected
// This gives the AI context about everything learned from previous sessions
if req.TargetType == "" && s.knowledgeStore != nil {
@ -2877,6 +2942,7 @@ This is a 3-command job. Don't over-investigate.`
// Add past remediation history for this resource
prompt += s.buildRemediationContext(req.TargetID, req.Prompt)
}
// Add any provided context in a structured way
@ -3258,6 +3324,66 @@ func (s *Service) buildRemediationContext(resourceID, currentProblem string) str
return context
}
// buildIncidentContext adds incident timeline context for alerts/resources.
func (s *Service) buildIncidentContext(resourceID, alertID string) string {
s.mu.RLock()
store := s.incidentStore
s.mu.RUnlock()
if store == nil {
return ""
}
if alertID != "" {
return store.FormatForAlert(alertID, 8)
}
if resourceID != "" {
return store.FormatForResource(resourceID, 4)
}
return ""
}
// RecordIncidentAnalysis stores an AI analysis event for an alert.
func (s *Service) RecordIncidentAnalysis(alertID, summary string, details map[string]interface{}) {
if alertID == "" {
return
}
s.mu.RLock()
store := s.incidentStore
s.mu.RUnlock()
if store == nil {
return
}
store.RecordAnalysis(alertID, summary, details)
}
// RecordIncidentRunbook stores a runbook execution event for an alert.
func (s *Service) RecordIncidentRunbook(alertID, runbookID, title string, outcome memory.Outcome, automatic bool, message string) {
if alertID == "" || runbookID == "" {
return
}
s.mu.RLock()
store := s.incidentStore
s.mu.RUnlock()
if store == nil {
return
}
store.RecordRunbook(alertID, runbookID, title, string(outcome), automatic, message)
}
func extractAlertID(ctx map[string]interface{}) string {
if ctx == nil {
return ""
}
if alertID, ok := ctx["alertId"].(string); ok && alertID != "" {
return alertID
}
if alertID, ok := ctx["alert_id"].(string); ok && alertID != "" {
return alertID
}
return ""
}
// truncateString truncates a string to maxLen characters
func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
@ -3302,40 +3428,51 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
return 0, false
}
// Check CPU baseline
// Helper to format baseline comparison with status
// ALWAYS shows baseline info when available - this is Pulse Pro's value-add
formatBaselineComparison := func(metric string, currentVal float64, bl *baseline.MetricBaseline) string {
if bl.SampleCount < 10 {
return fmt.Sprintf("- %s: %.1f%% (baseline learning: %d/10 samples)", metric, currentVal, bl.SampleCount)
}
ratio := currentVal / bl.Mean
status := "✓ normal"
if ratio > 2.0 {
status = "🔴 **ANOMALY**"
} else if ratio > 1.5 {
status = "🟡 elevated"
} else if ratio < 0.3 {
status = "🔵 unusually low"
} else if ratio < 0.5 {
status = "low"
}
return fmt.Sprintf("- %s: %.1f%% vs baseline %.1f%% (σ=%.1f) → %s", metric, currentVal, bl.Mean, bl.StdDev, status)
}
// Check CPU baseline - always show if we have data
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists && bl.SampleCount >= 10 {
ratio := cpuVal / bl.Mean
if ratio > 1.5 {
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx higher** than baseline %.0f%% (σ=%.1f)", cpuVal, ratio, bl.Mean, bl.StdDev))
} else if ratio < 0.5 {
baselineInfo = append(baselineInfo, fmt.Sprintf("CPU %.0f%% is **%.1fx lower** than baseline %.0f%%", cpuVal, 1/ratio, bl.Mean))
}
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists {
baselineInfo = append(baselineInfo, formatBaselineComparison("CPU", cpuVal, bl))
}
}
// Check memory baseline
// Check memory baseline - always show if we have data
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists && bl.SampleCount >= 10 {
ratio := memVal / bl.Mean
if ratio > 1.3 {
baselineInfo = append(baselineInfo, fmt.Sprintf("Memory %.0f%% is **%.1fx higher** than baseline %.0f%%", memVal, ratio, bl.Mean))
}
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists {
baselineInfo = append(baselineInfo, formatBaselineComparison("Memory", memVal, bl))
}
}
// Check disk baseline
// Check disk baseline - always show if we have data
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists && bl.SampleCount >= 10 {
ratio := diskVal / bl.Mean
if ratio > 1.2 { // More sensitive for disk
baselineInfo = append(baselineInfo, fmt.Sprintf("Disk %.0f%% is **%.1fx higher** than baseline %.0f%%", diskVal, ratio, bl.Mean))
}
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists {
baselineInfo = append(baselineInfo, formatBaselineComparison("Disk", diskVal, bl))
}
}
if len(baselineInfo) > 0 {
sections = append(sections, "### Baseline Comparisons\n"+strings.Join(baselineInfo, "\n"))
sections = append(sections, "### Learned Baselines (7-day patterns)\n"+strings.Join(baselineInfo, "\n"))
}
}
@ -3382,21 +3519,21 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
// Only report significant trends
if metric == "disk" && ratePerDay > 0.5 { // Growing by >0.5GB/day
trendInfo = append(trendInfo, fmt.Sprintf("**Disk** growing %.1f GB/day over last %.0fh",
trendInfo = append(trendInfo, fmt.Sprintf("**Disk** growing %.1f GB/day over last %.0fh",
ratePerDay, hoursSpan))
} else if metric == "memory" && absFloat(ratePerDay) > 2 { // >2% change per day
direction := "growing"
if ratePerDay < 0 {
direction = "declining"
}
trendInfo = append(trendInfo, fmt.Sprintf("**Memory** %s %.1f%%/day over last %.0fh",
trendInfo = append(trendInfo, fmt.Sprintf("**Memory** %s %.1f%%/day over last %.0fh",
direction, absFloat(ratePerDay), hoursSpan))
} else if metric == "cpu" && absFloat(ratePerDay) > 5 { // >5% change per day
direction := "increasing"
if ratePerDay < 0 {
direction = "decreasing"
}
trendInfo = append(trendInfo, fmt.Sprintf("**CPU** %s %.1f%%/day over last %.0fh",
trendInfo = append(trendInfo, fmt.Sprintf("**CPU** %s %.1f%%/day over last %.0fh",
direction, absFloat(ratePerDay), hoursSpan))
}
}
@ -3414,7 +3551,7 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
var predInfo []string
for _, pred := range predictions {
if pred.DaysUntil < 30 { // Only show predictions within 30 days
predInfo = append(predInfo, fmt.Sprintf("**%s** predicted in %.0f days (%.0f%% confidence)",
predInfo = append(predInfo, fmt.Sprintf("**%s** predicted in %.0f days (%.0f%% confidence)",
pred.EventType, pred.DaysUntil, pred.Confidence*100))
}
}
@ -3470,7 +3607,7 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
if alertProvider != nil {
// Get active alerts for this resource
activeAlerts := alertProvider.GetAlertsByResource(resourceID)
// Get historical alerts (last 20)
historicalAlerts := alertProvider.GetAlertHistory(resourceID, 20)
@ -3480,7 +3617,7 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
if len(activeAlerts) > 0 {
alertInfo = append(alertInfo, fmt.Sprintf("**%d active alert(s)** right now", len(activeAlerts)))
for _, a := range activeAlerts {
alertInfo = append(alertInfo, fmt.Sprintf("- %s %s: %s (active %s)",
alertInfo = append(alertInfo, fmt.Sprintf("- %s %s: %s (active %s)",
strings.ToUpper(a.Level), a.Type, a.Message, a.Duration))
}
}
@ -3526,7 +3663,7 @@ func (s *Service) buildEnrichedResourceContext(resourceID, resourceType string,
learning++
}
}
if learning > 0 && ready == 0 {
return "\n\n## Historical Intelligence (Pulse Pro)\n*Learning baselines for this resource... Trends and anomaly detection will be available after more data is collected.*"
}

105
internal/alerts/alerts.go
View file

@ -58,7 +58,7 @@ const (
DefaultObservationWindow = 24 // hours
// Flapping detection
DefaultFlappingWindow = 300 // seconds (5 minutes)
DefaultFlappingWindow = 300 // seconds (5 minutes)
DefaultFlappingThreshold = 5 // state changes to trigger flapping
DefaultFlappingCooldown = 15 // minutes
@ -256,10 +256,10 @@ type GroupingConfig struct {
// ScheduleConfig represents alerting schedule configuration
type ScheduleConfig struct {
QuietHours QuietHours `json:"quietHours"`
Cooldown int `json:"cooldown"` // minutes
Cooldown int `json:"cooldown"` // minutes
GroupingWindow int `json:"groupingWindow,omitempty"` // Deprecated: use Grouping.Window instead. Will be auto-migrated on config update.
MaxAlertsHour int `json:"maxAlertsHour"` // max alerts per hour per resource
NotifyOnResolve bool `json:"notifyOnResolve"` // Send notification when alert clears
MaxAlertsHour int `json:"maxAlertsHour"` // max alerts per hour per resource
NotifyOnResolve bool `json:"notifyOnResolve"` // Send notification when alert clears
Escalation EscalationConfig `json:"escalation"`
Grouping GroupingConfig `json:"grouping"`
}
@ -484,15 +484,17 @@ func SetMetricHooks(fired func(*Alert), resolved func(*Alert), suppressed func(s
}
type Manager struct {
mu sync.RWMutex
config AlertConfig
activeAlerts map[string]*Alert
historyManager *HistoryManager
onAlert func(alert *Alert)
onResolved func(alertID string)
onEscalate func(alert *Alert, level int)
escalationStop chan struct{}
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
mu sync.RWMutex
config AlertConfig
activeAlerts map[string]*Alert
historyManager *HistoryManager
onAlert func(alert *Alert)
onResolved func(alertID string)
onAcknowledged func(alert *Alert, user string)
onUnacknowledged func(alert *Alert, user string)
onEscalate func(alert *Alert, level int)
escalationStop chan struct{}
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
// New fields for deduplication and suppression
recentAlerts map[string]*Alert // Track recent alerts for deduplication
suppressedUntil map[string]time.Time // Track suppression windows
@ -736,6 +738,20 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
m.onResolved = cb
}
// SetAcknowledgedCallback sets the callback for acknowledged alerts.
func (m *Manager) SetAcknowledgedCallback(cb func(alert *Alert, user string)) {
m.mu.Lock()
defer m.mu.Unlock()
m.onAcknowledged = cb
}
// SetUnacknowledgedCallback sets the callback for unacknowledged alerts.
func (m *Manager) SetUnacknowledgedCallback(cb func(alert *Alert, user string)) {
m.mu.Lock()
defer m.mu.Unlock()
m.onUnacknowledged = cb
}
// SetEscalateCallback sets the callback for escalated alerts
func (m *Manager) SetEscalateCallback(cb func(alert *Alert, level int)) {
m.mu.Lock()
@ -768,6 +784,46 @@ func (m *Manager) safeCallResolvedCallback(alertID string, async bool) {
}
}
// safeCallAcknowledgedCallback invokes onAcknowledged with panic recovery and alert cloning.
func (m *Manager) safeCallAcknowledgedCallback(alert *Alert, user string) {
if m.onAcknowledged == nil || alert == nil {
return
}
alertCopy := alert.Clone()
go func(a *Alert, u string) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", a.ID).
Msg("Panic in onAcknowledged callback")
}
}()
m.onAcknowledged(a, u)
}(alertCopy, user)
}
// safeCallUnacknowledgedCallback invokes onUnacknowledged with panic recovery and alert cloning.
func (m *Manager) safeCallUnacknowledgedCallback(alert *Alert, user string) {
if m.onUnacknowledged == nil || alert == nil {
return
}
alertCopy := alert.Clone()
go func(a *Alert, u string) {
defer func() {
if r := recover(); r != nil {
log.Error().
Interface("panic", r).
Str("alertID", a.ID).
Msg("Panic in onUnacknowledged callback")
}
}()
m.onUnacknowledged(a, u)
}(alertCopy, user)
}
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
if m.onEscalate == nil {
@ -2457,7 +2513,6 @@ func (m *Manager) hasHostAgentForNode(nodeName string) bool {
return exists
}
func hostResourceID(hostID string) string {
trimmed := strings.TrimSpace(hostID)
if trimmed == "" {
@ -5767,10 +5822,10 @@ func abs(x float64) float64 {
// AcknowledgeAlert acknowledges an alert
func (m *Manager) AcknowledgeAlert(alertID, user string) error {
m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.activeAlerts[alertID]
if !exists {
m.mu.Unlock()
return fmt.Errorf("alert not found: %s", alertID)
}
@ -5787,22 +5842,26 @@ func (m *Manager) AcknowledgeAlert(alertID, user string) error {
time: now,
}
alertCopy := alert.Clone()
m.mu.Unlock()
log.Debug().
Str("alertID", alertID).
Str("user", user).
Time("ackTime", now).
Msg("Alert acknowledgment recorded")
m.safeCallAcknowledgedCallback(alertCopy, user)
return nil
}
// UnacknowledgeAlert removes the acknowledged status from an alert
func (m *Manager) UnacknowledgeAlert(alertID string) error {
m.mu.Lock()
defer m.mu.Unlock()
alert, exists := m.activeAlerts[alertID]
if !exists {
m.mu.Unlock()
return fmt.Errorf("alert not found: %s", alertID)
}
@ -5814,10 +5873,14 @@ func (m *Manager) UnacknowledgeAlert(alertID string) error {
m.activeAlerts[alertID] = alert
delete(m.ackState, alertID)
alertCopy := alert.Clone()
m.mu.Unlock()
log.Info().
Str("alertID", alertID).
Msg("Alert unacknowledged")
m.safeCallUnacknowledgedCallback(alertCopy, "")
return nil
}
@ -7593,9 +7656,8 @@ func (m *Manager) ClearAlert(alertID string) bool {
// Cleanup removes old acknowledged alerts and cleans up tracking maps
func (m *Manager) Cleanup(maxAge time.Duration) {
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now()
var autoAcked []*Alert
// Auto-acknowledge old alerts if configured
if m.config.AutoAcknowledgeAfterHours > 0 {
@ -7610,6 +7672,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
ackTime := now
alert.AckTime = &ackTime
alert.AckUser = "system-auto"
autoAcked = append(autoAcked, alert.Clone())
if recordAlertAcknowledged != nil {
recordAlertAcknowledged()
@ -7776,6 +7839,12 @@ func (m *Manager) Cleanup(maxAge time.Duration) {
Msg("Cleaned up stale PMG quarantine history")
}
}
m.mu.Unlock()
for _, alert := range autoAcked {
m.safeCallAcknowledgedCallback(alert, "system-auto")
}
}
// convertLegacyThreshold converts a legacy float64 threshold to HysteresisThreshold

24
internal/api/ai_handlers.go
View file

@ -19,6 +19,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/license"
@ -118,6 +119,11 @@ func (h *AISettingsHandler) SetRemediationLog(remLog *ai.RemediationLog) {
h.aiService.SetRemediationLog(remLog)
}
// SetIncidentStore sets the incident store for alert timelines.
func (h *AISettingsHandler) SetIncidentStore(store *memory.IncidentStore) {
h.aiService.SetIncidentStore(store)
}
// SetPatternDetector sets the pattern detector for failure prediction
func (h *AISettingsHandler) SetPatternDetector(detector *ai.PatternDetector) {
h.aiService.SetPatternDetector(detector)
@ -1767,11 +1773,29 @@ func (h *AISettingsHandler) HandleInvestigateAlert(w http.ResponseWriter, r *htt
data, _ := json.Marshal(finalEvent)
safeWrite([]byte("data: " + string(data) + "\n\n"))
if req.AlertID != "" {
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI alert investigation completed", map[string]interface{}{
"model": resp.Model,
"tool_calls": len(resp.ToolCalls),
"input_tokens": resp.InputTokens,
"output_tokens": resp.OutputTokens,
})
}
log.Info().
Str("alert_id", req.AlertID).
Str("model", resp.Model).
Int("tool_calls", len(resp.ToolCalls)).
Msg("AI alert investigation completed")
if req.AlertID != "" {
h.aiService.RecordIncidentAnalysis(req.AlertID, "AI investigation completed", map[string]interface{}{
"model": resp.Model,
"input_tokens": resp.InputTokens,
"output_tokens": resp.OutputTokens,
"tool_calls": len(resp.ToolCalls),
})
}
}
// SetAlertProvider sets the alert provider for AI context

135
internal/api/alerts.go
View file

@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
@ -361,6 +362,130 @@ func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request)
}
}
// GetAlertIncidentTimeline returns the incident timeline for an alert or resource.
func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
store := h.monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
}
query := r.URL.Query()
alertID := strings.TrimSpace(query.Get("alert_id"))
resourceID := strings.TrimSpace(query.Get("resource_id"))
startedAtRaw := strings.TrimSpace(query.Get("started_at"))
if startedAtRaw == "" {
startedAtRaw = strings.TrimSpace(query.Get("start_time"))
}
limit := 0
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
if parsed, err := strconv.Atoi(rawLimit); err == nil && parsed > 0 {
limit = parsed
}
}
if alertID != "" {
if !validateAlertID(alertID) {
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
return
}
var startedAt time.Time
if startedAtRaw != "" {
parsed, err := time.Parse(time.RFC3339, startedAtRaw)
if err != nil {
http.Error(w, "Invalid started_at time", http.StatusBadRequest)
return
}
startedAt = parsed
}
var incident *memory.Incident
if !startedAt.IsZero() {
incident = store.GetTimelineByAlertAt(alertID, startedAt)
} else {
incident = store.GetTimelineByAlertID(alertID)
}
if err := utils.WriteJSONResponse(w, incident); err != nil {
log.Error().Err(err).Msg("Failed to write incident timeline response")
}
return
}
if resourceID != "" {
if len(resourceID) > 500 {
http.Error(w, "Invalid resource ID", http.StatusBadRequest)
return
}
incidents := store.ListIncidentsByResource(resourceID, limit)
if err := utils.WriteJSONResponse(w, incidents); err != nil {
log.Error().Err(err).Msg("Failed to write incident list response")
}
return
}
http.Error(w, "Missing alert_id or resource_id", http.StatusBadRequest)
}
// SaveAlertIncidentNote stores a user note in the incident timeline.
func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
store := h.monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 8*1024)
var req struct {
AlertID string `json:"alert_id"`
IncidentID string `json:"incident_id"`
Note string `json:"note"`
User string `json:"user,omitempty"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid request body", http.StatusBadRequest)
return
}
req.AlertID = strings.TrimSpace(req.AlertID)
req.IncidentID = strings.TrimSpace(req.IncidentID)
req.Note = strings.TrimSpace(req.Note)
req.User = strings.TrimSpace(req.User)
if req.AlertID == "" && req.IncidentID == "" {
http.Error(w, "alert_id or incident_id is required", http.StatusBadRequest)
return
}
if req.AlertID != "" && !validateAlertID(req.AlertID) {
http.Error(w, "Invalid alert ID", http.StatusBadRequest)
return
}
if req.Note == "" {
http.Error(w, "note is required", http.StatusBadRequest)
return
}
if ok := store.RecordNote(req.AlertID, req.IncidentID, req.Note, req.User); !ok {
http.Error(w, "Failed to save note", http.StatusBadRequest)
return
}
if err := utils.WriteJSONResponse(w, map[string]interface{}{
"success": true,
}); err != nil {
log.Error().Err(err).Msg("Failed to write incident note response")
}
}
// ClearAlertHistory clears all alert history
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
if err := h.monitor.GetAlertManager().ClearAlertHistory(); err != nil {
@ -737,6 +862,16 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
return
}
h.GetAlertHistory(w, r)
case path == "incidents" && r.Method == http.MethodGet:
if !ensureScope(w, r, config.ScopeMonitoringRead) {
return
}
h.GetAlertIncidentTimeline(w, r)
case path == "incidents/note" && r.Method == http.MethodPost:
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
return
}
h.SaveAlertIncidentNote(w, r)
case path == "history" && r.Method == http.MethodDelete:
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
return

3
internal/api/router.go
View file

@ -1109,6 +1109,9 @@ func (r *Router) setupRoutes() {
if alertManager := r.monitor.GetAlertManager(); alertManager != nil {
r.aiSettingsHandler.SetAlertProvider(ai.NewAlertManagerAdapter(alertManager))
}
if incidentStore := r.monitor.GetIncidentStore(); incidentStore != nil {
r.aiSettingsHandler.SetIncidentStore(incidentStore)
}
}
// Inject unified resource provider for Phase 2 AI context (cleaner, deduplicated view)
if r.resourceHandlers != nil {

106
internal/monitoring/monitor.go
View file

@ -20,6 +20,7 @@ import (
"sync/atomic"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
@ -604,6 +605,7 @@ type Monitor struct {
metricsHistory *MetricsHistory
metricsStore *metrics.Store // Persistent SQLite metrics storage
alertManager *alerts.Manager
incidentStore *memory.IncidentStore
notificationMgr *notifications.NotificationManager
configPersist *config.ConfigPersistence
discoveryService *discovery.Service // Background discovery service
@ -3050,6 +3052,10 @@ func New(cfg *config.Config) (*Monitor, error) {
Msg("Persistent metrics store initialized with configurable retention")
}
incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{
DataDir: cfg.DataPath,
})
m := &Monitor{
config: cfg,
state: models.NewState(),
@ -3076,6 +3082,7 @@ func New(cfg *config.Config) (*Monitor, error) {
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
metricsStore: metricsStore, // Persistent SQLite storage
alertManager: alerts.NewManager(),
incidentStore: incidentStore,
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
configPersist: config.NewConfigPersistence(cfg.DataPath),
discoveryService: nil, // Will be initialized in Start()
@ -3756,25 +3763,19 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
// Set up alert callbacks
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
wsHub.BroadcastAlert(alert)
// Send notifications
log.Debug().
Str("alertID", alert.ID).
Str("level", string(alert.Level)).
Msg("Alert raised, sending to notification manager")
go m.notificationMgr.SendAlert(alert)
m.handleAlertFired(alert)
})
m.alertManager.SetResolvedCallback(func(alertID string) {
wsHub.BroadcastAlertResolved(alertID)
m.notificationMgr.CancelAlert(alertID)
if m.notificationMgr.GetNotifyOnResolve() {
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
go m.notificationMgr.SendResolvedAlert(resolved)
}
}
m.handleAlertResolved(alertID)
// Don't broadcast full state here - it causes a cascade with many guests.
// The frontend will get the updated alerts through the regular broadcast ticker.
})
m.alertManager.SetAcknowledgedCallback(func(alert *alerts.Alert, user string) {
m.handleAlertAcknowledged(alert, user)
})
m.alertManager.SetUnacknowledgedCallback(func(alert *alerts.Alert, user string) {
m.handleAlertUnacknowledged(alert, user)
})
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
log.Info().
Str("alertID", alert.ID).
@ -6355,7 +6356,7 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
}
}
}
// Trigger guest metadata migration if old format exists
if m.guestMetadataStore != nil {
m.guestMetadataStore.GetWithLegacyMigration(guestID, instanceName, res.Node, res.VMID)
@ -7731,6 +7732,11 @@ func (m *Monitor) GetAlertManager() *alerts.Manager {
return m.alertManager
}
// GetIncidentStore returns the incident timeline store.
func (m *Monitor) GetIncidentStore() *memory.IncidentStore {
return m.incidentStore
}
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
// This enables token-efficient, real-time AI insights on specific resources
func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
@ -7738,33 +7744,69 @@ func (m *Monitor) SetAlertTriggeredAICallback(callback func(*alerts.Alert)) {
return
}
// Get the current callback
originalCallback := m.alertManager
// Wrap the existing callback to also call the AI callback
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
// Broadcast to WebSocket (this happens via the callback set in Start())
if m.wsHub != nil {
m.wsHub.BroadcastAlert(alert)
}
// Send notifications
log.Debug().
Str("alertID", alert.ID).
Str("level", string(alert.Level)).
Msg("Alert raised, sending to notification manager")
go m.notificationMgr.SendAlert(alert)
m.handleAlertFired(alert)
// Trigger AI analysis
go callback(alert)
})
// Avoid unused variable warning
_ = originalCallback
log.Info().Msg("Alert-triggered AI callback registered")
}
func (m *Monitor) handleAlertFired(alert *alerts.Alert) {
if alert == nil {
return
}
if m.wsHub != nil {
m.wsHub.BroadcastAlert(alert)
}
log.Debug().
Str("alertID", alert.ID).
Str("level", string(alert.Level)).
Msg("Alert raised, sending to notification manager")
go m.notificationMgr.SendAlert(alert)
if m.incidentStore != nil {
m.incidentStore.RecordAlertFired(alert)
}
}
func (m *Monitor) handleAlertResolved(alertID string) {
if m.wsHub != nil {
m.wsHub.BroadcastAlertResolved(alertID)
}
m.notificationMgr.CancelAlert(alertID)
if m.notificationMgr.GetNotifyOnResolve() {
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil {
go m.notificationMgr.SendResolvedAlert(resolved)
}
}
if m.incidentStore != nil {
if resolved := m.alertManager.GetResolvedAlert(alertID); resolved != nil && resolved.Alert != nil {
m.incidentStore.RecordAlertResolved(resolved.Alert, resolved.ResolvedTime)
}
}
}
func (m *Monitor) handleAlertAcknowledged(alert *alerts.Alert, user string) {
if m.incidentStore == nil || alert == nil {
return
}
m.incidentStore.RecordAlertAcknowledged(alert, user)
}
func (m *Monitor) handleAlertUnacknowledged(alert *alerts.Alert, user string) {
if m.incidentStore == nil || alert == nil {
return
}
m.incidentStore.RecordAlertUnacknowledged(alert, user)
}
// SetResourceStore sets the resource store for polling optimization.
// When set, the monitor will check if it should reduce polling frequency
// for nodes that have host agents providing data.