feat(ai): Add operational memory (Phase 3) - change detection and remediation logging
Phase 3 of Pulse AI differentiation: Create internal/ai/memory package with: 1. Change Detection (changes.go): - Tracks infrastructure changes: creation, deletion, config changes - Detects status changes (started, stopped) - Detects VM/container migrations between nodes - Detects CPU/memory configuration changes - Detects backup completions - Persists change history to ai_changes.json - GetChangesSummary for AI context 2. Remediation Logging (remediation.go): - Records actions taken to fix problems - Tracks command, output, and outcome - Links to AI findings via findingID - GetSimilar finds past similar problems - GetSuccessfulRemediations for learning - Persists to ai_remediations.json 3. Type exports (memory_exports.go): - Clean re-exports from ai package This enables the AI to say things like: - 'This VM was migrated 2 hours ago' - 'Memory was increased from 4GB to 8GB yesterday' - 'Last time this happened, restarting nginx resolved it' All tests passing.
This commit is contained in:
parent
ec8f306ad1
commit
286edee8aa
4 changed files with 1149 additions and 0 deletions
471
internal/ai/memory/changes.go
Normal file
471
internal/ai/memory/changes.go
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
// Package memory provides operational memory for AI context.
|
||||
// It tracks changes to infrastructure, remediation actions, and their outcomes
|
||||
// to enable the AI to provide more informed, experience-based recommendations.
|
||||
package memory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// ChangeType represents the type of infrastructure change detected
|
||||
type ChangeType string
|
||||
|
||||
const (
|
||||
ChangeCreated ChangeType = "created" // New resource appeared
|
||||
ChangeDeleted ChangeType = "deleted" // Resource removed
|
||||
ChangeConfig ChangeType = "config" // Configuration changed (RAM, CPU, etc)
|
||||
ChangeStatus ChangeType = "status" // Status changed (started, stopped, paused)
|
||||
ChangeMigrated ChangeType = "migrated" // Moved to different node
|
||||
ChangeRestarted ChangeType = "restarted" // Resource was restarted
|
||||
ChangeBackedUp ChangeType = "backed_up" // Backup completed
|
||||
)
|
||||
|
||||
// Change represents a detected change to infrastructure
|
||||
type Change struct {
|
||||
ID string `json:"id"`
|
||||
ResourceID string `json:"resource_id"`
|
||||
ResourceType string `json:"resource_type"` // vm, container, node, storage
|
||||
ResourceName string `json:"resource_name"`
|
||||
ChangeType ChangeType `json:"change_type"`
|
||||
Before interface{} `json:"before,omitempty"`
|
||||
After interface{} `json:"after,omitempty"`
|
||||
DetectedAt time.Time `json:"detected_at"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// ResourceSnapshot captures key attributes for change detection
|
||||
type ResourceSnapshot struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
Node string `json:"node,omitempty"`
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
MemoryBytes int64 `json:"memory_bytes,omitempty"`
|
||||
DiskBytes int64 `json:"disk_bytes,omitempty"`
|
||||
LastBackup time.Time `json:"last_backup,omitempty"`
|
||||
SnapshotTime time.Time `json:"snapshot_time"`
|
||||
}
|
||||
|
||||
// ChangeDetector tracks infrastructure changes over time
|
||||
type ChangeDetector struct {
|
||||
mu sync.RWMutex
|
||||
previousState map[string]ResourceSnapshot // resourceID -> snapshot
|
||||
changes []Change
|
||||
maxChanges int
|
||||
|
||||
// Persistence
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// ChangeDetectorConfig configures the change detector
|
||||
type ChangeDetectorConfig struct {
|
||||
MaxChanges int // Maximum changes to retain (default: 1000)
|
||||
DataDir string // Directory for persistence
|
||||
}
|
||||
|
||||
// NewChangeDetector creates a new change detector
|
||||
func NewChangeDetector(cfg ChangeDetectorConfig) *ChangeDetector {
|
||||
if cfg.MaxChanges <= 0 {
|
||||
cfg.MaxChanges = 1000
|
||||
}
|
||||
|
||||
d := &ChangeDetector{
|
||||
previousState: make(map[string]ResourceSnapshot),
|
||||
changes: make([]Change, 0),
|
||||
maxChanges: cfg.MaxChanges,
|
||||
dataDir: cfg.DataDir,
|
||||
}
|
||||
|
||||
// Load existing changes from disk
|
||||
if cfg.DataDir != "" {
|
||||
if err := d.loadFromDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to load change history from disk")
|
||||
} else if len(d.changes) > 0 {
|
||||
log.Info().Int("count", len(d.changes)).Msg("Loaded change history from disk")
|
||||
}
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
// DetectChanges compares current snapshots to previous state and detects changes
|
||||
func (d *ChangeDetector) DetectChanges(currentSnapshots []ResourceSnapshot) []Change {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
var newChanges []Change
|
||||
|
||||
// Track which resources we've seen in current snapshot
|
||||
currentIDs := make(map[string]bool)
|
||||
|
||||
for _, current := range currentSnapshots {
|
||||
currentIDs[current.ID] = true
|
||||
|
||||
prev, exists := d.previousState[current.ID]
|
||||
if !exists {
|
||||
// New resource
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: current.ID,
|
||||
ResourceType: current.Type,
|
||||
ResourceName: current.Name,
|
||||
ChangeType: ChangeCreated,
|
||||
After: current,
|
||||
DetectedAt: now,
|
||||
Description: formatCreateDescription(current),
|
||||
}
|
||||
newChanges = append(newChanges, change)
|
||||
} else {
|
||||
// Check for changes
|
||||
changes := d.detectResourceChanges(prev, current, now)
|
||||
newChanges = append(newChanges, changes...)
|
||||
}
|
||||
|
||||
// Update previous state
|
||||
d.previousState[current.ID] = current
|
||||
}
|
||||
|
||||
// Check for deleted resources
|
||||
for id, prev := range d.previousState {
|
||||
if !currentIDs[id] {
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: id,
|
||||
ResourceType: prev.Type,
|
||||
ResourceName: prev.Name,
|
||||
ChangeType: ChangeDeleted,
|
||||
Before: prev,
|
||||
DetectedAt: now,
|
||||
Description: formatDeleteDescription(prev),
|
||||
}
|
||||
newChanges = append(newChanges, change)
|
||||
delete(d.previousState, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Store new changes
|
||||
if len(newChanges) > 0 {
|
||||
d.changes = append(d.changes, newChanges...)
|
||||
d.trimChanges()
|
||||
|
||||
// Persist asynchronously
|
||||
go func() {
|
||||
if err := d.saveToDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to save change history")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
return newChanges
|
||||
}
|
||||
|
||||
// detectResourceChanges checks for changes between two snapshots of the same resource
|
||||
func (d *ChangeDetector) detectResourceChanges(prev, current ResourceSnapshot, now time.Time) []Change {
|
||||
var changes []Change
|
||||
|
||||
// Status change
|
||||
if prev.Status != current.Status {
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: current.ID,
|
||||
ResourceType: current.Type,
|
||||
ResourceName: current.Name,
|
||||
ChangeType: ChangeStatus,
|
||||
Before: prev.Status,
|
||||
After: current.Status,
|
||||
DetectedAt: now,
|
||||
Description: formatStatusDescription(current.Name, prev.Status, current.Status),
|
||||
}
|
||||
changes = append(changes, change)
|
||||
}
|
||||
|
||||
// Node change (migration)
|
||||
if prev.Node != "" && current.Node != "" && prev.Node != current.Node {
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: current.ID,
|
||||
ResourceType: current.Type,
|
||||
ResourceName: current.Name,
|
||||
ChangeType: ChangeMigrated,
|
||||
Before: prev.Node,
|
||||
After: current.Node,
|
||||
DetectedAt: now,
|
||||
Description: formatMigrationDescription(current.Name, prev.Node, current.Node),
|
||||
}
|
||||
changes = append(changes, change)
|
||||
}
|
||||
|
||||
// CPU change
|
||||
if prev.CPUCores > 0 && current.CPUCores > 0 && prev.CPUCores != current.CPUCores {
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: current.ID,
|
||||
ResourceType: current.Type,
|
||||
ResourceName: current.Name,
|
||||
ChangeType: ChangeConfig,
|
||||
Before: map[string]int{"cpu_cores": prev.CPUCores},
|
||||
After: map[string]int{"cpu_cores": current.CPUCores},
|
||||
DetectedAt: now,
|
||||
Description: formatCPUChangeDescription(current.Name, prev.CPUCores, current.CPUCores),
|
||||
}
|
||||
changes = append(changes, change)
|
||||
}
|
||||
|
||||
// Memory change (significant change > 5%)
|
||||
if prev.MemoryBytes > 0 && current.MemoryBytes > 0 {
|
||||
pctChange := float64(current.MemoryBytes-prev.MemoryBytes) / float64(prev.MemoryBytes)
|
||||
if pctChange > 0.05 || pctChange < -0.05 {
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: current.ID,
|
||||
ResourceType: current.Type,
|
||||
ResourceName: current.Name,
|
||||
ChangeType: ChangeConfig,
|
||||
Before: map[string]int64{"memory_bytes": prev.MemoryBytes},
|
||||
After: map[string]int64{"memory_bytes": current.MemoryBytes},
|
||||
DetectedAt: now,
|
||||
Description: formatMemoryChangeDescription(current.Name, prev.MemoryBytes, current.MemoryBytes),
|
||||
}
|
||||
changes = append(changes, change)
|
||||
}
|
||||
}
|
||||
|
||||
// Backup completed
|
||||
if !prev.LastBackup.IsZero() && !current.LastBackup.IsZero() &&
|
||||
current.LastBackup.After(prev.LastBackup) {
|
||||
change := Change{
|
||||
ID: generateChangeID(),
|
||||
ResourceID: current.ID,
|
||||
ResourceType: current.Type,
|
||||
ResourceName: current.Name,
|
||||
ChangeType: ChangeBackedUp,
|
||||
Before: prev.LastBackup,
|
||||
After: current.LastBackup,
|
||||
DetectedAt: now,
|
||||
Description: formatBackupDescription(current.Name, current.LastBackup),
|
||||
}
|
||||
changes = append(changes, change)
|
||||
}
|
||||
|
||||
return changes
|
||||
}
|
||||
|
||||
// GetChangesForResource returns recent changes for a specific resource
|
||||
func (d *ChangeDetector) GetChangesForResource(resourceID string, limit int) []Change {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
var result []Change
|
||||
// Iterate in reverse to get most recent first
|
||||
for i := len(d.changes) - 1; i >= 0 && len(result) < limit; i-- {
|
||||
if d.changes[i].ResourceID == resourceID {
|
||||
result = append(result, d.changes[i])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetRecentChanges returns the most recent changes across all resources
|
||||
func (d *ChangeDetector) GetRecentChanges(limit int, since time.Time) []Change {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
var result []Change
|
||||
for i := len(d.changes) - 1; i >= 0 && len(result) < limit; i-- {
|
||||
if d.changes[i].DetectedAt.After(since) {
|
||||
result = append(result, d.changes[i])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetChangesSummary returns a formatted summary of recent changes for AI context
|
||||
func (d *ChangeDetector) GetChangesSummary(since time.Time, maxChanges int) string {
|
||||
changes := d.GetRecentChanges(maxChanges, since)
|
||||
if len(changes) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var result string
|
||||
for _, c := range changes {
|
||||
ago := time.Since(c.DetectedAt)
|
||||
result += "- " + c.Description + " (" + formatDuration(ago) + " ago)\n"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// trimChanges removes old changes beyond maxChanges
|
||||
func (d *ChangeDetector) trimChanges() {
|
||||
if len(d.changes) > d.maxChanges {
|
||||
// Keep most recent
|
||||
d.changes = d.changes[len(d.changes)-d.maxChanges:]
|
||||
}
|
||||
}
|
||||
|
||||
// saveToDisk persists changes to JSON file
|
||||
func (d *ChangeDetector) saveToDisk() error {
|
||||
if d.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.mu.RLock()
|
||||
changes := make([]Change, len(d.changes))
|
||||
copy(changes, d.changes)
|
||||
d.mu.RUnlock()
|
||||
|
||||
path := filepath.Join(d.dataDir, "ai_changes.json")
|
||||
data, err := json.MarshalIndent(changes, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpPath := path + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(tmpPath, path)
|
||||
}
|
||||
|
||||
// loadFromDisk loads changes from JSON file
|
||||
func (d *ChangeDetector) loadFromDisk() error {
|
||||
if d.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
path := filepath.Join(d.dataDir, "ai_changes.json")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var changes []Change
|
||||
if err := json.Unmarshal(data, &changes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort by time
|
||||
sort.Slice(changes, func(i, j int) bool {
|
||||
return changes[i].DetectedAt.Before(changes[j].DetectedAt)
|
||||
})
|
||||
|
||||
d.changes = changes
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
var changeCounter int64
|
||||
|
||||
func generateChangeID() string {
|
||||
changeCounter++
|
||||
return time.Now().Format("20060102150405") + "-" + string(rune('0'+changeCounter%10))
|
||||
}
|
||||
|
||||
func formatDuration(d time.Duration) string {
|
||||
if d < time.Minute {
|
||||
return "just now"
|
||||
}
|
||||
if d < time.Hour {
|
||||
return formatUnit(int(d.Minutes()), "minute")
|
||||
}
|
||||
if d < 24*time.Hour {
|
||||
return formatUnit(int(d.Hours()), "hour")
|
||||
}
|
||||
return formatUnit(int(d.Hours()/24), "day")
|
||||
}
|
||||
|
||||
func formatUnit(n int, unit string) string {
|
||||
if n == 1 {
|
||||
return "1 " + unit
|
||||
}
|
||||
return string(rune('0'+n/10)) + string(rune('0'+n%10)) + " " + unit + "s"
|
||||
}
|
||||
|
||||
func formatBytes(bytes int64) string {
|
||||
const (
|
||||
KB = 1024
|
||||
MB = KB * 1024
|
||||
GB = MB * 1024
|
||||
)
|
||||
switch {
|
||||
case bytes >= GB:
|
||||
return formatFloat(float64(bytes)/GB) + " GB"
|
||||
case bytes >= MB:
|
||||
return formatFloat(float64(bytes)/MB) + " MB"
|
||||
case bytes >= KB:
|
||||
return formatFloat(float64(bytes)/KB) + " KB"
|
||||
default:
|
||||
return string(rune('0'+bytes/10)) + string(rune('0'+bytes%10)) + " B"
|
||||
}
|
||||
}
|
||||
|
||||
func formatFloat(v float64) string {
|
||||
// Simple formatting without fmt dependency
|
||||
whole := int(v)
|
||||
frac := int((v - float64(whole)) * 10)
|
||||
if frac == 0 {
|
||||
return intToString(whole)
|
||||
}
|
||||
return intToString(whole) + "." + string(rune('0'+frac))
|
||||
}
|
||||
|
||||
func intToString(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var result string
|
||||
for n > 0 {
|
||||
result = string(rune('0'+n%10)) + result
|
||||
n /= 10
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func formatCreateDescription(r ResourceSnapshot) string {
|
||||
return r.Type + " '" + r.Name + "' created"
|
||||
}
|
||||
|
||||
func formatDeleteDescription(r ResourceSnapshot) string {
|
||||
return r.Type + " '" + r.Name + "' deleted"
|
||||
}
|
||||
|
||||
func formatStatusDescription(name, before, after string) string {
|
||||
return "'" + name + "' status changed: " + before + " → " + after
|
||||
}
|
||||
|
||||
func formatMigrationDescription(name, fromNode, toNode string) string {
|
||||
return "'" + name + "' migrated from " + fromNode + " to " + toNode
|
||||
}
|
||||
|
||||
func formatCPUChangeDescription(name string, before, after int) string {
|
||||
direction := "increased"
|
||||
if after < before {
|
||||
direction = "decreased"
|
||||
}
|
||||
return "'" + name + "' CPU " + direction + ": " + intToString(before) + " → " + intToString(after) + " cores"
|
||||
}
|
||||
|
||||
func formatMemoryChangeDescription(name string, before, after int64) string {
|
||||
direction := "increased"
|
||||
if after < before {
|
||||
direction = "decreased"
|
||||
}
|
||||
return "'" + name + "' memory " + direction + ": " + formatBytes(before) + " → " + formatBytes(after)
|
||||
}
|
||||
|
||||
func formatBackupDescription(name string, backupTime time.Time) string {
|
||||
return "'" + name + "' backup completed at " + backupTime.Format("2006-01-02 15:04")
|
||||
}
|
||||
232
internal/ai/memory/memory_test.go
Normal file
232
internal/ai/memory/memory_test.go
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
package memory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestChangeDetector_DetectNew(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
// First detection - should see new resource
|
||||
snapshots := []ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "running", Node: "node1"},
|
||||
}
|
||||
|
||||
changes := d.DetectChanges(snapshots)
|
||||
if len(changes) != 1 {
|
||||
t.Errorf("Expected 1 change (creation), got %d", len(changes))
|
||||
}
|
||||
if changes[0].ChangeType != ChangeCreated {
|
||||
t.Errorf("Expected ChangeCreated, got %s", changes[0].ChangeType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeDetector_DetectStatusChange(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
// Initial state
|
||||
d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "running", Node: "node1"},
|
||||
})
|
||||
|
||||
// Status change
|
||||
changes := d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "stopped", Node: "node1"},
|
||||
})
|
||||
|
||||
if len(changes) != 1 {
|
||||
t.Errorf("Expected 1 status change, got %d", len(changes))
|
||||
}
|
||||
if changes[0].ChangeType != ChangeStatus {
|
||||
t.Errorf("Expected ChangeStatus, got %s", changes[0].ChangeType)
|
||||
}
|
||||
if changes[0].Before != "running" || changes[0].After != "stopped" {
|
||||
t.Errorf("Expected running->stopped, got %v->%v", changes[0].Before, changes[0].After)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeDetector_DetectMigration(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
// Initial state
|
||||
d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "running", Node: "node1"},
|
||||
})
|
||||
|
||||
// Migration
|
||||
changes := d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "running", Node: "node2"},
|
||||
})
|
||||
|
||||
if len(changes) != 1 {
|
||||
t.Errorf("Expected 1 migration change, got %d", len(changes))
|
||||
}
|
||||
if changes[0].ChangeType != ChangeMigrated {
|
||||
t.Errorf("Expected ChangeMigrated, got %s", changes[0].ChangeType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeDetector_DetectDeleted(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
// Initial state
|
||||
d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "running", Node: "node1"},
|
||||
})
|
||||
|
||||
// Delete (empty snapshot)
|
||||
changes := d.DetectChanges([]ResourceSnapshot{})
|
||||
|
||||
if len(changes) != 1 {
|
||||
t.Errorf("Expected 1 deletion change, got %d", len(changes))
|
||||
}
|
||||
if changes[0].ChangeType != ChangeDeleted {
|
||||
t.Errorf("Expected ChangeDeleted, got %s", changes[0].ChangeType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeDetector_NoChanges(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
snapshot := []ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web-server", Type: "vm", Status: "running", Node: "node1"},
|
||||
}
|
||||
|
||||
// First time - creates
|
||||
d.DetectChanges(snapshot)
|
||||
|
||||
// Second time - no changes
|
||||
changes := d.DetectChanges(snapshot)
|
||||
if len(changes) != 0 {
|
||||
t.Errorf("Expected 0 changes, got %d", len(changes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeDetector_GetChangesForResource(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
// Create and change a few times
|
||||
d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web", Type: "vm", Status: "stopped", Node: "node1"},
|
||||
{ID: "vm-200", Name: "db", Type: "vm", Status: "running", Node: "node1"},
|
||||
})
|
||||
d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web", Type: "vm", Status: "running", Node: "node1"},
|
||||
{ID: "vm-200", Name: "db", Type: "vm", Status: "running", Node: "node1"},
|
||||
})
|
||||
|
||||
// Get changes for vm-100 only
|
||||
changes := d.GetChangesForResource("vm-100", 10)
|
||||
for _, c := range changes {
|
||||
if c.ResourceID != "vm-100" {
|
||||
t.Errorf("Got change for wrong resource: %s", c.ResourceID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediationLog_LogAndRetrieve(t *testing.T) {
|
||||
r := NewRemediationLog(RemediationLogConfig{MaxRecords: 100})
|
||||
|
||||
r.Log(RemediationRecord{
|
||||
ResourceID: "vm-100",
|
||||
Problem: "High memory usage",
|
||||
Action: "systemctl restart nginx",
|
||||
Outcome: OutcomeResolved,
|
||||
})
|
||||
|
||||
records := r.GetForResource("vm-100", 10)
|
||||
if len(records) != 1 {
|
||||
t.Errorf("Expected 1 record, got %d", len(records))
|
||||
}
|
||||
if records[0].Action != "systemctl restart nginx" {
|
||||
t.Errorf("Wrong action: %s", records[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediationLog_GetSimilar(t *testing.T) {
|
||||
r := NewRemediationLog(RemediationLogConfig{MaxRecords: 100})
|
||||
|
||||
// Log some remediations
|
||||
r.Log(RemediationRecord{
|
||||
ResourceID: "vm-100",
|
||||
Problem: "High memory usage causing OOM",
|
||||
Action: "Restart service",
|
||||
Outcome: OutcomeResolved,
|
||||
})
|
||||
r.Log(RemediationRecord{
|
||||
ResourceID: "vm-200",
|
||||
Problem: "Memory leak detected",
|
||||
Action: "Cleared cache",
|
||||
Outcome: OutcomePartial,
|
||||
})
|
||||
r.Log(RemediationRecord{
|
||||
ResourceID: "vm-300",
|
||||
Problem: "CPU spike from backup",
|
||||
Action: "Rescheduled backup",
|
||||
Outcome: OutcomeResolved,
|
||||
})
|
||||
|
||||
// Search for similar memory issues
|
||||
similar := r.GetSimilar("High memory usage causing slowdown", 5)
|
||||
if len(similar) < 1 {
|
||||
t.Errorf("Expected at least 1 similar record")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediationLog_GetSuccessfulRemediations(t *testing.T) {
|
||||
r := NewRemediationLog(RemediationLogConfig{MaxRecords: 100})
|
||||
|
||||
r.Log(RemediationRecord{
|
||||
Problem: "Memory usage high",
|
||||
Action: "Restart service",
|
||||
Outcome: OutcomeResolved,
|
||||
})
|
||||
r.Log(RemediationRecord{
|
||||
Problem: "Memory usage high",
|
||||
Action: "Kill process",
|
||||
Outcome: OutcomeFailed,
|
||||
})
|
||||
|
||||
successful := r.GetSuccessfulRemediations("Memory usage issue", 5)
|
||||
for _, rec := range successful {
|
||||
if rec.Outcome != OutcomeResolved && rec.Outcome != OutcomePartial {
|
||||
t.Errorf("Got unsuccessful remediation in successful list: %s", rec.Outcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemediationLog_Stats(t *testing.T) {
|
||||
r := NewRemediationLog(RemediationLogConfig{MaxRecords: 100})
|
||||
|
||||
r.Log(RemediationRecord{Problem: "p1", Action: "a1", Outcome: OutcomeResolved})
|
||||
r.Log(RemediationRecord{Problem: "p2", Action: "a2", Outcome: OutcomeResolved})
|
||||
r.Log(RemediationRecord{Problem: "p3", Action: "a3", Outcome: OutcomeFailed})
|
||||
|
||||
stats := r.GetRemediationStats()
|
||||
if stats["total"] != 3 {
|
||||
t.Errorf("Expected 3 total, got %d", stats["total"])
|
||||
}
|
||||
if stats["resolved"] != 2 {
|
||||
t.Errorf("Expected 2 resolved, got %d", stats["resolved"])
|
||||
}
|
||||
if stats["failed"] != 1 {
|
||||
t.Errorf("Expected 1 failed, got %d", stats["failed"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeDetector_GetRecentChanges(t *testing.T) {
|
||||
d := NewChangeDetector(ChangeDetectorConfig{MaxChanges: 100})
|
||||
|
||||
// Create some changes
|
||||
d.DetectChanges([]ResourceSnapshot{
|
||||
{ID: "vm-100", Name: "web", Type: "vm", Status: "running", Node: "node1"},
|
||||
})
|
||||
|
||||
// Get recent changes
|
||||
since := time.Now().Add(-1 * time.Hour)
|
||||
changes := d.GetRecentChanges(10, since)
|
||||
if len(changes) == 0 {
|
||||
t.Error("Expected at least 1 recent change")
|
||||
}
|
||||
}
|
||||
385
internal/ai/memory/remediation.go
Normal file
385
internal/ai/memory/remediation.go
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
package memory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Outcome represents the result of a remediation action
|
||||
type Outcome string
|
||||
|
||||
const (
|
||||
OutcomeResolved Outcome = "resolved" // Problem was fixed
|
||||
OutcomePartial Outcome = "partial" // Partially fixed
|
||||
OutcomeFailed Outcome = "failed" // Action failed
|
||||
OutcomeUnknown Outcome = "unknown" // Outcome not determined
|
||||
)
|
||||
|
||||
// RemediationRecord represents a logged remediation action
|
||||
type RemediationRecord struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
ResourceID string `json:"resource_id"`
|
||||
ResourceType string `json:"resource_type,omitempty"`
|
||||
ResourceName string `json:"resource_name,omitempty"`
|
||||
FindingID string `json:"finding_id,omitempty"` // Linked AI finding if any
|
||||
Problem string `json:"problem"` // What was wrong
|
||||
Action string `json:"action"` // What was done (command or action)
|
||||
Output string `json:"output,omitempty"` // Command output if any
|
||||
Outcome Outcome `json:"outcome"` // Did it work?
|
||||
Duration time.Duration `json:"duration,omitempty"` // How long until resolved
|
||||
Note string `json:"note,omitempty"` // Optional user/AI note
|
||||
Automatic bool `json:"automatic"` // Was this triggered automatically by AI
|
||||
}
|
||||
|
||||
// RemediationLog stores remediation history
|
||||
type RemediationLog struct {
|
||||
mu sync.RWMutex
|
||||
records []RemediationRecord
|
||||
maxRecords int
|
||||
|
||||
// Persistence
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// RemediationLogConfig configures the remediation log
|
||||
type RemediationLogConfig struct {
|
||||
MaxRecords int
|
||||
DataDir string
|
||||
}
|
||||
|
||||
// NewRemediationLog creates a new remediation log
|
||||
func NewRemediationLog(cfg RemediationLogConfig) *RemediationLog {
|
||||
if cfg.MaxRecords <= 0 {
|
||||
cfg.MaxRecords = 500
|
||||
}
|
||||
|
||||
r := &RemediationLog{
|
||||
records: make([]RemediationRecord, 0),
|
||||
maxRecords: cfg.MaxRecords,
|
||||
dataDir: cfg.DataDir,
|
||||
}
|
||||
|
||||
// Load existing records from disk
|
||||
if cfg.DataDir != "" {
|
||||
if err := r.loadFromDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to load remediation log from disk")
|
||||
} else if len(r.records) > 0 {
|
||||
log.Info().Int("count", len(r.records)).Msg("Loaded remediation log from disk")
|
||||
}
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// Log records a remediation action
|
||||
func (r *RemediationLog) Log(record RemediationRecord) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if record.ID == "" {
|
||||
record.ID = generateRecordID()
|
||||
}
|
||||
if record.Timestamp.IsZero() {
|
||||
record.Timestamp = time.Now()
|
||||
}
|
||||
|
||||
r.records = append(r.records, record)
|
||||
r.trimRecords()
|
||||
|
||||
// Persist asynchronously
|
||||
go func() {
|
||||
if err := r.saveToDisk(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to save remediation log")
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LogCommand is a convenience method for logging a command execution
|
||||
func (r *RemediationLog) LogCommand(resourceID, resourceType, resourceName, findingID, problem, command, output string, success bool, automatic bool) {
|
||||
outcome := OutcomeUnknown
|
||||
if success {
|
||||
outcome = OutcomeResolved
|
||||
} else {
|
||||
outcome = OutcomeFailed
|
||||
}
|
||||
|
||||
r.Log(RemediationRecord{
|
||||
ResourceID: resourceID,
|
||||
ResourceType: resourceType,
|
||||
ResourceName: resourceName,
|
||||
FindingID: findingID,
|
||||
Problem: problem,
|
||||
Action: command,
|
||||
Output: truncateOutput(output, 1000),
|
||||
Outcome: outcome,
|
||||
Automatic: automatic,
|
||||
})
|
||||
}
|
||||
|
||||
// GetForResource returns remediation history for a specific resource
|
||||
func (r *RemediationLog) GetForResource(resourceID string, limit int) []RemediationRecord {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []RemediationRecord
|
||||
// Iterate in reverse to get most recent first
|
||||
for i := len(r.records) - 1; i >= 0 && len(result) < limit; i-- {
|
||||
if r.records[i].ResourceID == resourceID {
|
||||
result = append(result, r.records[i])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetForFinding returns remediation history linked to a specific finding
|
||||
func (r *RemediationLog) GetForFinding(findingID string, limit int) []RemediationRecord {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []RemediationRecord
|
||||
for i := len(r.records) - 1; i >= 0 && len(result) < limit; i-- {
|
||||
if r.records[i].FindingID == findingID {
|
||||
result = append(result, r.records[i])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetSimilar finds remediation records with similar problems
|
||||
func (r *RemediationLog) GetSimilar(problem string, limit int) []RemediationRecord {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
// Simple keyword-based matching
|
||||
keywords := extractKeywords(problem)
|
||||
if len(keywords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
type scored struct {
|
||||
record RemediationRecord
|
||||
score int
|
||||
}
|
||||
|
||||
var matches []scored
|
||||
for _, record := range r.records {
|
||||
recordKeywords := extractKeywords(record.Problem)
|
||||
score := countMatches(keywords, recordKeywords)
|
||||
if score > 0 {
|
||||
matches = append(matches, scored{record: record, score: score})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score descending
|
||||
sort.Slice(matches, func(i, j int) bool {
|
||||
return matches[i].score > matches[j].score
|
||||
})
|
||||
|
||||
var result []RemediationRecord
|
||||
for i := 0; i < len(matches) && len(result) < limit; i++ {
|
||||
result = append(result, matches[i].record)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetSuccessfulRemediations returns successful remediations for similar problems
|
||||
func (r *RemediationLog) GetSuccessfulRemediations(problem string, limit int) []RemediationRecord {
|
||||
similar := r.GetSimilar(problem, limit*3) // Get more to filter
|
||||
var result []RemediationRecord
|
||||
for _, rec := range similar {
|
||||
if rec.Outcome == OutcomeResolved || rec.Outcome == OutcomePartial {
|
||||
result = append(result, rec)
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetRecentRemediations returns the most recent remediations
|
||||
func (r *RemediationLog) GetRecentRemediations(limit int, since time.Time) []RemediationRecord {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
var result []RemediationRecord
|
||||
for i := len(r.records) - 1; i >= 0 && len(result) < limit; i-- {
|
||||
if r.records[i].Timestamp.After(since) {
|
||||
result = append(result, r.records[i])
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// FormatForContext creates AI-consumable summary of remediation history
|
||||
func (r *RemediationLog) FormatForContext(resourceID string, limit int) string {
|
||||
records := r.GetForResource(resourceID, limit)
|
||||
if len(records) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var result string
|
||||
for _, rec := range records {
|
||||
ago := time.Since(rec.Timestamp)
|
||||
outcomeStr := string(rec.Outcome)
|
||||
result += "- " + formatDuration(ago) + " ago: " + rec.Problem + "\n"
|
||||
result += " Action: " + rec.Action + " (" + outcomeStr + ")\n"
|
||||
if rec.Note != "" {
|
||||
result += " Note: " + rec.Note + "\n"
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetRemediationStats returns statistics about remediation effectiveness
|
||||
func (r *RemediationLog) GetRemediationStats() map[string]int {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
stats := map[string]int{
|
||||
"total": len(r.records),
|
||||
"resolved": 0,
|
||||
"partial": 0,
|
||||
"failed": 0,
|
||||
"unknown": 0,
|
||||
}
|
||||
|
||||
for _, rec := range r.records {
|
||||
switch rec.Outcome {
|
||||
case OutcomeResolved:
|
||||
stats["resolved"]++
|
||||
case OutcomePartial:
|
||||
stats["partial"]++
|
||||
case OutcomeFailed:
|
||||
stats["failed"]++
|
||||
default:
|
||||
stats["unknown"]++
|
||||
}
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// trimRecords removes old records beyond maxRecords
|
||||
func (r *RemediationLog) trimRecords() {
|
||||
if len(r.records) > r.maxRecords {
|
||||
r.records = r.records[len(r.records)-r.maxRecords:]
|
||||
}
|
||||
}
|
||||
|
||||
// saveToDisk persists records to JSON file
|
||||
func (r *RemediationLog) saveToDisk() error {
|
||||
if r.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
r.mu.RLock()
|
||||
records := make([]RemediationRecord, len(r.records))
|
||||
copy(records, r.records)
|
||||
r.mu.RUnlock()
|
||||
|
||||
path := filepath.Join(r.dataDir, "ai_remediations.json")
|
||||
data, err := json.MarshalIndent(records, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpPath := path + ".tmp"
|
||||
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(tmpPath, path)
|
||||
}
|
||||
|
||||
// loadFromDisk loads records from JSON file
|
||||
func (r *RemediationLog) loadFromDisk() error {
|
||||
if r.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
path := filepath.Join(r.dataDir, "ai_remediations.json")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var records []RemediationRecord
|
||||
if err := json.Unmarshal(data, &records); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort by timestamp
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].Timestamp.Before(records[j].Timestamp)
|
||||
})
|
||||
|
||||
r.records = records
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
var recordCounter int64
|
||||
|
||||
func generateRecordID() string {
|
||||
recordCounter++
|
||||
return "rem-" + time.Now().Format("20060102150405") + "-" + intToString(int(recordCounter%1000))
|
||||
}
|
||||
|
||||
func truncateOutput(output string, maxLen int) string {
|
||||
if len(output) <= maxLen {
|
||||
return output
|
||||
}
|
||||
return output[:maxLen-3] + "..."
|
||||
}
|
||||
|
||||
func extractKeywords(text string) []string {
|
||||
// Simple keyword extraction - split by common delimiters
|
||||
// In production, this could use NLP or embeddings
|
||||
var keywords []string
|
||||
var current string
|
||||
|
||||
for _, c := range text {
|
||||
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' {
|
||||
current += string(c)
|
||||
} else {
|
||||
if len(current) > 3 { // Only keep words > 3 chars
|
||||
keywords = append(keywords, current)
|
||||
}
|
||||
current = ""
|
||||
}
|
||||
}
|
||||
if len(current) > 3 {
|
||||
keywords = append(keywords, current)
|
||||
}
|
||||
|
||||
return keywords
|
||||
}
|
||||
|
||||
func countMatches(a, b []string) int {
|
||||
bSet := make(map[string]bool)
|
||||
for _, s := range b {
|
||||
bSet[s] = true
|
||||
}
|
||||
|
||||
count := 0
|
||||
for _, s := range a {
|
||||
if bSet[s] {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
61
internal/ai/memory_exports.go
Normal file
61
internal/ai/memory_exports.go
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||
)
|
||||
|
||||
// ChangeDetector is an alias for memory.ChangeDetector
|
||||
type ChangeDetector = memory.ChangeDetector
|
||||
|
||||
// ChangeDetectorConfig is an alias for memory.ChangeDetectorConfig
|
||||
type ChangeDetectorConfig = memory.ChangeDetectorConfig
|
||||
|
||||
// Change is an alias for memory.Change
|
||||
type Change = memory.Change
|
||||
|
||||
// ResourceSnapshot is an alias for memory.ResourceSnapshot
|
||||
type ResourceSnapshot = memory.ResourceSnapshot
|
||||
|
||||
// ChangeType is an alias for memory.ChangeType
|
||||
type ChangeType = memory.ChangeType
|
||||
|
||||
// RemediationLog is an alias for memory.RemediationLog
|
||||
type RemediationLog = memory.RemediationLog
|
||||
|
||||
// RemediationLogConfig is an alias for memory.RemediationLogConfig
|
||||
type RemediationLogConfig = memory.RemediationLogConfig
|
||||
|
||||
// RemediationRecord is an alias for memory.RemediationRecord
|
||||
type RemediationRecord = memory.RemediationRecord
|
||||
|
||||
// Outcome is an alias for memory.Outcome
|
||||
type Outcome = memory.Outcome
|
||||
|
||||
// Change type constants
|
||||
const (
|
||||
ChangeCreated = memory.ChangeCreated
|
||||
ChangeDeleted = memory.ChangeDeleted
|
||||
ChangeConfig = memory.ChangeConfig
|
||||
ChangeStatus = memory.ChangeStatus
|
||||
ChangeMigrated = memory.ChangeMigrated
|
||||
ChangeRestarted = memory.ChangeRestarted
|
||||
ChangeBackedUp = memory.ChangeBackedUp
|
||||
)
|
||||
|
||||
// Outcome constants
|
||||
const (
|
||||
OutcomeResolved = memory.OutcomeResolved
|
||||
OutcomePartial = memory.OutcomePartial
|
||||
OutcomeFailed = memory.OutcomeFailed
|
||||
OutcomeUnknown = memory.OutcomeUnknown
|
||||
)
|
||||
|
||||
// NewChangeDetector creates a new change detector
|
||||
func NewChangeDetector(cfg ChangeDetectorConfig) *ChangeDetector {
|
||||
return memory.NewChangeDetector(cfg)
|
||||
}
|
||||
|
||||
// NewRemediationLog creates a new remediation log
|
||||
func NewRemediationLog(cfg RemediationLogConfig) *RemediationLog {
|
||||
return memory.NewRemediationLog(cfg)
|
||||
}
|
||||
Loading…
Reference in a new issue