refactor: remove runbooks feature entirely
Runbooks were a half-built feature that provided no value: - Only 3 runbooks existed - AI dynamic remediation already covers the same ground - Added UI complexity without benefit Removed: - runbooks.go and runbooks_test.go - Handler functions in ai_handlers.go - Routes in router.go - Test cases in ai_handlers_test.go - Auto-fix call in patrol.go Kept (dead code but harmless): - Frontend types/API calls (will 404) - RecordIncidentRunbook function (unused) Less code = easier to maintain.
This commit is contained in:
parent
3c8d264283
commit
284ca0271b
6 changed files with 2 additions and 1441 deletions
|
|
@ -793,12 +793,7 @@ func (p *PatrolService) runPatrol(ctx context.Context) {
|
||||||
autoFixEnabled = aiCfg.PatrolAutoFix
|
autoFixEnabled = aiCfg.PatrolAutoFix
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if hasPatrolFeature && autoFixEnabled && p.aiService.HasLicenseFeature(FeatureAIAutoFix) {
|
_ = autoFixEnabled // Auto-fix via runbooks removed - dynamic AI remediation handles this
|
||||||
runbookResolved = p.AutoFixWithRunbooks(ctx, newFindings)
|
|
||||||
if runbookResolved > 0 {
|
|
||||||
log.Info().Int("resolved", runbookResolved).Msg("AI Patrol: Auto-fix runbooks resolved findings")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Auto-resolve findings that weren't seen in this patrol run
|
// Auto-resolve findings that weren't seen in this patrol run
|
||||||
var resolvedCount int
|
var resolvedCount int
|
||||||
|
|
|
||||||
|
|
@ -1,706 +0,0 @@
|
||||||
package ai
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
type RunbookRisk string
|
|
||||||
|
|
||||||
const (
|
|
||||||
RunbookRiskLow RunbookRisk = "low"
|
|
||||||
RunbookRiskMedium RunbookRisk = "medium"
|
|
||||||
RunbookRiskHigh RunbookRisk = "high"
|
|
||||||
)
|
|
||||||
|
|
||||||
const runbookVerifierDiskUsage = "disk-usage"
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrRunbookNotFound = errors.New("runbook not found")
|
|
||||||
ErrRunbookNotApplicable = errors.New("runbook does not apply to finding")
|
|
||||||
)
|
|
||||||
|
|
||||||
type RunbookInfo struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
Risk RunbookRisk `json:"risk"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RunbookStep struct {
|
|
||||||
Name string
|
|
||||||
Command string
|
|
||||||
RunOnHost bool
|
|
||||||
AllowFailure bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type RunbookVerification struct {
|
|
||||||
Name string
|
|
||||||
Command string
|
|
||||||
RunOnHost bool
|
|
||||||
SuccessRegex string
|
|
||||||
FailureRegex string
|
|
||||||
Verifier string
|
|
||||||
}
|
|
||||||
|
|
||||||
type Runbook struct {
|
|
||||||
ID string
|
|
||||||
Title string
|
|
||||||
Description string
|
|
||||||
Risk RunbookRisk
|
|
||||||
FindingKeys []string
|
|
||||||
ResourceTypes []string
|
|
||||||
Steps []RunbookStep
|
|
||||||
Verification *RunbookVerification
|
|
||||||
ResolutionNote string
|
|
||||||
}
|
|
||||||
|
|
||||||
type RunbookStepResult struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Command string `json:"command"`
|
|
||||||
Output string `json:"output"`
|
|
||||||
Success bool `json:"success"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RunbookExecutionResult struct {
|
|
||||||
RunbookID string `json:"runbook_id"`
|
|
||||||
Outcome memory.Outcome `json:"outcome"`
|
|
||||||
Message string `json:"message"`
|
|
||||||
Steps []RunbookStepResult `json:"steps"`
|
|
||||||
VerifyStep *RunbookStepResult `json:"verification,omitempty"`
|
|
||||||
Resolved bool `json:"resolved"`
|
|
||||||
ExecutedAt time.Time `json:"executed_at"`
|
|
||||||
FindingID string `json:"finding_id"`
|
|
||||||
FindingKey string `json:"finding_key,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *PatrolService) GetRunbooksForFinding(findingID string) ([]RunbookInfo, error) {
|
|
||||||
finding := p.findings.Get(findingID)
|
|
||||||
if finding == nil {
|
|
||||||
return nil, fmt.Errorf("finding not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
runbooks := matchRunbooksForFinding(finding)
|
|
||||||
infos := make([]RunbookInfo, 0, len(runbooks))
|
|
||||||
for _, rb := range runbooks {
|
|
||||||
infos = append(infos, RunbookInfo{
|
|
||||||
ID: rb.ID,
|
|
||||||
Title: rb.Title,
|
|
||||||
Description: rb.Description,
|
|
||||||
Risk: rb.Risk,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return infos, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *PatrolService) ExecuteRunbook(ctx context.Context, findingID, runbookID string) (*RunbookExecutionResult, error) {
|
|
||||||
return p.executeRunbook(ctx, findingID, runbookID, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *PatrolService) executeRunbook(ctx context.Context, findingID, runbookID string, automatic bool) (*RunbookExecutionResult, error) {
|
|
||||||
finding := p.findings.Get(findingID)
|
|
||||||
if finding == nil {
|
|
||||||
return nil, fmt.Errorf("finding not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
runbook, ok := getRunbookByID(runbookID)
|
|
||||||
if !ok {
|
|
||||||
return nil, ErrRunbookNotFound
|
|
||||||
}
|
|
||||||
if !runbookApplies(runbook, finding) {
|
|
||||||
return nil, ErrRunbookNotApplicable
|
|
||||||
}
|
|
||||||
if p.aiService == nil {
|
|
||||||
return nil, fmt.Errorf("AI service not available")
|
|
||||||
}
|
|
||||||
|
|
||||||
context := buildRunbookContext(finding)
|
|
||||||
results := make([]RunbookStepResult, 0, len(runbook.Steps))
|
|
||||||
executionTime := time.Now()
|
|
||||||
|
|
||||||
for _, step := range runbook.Steps {
|
|
||||||
if step.RunOnHost && context.Node == "" {
|
|
||||||
return nil, fmt.Errorf("target host is required for runbook step %q", step.Name)
|
|
||||||
}
|
|
||||||
command, err := renderRunbookCommand(step.Command, context)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := p.aiService.RunCommand(ctx, RunCommandRequest{
|
|
||||||
Command: command,
|
|
||||||
TargetType: runbookTargetType(finding, step.RunOnHost),
|
|
||||||
TargetID: finding.ResourceID,
|
|
||||||
RunOnHost: step.RunOnHost,
|
|
||||||
VMID: context.VMID,
|
|
||||||
TargetHost: context.Node,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
stepResult := RunbookStepResult{
|
|
||||||
Name: step.Name,
|
|
||||||
Command: command,
|
|
||||||
Output: strings.TrimSpace(resp.Output),
|
|
||||||
Success: resp.Success,
|
|
||||||
}
|
|
||||||
results = append(results, stepResult)
|
|
||||||
|
|
||||||
if !resp.Success && !step.AllowFailure {
|
|
||||||
outcome := memory.OutcomeFailed
|
|
||||||
message := fmt.Sprintf("Runbook step failed: %s", step.Name)
|
|
||||||
p.logRunbookExecution(finding, runbook, results, nil, outcome, message, executionTime, automatic)
|
|
||||||
return &RunbookExecutionResult{
|
|
||||||
RunbookID: runbook.ID,
|
|
||||||
Outcome: outcome,
|
|
||||||
Message: message,
|
|
||||||
Steps: results,
|
|
||||||
Resolved: false,
|
|
||||||
ExecutedAt: executionTime,
|
|
||||||
FindingID: finding.ID,
|
|
||||||
FindingKey: finding.Key,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var verifyResult *RunbookStepResult
|
|
||||||
outcome := memory.OutcomeUnknown
|
|
||||||
message := "Runbook completed"
|
|
||||||
|
|
||||||
if runbook.Verification != nil {
|
|
||||||
if runbook.Verification.RunOnHost && context.Node == "" {
|
|
||||||
return nil, fmt.Errorf("target host is required for runbook verification")
|
|
||||||
}
|
|
||||||
command, err := renderRunbookCommand(runbook.Verification.Command, context)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := p.aiService.RunCommand(ctx, RunCommandRequest{
|
|
||||||
Command: command,
|
|
||||||
TargetType: runbookTargetType(finding, runbook.Verification.RunOnHost),
|
|
||||||
TargetID: finding.ResourceID,
|
|
||||||
RunOnHost: runbook.Verification.RunOnHost,
|
|
||||||
VMID: context.VMID,
|
|
||||||
TargetHost: context.Node,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
verifyResult = &RunbookStepResult{
|
|
||||||
Name: runbook.Verification.Name,
|
|
||||||
Command: command,
|
|
||||||
Output: strings.TrimSpace(resp.Output),
|
|
||||||
Success: resp.Success,
|
|
||||||
}
|
|
||||||
|
|
||||||
outcome, message = evaluateRunbookVerification(runbook.Verification, verifyResult.Output, finding, p.thresholds)
|
|
||||||
if !resp.Success && outcome == memory.OutcomeResolved {
|
|
||||||
outcome = memory.OutcomePartial
|
|
||||||
message = "Verification command failed"
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
outcome = memory.OutcomeUnknown
|
|
||||||
message = "Runbook completed without verification"
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved := false
|
|
||||||
if outcome == memory.OutcomeResolved {
|
|
||||||
note := runbook.ResolutionNote
|
|
||||||
if note == "" {
|
|
||||||
note = fmt.Sprintf("Applied runbook: %s", runbook.Title)
|
|
||||||
}
|
|
||||||
if err := p.ResolveFinding(finding.ID, note); err == nil {
|
|
||||||
resolved = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p.logRunbookExecution(finding, runbook, results, verifyResult, outcome, message, executionTime, automatic)
|
|
||||||
|
|
||||||
return &RunbookExecutionResult{
|
|
||||||
RunbookID: runbook.ID,
|
|
||||||
Outcome: outcome,
|
|
||||||
Message: message,
|
|
||||||
Steps: results,
|
|
||||||
VerifyStep: verifyResult,
|
|
||||||
Resolved: resolved,
|
|
||||||
ExecutedAt: executionTime,
|
|
||||||
FindingID: finding.ID,
|
|
||||||
FindingKey: finding.Key,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *PatrolService) AutoFixWithRunbooks(ctx context.Context, findings []*Finding) int {
|
|
||||||
if ctx == nil {
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
|
||||||
if p.aiService == nil || len(findings) == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved := 0
|
|
||||||
for _, finding := range findings {
|
|
||||||
if finding == nil || finding.Key == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return resolved
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
runbook, ok := selectRunbookForAutoFix(finding)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !p.shouldAutoFixFinding(finding) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := p.executeRunbook(ctx, finding.ID, runbook.ID, true)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn().Err(err).Str("runbook_id", runbook.ID).Str("finding_id", finding.ID).Msg("Runbook auto-fix failed")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if result != nil && result.Resolved {
|
|
||||||
resolved++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return resolved
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *PatrolService) logRunbookExecution(finding *Finding, runbook Runbook, steps []RunbookStepResult, verify *RunbookStepResult, outcome memory.Outcome, message string, executedAt time.Time, automatic bool) {
|
|
||||||
if p.remediationLog == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
noteParts := []string{}
|
|
||||||
for _, step := range steps {
|
|
||||||
status := "ok"
|
|
||||||
if !step.Success {
|
|
||||||
status = "failed"
|
|
||||||
}
|
|
||||||
noteParts = append(noteParts, fmt.Sprintf("%s (%s)", step.Name, status))
|
|
||||||
}
|
|
||||||
if verify != nil {
|
|
||||||
noteParts = append(noteParts, fmt.Sprintf("verification: %s", verify.Name))
|
|
||||||
}
|
|
||||||
note := strings.Join(noteParts, "; ")
|
|
||||||
if message != "" {
|
|
||||||
note = message + ". " + note
|
|
||||||
}
|
|
||||||
|
|
||||||
output := ""
|
|
||||||
if verify != nil {
|
|
||||||
output = verify.Output
|
|
||||||
}
|
|
||||||
|
|
||||||
record := memory.RemediationRecord{
|
|
||||||
Timestamp: executedAt,
|
|
||||||
ResourceID: finding.ResourceID,
|
|
||||||
ResourceType: finding.ResourceType,
|
|
||||||
ResourceName: finding.ResourceName,
|
|
||||||
FindingID: finding.ID,
|
|
||||||
Problem: finding.Title,
|
|
||||||
Action: runbook.Title,
|
|
||||||
Output: truncateRunbookOutput(output, 1000),
|
|
||||||
Outcome: outcome,
|
|
||||||
Note: truncateRunbookOutput(note, 500),
|
|
||||||
Automatic: automatic,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := p.remediationLog.Log(record); err != nil {
|
|
||||||
log.Warn().Err(err).Msg("Failed to log runbook execution")
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.aiService != nil && finding != nil && finding.AlertID != "" {
|
|
||||||
p.aiService.RecordIncidentRunbook(finding.AlertID, runbook.ID, runbook.Title, outcome, automatic, message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type runbookContext struct {
|
|
||||||
ResourceID string
|
|
||||||
ResourceName string
|
|
||||||
Node string
|
|
||||||
VMID string
|
|
||||||
PBSID string
|
|
||||||
Datastore string
|
|
||||||
JobID string
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildRunbookContext(finding *Finding) runbookContext {
|
|
||||||
vmid := parseVMID(finding.ResourceID)
|
|
||||||
pbsID, datastore, jobID := parsePBSResourceParts(finding.ResourceID)
|
|
||||||
|
|
||||||
return runbookContext{
|
|
||||||
ResourceID: finding.ResourceID,
|
|
||||||
ResourceName: finding.ResourceName,
|
|
||||||
Node: finding.Node,
|
|
||||||
VMID: vmid,
|
|
||||||
PBSID: pbsID,
|
|
||||||
Datastore: datastore,
|
|
||||||
JobID: jobID,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runbookTargetType(finding *Finding, runOnHost bool) string {
|
|
||||||
if runOnHost {
|
|
||||||
return "host"
|
|
||||||
}
|
|
||||||
switch finding.ResourceType {
|
|
||||||
case "vm":
|
|
||||||
return "vm"
|
|
||||||
case "container":
|
|
||||||
return "container"
|
|
||||||
default:
|
|
||||||
return "host"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func renderRunbookCommand(command string, ctx runbookContext) (string, error) {
|
|
||||||
placeholders := map[string]string{
|
|
||||||
"resource_id": ctx.ResourceID,
|
|
||||||
"resource_name": ctx.ResourceName,
|
|
||||||
"node": ctx.Node,
|
|
||||||
"vmid": ctx.VMID,
|
|
||||||
"pbs_id": ctx.PBSID,
|
|
||||||
"datastore": ctx.Datastore,
|
|
||||||
"job_id": ctx.JobID,
|
|
||||||
}
|
|
||||||
|
|
||||||
result := command
|
|
||||||
for key, value := range placeholders {
|
|
||||||
placeholder := "{{" + key + "}}"
|
|
||||||
if strings.Contains(result, placeholder) {
|
|
||||||
if value == "" {
|
|
||||||
return "", fmt.Errorf("missing value for %s", key)
|
|
||||||
}
|
|
||||||
result = strings.ReplaceAll(result, placeholder, escapeShellArg(value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func escapeShellArg(value string) string {
|
|
||||||
if value == "" {
|
|
||||||
return "''"
|
|
||||||
}
|
|
||||||
if !strings.ContainsAny(value, " \t\n'\"") {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
escaped := strings.ReplaceAll(value, `'`, `'\''`)
|
|
||||||
return "'" + escaped + "'"
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseVMID(resourceID string) string {
|
|
||||||
if resourceID == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
parts := strings.Split(resourceID, "-")
|
|
||||||
last := parts[len(parts)-1]
|
|
||||||
if _, err := strconv.Atoi(last); err == nil {
|
|
||||||
return last
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func parsePBSResourceParts(resourceID string) (string, string, string) {
|
|
||||||
if resourceID == "" {
|
|
||||||
return "", "", ""
|
|
||||||
}
|
|
||||||
parts := strings.Split(resourceID, ":")
|
|
||||||
if len(parts) < 2 {
|
|
||||||
return resourceID, "", ""
|
|
||||||
}
|
|
||||||
pbsID := parts[0]
|
|
||||||
if len(parts) >= 3 && (parts[1] == "job" || parts[1] == "verify") {
|
|
||||||
return pbsID, "", parts[2]
|
|
||||||
}
|
|
||||||
return pbsID, parts[1], ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func evaluateRunbookVerification(verification *RunbookVerification, output string, finding *Finding, thresholds PatrolThresholds) (memory.Outcome, string) {
|
|
||||||
if verification == nil {
|
|
||||||
return memory.OutcomeUnknown, "No verification configured"
|
|
||||||
}
|
|
||||||
|
|
||||||
if verification.Verifier == runbookVerifierDiskUsage {
|
|
||||||
return verifyDiskUsage(output, finding, thresholds)
|
|
||||||
}
|
|
||||||
|
|
||||||
if verification.SuccessRegex != "" {
|
|
||||||
matched, _ := regexp.MatchString(verification.SuccessRegex, output)
|
|
||||||
if matched {
|
|
||||||
return memory.OutcomeResolved, "Verification passed"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if verification.FailureRegex != "" {
|
|
||||||
matched, _ := regexp.MatchString(verification.FailureRegex, output)
|
|
||||||
if matched {
|
|
||||||
return memory.OutcomeFailed, "Verification failed"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return memory.OutcomeUnknown, "Verification inconclusive"
|
|
||||||
}
|
|
||||||
|
|
||||||
func verifyDiskUsage(output string, finding *Finding, thresholds PatrolThresholds) (memory.Outcome, string) {
|
|
||||||
usage, ok := parseDFUsagePercent(output)
|
|
||||||
if !ok {
|
|
||||||
return memory.OutcomeUnknown, "Unable to parse disk usage"
|
|
||||||
}
|
|
||||||
|
|
||||||
threshold := thresholds.GuestDiskWatch
|
|
||||||
if finding.ResourceType == "storage" {
|
|
||||||
threshold = thresholds.StorageWatch
|
|
||||||
}
|
|
||||||
|
|
||||||
note := fmt.Sprintf("Disk usage now %d%% (threshold %.0f%%)", usage, threshold)
|
|
||||||
|
|
||||||
if float64(usage) < threshold {
|
|
||||||
return memory.OutcomeResolved, note
|
|
||||||
}
|
|
||||||
|
|
||||||
baseline := parsePercentFromFinding(finding.Evidence)
|
|
||||||
if baseline > 0 {
|
|
||||||
if float64(usage) < baseline {
|
|
||||||
return memory.OutcomePartial, note + fmt.Sprintf(", was %.0f%%", baseline)
|
|
||||||
}
|
|
||||||
return memory.OutcomeFailed, note + fmt.Sprintf(", was %.0f%%", baseline)
|
|
||||||
}
|
|
||||||
|
|
||||||
return memory.OutcomeUnknown, note
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseDFUsagePercent(output string) (int, bool) {
|
|
||||||
lines := strings.Split(output, "\n")
|
|
||||||
for _, line := range lines {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) < 6 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
mount := fields[len(fields)-1]
|
|
||||||
if mount != "/" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
usageField := fields[4]
|
|
||||||
usageField = strings.TrimSuffix(usageField, "%")
|
|
||||||
usage, err := strconv.Atoi(usageField)
|
|
||||||
if err != nil {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
return usage, true
|
|
||||||
}
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func parsePercentFromFinding(value string) float64 {
|
|
||||||
if value == "" {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
re := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)%`)
|
|
||||||
match := re.FindStringSubmatch(value)
|
|
||||||
if len(match) < 2 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
parsed, err := strconv.ParseFloat(match[1], 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return parsed
|
|
||||||
}
|
|
||||||
|
|
||||||
func truncateRunbookOutput(output string, limit int) string {
|
|
||||||
output = strings.TrimSpace(output)
|
|
||||||
if len(output) <= limit {
|
|
||||||
return output
|
|
||||||
}
|
|
||||||
return output[:limit] + "..."
|
|
||||||
}
|
|
||||||
|
|
||||||
func runbookApplies(runbook Runbook, finding *Finding) bool {
|
|
||||||
if len(runbook.FindingKeys) > 0 {
|
|
||||||
matched := false
|
|
||||||
for _, key := range runbook.FindingKeys {
|
|
||||||
if key == finding.Key {
|
|
||||||
matched = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !matched {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(runbook.ResourceTypes) > 0 {
|
|
||||||
matched := false
|
|
||||||
for _, rt := range runbook.ResourceTypes {
|
|
||||||
if rt == finding.ResourceType {
|
|
||||||
matched = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !matched {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectRunbookForAutoFix(finding *Finding) (Runbook, bool) {
|
|
||||||
for _, runbook := range runbooksCatalog {
|
|
||||||
if runbook.Risk != RunbookRiskLow {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if runbookApplies(runbook, finding) {
|
|
||||||
return runbook, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Runbook{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *PatrolService) shouldAutoFixFinding(finding *Finding) bool {
|
|
||||||
if p.remediationLog == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
records := p.remediationLog.GetForFinding(finding.ID, 1)
|
|
||||||
if len(records) == 0 {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
last := records[0]
|
|
||||||
if time.Since(last.Timestamp) < 6*time.Hour {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func matchRunbooksForFinding(finding *Finding) []Runbook {
|
|
||||||
var matches []Runbook
|
|
||||||
for _, runbook := range runbooksCatalog {
|
|
||||||
if runbookApplies(runbook, finding) {
|
|
||||||
matches = append(matches, runbook)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return matches
|
|
||||||
}
|
|
||||||
|
|
||||||
func getRunbookByID(id string) (Runbook, bool) {
|
|
||||||
for _, runbook := range runbooksCatalog {
|
|
||||||
if runbook.ID == id {
|
|
||||||
return runbook, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Runbook{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
var runbooksCatalog = []Runbook{
|
|
||||||
{
|
|
||||||
ID: "docker-restart-loop",
|
|
||||||
Title: "Restart Docker container and verify",
|
|
||||||
Description: "Collect recent logs, restart the container, then verify it is running.",
|
|
||||||
Risk: RunbookRiskLow,
|
|
||||||
FindingKeys: []string{"restart-loop"},
|
|
||||||
ResourceTypes: []string{
|
|
||||||
"docker_container",
|
|
||||||
},
|
|
||||||
Steps: []RunbookStep{
|
|
||||||
{
|
|
||||||
Name: "Fetch recent logs",
|
|
||||||
Command: "docker logs --tail 200 {{resource_name}}",
|
|
||||||
RunOnHost: true,
|
|
||||||
AllowFailure: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "Restart container",
|
|
||||||
Command: "docker restart {{resource_name}}",
|
|
||||||
RunOnHost: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Verification: &RunbookVerification{
|
|
||||||
Name: "Verify container running",
|
|
||||||
Command: "docker inspect -f '{{.State.Status}}' {{resource_name}}",
|
|
||||||
RunOnHost: true,
|
|
||||||
SuccessRegex: "(?i)running",
|
|
||||||
},
|
|
||||||
ResolutionNote: "Restarted docker container and verified it is running.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ID: "guest-disk-cleanup",
|
|
||||||
Title: "Clean package cache and old logs",
|
|
||||||
Description: "Vacuum systemd journal and clean package cache to free disk space.",
|
|
||||||
Risk: RunbookRiskMedium,
|
|
||||||
FindingKeys: []string{"high-disk"},
|
|
||||||
ResourceTypes: []string{
|
|
||||||
"vm",
|
|
||||||
"container",
|
|
||||||
},
|
|
||||||
Steps: []RunbookStep{
|
|
||||||
{
|
|
||||||
Name: "Vacuum systemd journal",
|
|
||||||
Command: "journalctl --vacuum-time=7d",
|
|
||||||
RunOnHost: false,
|
|
||||||
AllowFailure: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "Clean package cache",
|
|
||||||
Command: "apt-get clean",
|
|
||||||
RunOnHost: false,
|
|
||||||
AllowFailure: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Verification: &RunbookVerification{
|
|
||||||
Name: "Check root filesystem usage",
|
|
||||||
Command: "df -P /",
|
|
||||||
RunOnHost: false,
|
|
||||||
Verifier: runbookVerifierDiskUsage,
|
|
||||||
},
|
|
||||||
ResolutionNote: "Cleaned logs and package cache, then verified disk usage.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ID: "docker-high-memory-restart",
|
|
||||||
Title: "Restart Docker container to clear memory",
|
|
||||||
Description: "Capture current memory usage, restart the container, then verify it is running.",
|
|
||||||
Risk: RunbookRiskMedium,
|
|
||||||
FindingKeys: []string{"high-memory"},
|
|
||||||
ResourceTypes: []string{
|
|
||||||
"docker_container",
|
|
||||||
},
|
|
||||||
Steps: []RunbookStep{
|
|
||||||
{
|
|
||||||
Name: "Capture memory usage",
|
|
||||||
Command: "docker stats --no-stream {{resource_name}}",
|
|
||||||
RunOnHost: true,
|
|
||||||
AllowFailure: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "Restart container",
|
|
||||||
Command: "docker restart {{resource_name}}",
|
|
||||||
RunOnHost: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Verification: &RunbookVerification{
|
|
||||||
Name: "Verify container running",
|
|
||||||
Command: "docker inspect -f '{{.State.Status}}' {{resource_name}}",
|
|
||||||
RunOnHost: true,
|
|
||||||
SuccessRegex: "(?i)running",
|
|
||||||
},
|
|
||||||
ResolutionNote: "Restarted docker container to clear memory usage.",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
@ -1,561 +0,0 @@
|
||||||
package ai
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMatchRunbooksForFinding_DockerRestartLoop(t *testing.T) {
|
|
||||||
finding := &Finding{
|
|
||||||
Key: "restart-loop",
|
|
||||||
ResourceType: "docker_container",
|
|
||||||
}
|
|
||||||
|
|
||||||
runbooks := matchRunbooksForFinding(finding)
|
|
||||||
if len(runbooks) == 0 {
|
|
||||||
t.Fatal("expected at least one runbook")
|
|
||||||
}
|
|
||||||
|
|
||||||
found := false
|
|
||||||
for _, rb := range runbooks {
|
|
||||||
if rb.ID == "docker-restart-loop" {
|
|
||||||
found = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !found {
|
|
||||||
t.Fatal("expected docker-restart-loop runbook to match")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMatchRunbooksForFinding_DockerHighMemory(t *testing.T) {
|
|
||||||
finding := &Finding{
|
|
||||||
Key: "high-memory",
|
|
||||||
ResourceType: "docker_container",
|
|
||||||
}
|
|
||||||
|
|
||||||
runbooks := matchRunbooksForFinding(finding)
|
|
||||||
found := false
|
|
||||||
for _, rb := range runbooks {
|
|
||||||
if rb.ID == "docker-high-memory-restart" {
|
|
||||||
found = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("expected docker-high-memory-restart runbook to match")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVerifyDiskUsage(t *testing.T) {
|
|
||||||
finding := &Finding{
|
|
||||||
ResourceType: "container",
|
|
||||||
Evidence: "Disk: 95.0%",
|
|
||||||
}
|
|
||||||
thresholds := PatrolThresholds{
|
|
||||||
GuestDiskWatch: 85,
|
|
||||||
}
|
|
||||||
|
|
||||||
output := `Filesystem 1024-blocks Used Available Capacity Mounted on
|
|
||||||
/dev/sda1 20511356 1000000 19511356 80% /`
|
|
||||||
|
|
||||||
outcome, note := verifyDiskUsage(output, finding, thresholds)
|
|
||||||
if outcome != memory.OutcomeResolved {
|
|
||||||
t.Fatalf("expected resolved, got %s (%s)", outcome, note)
|
|
||||||
}
|
|
||||||
|
|
||||||
output = `Filesystem 1024-blocks Used Available Capacity Mounted on
|
|
||||||
/dev/sda1 20511356 1000000 19511356 93% /`
|
|
||||||
|
|
||||||
outcome, _ = verifyDiskUsage(output, finding, thresholds)
|
|
||||||
if outcome != memory.OutcomePartial {
|
|
||||||
t.Fatalf("expected partial, got %s", outcome)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// escapeShellArg tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestEscapeShellArg(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
input string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty string",
|
|
||||||
input: "",
|
|
||||||
expected: "''",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "simple string",
|
|
||||||
input: "hello",
|
|
||||||
expected: "hello",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "string with space",
|
|
||||||
input: "hello world",
|
|
||||||
expected: "'hello world'",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "string with single quote",
|
|
||||||
input: "it's",
|
|
||||||
expected: "'it'\\''s'",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "string with tab",
|
|
||||||
input: "hello\tworld",
|
|
||||||
expected: "'hello\tworld'",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "alphanumeric only",
|
|
||||||
input: "test123",
|
|
||||||
expected: "test123",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result := escapeShellArg(tt.input)
|
|
||||||
if result != tt.expected {
|
|
||||||
t.Errorf("escapeShellArg(%q) = %q, want %q", tt.input, result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// parseVMID tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestParseVMID(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
resourceID string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty string",
|
|
||||||
resourceID: "",
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "simple vmid",
|
|
||||||
resourceID: "node1-100",
|
|
||||||
expected: "100",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "complex id",
|
|
||||||
resourceID: "pve-cluster-node-200",
|
|
||||||
expected: "200",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "non-numeric ending",
|
|
||||||
resourceID: "node1-abc",
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "single number",
|
|
||||||
resourceID: "100",
|
|
||||||
expected: "100",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no dash prefix",
|
|
||||||
resourceID: "node",
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result := parseVMID(tt.resourceID)
|
|
||||||
if result != tt.expected {
|
|
||||||
t.Errorf("parseVMID(%q) = %q, want %q", tt.resourceID, result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// truncateRunbookOutput tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestTruncateRunbookOutput(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
output string
|
|
||||||
limit int
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty string",
|
|
||||||
output: "",
|
|
||||||
limit: 10,
|
|
||||||
expected: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "under limit",
|
|
||||||
output: "hello",
|
|
||||||
limit: 10,
|
|
||||||
expected: "hello",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "at limit",
|
|
||||||
output: "helloworld",
|
|
||||||
limit: 10,
|
|
||||||
expected: "helloworld",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "over limit",
|
|
||||||
output: "hello world example",
|
|
||||||
limit: 10,
|
|
||||||
expected: "hello worl...",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "with whitespace",
|
|
||||||
output: " hello ",
|
|
||||||
limit: 10,
|
|
||||||
expected: "hello",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result := truncateRunbookOutput(tt.output, tt.limit)
|
|
||||||
if result != tt.expected {
|
|
||||||
t.Errorf("truncateRunbookOutput(%q, %d) = %q, want %q", tt.output, tt.limit, result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// getRunbookByID tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestGetRunbookByID(t *testing.T) {
|
|
||||||
// Test finding a real runbook from the catalog
|
|
||||||
rb, found := getRunbookByID("docker-restart-loop")
|
|
||||||
if !found {
|
|
||||||
t.Fatal("expected to find docker-restart-loop runbook")
|
|
||||||
}
|
|
||||||
if rb.ID != "docker-restart-loop" {
|
|
||||||
t.Errorf("expected ID 'docker-restart-loop', got %q", rb.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test not finding a non-existent runbook
|
|
||||||
_, found = getRunbookByID("non-existent-runbook")
|
|
||||||
if found {
|
|
||||||
t.Error("expected not to find non-existent-runbook")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// parseDFUsagePercent tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestParseDFUsagePercent(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
output string
|
|
||||||
expected int
|
|
||||||
ok bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "standard df output",
|
|
||||||
output: "Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 20511356 1000000 19511356 80% /",
|
|
||||||
expected: 80,
|
|
||||||
ok: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no percentage",
|
|
||||||
output: "No disk info here",
|
|
||||||
expected: 0,
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result, ok := parseDFUsagePercent(tt.output)
|
|
||||||
if ok != tt.ok {
|
|
||||||
t.Errorf("parseDFUsagePercent(%q) ok = %v, want %v", tt.output, ok, tt.ok)
|
|
||||||
}
|
|
||||||
if ok && result != tt.expected {
|
|
||||||
t.Errorf("parseDFUsagePercent(%q) = %v, want %v", tt.output, result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// parsePercentFromFinding tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestParsePercentFromFinding(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
evidence string
|
|
||||||
expected float64
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "disk percentage",
|
|
||||||
evidence: "Disk: 95.0%",
|
|
||||||
expected: 95.0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cpu percentage",
|
|
||||||
evidence: "CPU usage is at 87.5% utilization",
|
|
||||||
expected: 87.5,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no percentage",
|
|
||||||
evidence: "No percentage here",
|
|
||||||
expected: 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result := parsePercentFromFinding(tt.evidence)
|
|
||||||
if result != tt.expected {
|
|
||||||
t.Errorf("parsePercentFromFinding(%q) = %v, want %v", tt.evidence, result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// runbookApplies tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestRunbookApplies(t *testing.T) {
|
|
||||||
// Test with matching resource type and exact key
|
|
||||||
rb := Runbook{
|
|
||||||
ResourceTypes: []string{"docker_container"},
|
|
||||||
FindingKeys: []string{"restart-loop"},
|
|
||||||
}
|
|
||||||
finding := &Finding{
|
|
||||||
Key: "restart-loop", // exact match required
|
|
||||||
ResourceType: "docker_container",
|
|
||||||
}
|
|
||||||
if !runbookApplies(rb, finding) {
|
|
||||||
t.Error("expected runbook to apply")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test with non-matching resource type
|
|
||||||
finding2 := &Finding{
|
|
||||||
Key: "restart-loop",
|
|
||||||
ResourceType: "vm",
|
|
||||||
}
|
|
||||||
if runbookApplies(rb, finding2) {
|
|
||||||
t.Error("expected runbook NOT to apply (wrong resource type)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test with non-matching key
|
|
||||||
finding3 := &Finding{
|
|
||||||
Key: "high-memory",
|
|
||||||
ResourceType: "docker_container",
|
|
||||||
}
|
|
||||||
if runbookApplies(rb, finding3) {
|
|
||||||
t.Error("expected runbook NOT to apply (wrong key)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test with empty FindingKeys (should match any key)
|
|
||||||
rb2 := Runbook{
|
|
||||||
ResourceTypes: []string{"vm"},
|
|
||||||
FindingKeys: []string{}, // empty means match any key
|
|
||||||
}
|
|
||||||
finding4 := &Finding{
|
|
||||||
Key: "any-key",
|
|
||||||
ResourceType: "vm",
|
|
||||||
}
|
|
||||||
if !runbookApplies(rb2, finding4) {
|
|
||||||
t.Error("expected runbook with empty FindingKeys to apply to any key")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// parsePBSResourceParts tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestParsePBSResourceParts(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
resourceID string
|
|
||||||
expectedPBS string
|
|
||||||
expectedDS string
|
|
||||||
expectedJobID string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty string",
|
|
||||||
resourceID: "",
|
|
||||||
expectedPBS: "",
|
|
||||||
expectedDS: "",
|
|
||||||
expectedJobID: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "simple pbs id only",
|
|
||||||
resourceID: "pbs1",
|
|
||||||
expectedPBS: "pbs1",
|
|
||||||
expectedDS: "",
|
|
||||||
expectedJobID: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "pbs with datastore",
|
|
||||||
resourceID: "pbs1:datastore1",
|
|
||||||
expectedPBS: "pbs1",
|
|
||||||
expectedDS: "datastore1",
|
|
||||||
expectedJobID: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "pbs with job",
|
|
||||||
resourceID: "pbs1:job:backup-job-1",
|
|
||||||
expectedPBS: "pbs1",
|
|
||||||
expectedDS: "",
|
|
||||||
expectedJobID: "backup-job-1",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "pbs with verify job",
|
|
||||||
resourceID: "pbs1:verify:verify-job-1",
|
|
||||||
expectedPBS: "pbs1",
|
|
||||||
expectedDS: "",
|
|
||||||
expectedJobID: "verify-job-1",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
pbsID, ds, jobID := parsePBSResourceParts(tt.resourceID)
|
|
||||||
if pbsID != tt.expectedPBS {
|
|
||||||
t.Errorf("pbsID = %q, want %q", pbsID, tt.expectedPBS)
|
|
||||||
}
|
|
||||||
if ds != tt.expectedDS {
|
|
||||||
t.Errorf("ds = %q, want %q", ds, tt.expectedDS)
|
|
||||||
}
|
|
||||||
if jobID != tt.expectedJobID {
|
|
||||||
t.Errorf("jobID = %q, want %q", jobID, tt.expectedJobID)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// runbookTargetType tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestRunbookTargetType(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
finding *Finding
|
|
||||||
runOnHost bool
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "run on host override",
|
|
||||||
finding: &Finding{ResourceType: "vm"},
|
|
||||||
runOnHost: true,
|
|
||||||
expected: "host",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "vm resource",
|
|
||||||
finding: &Finding{ResourceType: "vm"},
|
|
||||||
runOnHost: false,
|
|
||||||
expected: "vm",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "container resource",
|
|
||||||
finding: &Finding{ResourceType: "container"},
|
|
||||||
runOnHost: false,
|
|
||||||
expected: "container",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "other resource type",
|
|
||||||
finding: &Finding{ResourceType: "storage"},
|
|
||||||
runOnHost: false,
|
|
||||||
expected: "host",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result := runbookTargetType(tt.finding, tt.runOnHost)
|
|
||||||
if result != tt.expected {
|
|
||||||
t.Errorf("runbookTargetType() = %q, want %q", result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// renderRunbookCommand tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestRenderRunbookCommand(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
command string
|
|
||||||
ctx runbookContext
|
|
||||||
expected string
|
|
||||||
expectError bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "simple command no placeholders",
|
|
||||||
command: "echo hello",
|
|
||||||
ctx: runbookContext{},
|
|
||||||
expected: "echo hello",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "command with resource_id",
|
|
||||||
command: "restart {{resource_id}}",
|
|
||||||
ctx: runbookContext{ResourceID: "node1-100"},
|
|
||||||
expected: "restart node1-100",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "command with vmid",
|
|
||||||
command: "qm reboot {{vmid}}",
|
|
||||||
ctx: runbookContext{VMID: "100"},
|
|
||||||
expected: "qm reboot 100",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "command with node",
|
|
||||||
command: "ssh {{node}} hostname",
|
|
||||||
ctx: runbookContext{Node: "node1"},
|
|
||||||
expected: "ssh node1 hostname",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "missing required value",
|
|
||||||
command: "restart {{vmid}}",
|
|
||||||
ctx: runbookContext{VMID: ""},
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "command with value needing escaping",
|
|
||||||
command: "echo {{resource_name}}",
|
|
||||||
ctx: runbookContext{ResourceName: "my vm"},
|
|
||||||
expected: "echo 'my vm'",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
result, err := renderRunbookCommand(tt.command, tt.ctx)
|
|
||||||
if tt.expectError {
|
|
||||||
if err == nil {
|
|
||||||
t.Error("expected error but got none")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if result != tt.expected {
|
|
||||||
t.Errorf("renderRunbookCommand() = %q, want %q", result, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
@ -735,11 +735,6 @@ type AIKubernetesAnalyzeRequest struct {
|
||||||
ClusterID string `json:"cluster_id"`
|
ClusterID string `json:"cluster_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type AIRunbookExecuteRequest struct {
|
|
||||||
FindingID string `json:"finding_id"`
|
|
||||||
RunbookID string `json:"runbook_id"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleExecute executes an AI prompt (POST /api/ai/execute)
|
// HandleExecute executes an AI prompt (POST /api/ai/execute)
|
||||||
func (h *AISettingsHandler) HandleExecute(w http.ResponseWriter, r *http.Request) {
|
func (h *AISettingsHandler) HandleExecute(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Method != http.MethodPost {
|
if r.Method != http.MethodPost {
|
||||||
|
|
@ -881,91 +876,6 @@ func (h *AISettingsHandler) HandleAnalyzeKubernetesCluster(w http.ResponseWriter
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleGetRunbooksForFinding returns available runbooks for a finding (GET /api/ai/runbooks)
|
|
||||||
func (h *AISettingsHandler) HandleGetRunbooksForFinding(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !CheckAuth(h.config, w, r) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
findingID := strings.TrimSpace(r.URL.Query().Get("finding_id"))
|
|
||||||
if findingID == "" {
|
|
||||||
http.Error(w, "finding_id is required", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
patrol := h.aiService.GetPatrolService()
|
|
||||||
if patrol == nil {
|
|
||||||
http.Error(w, "AI patrol not available", http.StatusServiceUnavailable)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
runbooks, err := patrol.GetRunbooksForFinding(findingID)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := utils.WriteJSONResponse(w, runbooks); err != nil {
|
|
||||||
log.Error().Err(err).Msg("Failed to write runbooks response")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleExecuteRunbook executes a runbook for a finding (POST /api/ai/runbooks/execute)
|
|
||||||
func (h *AISettingsHandler) HandleExecuteRunbook(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !CheckAuth(h.config, w, r) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, 16*1024)
|
|
||||||
var req AIRunbookExecuteRequest
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
||||||
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.TrimSpace(req.FindingID) == "" || strings.TrimSpace(req.RunbookID) == "" {
|
|
||||||
http.Error(w, "finding_id and runbook_id are required", http.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
patrol := h.aiService.GetPatrolService()
|
|
||||||
if patrol == nil {
|
|
||||||
http.Error(w, "AI patrol not available", http.StatusServiceUnavailable)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 300*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
result, err := patrol.ExecuteRunbook(ctx, req.FindingID, req.RunbookID)
|
|
||||||
if err != nil {
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, ai.ErrRunbookNotFound):
|
|
||||||
http.Error(w, "Runbook not found", http.StatusNotFound)
|
|
||||||
case errors.Is(err, ai.ErrRunbookNotApplicable):
|
|
||||||
http.Error(w, "Runbook does not apply to finding", http.StatusBadRequest)
|
|
||||||
default:
|
|
||||||
log.Error().Err(err).Str("runbook_id", req.RunbookID).Msg("Runbook execution failed")
|
|
||||||
http.Error(w, "Runbook execution failed: "+err.Error(), http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := utils.WriteJSONResponse(w, result); err != nil {
|
|
||||||
log.Error().Err(err).Msg("Failed to write runbook execution response")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleExecuteStream executes an AI prompt with SSE streaming (POST /api/ai/execute/stream)
|
// HandleExecuteStream executes an AI prompt with SSE streaming (POST /api/ai/execute/stream)
|
||||||
func (h *AISettingsHandler) HandleExecuteStream(w http.ResponseWriter, r *http.Request) {
|
func (h *AISettingsHandler) HandleExecuteStream(w http.ResponseWriter, r *http.Request) {
|
||||||
// Handle CORS for dev mode (frontend on different port)
|
// Handle CORS for dev mode (frontend on different port)
|
||||||
|
|
|
||||||
|
|
@ -846,82 +846,6 @@ func TestHandleAnalyzeKubernetesCluster_InvalidBody(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// HandleGetRunbooksForFinding tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestHandleGetRunbooksForFinding_MethodNotAllowed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tmp := t.TempDir()
|
|
||||||
cfg := &config.Config{DataPath: tmp}
|
|
||||||
persistence := config.NewConfigPersistence(tmp)
|
|
||||||
handler := NewAISettingsHandler(cfg, persistence, nil)
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "/api/ai/runbooks", nil)
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
handler.HandleGetRunbooksForFinding(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusMethodNotAllowed {
|
|
||||||
t.Fatalf("expected status %d, got %d", http.StatusMethodNotAllowed, rec.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandleGetRunbooksForFinding_MissingFindingID(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tmp := t.TempDir()
|
|
||||||
cfg := &config.Config{DataPath: tmp}
|
|
||||||
persistence := config.NewConfigPersistence(tmp)
|
|
||||||
handler := NewAISettingsHandler(cfg, persistence, nil)
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/api/ai/runbooks", nil)
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
handler.HandleGetRunbooksForFinding(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusBadRequest {
|
|
||||||
t.Fatalf("expected status %d, got %d", http.StatusBadRequest, rec.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
|
||||||
// HandleExecuteRunbook tests
|
|
||||||
// ========================================
|
|
||||||
|
|
||||||
func TestHandleExecuteRunbook_MethodNotAllowed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tmp := t.TempDir()
|
|
||||||
cfg := &config.Config{DataPath: tmp}
|
|
||||||
persistence := config.NewConfigPersistence(tmp)
|
|
||||||
handler := NewAISettingsHandler(cfg, persistence, nil)
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/api/ai/runbooks/execute", nil)
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
handler.HandleExecuteRunbook(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusMethodNotAllowed {
|
|
||||||
t.Fatalf("expected status %d, got %d", http.StatusMethodNotAllowed, rec.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandleExecuteRunbook_InvalidBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tmp := t.TempDir()
|
|
||||||
cfg := &config.Config{DataPath: tmp}
|
|
||||||
persistence := config.NewConfigPersistence(tmp)
|
|
||||||
handler := NewAISettingsHandler(cfg, persistence, nil)
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "/api/ai/runbooks/execute", bytes.NewReader([]byte(`{invalid json}`)))
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
handler.HandleExecuteRunbook(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusBadRequest {
|
|
||||||
t.Fatalf("expected status %d, got %d", http.StatusBadRequest, rec.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ========================================
|
// ========================================
|
||||||
// HandleInvestigateAlert tests
|
// HandleInvestigateAlert tests
|
||||||
// ========================================
|
// ========================================
|
||||||
|
|
|
||||||
|
|
@ -1135,9 +1135,8 @@ func (r *Router) setupRoutes() {
|
||||||
r.mux.HandleFunc("/api/ai/execute", RequireAuth(r.config, r.aiSettingsHandler.HandleExecute))
|
r.mux.HandleFunc("/api/ai/execute", RequireAuth(r.config, r.aiSettingsHandler.HandleExecute))
|
||||||
r.mux.HandleFunc("/api/ai/execute/stream", RequireAuth(r.config, r.aiSettingsHandler.HandleExecuteStream))
|
r.mux.HandleFunc("/api/ai/execute/stream", RequireAuth(r.config, r.aiSettingsHandler.HandleExecuteStream))
|
||||||
r.mux.HandleFunc("/api/ai/kubernetes/analyze", RequireAuth(r.config, RequireLicenseFeature(r.licenseHandlers.Service(), license.FeatureKubernetesAI, r.aiSettingsHandler.HandleAnalyzeKubernetesCluster)))
|
r.mux.HandleFunc("/api/ai/kubernetes/analyze", RequireAuth(r.config, RequireLicenseFeature(r.licenseHandlers.Service(), license.FeatureKubernetesAI, r.aiSettingsHandler.HandleAnalyzeKubernetesCluster)))
|
||||||
r.mux.HandleFunc("/api/ai/runbooks", RequireAuth(r.config, RequireLicenseFeature(r.licenseHandlers.Service(), license.FeatureAIAutoFix, r.aiSettingsHandler.HandleGetRunbooksForFinding)))
|
|
||||||
r.mux.HandleFunc("/api/ai/runbooks/execute", RequireAuth(r.config, RequireLicenseFeature(r.licenseHandlers.Service(), license.FeatureAIAutoFix, r.aiSettingsHandler.HandleExecuteRunbook)))
|
|
||||||
r.mux.HandleFunc("/api/ai/investigate-alert", RequireAuth(r.config, RequireLicenseFeature(r.licenseHandlers.Service(), license.FeatureAIAlerts, r.aiSettingsHandler.HandleInvestigateAlert)))
|
r.mux.HandleFunc("/api/ai/investigate-alert", RequireAuth(r.config, RequireLicenseFeature(r.licenseHandlers.Service(), license.FeatureAIAlerts, r.aiSettingsHandler.HandleInvestigateAlert)))
|
||||||
|
|
||||||
r.mux.HandleFunc("/api/ai/run-command", RequireAuth(r.config, r.aiSettingsHandler.HandleRunCommand))
|
r.mux.HandleFunc("/api/ai/run-command", RequireAuth(r.config, r.aiSettingsHandler.HandleRunCommand))
|
||||||
r.mux.HandleFunc("/api/ai/knowledge", RequireAuth(r.config, r.aiSettingsHandler.HandleGetGuestKnowledge))
|
r.mux.HandleFunc("/api/ai/knowledge", RequireAuth(r.config, r.aiSettingsHandler.HandleGetGuestKnowledge))
|
||||||
r.mux.HandleFunc("/api/ai/knowledge/save", RequireAuth(r.config, r.aiSettingsHandler.HandleSaveGuestNote))
|
r.mux.HandleFunc("/api/ai/knowledge/save", RequireAuth(r.config, r.aiSettingsHandler.HandleSaveGuestNote))
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue