diff --git a/README.md b/README.md index 15f6236..0f3c152 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ ## πŸš€ Overview -Pulse is a modern, unified dashboard for your **Proxmox** and **Docker** estate. It consolidates metrics, logs, and alerts from Proxmox VE, Proxmox Backup Server, Proxmox Mail Gateway, and standalone Docker hosts into a single, beautiful interface. +Pulse is a modern, unified dashboard for monitoring your **infrastructure** across Proxmox, Docker, and Kubernetes. It consolidates metrics, alerts, and AI-powered insights from all your systems into a single, beautiful interface. Designed for homelabs, sysadmins, and MSPs who need a "single pane of glass" without the complexity of enterprise monitoring stacks. @@ -23,13 +23,30 @@ Designed for homelabs, sysadmins, and MSPs who need a "single pane of glass" wit ## ✨ Features -- **Unified Monitoring**: View health and metrics for PVE, PBS, PMG, and Docker containers in one place. -- **Smart Alerts**: Get notified via Discord, Slack, Telegram, Email, and more when things go wrong (e.g., "VM down", "Storage full"). -- **Auto-Discovery**: Automatically finds Proxmox nodes on your network. -- **Secure by Design**: Credentials encrypted at rest, no external dependencies, and strict API scoping. -- **Backup Explorer**: Visualize backup jobs and storage usage across your entire infrastructure. -- **Privacy Focused**: No telemetry, no phone-home, all data stays on your server. -- **Lightweight**: Built with Go and React, running as a single binary or container. +### Core Monitoring +- **Unified Monitoring**: View health and metrics for PVE, PBS, PMG, Docker, and Kubernetes in one place +- **Smart Alerts**: Get notified via Discord, Slack, Telegram, Email, and more +- **Auto-Discovery**: Automatically finds Proxmox nodes on your network +- **Metrics History**: Persistent storage with configurable retention +- **Backup Explorer**: Visualize backup jobs and storage usage + +### Pulse AI *(New in 5.0)* +- **Chat Assistant**: Ask questions about your infrastructure in natural language +- **Patrol Mode**: Automated health checks with proactive issue detection +- **Auto-Fix**: Automatically resolve common issues with AI-guided remediation +- **Predictive Intelligence**: Forecast problems before they happen + +### Multi-Platform +- **Proxmox VE/PBS/PMG**: Full monitoring and management +- **Kubernetes**: Complete K8s cluster monitoring via agents +- **Docker/Podman**: Container and Swarm service monitoring +- **OCI Containers**: Proxmox 9.1+ native container support + +### Security & Operations +- **Secure by Design**: Credentials encrypted at rest, strict API scoping +- **One-Click Updates**: Easy upgrades for supported deployments +- **OIDC/SSO**: Enterprise authentication support +- **Privacy Focused**: No telemetry, all data stays on your server ## ⚑ Quick Start @@ -65,7 +82,7 @@ Access the dashboard at `http://:7655`. Community-maintained integrations and addons: -- **[Home Assistant Addon](https://github.com/Kosztyk/pulse-docker-agent-addon)** - Run the Pulse Docker Agent as a Home Assistant addon. +- **[Home Assistant Addons](https://github.com/Kosztyk/homeassistant-addons)** - Run Pulse Agent and Pulse Server as Home Assistant addons. ## ❀️ Support Pulse Development diff --git a/cmd/pulse-agent/main.go b/cmd/pulse-agent/main.go index e6e221f..6b30525 100644 --- a/cmd/pulse-agent/main.go +++ b/cmd/pulse-agent/main.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "sync/atomic" "syscall" @@ -18,6 +19,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentupdate" "github.com/rcourtman/pulse-go-rewrite/internal/dockeragent" "github.com/rcourtman/pulse-go-rewrite/internal/hostagent" + "github.com/rcourtman/pulse-go-rewrite/internal/kubernetesagent" "github.com/rcourtman/pulse-go-rewrite/internal/utils" "github.com/rs/zerolog" "golang.org/x/sync/errgroup" @@ -30,7 +32,7 @@ var ( agentInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "pulse_agent_info", Help: "Information about the Pulse agent", - }, []string{"version", "host_enabled", "docker_enabled"}) + }, []string{"version", "host_enabled", "docker_enabled", "kubernetes_enabled"}) agentUp = promauto.NewGauge(prometheus.GaugeOpts{ Name: "pulse_agent_up", @@ -78,6 +80,8 @@ func main() { Str("pulse_url", cfg.PulseURL). Bool("host_agent", cfg.EnableHost). Bool("docker_agent", cfg.EnableDocker). + Bool("kubernetes_agent", cfg.EnableKubernetes). + Bool("proxmox_mode", cfg.EnableProxmox). Bool("auto_update", !cfg.DisableAutoUpdate). Msg("Starting Pulse Unified Agent") @@ -86,6 +90,7 @@ func main() { Version, fmt.Sprintf("%t", cfg.EnableHost), fmt.Sprintf("%t", cfg.EnableDocker), + fmt.Sprintf("%t", cfg.EnableKubernetes), ).Set(1) agentUp.Set(1) @@ -126,6 +131,8 @@ func main() { InsecureSkipVerify: cfg.InsecureSkipVerify, LogLevel: cfg.LogLevel, Logger: &logger, + EnableProxmox: cfg.EnableProxmox, + ProxmoxType: cfg.ProxmoxType, } agent, err := hostagent.New(hostCfg) @@ -163,19 +170,71 @@ func main() { dockerAgent, err = dockeragent.New(dockerCfg) if err != nil { - logger.Fatal().Err(err).Msg("Failed to initialize docker agent") + // Docker isn't available yet - start retry loop in background + logger.Warn().Err(err).Msg("Docker not available, will retry with exponential backoff") + + g.Go(func() error { + agent := initDockerWithRetry(ctx, dockerCfg, &logger) + if agent != nil { + dockerAgent = agent + logger.Info().Msg("Docker agent module started (after retry)") + return agent.Run(ctx) + } + // Docker never became available, continue without it + return nil + }) + } else { + g.Go(func() error { + logger.Info().Msg("Docker agent module started") + return dockerAgent.Run(ctx) + }) + } + } + + // 10. Start Kubernetes Agent (if enabled) + if cfg.EnableKubernetes { + kubeCfg := kubernetesagent.Config{ + PulseURL: cfg.PulseURL, + APIToken: cfg.APIToken, + Interval: cfg.Interval, + AgentID: cfg.AgentID, + AgentType: "unified", + AgentVersion: Version, + InsecureSkipVerify: cfg.InsecureSkipVerify, + LogLevel: cfg.LogLevel, + Logger: &logger, + KubeconfigPath: cfg.KubeconfigPath, + KubeContext: cfg.KubeContext, + IncludeNamespaces: cfg.KubeIncludeNamespaces, + ExcludeNamespaces: cfg.KubeExcludeNamespaces, + IncludeAllPods: cfg.KubeIncludeAllPods, + MaxPods: cfg.KubeMaxPods, } - g.Go(func() error { - logger.Info().Msg("Docker agent module started") - return dockerAgent.Run(ctx) - }) + agent, err := kubernetesagent.New(kubeCfg) + if err != nil { + logger.Warn().Err(err).Msg("Kubernetes not available, will retry with exponential backoff") + + g.Go(func() error { + retried := initKubernetesWithRetry(ctx, kubeCfg, &logger) + if retried != nil { + logger.Info().Msg("Kubernetes agent module started (after retry)") + return retried.Run(ctx) + } + return nil + }) + } else { + g.Go(func() error { + logger.Info().Msg("Kubernetes agent module started") + return agent.Run(ctx) + }) + } } // Mark as ready after all agents started ready.Store(true) - // 10. Wait for all agents to exit + // 11. Wait for all agents to exit if err := g.Wait(); err != nil && err != context.Canceled { logger.Error().Err(err).Msg("Agent terminated with error") agentUp.Set(0) @@ -183,7 +242,7 @@ func main() { os.Exit(1) } - // 11. Cleanup + // 12. Cleanup agentUp.Set(0) cleanupDockerAgent(dockerAgent, &logger) @@ -259,14 +318,25 @@ type Config struct { Logger *zerolog.Logger // Module flags - EnableHost bool - EnableDocker bool + EnableHost bool + EnableDocker bool + EnableKubernetes bool + EnableProxmox bool + ProxmoxType string // "pve", "pbs", or "" for auto-detect // Auto-update DisableAutoUpdate bool // Health/metrics server HealthAddr string + + // Kubernetes + KubeconfigPath string + KubeContext string + KubeIncludeNamespaces []string + KubeExcludeNamespaces []string + KubeIncludeAllPods bool + KubeMaxPods int } func loadConfig() Config { @@ -281,8 +351,17 @@ func loadConfig() Config { envLogLevel := utils.GetenvTrim("LOG_LEVEL") envEnableHost := utils.GetenvTrim("PULSE_ENABLE_HOST") envEnableDocker := utils.GetenvTrim("PULSE_ENABLE_DOCKER") + envEnableKubernetes := utils.GetenvTrim("PULSE_ENABLE_KUBERNETES") + envEnableProxmox := utils.GetenvTrim("PULSE_ENABLE_PROXMOX") + envProxmoxType := utils.GetenvTrim("PULSE_PROXMOX_TYPE") envDisableAutoUpdate := utils.GetenvTrim("PULSE_DISABLE_AUTO_UPDATE") envHealthAddr := utils.GetenvTrim("PULSE_HEALTH_ADDR") + envKubeconfig := utils.GetenvTrim("PULSE_KUBECONFIG") + envKubeContext := utils.GetenvTrim("PULSE_KUBE_CONTEXT") + envKubeIncludeNamespaces := utils.GetenvTrim("PULSE_KUBE_INCLUDE_NAMESPACES") + envKubeExcludeNamespaces := utils.GetenvTrim("PULSE_KUBE_EXCLUDE_NAMESPACES") + envKubeIncludeAllPods := utils.GetenvTrim("PULSE_KUBE_INCLUDE_ALL_PODS") + envKubeMaxPods := utils.GetenvTrim("PULSE_KUBE_MAX_PODS") // Defaults defaultInterval := 30 * time.Second @@ -302,6 +381,16 @@ func loadConfig() Config { defaultEnableDocker = utils.ParseBool(envEnableDocker) } + defaultEnableKubernetes := false + if envEnableKubernetes != "" { + defaultEnableKubernetes = utils.ParseBool(envEnableKubernetes) + } + + defaultEnableProxmox := false + if envEnableProxmox != "" { + defaultEnableProxmox = utils.ParseBool(envEnableProxmox) + } + defaultHealthAddr := envHealthAddr if defaultHealthAddr == "" { defaultHealthAddr = ":9191" @@ -318,12 +407,23 @@ func loadConfig() Config { enableHostFlag := flag.Bool("enable-host", defaultEnableHost, "Enable Host Agent module") enableDockerFlag := flag.Bool("enable-docker", defaultEnableDocker, "Enable Docker Agent module") + enableKubernetesFlag := flag.Bool("enable-kubernetes", defaultEnableKubernetes, "Enable Kubernetes Agent module") + enableProxmoxFlag := flag.Bool("enable-proxmox", defaultEnableProxmox, "Enable Proxmox mode (creates API token, registers node)") + proxmoxTypeFlag := flag.String("proxmox-type", envProxmoxType, "Proxmox type: pve or pbs (auto-detected if not specified)") disableAutoUpdateFlag := flag.Bool("disable-auto-update", utils.ParseBool(envDisableAutoUpdate), "Disable automatic updates") healthAddrFlag := flag.String("health-addr", defaultHealthAddr, "Health/metrics server address (empty to disable)") + kubeconfigFlag := flag.String("kubeconfig", envKubeconfig, "Path to kubeconfig (optional; uses in-cluster config if available)") + kubeContextFlag := flag.String("kube-context", envKubeContext, "Kubeconfig context (optional)") + kubeIncludeAllPodsFlag := flag.Bool("kube-include-all-pods", utils.ParseBool(envKubeIncludeAllPods), "Include all non-succeeded pods (may be large)") + kubeMaxPodsFlag := flag.Int("kube-max-pods", defaultInt(envKubeMaxPods, 200), "Max pods included in report") showVersion := flag.Bool("version", false, "Print the agent version and exit") var tagFlags multiValue flag.Var(&tagFlags, "tag", "Tag to apply (repeatable)") + var kubeIncludeNamespaceFlags multiValue + flag.Var(&kubeIncludeNamespaceFlags, "kube-include-namespace", "Namespace to include (repeatable; default is all)") + var kubeExcludeNamespaceFlags multiValue + flag.Var(&kubeExcludeNamespaceFlags, "kube-exclude-namespace", "Namespace to exclude (repeatable)") flag.Parse() @@ -350,20 +450,31 @@ func loadConfig() Config { } tags := gatherTags(envTags, tagFlags) + kubeIncludeNamespaces := gatherCSV(envKubeIncludeNamespaces, kubeIncludeNamespaceFlags) + kubeExcludeNamespaces := gatherCSV(envKubeExcludeNamespaces, kubeExcludeNamespaceFlags) return Config{ - PulseURL: pulseURL, - APIToken: token, - Interval: *intervalFlag, - HostnameOverride: strings.TrimSpace(*hostnameFlag), - AgentID: strings.TrimSpace(*agentIDFlag), - Tags: tags, - InsecureSkipVerify: *insecureFlag, - LogLevel: logLevel, - EnableHost: *enableHostFlag, - EnableDocker: *enableDockerFlag, - DisableAutoUpdate: *disableAutoUpdateFlag, - HealthAddr: strings.TrimSpace(*healthAddrFlag), + PulseURL: pulseURL, + APIToken: token, + Interval: *intervalFlag, + HostnameOverride: strings.TrimSpace(*hostnameFlag), + AgentID: strings.TrimSpace(*agentIDFlag), + Tags: tags, + InsecureSkipVerify: *insecureFlag, + LogLevel: logLevel, + EnableHost: *enableHostFlag, + EnableDocker: *enableDockerFlag, + EnableKubernetes: *enableKubernetesFlag, + EnableProxmox: *enableProxmoxFlag, + ProxmoxType: strings.TrimSpace(*proxmoxTypeFlag), + DisableAutoUpdate: *disableAutoUpdateFlag, + HealthAddr: strings.TrimSpace(*healthAddrFlag), + KubeconfigPath: strings.TrimSpace(*kubeconfigFlag), + KubeContext: strings.TrimSpace(*kubeContextFlag), + KubeIncludeNamespaces: kubeIncludeNamespaces, + KubeExcludeNamespaces: kubeExcludeNamespaces, + KubeIncludeAllPods: *kubeIncludeAllPodsFlag, + KubeMaxPods: *kubeMaxPodsFlag, } } @@ -386,6 +497,37 @@ func gatherTags(env string, flags []string) []string { return tags } +func gatherCSV(env string, flags []string) []string { + values := make([]string, 0) + if env != "" { + for _, value := range strings.Split(env, ",") { + value = strings.TrimSpace(value) + if value != "" { + values = append(values, value) + } + } + } + for _, value := range flags { + value = strings.TrimSpace(value) + if value != "" { + values = append(values, value) + } + } + return values +} + +func defaultInt(value string, fallback int) int { + value = strings.TrimSpace(value) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + func parseLogLevel(value string) (zerolog.Level, error) { normalized := strings.ToLower(strings.TrimSpace(value)) if normalized == "" { @@ -400,3 +542,114 @@ func defaultLogLevel(envValue string) string { } return envValue } + +// initDockerWithRetry attempts to initialize the Docker agent with exponential backoff. +// It returns the agent when Docker becomes available, or nil if the context is cancelled. +// Retry intervals: 5s, 10s, 20s, 40s, 80s, 160s, then cap at 5 minutes. +func initDockerWithRetry(ctx context.Context, cfg dockeragent.Config, logger *zerolog.Logger) *dockeragent.Agent { + const ( + initialDelay = 5 * time.Second + maxDelay = 5 * time.Minute + multiplier = 2.0 + ) + + delay := initialDelay + attempt := 0 + + for { + select { + case <-ctx.Done(): + logger.Info().Msg("Docker retry cancelled, context done") + return nil + default: + } + + attempt++ + logger.Info(). + Int("attempt", attempt). + Str("next_retry", delay.String()). + Msg("Waiting to retry Docker connection") + + select { + case <-ctx.Done(): + logger.Info().Msg("Docker retry cancelled during wait") + return nil + case <-time.After(delay): + } + + agent, err := dockeragent.New(cfg) + if err == nil { + logger.Info(). + Int("attempts", attempt). + Msg("Successfully connected to Docker after retry") + return agent + } + + logger.Warn(). + Err(err). + Int("attempt", attempt). + Str("next_retry", (time.Duration(float64(delay) * multiplier)).String()). + Msg("Docker still not available, will retry") + + // Calculate next delay with exponential backoff, capped at maxDelay + delay = time.Duration(float64(delay) * multiplier) + if delay > maxDelay { + delay = maxDelay + } + } +} + +// initKubernetesWithRetry attempts to initialize the Kubernetes agent with exponential backoff. +// It returns the agent when Kubernetes becomes available, or nil if the context is cancelled. +// Retry intervals: 5s, 10s, 20s, 40s, 80s, 160s, then cap at 5 minutes. +func initKubernetesWithRetry(ctx context.Context, cfg kubernetesagent.Config, logger *zerolog.Logger) *kubernetesagent.Agent { + const ( + initialDelay = 5 * time.Second + maxDelay = 5 * time.Minute + multiplier = 2.0 + ) + + delay := initialDelay + attempt := 0 + + for { + select { + case <-ctx.Done(): + logger.Info().Msg("Kubernetes retry cancelled, context done") + return nil + default: + } + + attempt++ + logger.Info(). + Int("attempt", attempt). + Str("next_retry", delay.String()). + Msg("Waiting to retry Kubernetes connection") + + select { + case <-ctx.Done(): + logger.Info().Msg("Kubernetes retry cancelled during wait") + return nil + case <-time.After(delay): + } + + agent, err := kubernetesagent.New(cfg) + if err == nil { + logger.Info(). + Int("attempts", attempt). + Msg("Successfully connected to Kubernetes after retry") + return agent + } + + logger.Warn(). + Err(err). + Int("attempt", attempt). + Str("next_retry", (time.Duration(float64(delay) * multiplier)).String()). + Msg("Kubernetes still not available, will retry") + + delay = time.Duration(float64(delay) * multiplier) + if delay > maxDelay { + delay = maxDelay + } + } +} diff --git a/cmd/pulse-docker-agent/main.go b/cmd/pulse-docker-agent/main.go index 6fc7142..60b9f99 100644 --- a/cmd/pulse-docker-agent/main.go +++ b/cmd/pulse-docker-agent/main.go @@ -55,6 +55,12 @@ func main() { logger := zerolog.New(os.Stdout).Level(cfg.LogLevel).With().Timestamp().Logger() cfg.Logger = &logger + // Deprecation warning + logger.Warn().Msg("pulse-docker-agent is DEPRECATED and will be removed in a future release") + logger.Warn().Msg("Please migrate to the unified 'pulse-agent' with --enable-docker flag") + logger.Warn().Msg("Example: pulse-agent --url --token --enable-docker") + logger.Warn().Msg("") + agent, err := dockeragent.New(cfg) if err != nil { logger.Fatal().Err(err).Msg("Failed to create docker agent") diff --git a/cmd/pulse-host-agent/main.go b/cmd/pulse-host-agent/main.go index e1fe2d6..e64dc46 100644 --- a/cmd/pulse-host-agent/main.go +++ b/cmd/pulse-host-agent/main.go @@ -61,6 +61,12 @@ func main() { g, ctx := errgroup.WithContext(ctx) + // Deprecation warning + logger.Warn().Msg("pulse-host-agent is DEPRECATED and will be removed in a future release") + logger.Warn().Msg("Please migrate to the unified 'pulse-agent' with --enable-host flag") + logger.Warn().Msg("Example: pulse-agent --url --token --enable-host") + logger.Warn().Msg("") + logger.Info(). Str("version", Version). Str("pulse_url", hostCfg.PulseURL). diff --git a/cmd/pulse-sensor-proxy/README.md b/cmd/pulse-sensor-proxy/README.md index fb26e92..3753310 100644 --- a/cmd/pulse-sensor-proxy/README.md +++ b/cmd/pulse-sensor-proxy/README.md @@ -1,5 +1,10 @@ # pulse-sensor-proxy +> **⚠️ Deprecated:** The sensor-proxy is deprecated in favor of the unified Pulse agent. +> For new installations, use `install.sh --enable-proxmox` instead. The agent provides +> temperature monitoring plus additional features (AI, automatic Proxmox API setup). +> See [TEMPERATURE_MONITORING.md](/docs/security/TEMPERATURE_MONITORING.md) for details. + The sensor proxy keeps SSH identities and temperature polling logic on the Proxmox host while presenting a small RPC surface (Unix socket or HTTPS) to the Pulse server. It protects SSH keys from container breakouts, enforces per-UID diff --git a/cmd/pulse/main.go b/cmd/pulse/main.go index afb0c32..53a6095 100644 --- a/cmd/pulse/main.go +++ b/cmd/pulse/main.go @@ -141,8 +141,15 @@ func runServer() { } // Set state getter for WebSocket hub + // IMPORTANT: Return StateFrontend (not StateSnapshot) to match broadcast format. + // StateSnapshot uses time.Time fields while StateFrontend uses Unix timestamps, + // and includes frontend-specific field transformations. Without this conversion, + // nodes/hosts would be missing on initial page load but appear after broadcasts. wsHub.SetStateGetter(func() interface{} { - return reloadableMonitor.GetState() + // GetMonitor().GetState() returns models.StateSnapshot + state := reloadableMonitor.GetMonitor().GetState() + // Convert to frontend format, matching what BroadcastState does + return state.ToFrontend() }) // Wire up Prometheus metrics for alert lifecycle @@ -172,15 +179,29 @@ func runServer() { return nil } router = api.NewRouter(cfg, reloadableMonitor.GetMonitor(), wsHub, reloadFunc, Version) + + // Inject resource store into monitor for WebSocket broadcasts + // This must be done after router creation since resourceHandlers is created in NewRouter + router.SetMonitor(reloadableMonitor.GetMonitor()) + + // Start AI patrol service for background infrastructure monitoring + router.StartPatrol(ctx) + + // Wire alert-triggered AI analysis (token-efficient real-time insights when alerts fire) + router.WireAlertTriggeredAI() // Create HTTP server with unified configuration // In production, serve everything (frontend + API) on the frontend port + // NOTE: We use ReadHeaderTimeout instead of ReadTimeout to avoid affecting + // WebSocket connections. ReadTimeout sets a deadline on the underlying connection + // that persists even after WebSocket upgrade, causing premature disconnections. + // ReadHeaderTimeout only applies during header reading, not the full request body. srv := &http.Server{ - Addr: fmt.Sprintf("%s:%d", cfg.BackendHost, cfg.FrontendPort), - Handler: router.Handler(), - ReadTimeout: 15 * time.Second, - WriteTimeout: 60 * time.Second, // Increased from 15s to 60s to support large JSON responses (e.g., mock data) - IdleTimeout: 60 * time.Second, + Addr: fmt.Sprintf("%s:%d", cfg.BackendHost, cfg.FrontendPort), + Handler: router.Handler(), + ReadHeaderTimeout: 15 * time.Second, + WriteTimeout: 0, // Disabled to support SSE/streaming - each handler manages its own deadline + IdleTimeout: 120 * time.Second, } // Start config watcher for .env file changes diff --git a/docs/AI.md b/docs/AI.md new file mode 100644 index 0000000..416abc3 --- /dev/null +++ b/docs/AI.md @@ -0,0 +1,123 @@ +# Pulse AI + +Pulse AI is an intelligent monitoring assistant that helps you understand and manage your infrastructure through natural language conversations, proactive monitoring, and automated issue resolution. + +## Features + +### Chat Assistant +Ask questions about your infrastructure in plain English: +- "What's using the most CPU right now?" +- "Show me VMs with high memory usage" +- "Why is my backup failing?" +- "Summarize the health of my cluster" + +### Patrol Mode +Automated monitoring that runs on a schedule to: +- Analyze resource utilization patterns +- Detect potential issues before they cause problems +- Generate actionable recommendations +- Track trends over time + +### Auto-Fix +Automatically resolve common issues: +- Restart stuck services +- Clear disk space +- Restart unresponsive containers +- Apply recommended optimizations + +### Predictive Intelligence +- Identify resources trending toward problems +- Forecast disk space exhaustion +- Detect unusual patterns +- Alert on anomalies + +## Configuration + +### Enable Pulse AI + +1. Navigate to **Settings β†’ AI** +2. Toggle **Enable Pulse AI** +3. Configure your AI provider + +### Supported Providers + +| Provider | Models | Notes | +|----------|--------|-------| +| **OpenAI** | GPT-4, GPT-4 Turbo, GPT-3.5 | Recommended for best results | +| **Anthropic** | Claude 3 Opus, Sonnet, Haiku | Excellent for complex analysis | +| **Ollama** | Llama 3, Mistral, etc. | Self-hosted, privacy-focused | +| **OpenRouter** | Multiple models | Pay-per-use routing | +| **Custom** | Any OpenAI-compatible API | For enterprise deployments | + +### API Key Setup + +```bash +# Environment variable (recommended for production) +export PULSE_AI_PROVIDER=openai +export PULSE_AI_API_KEY=sk-... + +# Or configure via Settings UI +``` + +### Patrol Configuration + +| Setting | Default | Description | +|---------|---------|-------------| +| Patrol Enabled | Off | Run automated checks | +| Patrol Interval | 30 minutes | How often to run patrol | +| Auto-Fix Enabled | Off | Allow automatic remediation | +| Auto-Fix Model | Same as chat | Model for auto-fix decisions | + +## Usage + +### Chat Interface +Access the AI chat from the bottom-right corner of any page. Type your question and press Enter. + +**Example queries:** +``` +"What VMs are using more than 80% memory?" +"Show me the status of all backups" +"Why is node pve1 showing high load?" +"Compare resource usage between this week and last week" +``` + +### Context Selection +Click on any resource (VM, container, host) to add it to the AI context. This helps the AI provide more specific answers. + +### Patrol Reports +When Patrol is enabled, Pulse AI will: +1. Run periodic health checks +2. Generate findings (issues, warnings, info) +3. Offer to auto-fix issues (if enabled) +4. Track trends over time + +## Cost Tracking + +Track AI API usage in **Settings β†’ AI β†’ Cost Dashboard**: +- Daily/monthly usage statistics +- Cost breakdown by feature +- Usage trends over time + +## Privacy & Security + +- **Data stays local**: Only resource metadata is sent to AI providers +- **No training**: Your data is never used for model training +- **Audit logging**: All AI interactions are logged +- **Self-hosted option**: Use Ollama for complete data privacy + +## Troubleshooting + +### AI not responding +1. Check API key is configured correctly +2. Verify network connectivity to AI provider +3. Check Settings β†’ AI for error messages + +### Patrol not running +1. Ensure Patrol is enabled in Settings +2. Check system resource availability +3. Review logs: `journalctl -u pulse -f` + +### Auto-fix not working +1. Enable Auto-Fix in Settings β†’ AI +2. Verify the connected agents have execute permissions +3. Check the Auto-Fix model is configured diff --git a/docs/API.md b/docs/API.md index 32f2584..3173216 100644 --- a/docs/API.md +++ b/docs/API.md @@ -112,23 +112,6 @@ Triggers a test alert to all configured channels. --- ---- -## πŸ–₯️ Host Agent - -### Submit Report -`POST /api/agents/host/report` -Used by the Pulse Host Agent to push system metrics. - -### Lookup Agent -`POST /api/agents/host/lookup` -Check if a host agent is already registered. - -### Delete Host -`DELETE /api/agents/host/` -Remove a host agent from monitoring. - ---- - ## βš™οΈ System Settings ### Get Settings @@ -157,15 +140,55 @@ Initiate OIDC login flow. --- -## 🐳 Docker Agent +## πŸ€– Pulse AI *(New in 5.0)* -### Submit Report -`POST /api/agents/docker/report` -Used by the Pulse Docker Agent to push container metrics. +### Get AI Settings +`GET /api/settings/ai` +Returns current AI configuration (providers, models, patrol status). -### Download Agent -`GET /download/pulse-docker-agent` -Downloads the binary for the current platform. +### Update AI Settings +`PUT /api/settings/ai` +Configure AI providers, API keys, and preferences. + +### Chat +`POST /api/ai/chat` +Send a message to the AI assistant. +```json +{ "message": "What VMs are using the most CPU?", "context": ["vm-100", "vm-101"] } +``` + +### Patrol Status +`GET /api/ai/patrol/status` +Get current patrol status and recent findings. + +### Patrol Findings +`GET /api/ai/patrol/findings` +List all patrol findings with severity and recommendations. + +### Cost Tracking +`GET /api/ai/cost?period=30d` +Get AI usage statistics and costs. + +--- + +## πŸ€– Agent Endpoints + +### Unified Agent (Recommended) +`GET /download/pulse-agent` +Downloads the unified agent binary for the current platform. + +The unified agent combines host, Docker, and Kubernetes monitoring. Use `--enable-docker` or `--enable-kubernetes` to enable additional metrics. + +See [UNIFIED_AGENT.md](UNIFIED_AGENT.md) for installation instructions. + +### Legacy Agents (Deprecated) +`GET /download/pulse-host-agent` - *Deprecated, use pulse-agent* +`GET /download/pulse-docker-agent` - *Deprecated, use pulse-agent --enable-docker* + +### Submit Reports +`POST /api/agents/host/report` - Host metrics +`POST /api/agents/docker/report` - Docker container metrics +`POST /api/agents/kubernetes/report` - Kubernetes cluster metrics --- diff --git a/docs/AUTO_UPDATE.md b/docs/AUTO_UPDATE.md new file mode 100644 index 0000000..791619d --- /dev/null +++ b/docs/AUTO_UPDATE.md @@ -0,0 +1,141 @@ +# Automatic Updates + +Pulse 5.0 introduces one-click updates for supported deployment types, making it easy to keep your monitoring system up to date. + +## Supported Deployment Types + +| Deployment | Auto-Update | Method | +|------------|-------------|--------| +| **ProxmoxVE LXC** | βœ… Yes | In-app update button | +| **Systemd Service** | βœ… Yes | In-app update button | +| **Docker** | ❌ Manual | Pull new image | +| **Source Build** | ❌ Manual | Git pull + rebuild | + +## Using One-Click Updates + +### When an Update is Available + +1. Navigate to **Settings β†’ System Updates** +2. If an update is available, you'll see an **"Install Update"** button +3. Click the button to open the confirmation dialog +4. Review the update details: + - Current version β†’ New version + - Estimated time + - Changelog highlights +5. Click **"Install Update"** to begin + +### Update Process + +1. **Download**: New version is downloaded +2. **Backup**: Current installation is backed up +3. **Apply**: Files are updated +4. **Restart**: Service restarts automatically +5. **Verify**: Health check confirms success + +### Progress Tracking + +A real-time progress modal shows: +- Current step +- Download progress +- Any warnings or errors +- Automatic page reload on success + +## Configuration + +### Update Preferences + +In **Settings β†’ System Updates**: + +| Setting | Description | +|---------|-------------| +| **Update Channel** | Stable (recommended) or Release Candidate | +| **Auto-Check** | Automatically check for updates daily | + +### Environment Variables + +```bash +# Disable auto-update check +PULSE_AUTO_UPDATE_CHECK=false + +# Use release candidate channel +PULSE_UPDATE_CHANNEL=rc +``` + +## Manual Update Methods + +### Docker + +```bash +# Pull latest image +docker pull ghcr.io/rcourtman/pulse:latest + +# Restart container +docker-compose down && docker-compose up -d +``` + +### ProxmoxVE LXC (Manual) + +```bash +# Inside the container +curl -fsSL https://raw.githubusercontent.com/rcourtman/Pulse/main/scripts/install.sh | bash +``` + +### Systemd Service (Manual) + +```bash +# Download new release +curl -fsSL https://raw.githubusercontent.com/rcourtman/Pulse/main/scripts/install.sh | bash +``` + +### Source Build + +```bash +cd /opt/pulse +git pull +make build +sudo systemctl restart pulse +``` + +## Rollback + +If an update causes issues: + +### Automatic Rollback +Pulse creates a backup before updating. If the update fails: +1. The previous version is automatically restored +2. Service restarts with the old version +3. Error details are logged + +### Manual Rollback +```bash +# Backups are stored in /etc/pulse/backups/ +ls /etc/pulse/backups/ + +# Restore a specific backup +sudo /opt/pulse/scripts/restore-backup.sh /etc/pulse/backups/pulse-backup-20250101.tar.gz +``` + +## Update History + +View past updates in **Settings β†’ System Updates β†’ Update History**: +- Previous versions installed +- Update timestamps +- Success/failure status + +## Troubleshooting + +### Update button not showing +1. Check if your deployment supports auto-update +2. Verify an update is actually available +3. Ensure you have the latest frontend loaded (hard refresh) + +### Update failed +1. Check the error message in the progress modal +2. Review logs: `journalctl -u pulse -n 100` +3. Verify disk space is available +4. Check network connectivity to GitHub + +### Service won't restart after update +1. Check systemd status: `systemctl status pulse` +2. View recent logs: `journalctl -u pulse -f` +3. Manually restore from backup if needed diff --git a/docs/MAIL_GATEWAY.md b/docs/MAIL_GATEWAY.md new file mode 100644 index 0000000..4fb623e --- /dev/null +++ b/docs/MAIL_GATEWAY.md @@ -0,0 +1,99 @@ +# Proxmox Mail Gateway (PMG) Monitoring + +Pulse 5.0 adds support for monitoring Proxmox Mail Gateway instances alongside your PVE and PBS infrastructure. + +## Features + +- **Mail Queue Monitoring**: Track active, deferred, and held messages +- **Spam Statistics**: View spam detection rates and virus blocks +- **Cluster Status**: Monitor PMG cluster node health +- **Quarantine Overview**: See quarantine size and pending reviews + +## Adding a PMG Instance + +### Via Settings UI + +1. Navigate to **Settings β†’ Proxmox** +2. Click **Add Node** +3. Select **Proxmox Mail Gateway** as the type +4. Enter connection details: + - Host: Your PMG IP or hostname + - Port: 8006 (default) + - API Token ID: e.g., `root@pam!pulse` + - API Token Secret: Your token secret + +### Via Discovery + +Pulse can automatically discover PMG instances on your network: + +1. Go to **Settings β†’ Discovery** +2. Enable network discovery +3. PMG instances on port 8006 will be detected +4. Click to add discovered instances + +## API Token Setup on PMG + +Create an API token on your PMG server: + +```bash +# SSH to your PMG server +pveum user token add root@pam pulse --privsep 0 + +# Note the token secret - it's only shown once! +``` + +Required permissions: +- `Sys.Audit` - Read system status +- `Datastore.Audit` - Read mail statistics + +## Dashboard + +The Mail Gateway tab shows: + +| Metric | Description | +|--------|-------------| +| **Mail Processed** | Total emails processed today | +| **Spam Rate** | Percentage of spam detected | +| **Virus Blocked** | Malicious emails caught | +| **Queue Depth** | Messages pending delivery | +| **Quarantine Size** | Emails in quarantine | + +### Status Indicators + +- 🟒 **Healthy**: Normal operation +- 🟑 **Warning**: Queue building up or high spam rate +- πŸ”΄ **Critical**: Delivery issues or cluster problems + +## Alerts + +Configure alerts for PMG metrics in **Settings β†’ Alerts**: + +- Queue depth exceeding threshold +- Spam rate spike +- Delivery failures +- Cluster node offline + +## Multi-Instance Support + +Monitor multiple PMG instances from a single Pulse dashboard: + +- Compare spam rates across gateways +- Aggregate mail statistics +- View cluster-wide health + +## Troubleshooting + +### Connection refused +1. Verify PMG is accessible on port 8006 +2. Check firewall rules +3. Ensure API token has correct permissions + +### No statistics showing +1. Wait for initial data collection (may take 1-2 polling cycles) +2. Verify PMG has mail activity +3. Check Pulse logs for API errors + +### Cluster nodes missing +1. PMG cluster must be properly configured +2. API token needs cluster-wide permissions +3. All nodes must be reachable from Pulse diff --git a/docs/METRICS_HISTORY.md b/docs/METRICS_HISTORY.md new file mode 100644 index 0000000..b79d2a1 --- /dev/null +++ b/docs/METRICS_HISTORY.md @@ -0,0 +1,87 @@ +# Metrics History + +Pulse 5.0 introduces persistent metrics history, allowing you to view historical resource usage data and trends over time. + +## Features + +- **Persistent Storage**: Metrics are saved to disk and survive restarts +- **Configurable Retention**: Set how long to keep different metric types +- **Trend Analysis**: View resource usage patterns over time +- **Spark Lines**: See at-a-glance trends in the dashboard + +## Configuration + +### Retention Settings + +Configure retention periods in **Settings β†’ General β†’ Metrics History**: + +| Metric Type | Default | Description | +|-------------|---------|-------------| +| **Host Metrics** | 7 days | CPU, memory, disk for hypervisors | +| **Guest Metrics** | 7 days | VM and container metrics | +| **Container Metrics** | 3 days | Docker/Podman container stats | +| **Aggregate Metrics** | 30 days | Cluster-wide summaries | + +### Environment Variables + +```bash +# Override via environment +PULSE_METRICS_HOST_RETENTION_DAYS=14 +PULSE_METRICS_GUEST_RETENTION_DAYS=14 +PULSE_METRICS_CONTAINER_RETENTION_DAYS=7 +PULSE_METRICS_AGGREGATE_RETENTION_DAYS=60 +``` + +## Storage + +Metrics are stored in `/etc/pulse/data/metrics/` (or your configured data directory). + +### Disk Usage + +Approximate storage requirements: +- ~1 KB per resource per hour +- 10 hosts Γ— 50 guests Γ— 7 days β‰ˆ 8 MB + +### Database Maintenance + +Pulse automatically: +- Compacts old data +- Prunes metrics beyond retention period +- Optimizes storage during low-usage periods + +## API Access + +Query historical metrics via the API: + +```bash +# Get metrics for a specific resource +curl -H "X-API-Token: $TOKEN" \ + "http://localhost:7655/api/metrics/history?resource=vm-100&hours=24" + +# Get aggregated cluster metrics +curl -H "X-API-Token: $TOKEN" \ + "http://localhost:7655/api/metrics/history?type=aggregate&days=7" +``` + +## Visualization + +### Dashboard Sparklines +The dashboard shows 24-hour trend sparklines for each resource, updating in real-time. + +### Detailed Charts +Click on any resource to see detailed historical charts with: +- Selectable time ranges (1h, 6h, 24h, 7d, 30d) +- Multiple metric overlays (CPU, memory, disk, network) +- Zoom and pan controls + +## Troubleshooting + +### Metrics not persisting +1. Check data directory permissions +2. Verify disk space availability +3. Check logs: `journalctl -u pulse | grep metrics` + +### High disk usage +1. Reduce retention periods in Settings +2. Exclude low-value resources from history +3. Run manual cleanup: Settings β†’ General β†’ Clear Old Metrics diff --git a/docs/README.md b/docs/README.md index 24b71c1..75f13c1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,6 +26,14 @@ Welcome to the Pulse documentation portal. Here you'll find everything you need - **[Security Policy](../SECURITY.md)** – The core security model (Encryption, Auth, API Scopes). +## ✨ New in 5.0 + +- **[Pulse AI](AI.md)** – Intelligent monitoring assistant with chat, patrol mode, and auto-fix. +- **[Metrics History](METRICS_HISTORY.md)** – Persistent metrics storage with configurable retention. +- **[Mail Gateway](MAIL_GATEWAY.md)** – Proxmox Mail Gateway (PMG) monitoring. +- **[Auto Updates](AUTO_UPDATE.md)** – One-click updates for supported deployments. +- **[Kubernetes](KUBERNETES.md)** – Complete K8s cluster monitoring via agents. + ## πŸ“‘ Monitoring & Agents - **[Unified Agent](UNIFIED_AGENT.md)** – Single binary for Host and Docker monitoring. diff --git a/docs/UNIFIED_AGENT.md b/docs/UNIFIED_AGENT.md index 6b0447e..72700a8 100644 --- a/docs/UNIFIED_AGENT.md +++ b/docs/UNIFIED_AGENT.md @@ -1,6 +1,6 @@ # Pulse Unified Agent -The unified agent (`pulse-agent`) combines host and Docker monitoring into a single binary. It replaces the separate `pulse-host-agent` and `pulse-docker-agent` for simpler deployment and management. +The unified agent (`pulse-agent`) combines host, Docker, and Kubernetes monitoring into a single binary. It replaces the separate `pulse-host-agent` and `pulse-docker-agent` for simpler deployment and management. ## Quick Start @@ -29,6 +29,7 @@ curl -fsSL http://:7655/install.sh | \ - **Host Metrics**: CPU, memory, disk, network I/O, temperatures - **Docker Monitoring**: Container metrics, health checks, Swarm support (when enabled) +- **Kubernetes Monitoring**: Cluster, node, pod, and deployment health (when enabled) - **Auto-Update**: Automatically updates when a new version is released - **Multi-Platform**: Linux, macOS, Windows support @@ -41,6 +42,13 @@ curl -fsSL http://:7655/install.sh | \ | `--interval` | `PULSE_INTERVAL` | Reporting interval | `30s` | | `--enable-host` | `PULSE_ENABLE_HOST` | Enable host metrics | `true` | | `--enable-docker` | `PULSE_ENABLE_DOCKER` | Enable Docker metrics | `false` | +| `--enable-kubernetes` | `PULSE_ENABLE_KUBERNETES` | Enable Kubernetes metrics | `false` | +| `--kubeconfig` | `PULSE_KUBECONFIG` | Kubeconfig path (optional) | *(auto)* | +| `--kube-context` | `PULSE_KUBE_CONTEXT` | Kubeconfig context (optional) | *(auto)* | +| `--kube-include-namespace` | `PULSE_KUBE_INCLUDE_NAMESPACES` | Limit namespaces (repeatable or CSV) | *(all)* | +| `--kube-exclude-namespace` | `PULSE_KUBE_EXCLUDE_NAMESPACES` | Exclude namespaces (repeatable or CSV) | *(none)* | +| `--kube-include-all-pods` | `PULSE_KUBE_INCLUDE_ALL_PODS` | Include all non-succeeded pods | `false` | +| `--kube-max-pods` | `PULSE_KUBE_MAX_PODS` | Max pods per report | `200` | | `--disable-auto-update` | `PULSE_DISABLE_AUTO_UPDATE` | Disable auto-updates | `false` | | `--insecure` | `PULSE_INSECURE_SKIP_VERIFY` | Skip TLS verification | `false` | | `--hostname` | `PULSE_HOSTNAME` | Override hostname | *(OS hostname)* | @@ -61,6 +69,12 @@ curl -fsSL http://:7655/install.sh | \ bash -s -- --url http://:7655 --token --enable-docker ``` +### Host + Kubernetes Monitoring +```bash +curl -fsSL http://:7655/install.sh | \ + bash -s -- --url http://:7655 --token --enable-kubernetes +``` + ### Docker Monitoring Only ```bash curl -fsSL http://:7655/install.sh | \ diff --git a/docs/development/MOCK_MODE.md b/docs/development/MOCK_MODE.md index 5f09f6a..4d4e2ab 100644 --- a/docs/development/MOCK_MODE.md +++ b/docs/development/MOCK_MODE.md @@ -25,6 +25,8 @@ Edit `mock.env` (or `mock.env.local` for overrides): | `PULSE_MOCK_LXCS_PER_NODE` | `8` | Containers per node. | | `PULSE_MOCK_RANDOM_METRICS` | `true` | Jitter metrics. | | `PULSE_MOCK_STOPPED_PERCENT` | `20` | % of offline guests. | +| `PULSE_MOCK_TRENDS_SEED_DURATION` | `1h` | Pre-seed backend chart history (improves demo β€œTrends” immediately). | +| `PULSE_MOCK_TRENDS_SAMPLE_INTERVAL` | `30s` | Backend chart sampling interval while in mock mode. | ## ℹ️ How it Works * **Data**: Swaps `PULSE_DATA_DIR` to `/opt/pulse/tmp/mock-data`. diff --git a/docs/operations/SENSOR_PROXY_CONFIG.md b/docs/operations/SENSOR_PROXY_CONFIG.md index eae06ae..9305690 100644 --- a/docs/operations/SENSOR_PROXY_CONFIG.md +++ b/docs/operations/SENSOR_PROXY_CONFIG.md @@ -1,5 +1,9 @@ # βš™οΈ Sensor Proxy Configuration +> **⚠️ Deprecated:** The sensor-proxy is deprecated in favor of the unified Pulse agent. +> For new installations, use `install.sh --enable-proxmox` instead. +> See [TEMPERATURE_MONITORING.md](/docs/security/TEMPERATURE_MONITORING.md). + Safe configuration management using the CLI (v4.31.1+). ## πŸ“‚ Files diff --git a/docs/operations/SENSOR_PROXY_LOGS.md b/docs/operations/SENSOR_PROXY_LOGS.md index 5c56b4d..06c5cf1 100644 --- a/docs/operations/SENSOR_PROXY_LOGS.md +++ b/docs/operations/SENSOR_PROXY_LOGS.md @@ -1,5 +1,9 @@ # πŸ“ Sensor Proxy Log Forwarding +> **⚠️ Deprecated:** The sensor-proxy is deprecated in favor of the unified Pulse agent. +> For new installations, use `install.sh --enable-proxmox` instead. +> See [TEMPERATURE_MONITORING.md](/docs/security/TEMPERATURE_MONITORING.md). + Forward `audit.log` and `proxy.log` to a central SIEM via RELP + TLS. ## πŸš€ Quick Start diff --git a/docs/security/SENSOR_PROXY_APPARMOR.md b/docs/security/SENSOR_PROXY_APPARMOR.md index 44fb5a5..6d25c06 100644 --- a/docs/security/SENSOR_PROXY_APPARMOR.md +++ b/docs/security/SENSOR_PROXY_APPARMOR.md @@ -1,5 +1,9 @@ # πŸ›‘οΈ Sensor Proxy Hardening +> **⚠️ Deprecated:** The sensor-proxy is deprecated in favor of the unified Pulse agent. +> For new installations, use `install.sh --enable-proxmox` instead. +> See [TEMPERATURE_MONITORING.md](/docs/security/TEMPERATURE_MONITORING.md). + Secure `pulse-sensor-proxy` with AppArmor and Seccomp. ## πŸ›‘οΈ AppArmor diff --git a/docs/security/SENSOR_PROXY_HARDENING.md b/docs/security/SENSOR_PROXY_HARDENING.md index c03f99c..12f1c65 100644 --- a/docs/security/SENSOR_PROXY_HARDENING.md +++ b/docs/security/SENSOR_PROXY_HARDENING.md @@ -1,5 +1,9 @@ # πŸ›‘οΈ Sensor Proxy Hardening +> **⚠️ Deprecated:** The sensor-proxy is deprecated in favor of the unified Pulse agent. +> For new installations, use `install.sh --enable-proxmox` instead. +> See [TEMPERATURE_MONITORING.md](/docs/security/TEMPERATURE_MONITORING.md). + The `pulse-sensor-proxy` runs on the host to securely collect temperatures, keeping SSH keys out of containers. ## πŸ—οΈ Architecture diff --git a/docs/security/SENSOR_PROXY_NETWORK.md b/docs/security/SENSOR_PROXY_NETWORK.md index 55c6139..9a5d1fa 100644 --- a/docs/security/SENSOR_PROXY_NETWORK.md +++ b/docs/security/SENSOR_PROXY_NETWORK.md @@ -1,5 +1,9 @@ # 🌐 Sensor Proxy Network Segmentation +> **⚠️ Deprecated:** The sensor-proxy is deprecated in favor of the unified Pulse agent. +> For new installations, use `install.sh --enable-proxmox` instead. +> See [TEMPERATURE_MONITORING.md](/docs/security/TEMPERATURE_MONITORING.md). + Isolate the proxy to prevent lateral movement. ## 🚧 Zones diff --git a/docs/security/TEMPERATURE_MONITORING.md b/docs/security/TEMPERATURE_MONITORING.md index 760419c..0276393 100644 --- a/docs/security/TEMPERATURE_MONITORING.md +++ b/docs/security/TEMPERATURE_MONITORING.md @@ -1,29 +1,57 @@ -# 🌑️ Temperature Monitoring Security +# 🌑️ Temperature Monitoring -Secure architecture for collecting hardware temperatures. +Pulse supports two methods for collecting hardware temperatures from Proxmox nodes. -## πŸ›‘οΈ Security Model +## Recommended: Pulse Agent + +The simplest and most feature-rich method is installing the Pulse agent on your Proxmox nodes: + +```bash +curl -fsSL http://your-pulse-server:7655/api/download/install.sh | bash -s -- \ + --url http://your-pulse-server:7655 \ + --token YOUR_TOKEN \ + --enable-proxmox +``` + +**Benefits:** +- βœ… One-command setup +- βœ… Automatic API token creation +- βœ… Temperature monitoring built-in +- βœ… Enables AI features for VM/container management +- βœ… No SSH keys or proxy configuration required + +The agent runs `sensors -j` locally and reports temperatures directly to Pulse. + +--- + +## Legacy: Sensor Proxy (SSH-based) + +For users who prefer not to install an agent on their hypervisor, the sensor-proxy method is still available. + +> **Note:** This method is deprecated and will be removed in a future release. Consider migrating to the agent-based approach. + +### πŸ›‘οΈ Security Model * **Isolation**: SSH keys live on the host, not in the container. * **Least Privilege**: Proxy runs as `pulse-sensor-proxy` (no shell). * **Verification**: Container identity verified via `SO_PEERCRED`. -## πŸ—οΈ Components +### πŸ—οΈ Components 1. **Pulse Backend**: Connects to Unix socket `/mnt/pulse-proxy/pulse-sensor-proxy.sock`. 2. **Sensor Proxy**: Validates request, executes SSH to node. 3. **Target Node**: Accepts SSH key restricted to `sensors -j`. -## πŸ”’ Key Restrictions +### πŸ”’ Key Restrictions SSH keys deployed to nodes are locked down: ``` command="sensors -j",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty ``` -## 🚦 Rate Limiting +### 🚦 Rate Limiting * **Per Peer**: ~12 req/min. * **Concurrency**: Max 2 parallel requests per peer. * **Global**: Max 8 concurrent requests. -## πŸ“ Auditing +### πŸ“ Auditing All requests logged to system journal: ```bash journalctl -u pulse-sensor-proxy diff --git a/frontend-modern/index.html b/frontend-modern/index.html index 78bd7f0..0ce2d45 100644 --- a/frontend-modern/index.html +++ b/frontend-modern/index.html @@ -6,6 +6,34 @@ Pulse + + + + diff --git a/frontend-modern/package-lock.json b/frontend-modern/package-lock.json index e09c7a6..b13a360 100644 Binary files a/frontend-modern/package-lock.json and b/frontend-modern/package-lock.json differ diff --git a/frontend-modern/package.json b/frontend-modern/package.json index 666c8a9..9c94d57 100644 --- a/frontend-modern/package.json +++ b/frontend-modern/package.json @@ -27,11 +27,14 @@ }, "dependencies": { "@solidjs/router": "^0.10.10", + "dompurify": "^3.3.1", "lucide-solid": "^0.545.0", + "marked": "^17.0.1", "solid-js": "^1.8.0" }, "devDependencies": { "@solidjs/testing-library": "^0.8.5", + "@tailwindcss/typography": "^0.5.19", "@testing-library/jest-dom": "^6.5.0", "@types/node": "^20.10.0", "@typescript-eslint/eslint-plugin": "^8.0.0", diff --git a/frontend-modern/src/App.tsx b/frontend-modern/src/App.tsx index 5deca70..4d335d0 100644 --- a/frontend-modern/src/App.tsx +++ b/frontend-modern/src/App.tsx @@ -35,14 +35,21 @@ import { createTooltipSystem } from './components/shared/Tooltip'; import type { State } from '@/types/api'; import { ProxmoxIcon } from '@/components/icons/ProxmoxIcon'; import { startMetricsSampler } from './stores/metricsSampler'; +import { seedFromBackend } from './stores/metricsHistory'; +import { getMetricsViewMode } from './stores/metricsViewMode'; import BoxesIcon from 'lucide-solid/icons/boxes'; import MonitorIcon from 'lucide-solid/icons/monitor'; import BellIcon from 'lucide-solid/icons/bell'; import SettingsIcon from 'lucide-solid/icons/settings'; +import NetworkIcon from 'lucide-solid/icons/network'; import { TokenRevealDialog } from './components/TokenRevealDialog'; import { useAlertsActivation } from './stores/alertsActivation'; import { UpdateProgressModal } from './components/UpdateProgressModal'; import type { UpdateStatus } from './api/updates'; +import { AIChat } from './components/AI/AIChat'; +import { AIStatusIndicator } from './components/AI/AIStatusIndicator'; +import { aiChatStore } from './stores/aiChat'; +import { useResourcesAsLegacy } from './hooks/useResources'; const Dashboard = lazy(() => import('./components/Dashboard/Dashboard').then((module) => ({ default: module.Dashboard })), @@ -51,6 +58,7 @@ const StorageComponent = lazy(() => import('./components/Storage/Storage')); const Backups = lazy(() => import('./components/Backups/Backups')); const Replication = lazy(() => import('./components/Replication/Replication')); const MailGateway = lazy(() => import('./components/PMG/MailGateway')); +const CephPage = lazy(() => import('./pages/Ceph')); const AlertsPage = lazy(() => import('./pages/Alerts').then((module) => ({ default: module.Alerts })), ); @@ -58,12 +66,18 @@ const SettingsPage = lazy(() => import('./components/Settings/Settings')); const DockerHosts = lazy(() => import('./components/Docker/DockerHosts').then((module) => ({ default: module.DockerHosts })), ); +const KubernetesClusters = lazy(() => + import('./components/Kubernetes/KubernetesClusters').then((module) => ({ + default: module.KubernetesClusters, + })), +); const HostsOverview = lazy(() => import('./components/Hosts/HostsOverview').then((module) => ({ default: module.HostsOverview, })), ); + // Enhanced store type with proper typing type EnhancedStore = ReturnType; @@ -87,26 +101,29 @@ export const useDarkMode = () => { return context; }; -// Docker route component that properly uses activeAlerts from useWebSocket +// Docker route component - uses unified resources via useResourcesAsLegacy hook function DockerRoute() { const wsContext = useContext(WebSocketContext); if (!wsContext) { return
Loading...
; } - const { state, activeAlerts } = wsContext; - const hosts = createMemo(() => state.dockerHosts ?? []); - return ; + const { activeAlerts } = wsContext; + const { asDockerHosts } = useResourcesAsLegacy(); + + return ; } +// Hosts route component - HostsOverview uses useResourcesAsLegacy directly for proper reactivity function HostsRoute() { + return ; +} + +function KubernetesRoute() { const wsContext = useContext(WebSocketContext); if (!wsContext) { return
Loading...
; } - const { state } = wsContext; - return ( - - ); + return ; } // Helper to detect if an update is actively in progress (not just checking for updates) @@ -211,6 +228,13 @@ function App() { // Start metrics sampler for sparklines onMount(() => { startMetricsSampler(); + + // If user already has sparklines mode enabled, seed historical data immediately + if (getMetricsViewMode() === 'sparklines') { + seedFromBackend('1h').catch(() => { + // Errors are already logged in seedFromBackend + }); + } }); let hasPreloadedRoutes = false; @@ -226,6 +250,7 @@ function App() { () => import('./components/Replication/Replication'), () => import('./components/PMG/MailGateway'), () => import('./components/Hosts/HostsOverview'), + () => import('./pages/Alerts'), () => import('./components/Settings/Settings'), () => import('./components/Docker/DockerHosts'), @@ -251,23 +276,23 @@ function App() { pmg: [], replicationJobs: [], metrics: [], - pveBackups: { - backupTasks: [], - storageBackups: [], - guestSnapshots: [], - }, - pbsBackups: [], - pmgBackups: [], - backups: { - pve: { + pveBackups: { backupTasks: [], storageBackups: [], guestSnapshots: [], }, - pbs: [], - pmg: [], - }, - performance: { + pbsBackups: [], + pmgBackups: [], + backups: { + pve: { + backupTasks: [], + storageBackups: [], + guestSnapshots: [], + }, + pbs: [], + pmg: [], + }, + performance: { apiCallDuration: {}, lastPollDuration: 0, pollingStartTime: '', @@ -498,9 +523,9 @@ function App() { // Detect legacy DISABLE_AUTH flag (now ignored) so we can surface a warning if (securityData.deprecatedDisableAuth === true) { - logger.warn( - '[App] Legacy DISABLE_AUTH flag detected; authentication remains enabled. Remove the flag and restart Pulse to silence this warning.', - ); + logger.warn( + '[App] Legacy DISABLE_AUTH flag detected; authentication remains enabled. Remove the flag and restart Pulse to silence this warning.', + ); } const authConfigured = securityData.hasAuthentication || false; @@ -713,9 +738,14 @@ function App() { // Pass through the store directly (only when initialized) const enhancedStore = () => wsStore(); - const DashboardView = () => ( - - ); + // Dashboard view - uses unified resources via useResourcesAsLegacy hook + const DashboardView = () => { + const { asVMs, asContainers, asNodes } = useResourcesAsLegacy(); + + return ( + + ); + }; const SettingsRoute = () => ( @@ -723,42 +753,120 @@ function App() { // Root layout component for Router const RootLayout = (props: { children?: JSX.Element }) => { + // Check AI settings on mount and setup keyboard shortcut + onMount(() => { + // Only check AI settings if already authenticated (not on login screen) + // Otherwise, the 401 response triggers a redirect loop + if (!needsAuth()) { + import('./api/ai').then(({ AIAPI }) => { + AIAPI.getSettings() + .then((settings) => { + aiChatStore.setEnabled(settings.enabled && settings.configured); + }) + .catch(() => { + aiChatStore.setEnabled(false); + }); + }); + } + + // Keyboard shortcut: Cmd/Ctrl+K to toggle AI + const handleKeyDown = (e: KeyboardEvent) => { + if ((e.metaKey || e.ctrlKey) && e.key === 'k') { + e.preventDefault(); + if (aiChatStore.enabled) { + aiChatStore.toggle(); + } + } + // Escape to close + if (e.key === 'Escape' && aiChatStore.isOpen) { + aiChatStore.close(); + } + }; + + document.addEventListener('keydown', handleKeyDown); + onCleanup(() => { + document.removeEventListener('keydown', handleKeyDown); + }); + }); + return ( +
Loading...
} > - }> + }> - Initializing...}> + +
Initializing...
+ + }> -
- - {props.children} - + {/* Main layout container - flexbox to allow AI panel to push content */} +
+ {/* Main content area - shrinks when AI panel is open, scrolls independently */} +
+ + {props.children} + +
+ {/* AI Panel - slides in from right, pushes content */} + aiChatStore.close()} />
+ {/* Fixed AI Assistant Button - always visible on the side when AI is enabled */} + + {/* This component only shows when chat is closed */} + + @@ -776,13 +884,16 @@ function App() { } /> + } /> } /> + + } /> @@ -797,13 +908,12 @@ function ConnectionStatusBadge(props: { }) { return (
@@ -829,11 +939,10 @@ function ConnectionStatusBadge(props: { {props.connected() ? 'Connected' @@ -904,6 +1013,7 @@ function AppLayout(props: { const path = location.pathname; if (path.startsWith('/proxmox')) return 'proxmox'; if (path.startsWith('/docker')) return 'docker'; + if (path.startsWith('/kubernetes')) return 'kubernetes'; if (path.startsWith('/hosts')) return 'hosts'; if (path.startsWith('/servers')) return 'hosts'; // Legacy redirect if (path.startsWith('/alerts')) return 'alerts'; @@ -911,6 +1021,7 @@ function AppLayout(props: { return 'proxmox'; }; const hasDockerHosts = createMemo(() => (props.state().dockerHosts?.length ?? 0) > 0); + const hasKubernetesClusters = createMemo(() => (props.state().kubernetesClusters?.length ?? 0) > 0); const hasHosts = createMemo(() => (props.state().hosts?.length ?? 0) > 0); const hasProxmoxHosts = createMemo( () => @@ -925,6 +1036,12 @@ function AppLayout(props: { } }); + createEffect(() => { + if (hasKubernetesClusters()) { + markPlatformSeen('kubernetes'); + } + }); + createEffect(() => { if (hasProxmoxHosts()) { markPlatformSeen('proxmox'); @@ -938,7 +1055,7 @@ function AppLayout(props: { }); const platformTabs = createMemo(() => { - return [ + const allPlatforms = [ { id: 'proxmox' as const, label: 'Proxmox', @@ -950,6 +1067,7 @@ function AppLayout(props: { icon: ( ), + alwaysShow: true, // Proxmox is the default, always show }, { id: 'docker' as const, @@ -962,6 +1080,20 @@ function AppLayout(props: { icon: ( ), + alwaysShow: true, // Docker is commonly used, keep visible + }, + { + id: 'kubernetes' as const, + label: 'Kubernetes', + route: '/kubernetes', + settingsRoute: '/settings/agents', + tooltip: 'Monitor Kubernetes clusters and workloads', + enabled: hasKubernetesClusters(), + live: hasKubernetesClusters(), + icon: ( + + ), + alwaysShow: false, // Only show when clusters exist }, { id: 'hosts' as const, @@ -974,8 +1106,12 @@ function AppLayout(props: { icon: ( ), + alwaysShow: true, // Hosts is commonly used, keep visible }, ]; + + // Filter out platforms that should be hidden when not configured + return allPlatforms.filter(p => p.alwaysShow || p.enabled); }); const utilityTabs = createMemo(() => { @@ -1074,6 +1210,8 @@ function AppLayout(props: {
+ {/* AI Patrol Status Indicator */} + {props.proxyAuthInfo()?.username} @@ -1168,12 +1306,12 @@ function AppLayout(props: { const baseClasses = 'tab relative px-2 sm:px-3 py-1.5 text-xs sm:text-sm font-medium flex items-center gap-1 sm:gap-1.5 rounded-t border border-transparent transition-colors whitespace-nowrap cursor-pointer'; - const className = () => { - if (isActive()) { - return `${baseClasses} bg-white dark:bg-gray-800 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-700 border-b border-b-white dark:border-b-gray-800 shadow-sm font-semibold`; - } - return `${baseClasses} text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-300 hover:bg-gray-200/60 dark:hover:bg-gray-700/60`; - }; + const className = () => { + if (isActive()) { + return `${baseClasses} bg-white dark:bg-gray-800 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-700 border-b border-b-white dark:border-b-gray-800 shadow-sm font-semibold`; + } + return `${baseClasses} text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-300 hover:bg-gray-200/60 dark:hover:bg-gray-700/60`; + }; return (
{ + describe('TimeRange', () => { + it('supports all expected time range values', () => { + const validRanges: TimeRange[] = ['5m', '15m', '30m', '1h', '4h', '12h', '24h', '7d']; + + validRanges.forEach(range => { + // This is a compile-time check - if it compiles, the types are correct + const r: TimeRange = range; + expect(r).toBe(range); + }); + }); + }); + + describe('MetricPoint', () => { + it('has required timestamp and value properties', () => { + const point: MetricPoint = { + timestamp: 1733700000000, + value: 45.7, + }; + + expect(point.timestamp).toBe(1733700000000); + expect(point.value).toBe(45.7); + }); + }); + + describe('ChartData', () => { + it('supports all metric types', () => { + const data: ChartData = { + cpu: [{ timestamp: 1000, value: 50 }], + memory: [{ timestamp: 1000, value: 60 }], + disk: [{ timestamp: 1000, value: 70 }], + diskread: [{ timestamp: 1000, value: 1000000 }], + diskwrite: [{ timestamp: 1000, value: 500000 }], + netin: [{ timestamp: 1000, value: 2000000 }], + netout: [{ timestamp: 1000, value: 1500000 }], + }; + + expect(data.cpu).toHaveLength(1); + expect(data.memory![0].value).toBe(60); + }); + + it('allows partial data (all fields optional)', () => { + const cpuOnly: ChartData = { + cpu: [{ timestamp: 1000, value: 25 }], + }; + + expect(cpuOnly.cpu).toBeDefined(); + expect(cpuOnly.memory).toBeUndefined(); + expect(cpuOnly.disk).toBeUndefined(); + }); + + it('allows empty arrays for metrics', () => { + const emptyData: ChartData = { + cpu: [], + memory: [], + }; + + expect(emptyData.cpu).toHaveLength(0); + }); + }); + + describe('ChartsResponse', () => { + it('includes all required fields', () => { + const response: ChartsResponse = { + data: {}, + nodeData: {}, + storageData: {}, + timestamp: Date.now(), + stats: { oldestDataTimestamp: Date.now() - 3600000 }, + }; + + expect(response.data).toBeDefined(); + expect(response.nodeData).toBeDefined(); + expect(response.timestamp).toBeGreaterThan(0); + }); + + it('supports optional dockerData field', () => { + const response: ChartsResponse = { + data: {}, + nodeData: {}, + storageData: {}, + dockerData: { + 'container-abc123': { + cpu: [{ timestamp: 1000, value: 10 }], + memory: [{ timestamp: 1000, value: 20 }], + }, + }, + timestamp: Date.now(), + stats: { oldestDataTimestamp: Date.now() }, + }; + + expect(response.dockerData).toBeDefined(); + expect(response.dockerData!['container-abc123'].cpu).toHaveLength(1); + }); + + it('supports optional dockerHostData field', () => { + const response: ChartsResponse = { + data: {}, + nodeData: {}, + storageData: {}, + dockerHostData: { + 'host-1': { + cpu: [{ timestamp: 1000, value: 30 }], + memory: [{ timestamp: 1000, value: 40 }], + }, + }, + timestamp: Date.now(), + stats: { oldestDataTimestamp: Date.now() }, + }; + + expect(response.dockerHostData).toBeDefined(); + expect(response.dockerHostData!['host-1'].cpu![0].value).toBe(30); + }); + + it('supports optional guestTypes field for VM/container type mapping', () => { + const response: ChartsResponse = { + data: { + 'vm-100': { cpu: [{ timestamp: 1000, value: 50 }] }, + 'ct-200': { cpu: [{ timestamp: 1000, value: 30 }] }, + }, + nodeData: {}, + storageData: {}, + guestTypes: { + 'vm-100': 'vm', + 'ct-200': 'container', + }, + timestamp: Date.now(), + stats: { oldestDataTimestamp: Date.now() }, + }; + + expect(response.guestTypes!['vm-100']).toBe('vm'); + expect(response.guestTypes!['ct-200']).toBe('container'); + }); + + it('contains all data sources for comprehensive monitoring', () => { + const comprehensiveResponse: ChartsResponse = { + // VM and container metrics (legacy format) + data: { + 'pve1/qemu/100': { + cpu: [{ timestamp: 1000, value: 45 }], + memory: [{ timestamp: 1000, value: 55 }], + disk: [{ timestamp: 1000, value: 30 }], + }, + 'pve1/lxc/200': { + cpu: [{ timestamp: 1000, value: 15 }], + }, + }, + // Node metrics + nodeData: { + 'pve1': { + cpu: [{ timestamp: 1000, value: 35 }], + memory: [{ timestamp: 1000, value: 65 }], + }, + }, + // Storage metrics + storageData: { + 'local-zfs': { + disk: [{ timestamp: 1000, value: 50 }], + }, + }, + // Docker container metrics + dockerData: { + 'abc123': { + cpu: [{ timestamp: 1000, value: 5 }], + memory: [{ timestamp: 1000, value: 128 }], + }, + }, + // Docker host metrics + dockerHostData: { + 'docker-host-1': { + cpu: [{ timestamp: 1000, value: 25 }], + }, + }, + // Guest type mapping + guestTypes: { + 'pve1/qemu/100': 'vm', + 'pve1/lxc/200': 'container', + }, + timestamp: 1733700000000, + stats: { + oldestDataTimestamp: 1733696400000, // 1 hour ago + }, + }; + + expect(Object.keys(comprehensiveResponse.data)).toHaveLength(2); + expect(Object.keys(comprehensiveResponse.nodeData)).toHaveLength(1); + expect(Object.keys(comprehensiveResponse.dockerData!)).toHaveLength(1); + expect(Object.keys(comprehensiveResponse.dockerHostData!)).toHaveLength(1); + }); + }); + + describe('ChartStats', () => { + it('contains oldest data timestamp for data availability', () => { + const stats: ChartStats = { + oldestDataTimestamp: 1733696400000, + }; + + expect(stats.oldestDataTimestamp).toBe(1733696400000); + }); + + it('can be used to determine available data range', () => { + const now = Date.now(); + const oneHourAgo = now - 3600000; + + const stats: ChartStats = { + oldestDataTimestamp: oneHourAgo, + }; + + const availableRangeMs = now - stats.oldestDataTimestamp; + expect(availableRangeMs).toBeCloseTo(3600000, -2); // Allow 100ms tolerance + }); + }); +}); + +describe('Time Range to Milliseconds Conversion', () => { + // This tests the concept matching metricsHistory.ts timeRangeToMs + function timeRangeToMs(range: TimeRange): number { + switch (range) { + case '5m': return 5 * 60 * 1000; + case '15m': return 15 * 60 * 1000; + case '30m': return 30 * 60 * 1000; + case '1h': return 60 * 60 * 1000; + case '4h': return 4 * 60 * 60 * 1000; + case '12h': return 12 * 60 * 60 * 1000; + case '24h': return 24 * 60 * 60 * 1000; + case '7d': return 7 * 24 * 60 * 60 * 1000; + default: return 60 * 60 * 1000; + } + } + + const expectedValues: [TimeRange, number][] = [ + ['5m', 300000], + ['15m', 900000], + ['30m', 1800000], + ['1h', 3600000], + ['4h', 14400000], + ['12h', 43200000], + ['24h', 86400000], + ['7d', 604800000], + ]; + + it.each(expectedValues)('converts %s to %d ms', (range, expectedMs) => { + expect(timeRangeToMs(range)).toBe(expectedMs); + }); + + it('defaults to 1 hour for unknown range', () => { + // @ts-expect-error - Testing invalid input + expect(timeRangeToMs('invalid')).toBe(3600000); + }); +}); diff --git a/frontend-modern/src/api/ai.ts b/frontend-modern/src/api/ai.ts new file mode 100644 index 0000000..5c3e690 --- /dev/null +++ b/frontend-modern/src/api/ai.ts @@ -0,0 +1,417 @@ +import { apiFetchJSON, apiFetch } from '@/utils/apiClient'; +import { logger } from '@/utils/logger'; +import type { + AISettings, + AISettingsUpdateRequest, + AITestResult, + AIExecuteRequest, + AIExecuteResponse, + AIStreamEvent, + AICostSummary, +} from '@/types/ai'; +import type { + PatternsResponse, + PredictionsResponse, + CorrelationsResponse, + ChangesResponse, + BaselinesResponse, +} from '@/types/aiIntelligence'; + +export class AIAPI { + private static baseUrl = '/api'; + + // Get AI settings + static async getSettings(): Promise { + return apiFetchJSON(`${this.baseUrl}/settings/ai`) as Promise; + } + + // Update AI settings + static async updateSettings(settings: AISettingsUpdateRequest): Promise { + return apiFetchJSON(`${this.baseUrl}/settings/ai/update`, { + method: 'PUT', + body: JSON.stringify(settings), + }) as Promise; + } + + // Test AI connection + static async testConnection(): Promise { + return apiFetchJSON(`${this.baseUrl}/ai/test`, { + method: 'POST', + }) as Promise; + } + + // Test a specific provider connection + static async testProvider(provider: string): Promise<{ success: boolean; message: string; provider: string }> { + return apiFetchJSON(`${this.baseUrl}/ai/test/${provider}`, { + method: 'POST', + }) as Promise<{ success: boolean; message: string; provider: string }>; + } + + // Get available models from the AI provider + static async getModels(): Promise<{ models: { id: string; name: string; description?: string }[]; error?: string }> { + return apiFetchJSON(`${this.baseUrl}/ai/models`) as Promise<{ models: { id: string; name: string; description?: string }[]; error?: string }>; + } + + // Get AI cost/usage summary + static async getCostSummary(days = 30): Promise { + return apiFetchJSON(`${this.baseUrl}/ai/cost/summary?days=${days}`) as Promise; + } + + // Reset AI usage history (admin-only) + static async resetCostHistory(): Promise<{ ok: boolean; backup_file?: string }> { + return apiFetchJSON(`${this.baseUrl}/ai/cost/reset`, { + method: 'POST', + body: JSON.stringify({}), + }) as Promise<{ ok: boolean; backup_file?: string }>; + } + + static async exportCostHistory(days = 30, format: 'json' | 'csv' = 'csv'): Promise { + return apiFetch(`${this.baseUrl}/ai/cost/export?days=${days}&format=${format}`, { method: 'GET' }); + } + + // ============================================ + // AI Intelligence API - Patterns, Predictions, Correlations + // ============================================ + + // Get detected failure patterns + static async getPatterns(resourceId?: string): Promise { + const params = resourceId ? `?resource_id=${encodeURIComponent(resourceId)}` : ''; + return apiFetchJSON(`${this.baseUrl}/ai/intelligence/patterns${params}`) as Promise; + } + + // Get failure predictions + static async getPredictions(resourceId?: string): Promise { + const params = resourceId ? `?resource_id=${encodeURIComponent(resourceId)}` : ''; + return apiFetchJSON(`${this.baseUrl}/ai/intelligence/predictions${params}`) as Promise; + } + + // Get resource correlations + static async getCorrelations(resourceId?: string): Promise { + const params = resourceId ? `?resource_id=${encodeURIComponent(resourceId)}` : ''; + return apiFetchJSON(`${this.baseUrl}/ai/intelligence/correlations${params}`) as Promise; + } + + // Get recent infrastructure changes + static async getRecentChanges(hours = 24): Promise { + return apiFetchJSON(`${this.baseUrl}/ai/intelligence/changes?hours=${hours}`) as Promise; + } + + // Get learned baselines + static async getBaselines(resourceId?: string): Promise { + const params = resourceId ? `?resource_id=${encodeURIComponent(resourceId)}` : ''; + return apiFetchJSON(`${this.baseUrl}/ai/intelligence/baselines${params}`) as Promise; + } + + + // Start OAuth flow for Claude Pro/Max subscription + // Returns the authorization URL to redirect the user to + static async startOAuth(): Promise<{ auth_url: string; state: string }> { + return apiFetchJSON(`${this.baseUrl}/ai/oauth/start`, { + method: 'POST', + }) as Promise<{ auth_url: string; state: string }>; + } + + // Exchange manually-pasted authorization code for tokens + static async exchangeOAuthCode(code: string, state: string): Promise<{ success: boolean; message: string }> { + return apiFetchJSON(`${this.baseUrl}/ai/oauth/exchange`, { + method: 'POST', + body: JSON.stringify({ code, state }), + }) as Promise<{ success: boolean; message: string }>; + } + + // Disconnect OAuth and clear tokens + static async disconnectOAuth(): Promise<{ success: boolean; message: string }> { + return apiFetchJSON(`${this.baseUrl}/ai/oauth/disconnect`, { + method: 'POST', + }) as Promise<{ success: boolean; message: string }>; + } + + + // Execute an AI prompt + static async execute(request: AIExecuteRequest): Promise { + return apiFetchJSON(`${this.baseUrl}/ai/execute`, { + method: 'POST', + body: JSON.stringify(request), + }) as Promise; + } + + // Run a single command (for approved commands) + static async runCommand(request: { + command: string; + target_type: string; + target_id: string; + run_on_host: boolean; + vmid?: string; + target_host?: string; // Explicit host for command routing + }): Promise<{ output: string; success: boolean; error?: string }> { + // Ensure run_on_host is explicitly a boolean (not undefined) + const sanitizedRequest = { + command: request.command, + target_type: request.target_type, + target_id: request.target_id, + run_on_host: Boolean(request.run_on_host), + ...(request.vmid ? { vmid: String(request.vmid) } : {}), + ...(request.target_host ? { target_host: request.target_host } : {}), + }; + const body = JSON.stringify(sanitizedRequest); + logger.debug('[AI] runCommand', { request: sanitizedRequest, bodyLength: body.length }); + return apiFetchJSON(`${this.baseUrl}/ai/run-command`, { + method: 'POST', + body, + }) as Promise<{ output: string; success: boolean; error?: string }>; + } + + + // Investigate an alert with AI (one-click investigation) + static async investigateAlert( + request: { + alert_id: string; + resource_id: string; + resource_name: string; + resource_type: string; + alert_type: string; + level: string; + value: number; + threshold: number; + message: string; + duration: string; + node?: string; + vmid?: number; + }, + onEvent: (event: AIStreamEvent) => void, + signal?: AbortSignal + ): Promise { + logger.debug('[AI] Starting alert investigation', request); + + const response = await apiFetch(`${this.baseUrl}/ai/investigate-alert`, { + method: 'POST', + body: JSON.stringify(request), + headers: { + 'Content-Type': 'application/json', + Accept: 'text/event-stream', + }, + signal, + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(text || `Request failed with status ${response.status}`); + } + + const reader = response.body?.getReader(); + if (!reader) { + throw new Error('No response body'); + } + + const decoder = new TextDecoder(); + let buffer = ''; + // 5 minutes timeout - Opus models can take a long time + const STREAM_TIMEOUT_MS = 300000; + let lastEventTime = Date.now(); + + try { + for (;;) { + if (Date.now() - lastEventTime > STREAM_TIMEOUT_MS) { + console.warn('[AI] Alert investigation stream timeout'); + break; + } + + const readPromise = reader.read(); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Read timeout')), STREAM_TIMEOUT_MS); + }); + + let result: ReadableStreamReadResult; + try { + result = await Promise.race([readPromise, timeoutPromise]); + } catch (e) { + if ((e as Error).message === 'Read timeout') break; + throw e; + } + + const { done, value } = result; + if (done) break; + + lastEventTime = Date.now(); + buffer += decoder.decode(value, { stream: true }); + + const normalizedBuffer = buffer.replace(/\r\n/g, '\n'); + const messages = normalizedBuffer.split('\n\n'); + buffer = messages.pop() || ''; + + for (const message of messages) { + if (!message.trim() || message.trim().startsWith(':')) continue; + + const dataLines = message.split('\n').filter((line) => line.startsWith('data: ')); + for (const line of dataLines) { + try { + const jsonStr = line.slice(6); + if (!jsonStr.trim()) continue; + const data = JSON.parse(jsonStr); + onEvent(data as AIStreamEvent); + } catch (e) { + console.error('[AI] Failed to parse investigation event:', e); + } + } + } + } + } finally { + reader.releaseLock(); + } + } + + // Execute an AI prompt with streaming + // Returns an abort function to cancel the request + static async executeStream( + request: AIExecuteRequest, + onEvent: (event: AIStreamEvent) => void, + signal?: AbortSignal + ): Promise { + logger.debug('[AI SSE] Starting streaming request', request); + + const response = await apiFetch(`${this.baseUrl}/ai/execute/stream`, { + method: 'POST', + body: JSON.stringify(request), + headers: { + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream', + }, + signal, + }); + + logger.debug('[AI SSE] Response status', { status: response.status, contentType: response.headers.get('content-type') }); + + if (!response.ok) { + const text = await response.text(); + logger.error('[AI SSE] Request failed', text); + throw new Error(text || `Request failed with status ${response.status}`); + } + + const reader = response.body?.getReader(); + if (!reader) { + logger.error('[AI SSE] No response body'); + throw new Error('No response body'); + } + + const decoder = new TextDecoder(); + let buffer = ''; + let lastEventTime = Date.now(); + let receivedComplete = false; + let receivedDone = false; + + // Timeout to detect stalled streams (5 minutes - Opus models can take a long time) + const STREAM_TIMEOUT_MS = 300000; + + logger.debug('[AI SSE] Starting to read stream'); + + try { + for (;;) { + // Check for stream timeout + if (Date.now() - lastEventTime > STREAM_TIMEOUT_MS) { + logger.warn('[AI SSE] Stream timeout', { seconds: STREAM_TIMEOUT_MS / 1000 }); + break; + } + + // Create a promise with timeout for the read operation + const readPromise = reader.read(); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Read timeout')), STREAM_TIMEOUT_MS); + }); + + let result: ReadableStreamReadResult; + try { + result = await Promise.race([readPromise, timeoutPromise]); + } catch (e) { + if ((e as Error).message === 'Read timeout') { + logger.warn('[AI SSE] Read timeout, ending stream'); + break; + } + throw e; + } + + const { done, value } = result; + if (done) { + logger.debug('[AI SSE] Stream ended normally'); + break; + } + + lastEventTime = Date.now(); + const chunk = decoder.decode(value, { stream: true }); + + // Log chunk info only if it's not just a heartbeat + if (!chunk.includes(': heartbeat')) { + logger.debug('[AI SSE] Received chunk', { bytes: chunk.length }); + } + + buffer += chunk; + + // Process complete SSE messages (separated by double newlines) + // Handle both \n\n and \r\n\r\n for cross-platform compatibility + const normalizedBuffer = buffer.replace(/\r\n/g, '\n'); + const messages = normalizedBuffer.split('\n\n'); + buffer = messages.pop() || ''; // Keep incomplete message in buffer + + for (const message of messages) { + // Skip empty messages and heartbeat comments + if (!message.trim() || message.trim().startsWith(':')) { + if (message.includes('heartbeat')) { + logger.debug('[AI SSE] Received heartbeat'); + } + continue; + } + + // Parse SSE message (can have multiple lines, look for data: prefix) + const dataLines = message.split('\n').filter(line => line.startsWith('data: ')); + for (const line of dataLines) { + try { + const jsonStr = line.slice(6); // Remove 'data: ' prefix + if (!jsonStr.trim()) continue; + + const data = JSON.parse(jsonStr); + logger.debug('[AI SSE] Parsed event', { type: data.type, data }); + + // Track completion events + if (data.type === 'complete') { + receivedComplete = true; + } + if (data.type === 'done') { + receivedDone = true; + } + + onEvent(data as AIStreamEvent); + } catch (e) { + logger.error('[AI SSE] Failed to parse event', { error: e, line }); + } + } + } + } + + // Process any remaining buffer content + if (buffer.trim() && buffer.trim().startsWith('data: ')) { + try { + const jsonStr = buffer.slice(6); + if (jsonStr.trim()) { + const data = JSON.parse(jsonStr); + logger.debug('[AI SSE] Parsed final buffered event', { type: data.type }); + onEvent(data as AIStreamEvent); + if (data.type === 'complete') receivedComplete = true; + if (data.type === 'done') receivedDone = true; + } + } catch { + logger.warn('[AI SSE] Could not parse remaining buffer', { preview: buffer.substring(0, 100) }); + } + } + + // If we ended without receiving a done event, send a synthetic one + // This ensures the UI properly clears the streaming state + if (!receivedDone) { + logger.warn('[AI SSE] Stream ended without done event, sending synthetic done'); + onEvent({ type: 'done', data: undefined }); + } + + } finally { + reader.releaseLock(); + logger.debug('[AI SSE] Reader released', { receivedComplete, receivedDone }); + } + } +} diff --git a/frontend-modern/src/api/charts.ts b/frontend-modern/src/api/charts.ts new file mode 100644 index 0000000..0c0dd9e --- /dev/null +++ b/frontend-modern/src/api/charts.ts @@ -0,0 +1,151 @@ +/** + * Charts API + * + * Fetches historical metrics data from the backend for sparkline visualizations. + * The backend maintains proper historical data with 30s sample intervals. + */ + +import { apiFetchJSON } from '@/utils/apiClient'; + +// Types matching backend response format +export interface MetricPoint { + timestamp: number; // Unix timestamp in milliseconds + value: number; +} + +// Extended metric point with min/max for aggregated data +export interface AggregatedMetricPoint { + timestamp: number; // Unix timestamp in milliseconds + value: number; + min: number; + max: number; +} + +export interface ChartData { + cpu?: MetricPoint[]; + memory?: MetricPoint[]; + disk?: MetricPoint[]; + diskread?: MetricPoint[]; + diskwrite?: MetricPoint[]; + netin?: MetricPoint[]; + netout?: MetricPoint[]; +} + +export interface ChartStats { + oldestDataTimestamp: number; +} + +export interface ChartsResponse { + data: Record; // VM/Container data keyed by ID + nodeData: Record; // Node data keyed by ID + storageData: Record; // Storage data keyed by ID + dockerData?: Record; // Docker container data keyed by container ID + dockerHostData?: Record; // Docker host data keyed by host ID + guestTypes?: Record; // Maps guest ID to type + timestamp: number; + stats: ChartStats; +} + +// Persistent metrics history types (SQLite-backed, longer retention) +export type HistoryTimeRange = '1h' | '6h' | '12h' | '24h' | '7d' | '30d' | '90d'; +export type ResourceType = 'node' | 'guest' | 'storage' | 'docker' | 'dockerHost'; + +export interface MetricsHistoryParams { + resourceType: ResourceType; + resourceId: string; + metric?: string; // Optional: 'cpu', 'memory', 'disk', etc. Omit for all metrics + range?: HistoryTimeRange; // Default: '24h' +} + +export interface SingleMetricHistoryResponse { + resourceType: string; + resourceId: string; + metric: string; + range: string; + start: number; // Unix timestamp in milliseconds + end: number; // Unix timestamp in milliseconds + points: AggregatedMetricPoint[]; +} + +export interface AllMetricsHistoryResponse { + resourceType: string; + resourceId: string; + range: string; + start: number; // Unix timestamp in milliseconds + end: number; // Unix timestamp in milliseconds + metrics: Record; +} + +export interface MetricsStoreStats { + enabled: boolean; + dbPath?: string; + dbSize?: number; + rawCount?: number; + minuteCount?: number; + hourlyCount?: number; + dailyCount?: number; + totalWrites?: number; + bufferSize?: number; + lastFlush?: string; + lastRollup?: string; + lastRetention?: string; + error?: string; +} + +export type TimeRange = '5m' | '15m' | '30m' | '1h' | '4h' | '12h' | '24h' | '7d'; + +export class ChartsAPI { + private static baseUrl = '/api'; + + /** + * Fetch historical chart data for all resources + * @param range Time range to fetch (default: 1h) + */ + static async getCharts(range: TimeRange = '1h'): Promise { + const url = `${this.baseUrl}/charts?range=${range}`; + return apiFetchJSON(url); + } + + /** + * Fetch storage-specific chart data + * @param rangeMinutes Range in minutes (default: 60) + */ + static async getStorageCharts(rangeMinutes: number = 60): Promise> { + const url = `${this.baseUrl}/storage/charts?range=${rangeMinutes}`; + return apiFetchJSON(url); + } + + /** + * Fetch persistent metrics history for a specific resource + * This uses the SQLite-backed store with longer retention (up to 90 days) + * @param params Query parameters + */ + static async getMetricsHistory(params: MetricsHistoryParams): Promise { + const searchParams = new URLSearchParams({ + resourceType: params.resourceType, + resourceId: params.resourceId, + }); + if (params.metric) { + searchParams.set('metric', params.metric); + } + if (params.range) { + searchParams.set('range', params.range); + } + const url = `${this.baseUrl}/metrics-store/history?${searchParams.toString()}`; + return apiFetchJSON(url); + } + + /** + * Fetch statistics about the persistent metrics store + */ + static async getMetricsStoreStats(): Promise { + const url = `${this.baseUrl}/metrics-store/stats`; + return apiFetchJSON(url); + } +} + diff --git a/frontend-modern/src/api/dockerMetadata.ts b/frontend-modern/src/api/dockerMetadata.ts index ab3d4ab..2b9d929 100644 --- a/frontend-modern/src/api/dockerMetadata.ts +++ b/frontend-modern/src/api/dockerMetadata.ts @@ -6,6 +6,7 @@ export interface DockerMetadata { customUrl?: string; description?: string; tags?: string[]; + notes?: string[]; // User annotations for AI context } export class DockerMetadataAPI { diff --git a/frontend-modern/src/api/guestMetadata.ts b/frontend-modern/src/api/guestMetadata.ts index 3a29ecb..b3d1971 100644 --- a/frontend-modern/src/api/guestMetadata.ts +++ b/frontend-modern/src/api/guestMetadata.ts @@ -6,6 +6,7 @@ export interface GuestMetadata { customUrl?: string; description?: string; tags?: string[]; + notes?: string[]; // User annotations for AI context } export class GuestMetadataAPI { diff --git a/frontend-modern/src/api/hostMetadata.ts b/frontend-modern/src/api/hostMetadata.ts new file mode 100644 index 0000000..73026c0 --- /dev/null +++ b/frontend-modern/src/api/hostMetadata.ts @@ -0,0 +1,42 @@ +// Host Metadata API +import { apiFetchJSON } from '@/utils/apiClient'; + +export interface HostMetadata { + id: string; + customUrl?: string; + description?: string; + tags?: string[]; + notes?: string[]; // User annotations for AI context +} + +export class HostMetadataAPI { + private static baseUrl = '/api/hosts/metadata'; + + // Get metadata for a specific host + static async getMetadata(hostId: string): Promise { + return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(hostId)}`); + } + + // Get all host metadata + static async getAllMetadata(): Promise> { + return apiFetchJSON(this.baseUrl); + } + + // Update metadata for a host + static async updateMetadata( + hostId: string, + metadata: Partial, + ): Promise { + return apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(hostId)}`, { + method: 'PUT', + body: JSON.stringify(metadata), + }); + } + + // Delete metadata for a host + static async deleteMetadata(hostId: string): Promise { + await apiFetchJSON(`${this.baseUrl}/${encodeURIComponent(hostId)}`, { + method: 'DELETE', + }); + } +} diff --git a/frontend-modern/src/api/monitoring.ts b/frontend-modern/src/api/monitoring.ts index 04a140e..6a9eaf4 100644 --- a/frontend-modern/src/api/monitoring.ts +++ b/frontend-modern/src/api/monitoring.ts @@ -216,6 +216,191 @@ export class MonitoringAPI { } } + static async deleteKubernetesCluster(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}`; + + const response = await apiFetch(url, { + method: 'DELETE', + }); + + if (!response.ok) { + if (response.status === 404) { + return {}; + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + + if (response.status === 204) { + return {}; + } + + const text = await response.text(); + if (!text?.trim()) { + return {}; + } + + try { + return JSON.parse(text) as DeleteKubernetesClusterResponse; + } catch (err) { + throw new Error((err as Error).message || 'Failed to parse delete kubernetes cluster response'); + } + } + + static async unhideKubernetesCluster(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/unhide`; + + const response = await apiFetch(url, { + method: 'PUT', + }); + + if (!response.ok) { + if (response.status === 404) { + return; + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + + static async markKubernetesClusterPendingUninstall(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/pending-uninstall`; + + const response = await apiFetch(url, { + method: 'PUT', + }); + + if (!response.ok) { + if (response.status === 404) { + return; + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + + static async setKubernetesClusterDisplayName(clusterId: string, displayName: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/display-name`; + + const response = await apiFetch(url, { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ displayName }), + }); + + if (!response.ok) { + if (response.status === 404) { + throw new Error('Kubernetes cluster not found'); + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + + static async allowKubernetesClusterReenroll(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/allow-reenroll`; + + const response = await apiFetch(url, { + method: 'POST', + }); + + if (!response.ok) { + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_err) { + // ignore parse error + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + static async deleteHostAgent(hostId: string): Promise { if (!hostId) { throw new Error('Host ID is required to remove a host agent.'); @@ -313,3 +498,9 @@ export interface DeleteDockerHostResponse { message?: string; command?: DockerHostCommand; } + +export interface DeleteKubernetesClusterResponse { + success?: boolean; + clusterId?: string; + message?: string; +} diff --git a/frontend-modern/src/api/patrol.ts b/frontend-modern/src/api/patrol.ts new file mode 100644 index 0000000..b927a51 --- /dev/null +++ b/frontend-modern/src/api/patrol.ts @@ -0,0 +1,379 @@ +/** + * AI Patrol API client + * Provides access to background AI monitoring findings and status + */ + +import { apiFetchJSON } from '@/utils/apiClient'; + +export type FindingSeverity = 'info' | 'watch' | 'warning' | 'critical'; +export type FindingCategory = 'performance' | 'capacity' | 'reliability' | 'backup' | 'security' | 'general'; + +export interface Finding { + id: string; + severity: FindingSeverity; + category: FindingCategory; + resource_id: string; + resource_name: string; + resource_type: string; // node, vm, container, docker_host, docker_container, storage, pbs, host_raid + node?: string; + title: string; + description: string; + recommendation?: string; + evidence?: string; + detected_at: string; + last_seen_at: string; + resolved_at?: string; + auto_resolved: boolean; + acknowledged_at?: string; + snoozed_until?: string; // Finding hidden until this time + alert_id?: string; + // User feedback fields (LLM memory system) + dismissed_reason?: 'not_an_issue' | 'expected_behavior' | 'will_fix_later'; + user_note?: string; + times_raised: number; + suppressed: boolean; +} + +export interface FindingsSummary { + critical: number; + warning: number; + watch: number; + info: number; +} + +export interface PatrolStatus { + running: boolean; + enabled: boolean; + last_patrol_at?: string; + last_deep_analysis_at?: string; + next_patrol_at?: string; + last_duration_ms: number; + resources_checked: number; + findings_count: number; + error_count: number; + healthy: boolean; + interval_ms: number; // Patrol interval in milliseconds + summary: FindingsSummary; +} + +export interface PatrolRunRecord { + id: string; + started_at: string; + completed_at: string; + duration_ms: number; + type: string; // Always 'patrol' now (kept for backwards compat with old records) + resources_checked: number; + // Breakdown by resource type + nodes_checked: number; + guests_checked: number; + docker_checked: number; + storage_checked: number; + hosts_checked: number; + pbs_checked: number; + // Findings from this run + new_findings: number; + existing_findings: number; + resolved_findings: number; + findings_summary: string; + finding_ids: string[]; + error_count: number; + status: 'healthy' | 'issues_found' | 'critical' | 'error'; + // AI Analysis details + ai_analysis?: string; // The AI's raw response/analysis + input_tokens?: number; // Tokens sent to AI + output_tokens?: number; // Tokens received from AI +} + +/** + * Get the current AI patrol status + */ +export async function getPatrolStatus(): Promise { + const resp = await fetch('/api/ai/patrol/status', { + credentials: 'include', + }); + if (!resp.ok) { + throw new Error(`Failed to get patrol status: ${resp.status}`); + } + return resp.json(); +} + +/** + * Get all active findings from the patrol service + * Optionally filter by resource ID + */ +export async function getFindings(resourceId?: string): Promise { + const url = resourceId + ? `/api/ai/patrol/findings?resource_id=${encodeURIComponent(resourceId)}` + : '/api/ai/patrol/findings'; + + const resp = await fetch(url, { + credentials: 'include', + }); + if (!resp.ok) { + throw new Error(`Failed to get findings: ${resp.status}`); + } + return resp.json(); +} + +/** + * Trigger an immediate patrol run + */ +export async function forcePatrol(deep: boolean = false): Promise<{ success: boolean; message: string }> { + const url = deep ? '/api/ai/patrol/run?deep=true' : '/api/ai/patrol/run'; + return apiFetchJSON(url, { method: 'POST' }); +} + +/** + * Get AI findings history including resolved findings + * @param startTime Optional ISO timestamp to filter findings from + */ +export async function getFindingsHistory(startTime?: string): Promise { + const url = startTime + ? `/api/ai/patrol/history?start_time=${encodeURIComponent(startTime)}` + : '/api/ai/patrol/history'; + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + }); + if (!resp.ok) { + throw new Error(`Failed to get findings history: ${resp.status}`); + } + return resp.json(); +} + +/** + * Get the history of patrol check runs + * @param limit Maximum number of records to return (default: 50, max: 100) + */ +export async function getPatrolRunHistory(limit?: number): Promise { + const url = limit + ? `/api/ai/patrol/runs?limit=${limit}` + : '/api/ai/patrol/runs'; + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + }); + if (!resp.ok) { + throw new Error(`Failed to get patrol run history: ${resp.status}`); + } + return resp.json(); +} + +/** + * Acknowledge a finding (marks as seen but keeps visible, like alert acknowledgement) + * Finding will auto-resolve when the underlying condition clears. + */ +export async function acknowledgeFinding(findingId: string): Promise<{ success: boolean; message: string }> { + return apiFetchJSON('/api/ai/patrol/acknowledge', { + method: 'POST', + body: JSON.stringify({ finding_id: findingId }), + }); +} + +/** + * Snooze a finding for a specified duration + * @param findingId The ID of the finding to snooze + * @param durationHours Duration in hours (e.g., 1, 24, 168 for 7 days) + */ +export async function snoozeFinding(findingId: string, durationHours: number): Promise<{ success: boolean; message: string }> { + return apiFetchJSON('/api/ai/patrol/snooze', { + method: 'POST', + body: JSON.stringify({ finding_id: findingId, duration_hours: durationHours }), + }); +} + +/** + * Manually resolve a finding (mark as fixed) + * @param findingId The ID of the finding to resolve + */ +export async function resolveFinding(findingId: string): Promise<{ success: boolean; message: string }> { + return apiFetchJSON('/api/ai/patrol/resolve', { + method: 'POST', + body: JSON.stringify({ finding_id: findingId }), + }); +} + +/** + * Dismiss a finding with a reason (LLM memory feature) + * The LLM will be told not to re-raise this issue in future patrols. + * @param findingId The ID of the finding to dismiss + * @param reason One of: "not_an_issue", "expected_behavior", "will_fix_later" + * @param note Optional freeform explanation + */ +export async function dismissFinding( + findingId: string, + reason: 'not_an_issue' | 'expected_behavior' | 'will_fix_later', + note?: string +): Promise<{ success: boolean; message: string }> { + return apiFetchJSON('/api/ai/patrol/dismiss', { + method: 'POST', + body: JSON.stringify({ finding_id: findingId, reason, note }), + }); +} + +/** + * Permanently suppress a finding type (LLM memory feature) + * The LLM will never re-raise this type of finding for this resource. + * @param findingId The ID of the finding to suppress + */ +export async function suppressFinding(findingId: string): Promise<{ success: boolean; message: string }> { + return apiFetchJSON('/api/ai/patrol/suppress', { + method: 'POST', + body: JSON.stringify({ finding_id: findingId }), + }); +} + +/** + * Severity color mapping for UI + */ +export const severityColors: Record = { + critical: { bg: 'rgba(220, 38, 38, 0.15)', text: '#ef4444', border: 'rgba(220, 38, 38, 0.3)' }, + warning: { bg: 'rgba(234, 179, 8, 0.15)', text: '#eab308', border: 'rgba(234, 179, 8, 0.3)' }, + watch: { bg: 'rgba(59, 130, 246, 0.15)', text: '#3b82f6', border: 'rgba(59, 130, 246, 0.3)' }, + info: { bg: 'rgba(107, 114, 128, 0.15)', text: '#9ca3af', border: 'rgba(107, 114, 128, 0.3)' }, +}; + +/** + * Category labels for UI + */ +export const categoryLabels: Record = { + performance: 'Performance', + capacity: 'Capacity', + reliability: 'Reliability', + backup: 'Backup', + security: 'Security', + general: 'General', +}; + +/** + * Format a timestamp for display + */ +export function formatTimestamp(ts: string): string { + const date = new Date(ts); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMs / 3600000); + const diffDays = Math.floor(diffMs / 86400000); + + if (diffMins < 1) return 'just now'; + if (diffMins < 60) return `${diffMins}m ago`; + if (diffHours < 24) return `${diffHours}h ago`; + if (diffDays < 7) return `${diffDays}d ago`; + return date.toLocaleDateString(); +} + +/** + * Event types from patrol stream + */ +export interface PatrolStreamEvent { + type: 'start' | 'content' | 'thinking' | 'phase' | 'complete' | 'error'; + content?: string; + phase?: string; + tokens?: number; +} + +/** + * Subscribe to live patrol stream via SSE + * Returns an unsubscribe function + */ +export function subscribeToPatrolStream( + onEvent: (event: PatrolStreamEvent) => void, + onError?: (error: Error) => void +): () => void { + const eventSource = new EventSource('/api/ai/patrol/stream', { withCredentials: true }); + + eventSource.onmessage = (event) => { + try { + const data = JSON.parse(event.data) as PatrolStreamEvent; + onEvent(data); + } catch (e) { + console.error('Failed to parse patrol stream event:', e); + } + }; + + eventSource.onerror = () => { + if (onError) { + onError(new Error('Patrol stream connection error')); + } + eventSource.close(); + }; + + // Return unsubscribe function + return () => { + eventSource.close(); + }; +} + +// === Suppression Rules === + +export interface SuppressionRule { + id: string; + resource_id?: string; // Empty means "any resource" + resource_name?: string; // Human-readable name + category?: FindingCategory; // Empty means "any category" + description: string; // User's reason + created_at: string; + created_from: 'finding' | 'manual'; + finding_id?: string; // Original finding ID if created from dismissal +} + +/** + * Get all suppression rules (both manual and from dismissed findings) + */ +export async function getSuppressionRules(): Promise { + const resp = await fetch('/api/ai/patrol/suppressions', { + credentials: 'include', + }); + if (!resp.ok) { + throw new Error(`Failed to get suppression rules: ${resp.status}`); + } + return resp.json(); +} + +/** + * Create a new manual suppression rule + * @param resourceId Resource ID (empty for "any resource") + * @param resourceName Human-readable name for display + * @param category Category (empty for "any category") + * @param description User's reason for the rule + */ +export async function addSuppressionRule( + resourceId: string, + resourceName: string, + category: FindingCategory | '', + description: string +): Promise<{ success: boolean; message: string; rule: SuppressionRule }> { + return apiFetchJSON('/api/ai/patrol/suppressions', { + method: 'POST', + body: JSON.stringify({ + resource_id: resourceId, + resource_name: resourceName, + category: category, + description: description, + }), + }); +} + +/** + * Delete a suppression rule + * @param ruleId The ID of the rule to delete + */ +export async function deleteSuppressionRule(ruleId: string): Promise<{ success: boolean; message: string }> { + return apiFetchJSON(`/api/ai/patrol/suppressions/${encodeURIComponent(ruleId)}`, { + method: 'DELETE', + }); +} + +/** + * Get all dismissed/suppressed findings + */ +export async function getDismissedFindings(): Promise { + const resp = await fetch('/api/ai/patrol/dismissed', { + credentials: 'include', + }); + if (!resp.ok) { + throw new Error(`Failed to get dismissed findings: ${resp.status}`); + } + return resp.json(); +} diff --git a/frontend-modern/src/components/AI/AIChat.tsx b/frontend-modern/src/components/AI/AIChat.tsx new file mode 100644 index 0000000..8632781 --- /dev/null +++ b/frontend-modern/src/components/AI/AIChat.tsx @@ -0,0 +1,1549 @@ +import { Component, Show, createSignal, For, createEffect, createMemo, onMount, Switch, Match } from 'solid-js'; +import { marked } from 'marked'; +import DOMPurify from 'dompurify'; +import { AIAPI } from '@/api/ai'; +import { notificationStore } from '@/stores/notifications'; +import { logger } from '@/utils/logger'; +import { aiChatStore } from '@/stores/aiChat'; +import { useWebSocket } from '@/App'; +import { GuestNotes } from './GuestNotes'; +import type { + AIToolExecution, + AIStreamEvent, + AIStreamToolStartData, + AIStreamToolEndData, + AIStreamCompleteData, + AIStreamApprovalNeededData, + ModelInfo, +} from '@/types/ai'; + +// Provider display names for grouped model selection +const PROVIDER_DISPLAY_NAMES: Record = { + anthropic: 'Anthropic', + openai: 'OpenAI', + deepseek: 'DeepSeek', + ollama: 'Ollama', +}; + +// Parse provider from model ID (format: "provider:model-name") +function getProviderFromModelId(modelId: string): string { + const colonIndex = modelId.indexOf(':'); + if (colonIndex > 0) { + return modelId.substring(0, colonIndex); + } + // Default detection for models without prefix + if (modelId.includes('claude') || modelId.includes('opus') || modelId.includes('sonnet') || modelId.includes('haiku')) { + return 'anthropic'; + } + if (modelId.includes('gpt') || modelId.includes('o1') || modelId.includes('o3')) { + return 'openai'; + } + if (modelId.includes('deepseek')) { + return 'deepseek'; + } + return 'ollama'; +} + +// Group models by provider for grouped rendering +function groupModelsByProvider(models: ModelInfo[]): Map { + const grouped = new Map(); + + for (const model of models) { + const provider = getProviderFromModelId(model.id); + const existing = grouped.get(provider) || []; + existing.push(model); + grouped.set(provider, existing); + } + + return grouped; +} + +// Configure marked for safe rendering +marked.setOptions({ + breaks: true, // Convert \n to
+ gfm: true, // GitHub Flavored Markdown +}); + +let domPurifyConfigured = false; +const configureDOMPurify = () => { + if (domPurifyConfigured) return; + domPurifyConfigured = true; + + DOMPurify.addHook('afterSanitizeAttributes', (node) => { + const element = node as Element | null; + if (!element || element.tagName !== 'A') return; + element.setAttribute('target', '_blank'); + element.setAttribute('rel', 'noopener noreferrer'); + }); +}; + +// Helper to render markdown safely with XSS protection +// LLM output should NEVER be trusted - always sanitize before rendering as HTML +const renderMarkdown = (content: string): string => { + try { + configureDOMPurify(); + const rawHtml = marked.parse(content) as string; + // Sanitize to prevent XSS from malicious LLM output or injected content + return DOMPurify.sanitize(rawHtml, { + // Allow common formatting tags but block scripts, iframes, etc. + ALLOWED_TAGS: ['p', 'br', 'strong', 'em', 'b', 'i', 'u', 'code', 'pre', 'blockquote', + 'ul', 'ol', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'a', 'hr', 'table', + 'thead', 'tbody', 'tr', 'th', 'td', 'span', 'div'], + ALLOWED_ATTR: ['href', 'target', 'rel', 'class'], + // Force all links to open in new tab and prevent opener attacks + ADD_ATTR: ['target', 'rel'], + }); + } catch { + // If parsing fails, escape HTML entities as fallback + return content.replace(/[&<>"']/g, (char) => { + const entities: Record = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; + return entities[char] || char; + }); + } +}; + +// Helper to sanitize thinking/reasoning content for display +// Removes raw network errors with IP addresses that are not user-friendly +const sanitizeThinking = (content: string): string => { + // Replace raw TCP connection details like "write tcp 192.168.0.123:7655->192.168.0.134:58004: i/o timeout" + // with friendlier messages + let sanitized = content.replace( + /write tcp [\d.:]+->[\d.:]+: i\/o timeout/g, + 'connection timed out' + ); + sanitized = sanitized.replace( + /read tcp [\d.:]+: i\/o timeout/g, + 'connection timed out' + ); + sanitized = sanitized.replace( + /dial tcp [\d.:]+: connection refused/g, + 'connection refused' + ); + // Replace "failed to send command: " patterns + sanitized = sanitized.replace( + /failed to send command: write tcp [\d.:->\s]+/g, + 'failed to send command: connection error' + ); + return sanitized; +}; + +// In-progress tool execution (before completion) +interface PendingTool { + name: string; + input: string; +} + +// Command awaiting user approval +interface PendingApproval { + command: string; + toolId: string; + toolName: string; + runOnHost: boolean; + targetHost?: string; // Explicit host for command routing + isExecuting?: boolean; +} + + +// Unified event type for chronological display +interface StreamDisplayEvent { + type: 'thinking' | 'tool' | 'content'; + thinking?: string; + tool?: AIToolExecution; + content?: string; // Text content chunk for chronological display +} + +interface Message { + id: string; + role: 'user' | 'assistant'; + content: string; + thinking?: string; // DeepSeek reasoning/thinking content (accumulated) + thinkingChunks?: string[]; // Thinking split into sequential blocks for display + streamEvents?: StreamDisplayEvent[]; // All events in chronological order + timestamp: Date; + model?: string; + tokens?: { input: number; output: number }; + toolCalls?: AIToolExecution[]; + // Streaming state + isStreaming?: boolean; + pendingTools?: PendingTool[]; + pendingApprovals?: PendingApproval[]; +} + +interface AIChatProps { + onClose: () => void; +} + +// Extract guest name from context if available +const getGuestName = (context?: Record): string | undefined => { + if (!context) return undefined; + if (typeof context.guestName === 'string') return context.guestName; + if (typeof context.name === 'string') return context.name; + return undefined; +}; + +export const AIChat: Component = (props) => { + // Read all context from store for proper SolidJS reactivity + const isOpen = () => aiChatStore.isOpen; + const context = () => aiChatStore.context; + const targetType = () => context().targetType; + const targetId = () => context().targetId; + const contextData = () => context().context; + const initialPrompt = () => context().initialPrompt; + const findingId = () => context().findingId; // For resolving patrol findings + + // Access WebSocket state for listing available resources + const wsContext = useWebSocket(); + + // Context picker state + const [showContextPicker, setShowContextPicker] = createSignal(false); + const [contextSearch, setContextSearch] = createSignal(''); + + // Build a list of all available resources for the context picker + const availableResources = createMemo(() => { + const resources: Array<{ + id: string; + type: 'vm' | 'container' | 'node' | 'host' | 'docker'; + name: string; + status: string; + node?: string; + data: Record; + }> = []; + + // Add VMs + for (const vm of wsContext.state.vms || []) { + resources.push({ + id: `${vm.node}-${vm.vmid}`, + type: 'vm', + name: vm.name || `VM ${vm.vmid}`, + status: vm.status, + node: vm.node, + data: { + guest_id: `${vm.node}-${vm.vmid}`, + guest_name: vm.name, + guest_vmid: vm.vmid, + guest_type: 'qemu', + guest_node: vm.node, + guest_status: vm.status, + cpu: vm.cpu, + mem: vm.memory?.used, + maxmem: vm.memory?.total, + disk: vm.disk?.used, + maxdisk: vm.disk?.total, + }, + }); + } + + // Add containers + for (const ct of wsContext.state.containers || []) { + resources.push({ + id: `${ct.node}-${ct.vmid}`, + type: 'container', + name: ct.name || `CT ${ct.vmid}`, + status: ct.status, + node: ct.node, + data: { + guest_id: `${ct.node}-${ct.vmid}`, + guest_name: ct.name, + guest_vmid: ct.vmid, + guest_type: 'lxc', + guest_node: ct.node, + guest_status: ct.status, + cpu: ct.cpu, + mem: ct.memory?.used, + maxmem: ct.memory?.total, + disk: ct.disk?.used, + maxdisk: ct.disk?.total, + }, + }); + } + + // Add Proxmox nodes + for (const node of wsContext.state.nodes || []) { + resources.push({ + id: `node-${node.name}`, + type: 'node', + name: node.name, + status: node.status, + data: { + node_name: node.name, + node_status: node.status, + cpu: node.cpu, + mem: node.memory?.used, + maxmem: node.memory?.total, + disk: node.disk?.used, + maxdisk: node.disk?.total, + }, + }); + } + + // Add host agents + for (const host of wsContext.state.hosts || []) { + resources.push({ + id: `host-${host.hostname}`, + type: 'host', + name: host.hostname, + status: host.status === 'online' ? 'online' : 'offline', + data: { + host_name: host.hostname, + host_platform: host.platform, + host_version: host.agentVersion, + connected: host.status === 'online', + }, + }); + } + + return resources; + }); + + // Filtered resources based on search + const filteredResources = createMemo(() => { + const search = contextSearch().toLowerCase(); + if (!search) return availableResources(); + return availableResources().filter( + (r) => + r.name.toLowerCase().includes(search) || + r.type.toLowerCase().includes(search) || + (r.node && r.node.toLowerCase().includes(search)) + ); + }); + + // Add a resource to context + const addResourceToContext = (resource: ReturnType[number]) => { + aiChatStore.addContextItem(resource.type, resource.id, resource.name, resource.data); + setShowContextPicker(false); + setContextSearch(''); + }; + + // Initialize messages from store (for persistence across navigation) + const [messages, setMessagesLocal] = createSignal( + aiChatStore.messages as Message[] || [] + ); + const [input, setInput] = createSignal(''); + const [isLoading, setIsLoading] = createSignal(false); + const [queuedMessage, setQueuedMessage] = createSignal(null); + + // Model selection + const [availableModels, setAvailableModels] = createSignal([]); + const [selectedModel, setSelectedModel] = createSignal(''); // Empty = use default + const [showModelSelector, setShowModelSelector] = createSignal(false); + + let messagesEndRef: HTMLDivElement | undefined; + let inputRef: HTMLTextAreaElement | undefined; + let abortControllerRef: AbortController | null = null; + + // Fetch available models on mount using the dynamic API + onMount(async () => { + try { + const result = await AIAPI.getModels(); + if (result.models && result.models.length > 0) { + setAvailableModels(result.models.map(m => ({ + id: m.id, + name: m.name || m.id, + description: m.description, + }))); + } + } catch (_e) { + // Silently fail - models will just not be selectable + } + }); + + // Wrapper to sync messages to global store + const setMessages = (updater: Message[] | ((prev: Message[]) => Message[])) => { + setMessagesLocal((prev) => { + const newMsgs = typeof updater === 'function' ? updater(prev) : updater; + // Sync to global store for persistence (debounce or defer to avoid too many updates) + setTimeout(() => aiChatStore.setMessages(newMsgs as any), 0); + return newMsgs; + }); + }; + + // Auto-scroll to bottom when new messages arrive + createEffect(() => { + if (messages().length > 0 && messagesEndRef) { + messagesEndRef.scrollIntoView({ behavior: 'smooth' }); + } + }); + + // Focus input when drawer opens and register with store for keyboard shortcuts + createEffect(() => { + if (isOpen() && inputRef) { + setTimeout(() => inputRef?.focus(), 100); + aiChatStore.registerInput(inputRef); + } else { + aiChatStore.registerInput(null); + } + }); + + // Handle initial prompt if provided + createEffect(() => { + if (initialPrompt() && isOpen()) { + setInput(initialPrompt()!); + } + }); + + // Auto-send queued message when AI finishes processing + createEffect(() => { + const loading = isLoading(); + const queued = queuedMessage(); + + // When loading finishes and we have a queued message, send it + if (!loading && queued) { + logger.info('[AIChat] AI finished, auto-sending queued message', { prompt: queued.substring(0, 50) }); + setQueuedMessage(null); + // Small delay to let the UI update first + setTimeout(() => { + handleSubmit(undefined, queued); + }, 100); + } + }); + + const generateId = () => Math.random().toString(36).substring(2, 9); + + // Stop/cancel the current AI request + const handleStop = () => { + if (abortControllerRef) { + abortControllerRef.abort(); + abortControllerRef = null; + } + // Mark any streaming message as stopped + setMessages((prev) => + prev.map((msg) => + msg.isStreaming + ? { ...msg, isStreaming: false, content: msg.content || '(Stopped by user)' } + : msg + ) + ); + // Clear queued message when stopping - user likely doesn't want it sent + setQueuedMessage(null); + setIsLoading(false); + }; + + const handleSubmit = async (e?: Event, forcePrompt?: string) => { + e?.preventDefault(); + const prompt = forcePrompt || input().trim(); + if (!prompt) return; + + // If AI is currently working, queue this message for later + if (isLoading() && !forcePrompt) { + setQueuedMessage(prompt); + setInput(''); + logger.info('[AIChat] Message queued while AI is working', { prompt: prompt.substring(0, 50) }); + return; + } + + // IMPORTANT: Capture the current messages BEFORE adding new ones to avoid race conditions + // SolidJS batches updates, so messages() may not be updated synchronously + const previousMessages = messages(); + + // Build conversation history from previous messages (before we add new ones) + // Include tool call outputs so the AI remembers what commands were run + const history = previousMessages + .filter((m) => !m.isStreaming) // Only include completed messages + .filter((m) => m.content || (m.toolCalls && m.toolCalls.length > 0)) // Must have content or tool calls + .map((m) => { + let content = m.content || ''; + + // For assistant messages, prepend tool call outputs so AI has full context + if (m.role === 'assistant' && m.toolCalls && m.toolCalls.length > 0) { + const toolSummary = m.toolCalls + .map((tc) => `Command: ${tc.input}\nOutput: ${tc.output}`) + .join('\n\n'); + content = toolSummary + (content ? '\n\n' + content : ''); + } + + return { + role: m.role, + content, + }; + }) + .filter((m) => m.content); // Filter out any empty messages + + // Add user message + const userMessage: Message = { + id: generateId(), + role: 'user', + content: prompt, + timestamp: new Date(), + }; + setMessages((prev) => [...prev, userMessage]); + setInput(''); + setIsLoading(true); + + // Create abort controller for this request + abortControllerRef = new AbortController(); + + // Create a streaming assistant message + const assistantId = generateId(); + const streamingMessage: Message = { + id: assistantId, + role: 'assistant', + content: '', + timestamp: new Date(), + isStreaming: true, + pendingTools: [], + pendingApprovals: [], + toolCalls: [], + thinkingChunks: [], + streamEvents: [], + }; + setMessages((prev) => [...prev, streamingMessage]); + + // Safety timeout - clear streaming state if we don't get any completion event + // This prevents the UI from getting stuck in a streaming state + let lastEventTime = Date.now(); + const SAFETY_TIMEOUT_MS = 120000; // 2 minutes + + const safetyCheckInterval = setInterval(() => { + const timeSinceLastEvent = Date.now() - lastEventTime; + if (timeSinceLastEvent > SAFETY_TIMEOUT_MS) { + logger.warn('[AIChat] Safety timeout - forcing stream completion', { seconds: SAFETY_TIMEOUT_MS / 1000 }); + clearInterval(safetyCheckInterval); + setMessages((prev) => + prev.map((msg) => + msg.id === assistantId && msg.isStreaming + ? { ...msg, isStreaming: false, content: msg.content || '(Request timed out - no response received)' } + : msg + ) + ); + setIsLoading(false); + if (abortControllerRef) { + abortControllerRef.abort(); + abortControllerRef = null; + } + } + }, 10000); // Check every 10 seconds + + try { + await AIAPI.executeStream( + { + prompt, + target_type: targetType(), + target_id: targetId(), + context: contextData(), + history: history.length > 0 ? history : undefined, + finding_id: findingId(), // Pass finding ID so AI can resolve it when fixed + model: selectedModel() || undefined, // Use selected model or default + }, + (event: AIStreamEvent) => { + lastEventTime = Date.now(); // Update last event time + logger.debug('[AIChat] Received event', { type: event.type, event }); + // Update the streaming message based on event type + setMessages((prev) => + prev.map((msg) => { + if (msg.id !== assistantId) return msg; + + switch (event.type) { + case 'tool_start': { + const data = event.data as AIStreamToolStartData; + return { + ...msg, + pendingTools: [...(msg.pendingTools || []), { name: data.name, input: data.input }], + }; + } + case 'tool_end': { + const data = event.data as AIStreamToolEndData; + // Remove one pending tool with matching name + const pendingTools = msg.pendingTools || []; + const matchingIndex = pendingTools.findIndex((t) => t.name === data.name); + const updatedPending = matchingIndex >= 0 + ? [...pendingTools.slice(0, matchingIndex), ...pendingTools.slice(matchingIndex + 1)] + : pendingTools; + // Use input directly from tool_end event (authoritative) + const newToolCall: AIToolExecution = { + name: data.name, + input: data.input, + output: data.output, + success: data.success, + }; + // Add to both toolCalls and streamEvents for chronological display + const events = msg.streamEvents || []; + return { + ...msg, + pendingTools: updatedPending, + toolCalls: [...(msg.toolCalls || []), newToolCall], + streamEvents: [...events, { type: 'tool' as const, tool: newToolCall }], + }; + } + case 'thinking': { + const chunk = event.data as string; + if (!chunk.trim()) return msg; // Skip empty chunks + // Each thinking event is a new chunk - add to both arrays + const chunks = msg.thinkingChunks || []; + const events = msg.streamEvents || []; + return { + ...msg, + thinking: (msg.thinking || '') + chunk, + thinkingChunks: [...chunks, chunk.trim()], + streamEvents: [...events, { type: 'thinking' as const, thinking: chunk.trim() }], + }; + } + case 'content': { + const content = event.data as string; + if (!content.trim()) return msg; // Skip empty content + // Track content in streamEvents for chronological display + const events = msg.streamEvents || []; + // Also accumulate in content for backwards compatibility / summary + const existingContent = msg.content || ''; + const separator = existingContent && !existingContent.endsWith('\n') ? '\n\n' : ''; + return { + ...msg, + content: existingContent + separator + content, + streamEvents: [...events, { type: 'content' as const, content: content.trim() }], + }; + } + case 'complete': { + // Complete event has flat structure (model, input_tokens at top level, not under data) + const completeEvent = event as unknown as AIStreamCompleteData & { type: string }; + return { + ...msg, + isStreaming: false, + pendingTools: [], + model: completeEvent.model, + tokens: { + input: completeEvent.input_tokens, + output: completeEvent.output_tokens, + }, + // Use tool_calls from complete if we missed any + toolCalls: msg.toolCalls?.length ? msg.toolCalls : completeEvent.tool_calls, + }; + } + case 'done': { + return { + ...msg, + isStreaming: false, + pendingTools: [], + }; + } + case 'error': { + const errorMsg = event.data as string; + return { + ...msg, + isStreaming: false, + pendingTools: [], + content: `Error: ${errorMsg}`, + }; + } + case 'processing': { + // Show processing status for multi-iteration calls + const status = event.data as string; + logger.debug('[AIChat] Processing', status); + // Add as a pending tool for visual feedback + return { + ...msg, + pendingTools: [{ name: 'processing', input: status }], + }; + } + case 'approval_needed': { + const data = event.data as AIStreamApprovalNeededData; + return { + ...msg, + pendingApprovals: [...(msg.pendingApprovals || []), { + command: data.command, + toolId: data.tool_id, + toolName: data.tool_name, + runOnHost: data.run_on_host ?? false, // Default to false if undefined + targetHost: data.target_host, // Pass through the explicit routing target + }], + }; + } + + default: + return msg; + } + }) + ); + }, + abortControllerRef?.signal + ); + } catch (error) { + // Don't show error for user-initiated abort + if (error instanceof Error && error.name === 'AbortError') { + logger.debug('[AIChat] Request aborted by user'); + return; + } + logger.error('[AIChat] Execute failed:', error); + const errorMessage = error instanceof Error ? error.message : 'Failed to get AI response'; + notificationStore.error(errorMessage); + + // Update the streaming message to show error + setMessages((prev) => + prev.map((msg) => + msg.id === assistantId + ? { ...msg, isStreaming: false, content: `Error: ${errorMessage}` } + : msg + ) + ); + } finally { + clearInterval(safetyCheckInterval); + abortControllerRef = null; + setIsLoading(false); + } + }; + + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + handleSubmit(); + } + }; + + const clearChat = () => { + setMessages([]); + aiChatStore.clearConversation(); + }; + + // Execute an approved command + const executeApprovedCommand = async (messageId: string, approval: PendingApproval) => { + // Mark as executing + setMessages((prev) => + prev.map((m) => + m.id === messageId + ? { + ...m, + pendingApprovals: m.pendingApprovals?.map((a) => + a.toolId === approval.toolId ? { ...a, isExecuting: true } : a + ), + } + : m + ) + ); + + try { + // Extract VMID from context if available + const vmid = contextData()?.vmid as string | undefined; + + const result = await AIAPI.runCommand({ + command: approval.command, + target_type: targetType() || '', + target_id: targetId() || '', + run_on_host: approval.runOnHost, + vmid, + target_host: approval.targetHost, // Pass through the explicit routing target + }); + + + // Move from pending approvals to completed tool calls + const currentMessages = messages(); + const targetMessage = currentMessages.find((m) => m.id === messageId); + const pendingCount = targetMessage?.pendingApprovals?.length || 0; + const remainingAfterThis = (targetMessage?.pendingApprovals?.filter((a) => a.toolId !== approval.toolId) || []).length; + + logger.info('[AIChat] Approval processed', { + messageId, + toolId: approval.toolId, + pendingCount, + remainingAfterThis, + pendingApprovals: targetMessage?.pendingApprovals?.map(a => a.toolId) + }); + + setMessages((prev) => + prev.map((m) => { + if (m.id !== messageId) return m; + + const newToolCall: AIToolExecution = { + name: approval.toolName, + input: approval.command, + output: result.output || result.error || '', + success: result.success, + }; + + const remainingApprovals = m.pendingApprovals?.filter((a) => a.toolId !== approval.toolId) || []; + + return { + ...m, + pendingApprovals: remainingApprovals, + toolCalls: [...(m.toolCalls || []), newToolCall], + // Clear the stale "I need approval" content after the last approval is processed + // The tool output will show the result instead + content: remainingApprovals.length === 0 ? '' : m.content, + }; + }) + ); + + // No toast for success - the tool output shows the result inline + // Only show error toast for failures since they might need attention + if (!result.success && result.error) { + notificationStore.error(result.error); + } + + // After the last approval is processed, automatically continue the conversation + // This lets the AI analyze the command output and provide a summary + if (remainingAfterThis === 0) { + logger.info('[AIChat] Last approval processed, triggering auto-continuation'); + // Small delay to let the UI update first + setTimeout(async () => { + logger.info('[AIChat] Starting auto-continuation'); + setIsLoading(true); + + // Build history including the just-executed command + const currentMsgs = messages(); + logger.debug('[AIChat] Building history for continuation', { messageCount: currentMsgs.length }); + + const historyForContinuation = currentMsgs + .filter((m) => !m.isStreaming) + .filter((m) => m.content || (m.toolCalls && m.toolCalls.length > 0)) + .map((m) => { + let content = m.content || ''; + if (m.role === 'assistant' && m.toolCalls && m.toolCalls.length > 0) { + const toolSummary = m.toolCalls + .map((tc) => `Command: ${tc.input}\nOutput: ${tc.output}`) + .join('\n\n'); + content = toolSummary + (content ? '\n\n' + content : ''); + } + return { role: m.role, content }; + }) + .filter((m) => m.content); + + logger.debug('[AIChat] History for continuation built', { historyLength: historyForContinuation.length }); + + // Add a hidden continuation prompt - the AI will see it but user won't + const continuationPrompt = 'Continue analyzing the command output above and provide a summary.'; + + // Create the streaming assistant response message (no visible user message) + // Show "Analyzing..." as initial content so user sees inline feedback + const assistantId = generateId(); + const streamingMessage: Message = { + id: assistantId, + role: 'assistant', + content: '*Analyzing results...*', + timestamp: new Date(), + isStreaming: true, + pendingTools: [], + pendingApprovals: [], + toolCalls: [], + }; + setMessages((prev) => [...prev, streamingMessage]); + + try { + logger.info('[AIChat] Calling executeStream for continuation'); + await AIAPI.executeStream( + { + prompt: continuationPrompt, + target_type: targetType(), + target_id: targetId(), + context: contextData(), + history: historyForContinuation, + }, + (event: AIStreamEvent) => { + logger.debug('[AIChat] Continuation event received', { type: event.type }); + setMessages((prev) => + prev.map((msg) => { + if (msg.id !== assistantId) return msg; + switch (event.type) { + case 'content': { + // Append content, but filter out the initial placeholder + const content = event.data as string; + const existingContent = msg.content === '*Analyzing results...*' ? '' : (msg.content || ''); + const separator = existingContent && !existingContent.endsWith('\n') ? '\n\n' : ''; + return { ...msg, content: existingContent + separator + content, isStreaming: false }; + } + case 'done': + return { ...msg, isStreaming: false }; + case 'error': + return { ...msg, content: `Error: ${event.data}`, isStreaming: false }; + case 'thinking': + // Ignore thinking events for now + return msg; + case 'processing': + // Ignore processing events + return msg; + case 'tool_start': { + const data = event.data as { name: string; input: string }; + return { + ...msg, + pendingTools: [...(msg.pendingTools || []), { name: data.name, input: data.input }], + }; + } + case 'tool_end': { + const data = event.data as { name: string; input: string; output: string; success: boolean }; + const pendingTools = msg.pendingTools || []; + const matchingIndex = pendingTools.findIndex((t) => t.name === data.name); + const updatedPending = matchingIndex >= 0 + ? [...pendingTools.slice(0, matchingIndex), ...pendingTools.slice(matchingIndex + 1)] + : pendingTools; + return { + ...msg, + pendingTools: updatedPending, + toolCalls: [...(msg.toolCalls || []), { + name: data.name, + input: data.input, + output: data.output, + success: data.success, + }], + }; + } + case 'approval_needed': { + const data = event.data as AIStreamApprovalNeededData; + logger.info('[AIChat] Approval needed in continuation', { command: data.command }); + return { + ...msg, + pendingApprovals: [...(msg.pendingApprovals || []), { + command: data.command, + toolId: data.tool_id, + toolName: data.tool_name, + runOnHost: data.run_on_host, + targetHost: data.target_host, + }], + isStreaming: false, // Stop streaming when approval is needed + }; + } + default: + logger.debug('[AIChat] Unhandled continuation event', { type: event.type, event }); + return msg; + } + }) + ); + } + ); + logger.info('[AIChat] Continuation executeStream completed'); + } catch (err) { + logger.error('[AIChat] Failed to continue after approval:', err); + setMessages((prev) => + prev.map((msg) => + msg.id === assistantId + ? { ...msg, content: 'Failed to analyze results.', isStreaming: false } + : msg + ) + ); + } finally { + setIsLoading(false); + } + }, 200); + } else { + logger.debug('[AIChat] Approvals remaining, not triggering continuation', { remainingAfterThis }); + } + } catch (error) { + logger.error('[AIChat] Failed to execute approved command:', error); + const errorMsg = error instanceof Error ? error.message : 'Failed to execute command'; + notificationStore.error(errorMsg); + + // Mark as no longer executing + setMessages((prev) => + prev.map((m) => + m.id === messageId + ? { + ...m, + pendingApprovals: m.pendingApprovals?.map((a) => + a.toolId === approval.toolId ? { ...a, isExecuting: false } : a + ), + } + : m + ) + ); + } + }; + + // Panel renders as flex child, width controlled by isOpen state + return ( +
+ + {/* Header */} +
+
+
+ + + +
+
+

+ + Ask AI about {getGuestName(contextData())} + +

+ +

+ {targetType() === 'vm' ? 'Virtual Machine' : targetType() === 'container' ? 'LXC Container' : targetType()} +

+
+
+
+
+ {/* Model selector dropdown */} + 0}> +
+ + +
+ + + {([provider, models]) => ( + <> +
+ {PROVIDER_DISPLAY_NAMES[provider] || provider} +
+ + {(model) => ( + + )} + + + )} +
+
+
+
+
+ + +
+
+ + {/* Messages Area */} +
+ +
+ + + + +

Start a conversation

+

+ Ask about your infrastructure, diagnose issues, or get remediation suggestions. +

+ + }> +

Ask about {getGuestName(contextData())}

+

+ AI has access to this guest's current metrics and state. Try asking: +

+
+ + + +
+
+
+
+ + + {(message) => ( +
+
+ {/* Render all events in chronological order - thinking, tools, and content interleaved */} + 0}> +
+ + {(evt) => ( + + + {/* Tool call */} +
+
+ + + + {evt.tool!.input} +
+ +
+                                    {evt.tool!.output.length > 500 ? evt.tool!.output.substring(0, 500) + '...' : evt.tool!.output}
+                                  
+
+
+
+ + {/* Thinking chunk */} +
+ {sanitizeThinking(evt.thinking!.length > 500 ? evt.thinking!.substring(0, 500) + '...' : evt.thinking!)} +
+
+ + {/* Content chunk - rendered as markdown */} + {/* DOMPurify sanitizes LLM output in renderMarkdown before HTML rendering. */} +
+ + + )} + +
+ + + {/* Show in-progress tool executions - at the bottom */} + 0}> +
+ + {(tool) => ( +
+
+ + + + + {tool.input} + Running... +
+
+ )} +
+
+
+ + {/* Show AI's response text - only if no streamEvents (fallback for old messages or messages without streaming) */} + +
+ + + {/* Show commands awaiting approval */} + 0}> +
+ + {(approval) => ( +
+
+ + + + Approval Required + + HOST + +
+
+ {approval.command} +
+ + +
+
+
+ )} +
+
+
+ {/* Minimal footer - no model/token info shown */} +
+
+ )} + + +
+
+ + {/* Processing indicator - sticky above input */} + +
+ + + + + Analyzing... +
+
+ + {/* Input Area */} +
+ {/* Context section - always show with Add button */} +
+
+
+ + + + + Context {aiChatStore.contextItems.length > 0 ? `(${aiChatStore.contextItems.length})` : ''} + +
+
+ 0}> + + +
+
+ + {/* Context items */} +
+ + {(item) => ( + + {item.type} + {item.name} + + + )} + + + {/* Add context button */} +
+ + + {/* Context picker dropdown */} + +
+ {/* Search input */} +
+ setContextSearch(e.currentTarget.value)} + placeholder="Search VMs, containers, hosts..." + class="w-full px-2 py-1.5 text-xs rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-900 text-gray-900 dark:text-gray-100 placeholder-gray-400 focus:outline-none focus:ring-1 focus:ring-purple-500" + autofocus + /> +
+ + {/* Resource list */} +
+ 0} fallback={ +
+ No resources found +
+ }> + + {(resource) => { + const isAlreadyAdded = () => aiChatStore.hasContextItem(resource.id); + return ( + + ); + }} + +
+
+ + {/* Close button */} +
+ +
+
+
+
+
+ + {/* Empty state hint */} + +

+ Add VMs, containers, or hosts to provide context for your questions +

+
+ + {/* Guest Notes - show for first context item */} + 0}> + + +
+ {/* Queued message indicator */} + +
+ + + + + Queued: "{queuedMessage()!.substring(0, 50)}{queuedMessage()!.length > 50 ? '...' : ''}" + + +
+
+
+