From d96942596fa8b96b7bd91bac7d72a3ddf5aa6f8d Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 12 Dec 2025 21:31:11 +0000 Subject: [PATCH] feat: add Kubernetes platform support --- cmd/pulse-agent/main.go | 211 +++++- docs/API.md | 3 +- docs/UNIFIED_AGENT.md | 16 +- .../src/api/__tests__/charts.test.ts | 2 +- frontend-modern/src/api/monitoring.ts | 191 +++++ .../src/components/Settings/UnifiedAgents.tsx | 167 ++++- .../Settings/__tests__/UnifiedAgents.test.tsx | 2 +- .../src/components/shared/Sparkline.tsx | 14 +- frontend-modern/src/constants/apiScopes.ts | 14 + .../stores/__tests__/metricsHistory.test.ts | 2 +- frontend-modern/src/types/api.ts | 30 + frontend-modern/src/types/resource.ts | 3 +- go.mod | 35 +- go.sum | Bin 19758 -> 31467 bytes internal/api/kubernetes_agents.go | 286 ++++++++ internal/api/router.go | 20 +- internal/config/api_tokens.go | 22 +- internal/config/api_tokens_test.go | 2 + internal/kubernetesagent/agent.go | 683 ++++++++++++++++++ internal/kubernetesagent/version.go | 4 + internal/models/converters.go | 145 ++++ internal/models/models.go | 349 ++++++++- internal/models/models_frontend.go | 164 ++++- internal/models/state_snapshot.go | 90 ++- internal/monitoring/kubernetes_agents.go | 390 ++++++++++ internal/monitoring/monitor.go | 88 ++- internal/resources/converters.go | 282 +++++++- internal/resources/platform_data.go | 210 ++++-- internal/resources/resource.go | 19 +- internal/resources/store.go | 288 ++++---- pkg/agents/kubernetes/report.go | 98 +++ scripts/install.ps1 | 41 +- scripts/install.sh | 11 + 33 files changed, 3437 insertions(+), 445 deletions(-) create mode 100644 internal/api/kubernetes_agents.go create mode 100644 internal/kubernetesagent/agent.go create mode 100644 internal/kubernetesagent/version.go create mode 100644 internal/monitoring/kubernetes_agents.go create mode 100644 pkg/agents/kubernetes/report.go diff --git a/cmd/pulse-agent/main.go b/cmd/pulse-agent/main.go index 7a20cb8..6b30525 100644 --- a/cmd/pulse-agent/main.go +++ b/cmd/pulse-agent/main.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "sync/atomic" "syscall" @@ -18,6 +19,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentupdate" "github.com/rcourtman/pulse-go-rewrite/internal/dockeragent" "github.com/rcourtman/pulse-go-rewrite/internal/hostagent" + "github.com/rcourtman/pulse-go-rewrite/internal/kubernetesagent" "github.com/rcourtman/pulse-go-rewrite/internal/utils" "github.com/rs/zerolog" "golang.org/x/sync/errgroup" @@ -30,7 +32,7 @@ var ( agentInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "pulse_agent_info", Help: "Information about the Pulse agent", - }, []string{"version", "host_enabled", "docker_enabled"}) + }, []string{"version", "host_enabled", "docker_enabled", "kubernetes_enabled"}) agentUp = promauto.NewGauge(prometheus.GaugeOpts{ Name: "pulse_agent_up", @@ -78,6 +80,7 @@ func main() { Str("pulse_url", cfg.PulseURL). Bool("host_agent", cfg.EnableHost). Bool("docker_agent", cfg.EnableDocker). + Bool("kubernetes_agent", cfg.EnableKubernetes). Bool("proxmox_mode", cfg.EnableProxmox). Bool("auto_update", !cfg.DisableAutoUpdate). Msg("Starting Pulse Unified Agent") @@ -87,6 +90,7 @@ func main() { Version, fmt.Sprintf("%t", cfg.EnableHost), fmt.Sprintf("%t", cfg.EnableDocker), + fmt.Sprintf("%t", cfg.EnableKubernetes), ).Set(1) agentUp.Set(1) @@ -187,10 +191,50 @@ func main() { } } + // 10. Start Kubernetes Agent (if enabled) + if cfg.EnableKubernetes { + kubeCfg := kubernetesagent.Config{ + PulseURL: cfg.PulseURL, + APIToken: cfg.APIToken, + Interval: cfg.Interval, + AgentID: cfg.AgentID, + AgentType: "unified", + AgentVersion: Version, + InsecureSkipVerify: cfg.InsecureSkipVerify, + LogLevel: cfg.LogLevel, + Logger: &logger, + KubeconfigPath: cfg.KubeconfigPath, + KubeContext: cfg.KubeContext, + IncludeNamespaces: cfg.KubeIncludeNamespaces, + ExcludeNamespaces: cfg.KubeExcludeNamespaces, + IncludeAllPods: cfg.KubeIncludeAllPods, + MaxPods: cfg.KubeMaxPods, + } + + agent, err := kubernetesagent.New(kubeCfg) + if err != nil { + logger.Warn().Err(err).Msg("Kubernetes not available, will retry with exponential backoff") + + g.Go(func() error { + retried := initKubernetesWithRetry(ctx, kubeCfg, &logger) + if retried != nil { + logger.Info().Msg("Kubernetes agent module started (after retry)") + return retried.Run(ctx) + } + return nil + }) + } else { + g.Go(func() error { + logger.Info().Msg("Kubernetes agent module started") + return agent.Run(ctx) + }) + } + } + // Mark as ready after all agents started ready.Store(true) - // 10. Wait for all agents to exit + // 11. Wait for all agents to exit if err := g.Wait(); err != nil && err != context.Canceled { logger.Error().Err(err).Msg("Agent terminated with error") agentUp.Set(0) @@ -198,7 +242,7 @@ func main() { os.Exit(1) } - // 11. Cleanup + // 12. Cleanup agentUp.Set(0) cleanupDockerAgent(dockerAgent, &logger) @@ -274,16 +318,25 @@ type Config struct { Logger *zerolog.Logger // Module flags - EnableHost bool - EnableDocker bool - EnableProxmox bool - ProxmoxType string // "pve", "pbs", or "" for auto-detect + EnableHost bool + EnableDocker bool + EnableKubernetes bool + EnableProxmox bool + ProxmoxType string // "pve", "pbs", or "" for auto-detect // Auto-update DisableAutoUpdate bool // Health/metrics server HealthAddr string + + // Kubernetes + KubeconfigPath string + KubeContext string + KubeIncludeNamespaces []string + KubeExcludeNamespaces []string + KubeIncludeAllPods bool + KubeMaxPods int } func loadConfig() Config { @@ -298,10 +351,17 @@ func loadConfig() Config { envLogLevel := utils.GetenvTrim("LOG_LEVEL") envEnableHost := utils.GetenvTrim("PULSE_ENABLE_HOST") envEnableDocker := utils.GetenvTrim("PULSE_ENABLE_DOCKER") + envEnableKubernetes := utils.GetenvTrim("PULSE_ENABLE_KUBERNETES") envEnableProxmox := utils.GetenvTrim("PULSE_ENABLE_PROXMOX") envProxmoxType := utils.GetenvTrim("PULSE_PROXMOX_TYPE") envDisableAutoUpdate := utils.GetenvTrim("PULSE_DISABLE_AUTO_UPDATE") envHealthAddr := utils.GetenvTrim("PULSE_HEALTH_ADDR") + envKubeconfig := utils.GetenvTrim("PULSE_KUBECONFIG") + envKubeContext := utils.GetenvTrim("PULSE_KUBE_CONTEXT") + envKubeIncludeNamespaces := utils.GetenvTrim("PULSE_KUBE_INCLUDE_NAMESPACES") + envKubeExcludeNamespaces := utils.GetenvTrim("PULSE_KUBE_EXCLUDE_NAMESPACES") + envKubeIncludeAllPods := utils.GetenvTrim("PULSE_KUBE_INCLUDE_ALL_PODS") + envKubeMaxPods := utils.GetenvTrim("PULSE_KUBE_MAX_PODS") // Defaults defaultInterval := 30 * time.Second @@ -321,6 +381,11 @@ func loadConfig() Config { defaultEnableDocker = utils.ParseBool(envEnableDocker) } + defaultEnableKubernetes := false + if envEnableKubernetes != "" { + defaultEnableKubernetes = utils.ParseBool(envEnableKubernetes) + } + defaultEnableProxmox := false if envEnableProxmox != "" { defaultEnableProxmox = utils.ParseBool(envEnableProxmox) @@ -342,14 +407,23 @@ func loadConfig() Config { enableHostFlag := flag.Bool("enable-host", defaultEnableHost, "Enable Host Agent module") enableDockerFlag := flag.Bool("enable-docker", defaultEnableDocker, "Enable Docker Agent module") + enableKubernetesFlag := flag.Bool("enable-kubernetes", defaultEnableKubernetes, "Enable Kubernetes Agent module") enableProxmoxFlag := flag.Bool("enable-proxmox", defaultEnableProxmox, "Enable Proxmox mode (creates API token, registers node)") proxmoxTypeFlag := flag.String("proxmox-type", envProxmoxType, "Proxmox type: pve or pbs (auto-detected if not specified)") disableAutoUpdateFlag := flag.Bool("disable-auto-update", utils.ParseBool(envDisableAutoUpdate), "Disable automatic updates") healthAddrFlag := flag.String("health-addr", defaultHealthAddr, "Health/metrics server address (empty to disable)") + kubeconfigFlag := flag.String("kubeconfig", envKubeconfig, "Path to kubeconfig (optional; uses in-cluster config if available)") + kubeContextFlag := flag.String("kube-context", envKubeContext, "Kubeconfig context (optional)") + kubeIncludeAllPodsFlag := flag.Bool("kube-include-all-pods", utils.ParseBool(envKubeIncludeAllPods), "Include all non-succeeded pods (may be large)") + kubeMaxPodsFlag := flag.Int("kube-max-pods", defaultInt(envKubeMaxPods, 200), "Max pods included in report") showVersion := flag.Bool("version", false, "Print the agent version and exit") var tagFlags multiValue flag.Var(&tagFlags, "tag", "Tag to apply (repeatable)") + var kubeIncludeNamespaceFlags multiValue + flag.Var(&kubeIncludeNamespaceFlags, "kube-include-namespace", "Namespace to include (repeatable; default is all)") + var kubeExcludeNamespaceFlags multiValue + flag.Var(&kubeExcludeNamespaceFlags, "kube-exclude-namespace", "Namespace to exclude (repeatable)") flag.Parse() @@ -376,22 +450,31 @@ func loadConfig() Config { } tags := gatherTags(envTags, tagFlags) + kubeIncludeNamespaces := gatherCSV(envKubeIncludeNamespaces, kubeIncludeNamespaceFlags) + kubeExcludeNamespaces := gatherCSV(envKubeExcludeNamespaces, kubeExcludeNamespaceFlags) return Config{ - PulseURL: pulseURL, - APIToken: token, - Interval: *intervalFlag, - HostnameOverride: strings.TrimSpace(*hostnameFlag), - AgentID: strings.TrimSpace(*agentIDFlag), - Tags: tags, - InsecureSkipVerify: *insecureFlag, - LogLevel: logLevel, - EnableHost: *enableHostFlag, - EnableDocker: *enableDockerFlag, - EnableProxmox: *enableProxmoxFlag, - ProxmoxType: strings.TrimSpace(*proxmoxTypeFlag), - DisableAutoUpdate: *disableAutoUpdateFlag, - HealthAddr: strings.TrimSpace(*healthAddrFlag), + PulseURL: pulseURL, + APIToken: token, + Interval: *intervalFlag, + HostnameOverride: strings.TrimSpace(*hostnameFlag), + AgentID: strings.TrimSpace(*agentIDFlag), + Tags: tags, + InsecureSkipVerify: *insecureFlag, + LogLevel: logLevel, + EnableHost: *enableHostFlag, + EnableDocker: *enableDockerFlag, + EnableKubernetes: *enableKubernetesFlag, + EnableProxmox: *enableProxmoxFlag, + ProxmoxType: strings.TrimSpace(*proxmoxTypeFlag), + DisableAutoUpdate: *disableAutoUpdateFlag, + HealthAddr: strings.TrimSpace(*healthAddrFlag), + KubeconfigPath: strings.TrimSpace(*kubeconfigFlag), + KubeContext: strings.TrimSpace(*kubeContextFlag), + KubeIncludeNamespaces: kubeIncludeNamespaces, + KubeExcludeNamespaces: kubeExcludeNamespaces, + KubeIncludeAllPods: *kubeIncludeAllPodsFlag, + KubeMaxPods: *kubeMaxPodsFlag, } } @@ -414,6 +497,37 @@ func gatherTags(env string, flags []string) []string { return tags } +func gatherCSV(env string, flags []string) []string { + values := make([]string, 0) + if env != "" { + for _, value := range strings.Split(env, ",") { + value = strings.TrimSpace(value) + if value != "" { + values = append(values, value) + } + } + } + for _, value := range flags { + value = strings.TrimSpace(value) + if value != "" { + values = append(values, value) + } + } + return values +} + +func defaultInt(value string, fallback int) int { + value = strings.TrimSpace(value) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + func parseLogLevel(value string) (zerolog.Level, error) { normalized := strings.ToLower(strings.TrimSpace(value)) if normalized == "" { @@ -484,3 +598,58 @@ func initDockerWithRetry(ctx context.Context, cfg dockeragent.Config, logger *ze } } } + +// initKubernetesWithRetry attempts to initialize the Kubernetes agent with exponential backoff. +// It returns the agent when Kubernetes becomes available, or nil if the context is cancelled. +// Retry intervals: 5s, 10s, 20s, 40s, 80s, 160s, then cap at 5 minutes. +func initKubernetesWithRetry(ctx context.Context, cfg kubernetesagent.Config, logger *zerolog.Logger) *kubernetesagent.Agent { + const ( + initialDelay = 5 * time.Second + maxDelay = 5 * time.Minute + multiplier = 2.0 + ) + + delay := initialDelay + attempt := 0 + + for { + select { + case <-ctx.Done(): + logger.Info().Msg("Kubernetes retry cancelled, context done") + return nil + default: + } + + attempt++ + logger.Info(). + Int("attempt", attempt). + Str("next_retry", delay.String()). + Msg("Waiting to retry Kubernetes connection") + + select { + case <-ctx.Done(): + logger.Info().Msg("Kubernetes retry cancelled during wait") + return nil + case <-time.After(delay): + } + + agent, err := kubernetesagent.New(cfg) + if err == nil { + logger.Info(). + Int("attempts", attempt). + Msg("Successfully connected to Kubernetes after retry") + return agent + } + + logger.Warn(). + Err(err). + Int("attempt", attempt). + Str("next_retry", (time.Duration(float64(delay) * multiplier)).String()). + Msg("Kubernetes still not available, will retry") + + delay = time.Duration(float64(delay) * multiplier) + if delay > maxDelay { + delay = maxDelay + } + } +} diff --git a/docs/API.md b/docs/API.md index 7fcda81..d9b06ff 100644 --- a/docs/API.md +++ b/docs/API.md @@ -146,7 +146,7 @@ Initiate OIDC login flow. `GET /download/pulse-agent` Downloads the unified agent binary for the current platform. -The unified agent combines host and Docker monitoring. Use `--enable-docker` to enable Docker metrics. +The unified agent combines host, Docker, and Kubernetes monitoring. Use `--enable-docker` or `--enable-kubernetes` to enable additional metrics. See [UNIFIED_AGENT.md](UNIFIED_AGENT.md) for installation instructions. @@ -157,6 +157,7 @@ See [UNIFIED_AGENT.md](UNIFIED_AGENT.md) for installation instructions. ### Submit Reports `POST /api/agents/host/report` - Host metrics `POST /api/agents/docker/report` - Docker container metrics +`POST /api/agents/kubernetes/report` - Kubernetes cluster metrics --- diff --git a/docs/UNIFIED_AGENT.md b/docs/UNIFIED_AGENT.md index 6b0447e..72700a8 100644 --- a/docs/UNIFIED_AGENT.md +++ b/docs/UNIFIED_AGENT.md @@ -1,6 +1,6 @@ # Pulse Unified Agent -The unified agent (`pulse-agent`) combines host and Docker monitoring into a single binary. It replaces the separate `pulse-host-agent` and `pulse-docker-agent` for simpler deployment and management. +The unified agent (`pulse-agent`) combines host, Docker, and Kubernetes monitoring into a single binary. It replaces the separate `pulse-host-agent` and `pulse-docker-agent` for simpler deployment and management. ## Quick Start @@ -29,6 +29,7 @@ curl -fsSL http://:7655/install.sh | \ - **Host Metrics**: CPU, memory, disk, network I/O, temperatures - **Docker Monitoring**: Container metrics, health checks, Swarm support (when enabled) +- **Kubernetes Monitoring**: Cluster, node, pod, and deployment health (when enabled) - **Auto-Update**: Automatically updates when a new version is released - **Multi-Platform**: Linux, macOS, Windows support @@ -41,6 +42,13 @@ curl -fsSL http://:7655/install.sh | \ | `--interval` | `PULSE_INTERVAL` | Reporting interval | `30s` | | `--enable-host` | `PULSE_ENABLE_HOST` | Enable host metrics | `true` | | `--enable-docker` | `PULSE_ENABLE_DOCKER` | Enable Docker metrics | `false` | +| `--enable-kubernetes` | `PULSE_ENABLE_KUBERNETES` | Enable Kubernetes metrics | `false` | +| `--kubeconfig` | `PULSE_KUBECONFIG` | Kubeconfig path (optional) | *(auto)* | +| `--kube-context` | `PULSE_KUBE_CONTEXT` | Kubeconfig context (optional) | *(auto)* | +| `--kube-include-namespace` | `PULSE_KUBE_INCLUDE_NAMESPACES` | Limit namespaces (repeatable or CSV) | *(all)* | +| `--kube-exclude-namespace` | `PULSE_KUBE_EXCLUDE_NAMESPACES` | Exclude namespaces (repeatable or CSV) | *(none)* | +| `--kube-include-all-pods` | `PULSE_KUBE_INCLUDE_ALL_PODS` | Include all non-succeeded pods | `false` | +| `--kube-max-pods` | `PULSE_KUBE_MAX_PODS` | Max pods per report | `200` | | `--disable-auto-update` | `PULSE_DISABLE_AUTO_UPDATE` | Disable auto-updates | `false` | | `--insecure` | `PULSE_INSECURE_SKIP_VERIFY` | Skip TLS verification | `false` | | `--hostname` | `PULSE_HOSTNAME` | Override hostname | *(OS hostname)* | @@ -61,6 +69,12 @@ curl -fsSL http://:7655/install.sh | \ bash -s -- --url http://:7655 --token --enable-docker ``` +### Host + Kubernetes Monitoring +```bash +curl -fsSL http://:7655/install.sh | \ + bash -s -- --url http://:7655 --token --enable-kubernetes +``` + ### Docker Monitoring Only ```bash curl -fsSL http://:7655/install.sh | \ diff --git a/frontend-modern/src/api/__tests__/charts.test.ts b/frontend-modern/src/api/__tests__/charts.test.ts index b50c7dd..79d1411 100644 --- a/frontend-modern/src/api/__tests__/charts.test.ts +++ b/frontend-modern/src/api/__tests__/charts.test.ts @@ -1,7 +1,7 @@ /** * Tests for Charts API types and interface */ -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; +import { describe, expect, it } from 'vitest'; import type { ChartData, ChartsResponse, TimeRange, MetricPoint, ChartStats } from '@/api/charts'; // Note: We test the types and interfaces here since the actual API calls diff --git a/frontend-modern/src/api/monitoring.ts b/frontend-modern/src/api/monitoring.ts index 04a140e..6a9eaf4 100644 --- a/frontend-modern/src/api/monitoring.ts +++ b/frontend-modern/src/api/monitoring.ts @@ -216,6 +216,191 @@ export class MonitoringAPI { } } + static async deleteKubernetesCluster(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}`; + + const response = await apiFetch(url, { + method: 'DELETE', + }); + + if (!response.ok) { + if (response.status === 404) { + return {}; + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + + if (response.status === 204) { + return {}; + } + + const text = await response.text(); + if (!text?.trim()) { + return {}; + } + + try { + return JSON.parse(text) as DeleteKubernetesClusterResponse; + } catch (err) { + throw new Error((err as Error).message || 'Failed to parse delete kubernetes cluster response'); + } + } + + static async unhideKubernetesCluster(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/unhide`; + + const response = await apiFetch(url, { + method: 'PUT', + }); + + if (!response.ok) { + if (response.status === 404) { + return; + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + + static async markKubernetesClusterPendingUninstall(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/pending-uninstall`; + + const response = await apiFetch(url, { + method: 'PUT', + }); + + if (!response.ok) { + if (response.status === 404) { + return; + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + + static async setKubernetesClusterDisplayName(clusterId: string, displayName: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/display-name`; + + const response = await apiFetch(url, { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ displayName }), + }); + + if (!response.ok) { + if (response.status === 404) { + throw new Error('Kubernetes cluster not found'); + } + + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_jsonErr) { + // ignore parse errors + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + + static async allowKubernetesClusterReenroll(clusterId: string): Promise { + const url = `${this.baseUrl}/agents/kubernetes/clusters/${encodeURIComponent(clusterId)}/allow-reenroll`; + + const response = await apiFetch(url, { + method: 'POST', + }); + + if (!response.ok) { + let message = `Failed with status ${response.status}`; + try { + const text = await response.text(); + if (text?.trim()) { + message = text.trim(); + try { + const parsed = JSON.parse(text); + if (typeof parsed?.error === 'string' && parsed.error.trim()) { + message = parsed.error.trim(); + } + } catch (_err) { + // ignore parse error + } + } + } catch (_err) { + // ignore read error + } + + throw new Error(message); + } + } + static async deleteHostAgent(hostId: string): Promise { if (!hostId) { throw new Error('Host ID is required to remove a host agent.'); @@ -313,3 +498,9 @@ export interface DeleteDockerHostResponse { message?: string; command?: DockerHostCommand; } + +export interface DeleteKubernetesClusterResponse { + success?: boolean; + clusterId?: string; + message?: string; +} diff --git a/frontend-modern/src/components/Settings/UnifiedAgents.tsx b/frontend-modern/src/components/Settings/UnifiedAgents.tsx index ac9195c..d161c6d 100644 --- a/frontend-modern/src/components/Settings/UnifiedAgents.tsx +++ b/frontend-modern/src/components/Settings/UnifiedAgents.tsx @@ -8,7 +8,7 @@ import { notificationStore } from '@/stores/notifications'; import type { SecurityStatus } from '@/types/config'; import type { HostLookupResponse } from '@/types/api'; import type { APITokenRecord } from '@/api/security'; -import { HOST_AGENT_SCOPE, DOCKER_REPORT_SCOPE } from '@/constants/apiScopes'; +import { HOST_AGENT_SCOPE, DOCKER_REPORT_SCOPE, KUBERNETES_REPORT_SCOPE } from '@/constants/apiScopes'; import { copyToClipboard } from '@/utils/clipboard'; import { getPulseBaseUrl } from '@/utils/url'; import { logger } from '@/utils/logger'; @@ -108,6 +108,7 @@ export const UnifiedAgents: Component = () => { const [lookupError, setLookupError] = createSignal(null); const [lookupLoading, setLookupLoading] = createSignal(false); const [enableDocker, setEnableDocker] = createSignal(false); // Default to false - user must opt-in for Docker monitoring + const [enableKubernetes, setEnableKubernetes] = createSignal(false); // Default to false - user must opt-in for Kubernetes monitoring const [enableProxmox, setEnableProxmox] = createSignal(false); // For Proxmox VE/PBS nodes - creates API token and auto-registers const [insecureMode, setInsecureMode] = createSignal(false); // For self-signed certificates (issue #806) @@ -183,15 +184,15 @@ export const UnifiedAgents: Component = () => { setIsGeneratingToken(true); try { const desiredName = tokenName().trim() || buildDefaultTokenName(); - // Generate token with BOTH scopes - const scopes = [HOST_AGENT_SCOPE, DOCKER_REPORT_SCOPE]; + // Generate token with unified agent reporting scopes + const scopes = [HOST_AGENT_SCOPE, DOCKER_REPORT_SCOPE, KUBERNETES_REPORT_SCOPE]; const { token, record } = await SecurityAPI.createToken(desiredName, scopes); setCurrentToken(token); setLatestRecord(record); setTokenName(''); setConfirmedNoToken(false); - notificationStore.success('Token generated with Host and Docker permissions.', 4000); + notificationStore.success('Token generated with Host, Docker, and Kubernetes permissions.', 4000); } catch (err) { logger.error('Failed to generate agent token', err); notificationStore.error('Failed to generate agent token. Confirm you are signed in as an administrator.', 6000); @@ -237,6 +238,7 @@ export const UnifiedAgents: Component = () => { }; const getDockerFlag = () => enableDocker() ? ' --enable-docker' : ''; + const getKubernetesFlag = () => enableKubernetes() ? ' --enable-kubernetes' : ''; const getProxmoxFlag = () => enableProxmox() ? ' --enable-proxmox' : ''; const getInsecureFlag = () => insecureMode() ? ' --insecure' : ''; const getCurlInsecureFlag = () => insecureMode() ? '-k' : ''; @@ -344,19 +346,32 @@ export const UnifiedAgents: Component = () => { }); const hasRemovedDockerHosts = createMemo(() => removedDockerHosts().length > 0); + const kubernetesClusters = createMemo(() => { + const clusters = state.kubernetesClusters || []; + return clusters.slice().sort((a, b) => (a.displayName || a.name || a.id).localeCompare(b.displayName || b.name || b.id)); + }); + + const removedKubernetesClusters = createMemo(() => { + const removed = state.removedKubernetesClusters || []; + return removed.sort((a, b) => b.removedAt - a.removedAt); + }); + const hasRemovedKubernetesClusters = createMemo(() => removedKubernetesClusters().length > 0); + const getUpgradeCommand = (_hostname: string) => { const token = resolvedToken(); return `curl ${getCurlInsecureFlag()}-fsSL ${pulseUrl()}/install.sh | sudo bash -s -- --url ${pulseUrl()} --token ${token}${getInsecureFlag()}`; }; - const handleRemoveAgent = async (id: string, type: 'host' | 'docker') => { + const handleRemoveAgent = async (id: string, type: 'host' | 'docker' | 'kubernetes') => { if (!confirm('Are you sure you want to remove this agent? This will stop monitoring but will not uninstall the agent from the remote machine.')) return; try { if (type === 'host') { await MonitoringAPI.deleteHostAgent(id); - } else { + } else if (type === 'docker') { await MonitoringAPI.deleteDockerHost(id); + } else { + await MonitoringAPI.deleteKubernetesCluster(id); } notificationStore.success('Agent removed from Pulse'); } catch (err) { @@ -375,6 +390,16 @@ export const UnifiedAgents: Component = () => { } }; + const handleAllowKubernetesReenroll = async (clusterId: string, name?: string) => { + try { + await MonitoringAPI.allowKubernetesClusterReenroll(clusterId); + notificationStore.success(`Re-enrollment allowed for ${name || clusterId}. Restart the agent to reconnect.`); + } catch (err) { + logger.error('Failed to allow kubernetes re-enrollment', err); + notificationStore.error('Failed to allow kubernetes re-enrollment'); + } + }; + return (
@@ -391,7 +416,7 @@ export const UnifiedAgents: Component = () => {

Generate API token

- Create a fresh token scoped for both Host and Docker monitoring. + Create a fresh token scoped for Host, Docker, and Kubernetes monitoring.

@@ -466,6 +491,15 @@ export const UnifiedAgents: Component = () => { /> Docker monitoring + + +
+

Kubernetes Clusters

+

+ Kubernetes clusters currently reporting to Pulse. +

+
+ + + + + + + + + + + + + + + + + }> + {(cluster) => ( + + + + + + + + )} + + +
ClusterStatusVersionLast SeenActions
+ No Kubernetes clusters reporting yet. +
+ {cluster.customDisplayName || cluster.displayName || cluster.name || cluster.id} + + + {cluster.status} + + + {cluster.version || cluster.agentVersion || '—'} + + {cluster.lastSeen ? formatRelativeTime(cluster.lastSeen) : '—'} + + +
+ + +
@@ -848,6 +952,55 @@ export const UnifiedAgents: Component = () => { + + + +
+

Removed Kubernetes Clusters

+

+ Kubernetes clusters that were removed and are blocked from re-enrolling. Allow re-enrollment to let them report again. +

+
+ + + + + + + + + + + + + + {(cluster) => ( + + + + + + + )} + + +
ClusterCluster IDRemovedActions
+ {cluster.displayName || cluster.name || 'Unknown'} + + {cluster.id.slice(0, 8)}... + + {formatRelativeTime(cluster.removedAt)} + + +
+ + +
); }; diff --git a/frontend-modern/src/components/Settings/__tests__/UnifiedAgents.test.tsx b/frontend-modern/src/components/Settings/__tests__/UnifiedAgents.test.tsx index 88562cf..c54492a 100644 --- a/frontend-modern/src/components/Settings/__tests__/UnifiedAgents.test.tsx +++ b/frontend-modern/src/components/Settings/__tests__/UnifiedAgents.test.tsx @@ -177,7 +177,7 @@ describe('UnifiedAgents token generation', () => { { interval: 0 }, ); expect(notificationSuccessMock).toHaveBeenCalledWith( - 'Token generated with Host and Docker permissions.', + 'Token generated with Host, Docker, and Kubernetes permissions.', 4000, ); }); diff --git a/frontend-modern/src/components/shared/Sparkline.tsx b/frontend-modern/src/components/shared/Sparkline.tsx index f745a3c..6fc3394 100644 --- a/frontend-modern/src/components/shared/Sparkline.tsx +++ b/frontend-modern/src/components/shared/Sparkline.tsx @@ -187,16 +187,10 @@ export const Sparkline: Component = (props) => { // Redraw when data or dimensions change createEffect(() => { - // Read reactive values to track dependencies - // This ensures the effect re-runs when data, metric, or dimensions change - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _data = props.data; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _metric = props.metric; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _w = width(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _h = height(); + void props.data; + void props.metric; + void width(); + void height(); // Unregister previous draw callback if it exists if (unregister) { diff --git a/frontend-modern/src/constants/apiScopes.ts b/frontend-modern/src/constants/apiScopes.ts index b078858..f2e5daa 100644 --- a/frontend-modern/src/constants/apiScopes.ts +++ b/frontend-modern/src/constants/apiScopes.ts @@ -8,6 +8,8 @@ export interface APIScopeOption { export const HOST_AGENT_SCOPE = 'host-agent:report'; export const DOCKER_REPORT_SCOPE = 'docker:report'; export const DOCKER_MANAGE_SCOPE = 'docker:manage'; +export const KUBERNETES_REPORT_SCOPE = 'kubernetes:report'; +export const KUBERNETES_MANAGE_SCOPE = 'kubernetes:manage'; export const MONITORING_READ_SCOPE = 'monitoring:read'; export const MONITORING_WRITE_SCOPE = 'monitoring:write'; export const SETTINGS_READ_SCOPE = 'settings:read'; @@ -38,6 +40,18 @@ export const API_SCOPE_OPTIONS: APIScopeOption[] = [ description: 'Enable agent-triggered runtime commands and host actions.', group: 'Agents', }, + { + value: KUBERNETES_REPORT_SCOPE, + label: 'Kubernetes agent reporting', + description: 'Allow the Kubernetes agent to submit cluster, node, and workload telemetry.', + group: 'Agents', + }, + { + value: KUBERNETES_MANAGE_SCOPE, + label: 'Kubernetes cluster management', + description: 'Allow administrative actions for Kubernetes cluster agents.', + group: 'Agents', + }, { value: HOST_AGENT_SCOPE, label: 'Host agent reporting', diff --git a/frontend-modern/src/stores/__tests__/metricsHistory.test.ts b/frontend-modern/src/stores/__tests__/metricsHistory.test.ts index 15790b5..65b9191 100644 --- a/frontend-modern/src/stores/__tests__/metricsHistory.test.ts +++ b/frontend-modern/src/stores/__tests__/metricsHistory.test.ts @@ -3,7 +3,7 @@ * * These tests focus on the ring buffer operations and metric recording logic. */ -import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; // We can't directly test the module due to side effects (localStorage, window), // so we test the core logic patterns used in the implementation. diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index f8d81d4..3e281d5 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -8,6 +8,8 @@ export interface State { containers: Container[]; dockerHosts: DockerHost[]; removedDockerHosts?: RemovedDockerHost[]; + kubernetesClusters?: KubernetesCluster[]; + removedKubernetesClusters?: RemovedKubernetesCluster[]; hosts: Host[]; replicationJobs: ReplicationJob[]; storage: Storage[]; @@ -38,6 +40,34 @@ export interface RemovedDockerHost { removedAt: number; } +export interface KubernetesCluster { + id: string; + agentId: string; + name?: string; + displayName?: string; + customDisplayName?: string; + server?: string; + context?: string; + version?: string; + status: string; + lastSeen: number; + intervalSeconds: number; + agentVersion?: string; + tokenId?: string; + tokenName?: string; + tokenHint?: string; + tokenLastUsedAt?: number; + hidden?: boolean; + pendingUninstall?: boolean; +} + +export interface RemovedKubernetesCluster { + id: string; + name?: string; + displayName?: string; + removedAt: number; +} + export interface Node { id: string; name: string; diff --git a/frontend-modern/src/types/resource.ts b/frontend-modern/src/types/resource.ts index 24df59c..78d1ae8 100644 --- a/frontend-modern/src/types/resource.ts +++ b/frontend-modern/src/types/resource.ts @@ -12,6 +12,7 @@ export type ResourceType = | 'node' // Proxmox VE node | 'host' // Standalone host (via host-agent) | 'docker-host' // Docker/Podman host + | 'k8s-cluster' // Kubernetes cluster | 'k8s-node' // Kubernetes node | 'truenas' // TrueNAS system | 'vm' // Proxmox VM @@ -134,7 +135,7 @@ export interface Resource { * Helper type guards */ export function isInfrastructure(r: Resource): boolean { - return ['node', 'host', 'docker-host', 'k8s-node', 'truenas'].includes(r.type); + return ['node', 'host', 'docker-host', 'k8s-cluster', 'k8s-node', 'truenas'].includes(r.type); } export function isWorkload(r: Resource): boolean { diff --git a/go.mod b/go.mod index 2fdd729..be0e260 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/rs/zerolog v1.34.0 github.com/shirou/gopsutil/v4 v4.25.11 github.com/spf13/cobra v1.10.1 - github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.45.0 golang.org/x/oauth2 v0.33.0 golang.org/x/sync v0.18.0 @@ -27,6 +26,10 @@ require ( golang.org/x/term v0.37.0 golang.org/x/time v0.14.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.32.0 + k8s.io/apimachinery v0.32.0 + k8s.io/client-go v0.32.0 + modernc.org/sqlite v1.40.1 ) require ( @@ -36,34 +39,48 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.9.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/common v0.67.4 // indirect github.com/prometheus/procfs v0.19.2 // indirect @@ -71,6 +88,7 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect @@ -80,11 +98,20 @@ require ( go.opentelemetry.io/otel/trace v1.38.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/text v0.31.0 // indirect google.golang.org/grpc v1.75.1 // indirect google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gotest.tools/v3 v3.5.2 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.40.1 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index fcd51f985870ac54e166243fc11678aa627ec2a0..46a8ff48aec630676396ed2a8ad33a9cb9e2c382 100644 GIT binary patch delta 10523 zcmbt)ORV!~nwF}kQE6aWREzHFZkU;_ZhFuN>m1*2|7e&>Y$vf}C$SU9u_Zvh-!HKp zJI;VYBg6t)^hjqh8WvnOECK0kkVb<*vqbEWUL)A^-5VKQzJrw^rC4a*rV> zgD^OPvM7o(cODbaEM*5c?z%StI(yt~MQkG)+bz29%Y(XOC%7@Nih>(qcif(5hF=^d z@1)!WeE(-aKK}lPKmPjF&to^u{1_;5SNlyHoHDq>Kk4v$6un1oV(W?OD~ld=>B}xJ z+%g;nX5VR6-==wL5wo+P<%&tvJQV)+pAO?TIyv`lnZsX={A@V*CV?vO!<5Ta)xk(k z!DMd2%`eO;6>F^?Ai{|{r!opWuO0W{um8$#JR-j```{X!?HrtiU&F3b)^LY?(qZ@X zTgtIQ8etK7+SD!u>y}$De-VzkKa_S5|&uS23J4Wl@!J z(fai#9deJd_tXfyY=%j0Uf_HRbGQf@&2G01dvv|J*s@g0+)&08e5tB3XyX#fygBSq51Sc5tv}E(KwG7p-e}&6X?m7l0md7xiJmW-xRb zBGFXg&0#+wP|ZJQP`bXRlMU}31hZ(U;G&yfC872MzxE5)zu5-^pPPr^%|3~j6a}3Q zOPDHE<{Y}((w&7Wxjfj|vMVI*lvm2!d3m7EhPi%b`Y(Ww6qsh{rJsum1A{X?Ce2!v zC3w4lgo2z@M7^Jq%e^b|)*MUU$G_?A@QF5dkB^Y`>%~4c=Xq>9bWruc){bvu`2^cn7uhh$n<9SxP0yK~i<iy^VT?|Jq}p5QVYw0CGe&D|WFwVGNtl49ykwv~IV1n=G`b z5$4m7U;>`1w+(%+num|R`E9N!n>KduKqd6E<`x0eJ#%9?l<;Ia2^3OPv<4Kd;*jj7 zo$gR*I};b>CCbcB3+J`ZiZ>kT%Wy`t$bvpDj|(Iq`&kRw#*E<%snp83t#i588WR5s@ua*Zf3PlPb!1?&i zXDHa)NH}W)F@dzmvg2hUq0T1gRi^PhZUUkUGhJ?BP`*n`N{rkdpNt+Knd28D@hzhQ z2M#D`(V6$fOJL2>jU6Znj>^im!_^F5%&qs5k++|zhLK5ZHOLD{s6Dj!_eet}SIKrAPKYTq>%Dmb+xx6myUSuW^O!{+!tg6- zQ10(yQ0(m%^eAC@V+;PCYT9PcRK7^q$j}t-r4AvvJr60!2*J{16`49=NpZG?qArwo z+3*S)t6LB8WuPO%K90!SV_%k*?ZbMX366#_BsTY$C{a79T;w}qS46A%w0wsRpwqaC z{Vaw3x^^SqO>cb3vEyIgRI_t9lv9p&w)V8&gu&8FWsf{y*t{i%&oA;>RG> zOf?ING8Xl^+L)^>>DS7!M3G%;MrbhLt{InDhC5uNKK$h4@3J5dEi~zS`sT~}+?;gF zvyV5^M&K`N#`2BzHWSsrm}O_@B3nKk^)p?okr!6Z?Jq+dF+5*D2Xe}AoRlWU@z==G zx(xYx?bNAvA@xEh(s>|BvdOM8_u4Vo08+4syYWAudfx z?pRLQm-X*fRDk4LwSJFi9H9^tqX^>8v0aaKX+J={rt(WBF`(sAGj|odoL!z}jzwNys{7?KM~43YR2PT_&1&YmEf{QW)#v(@G|p2y zEai6Aumm?P+H-bV`nD17l&X`@9C51gLa{k^enCb_0|fRvg6d{ zBsMb)rPDd4Z|EGBrEfHedEspYu9@tVKK|2R|DRvG77dsi<01uQ-_Boe#|`uiU579c z;PsSYDu*l`K3rM#YEi3%PZ?DLJScLj(^M<0mo&>SMI3RY`4jFESbFAkKfK0>DCQR0 z>*__B?tM2D7m0>)&?=2&^<`#zi(KY@Xy5FKllSN?&bZ$3XwsT)&a>^wqIOWdf(_1r z!?jD9;cR-4GfR#*nr~iq=1*{Z_yTW4bJJOl$GeO~9VDURm&AfQY15pssoAAsVLp#I zmZOJRR)!D%{O*swvqGQO0Vsx1C_)khc1L;y&Jey& z1kRHyNOso)XkzS?D?rldQj58I0LxdUhqj*&T+Dg7b;(QkbJXFxsz(SK%A zI?4GK)-TiAVwRdLN{KV5?ma$GCh4ADxBclj-Ct^fnQPR1c7}T4tOyVP7XJ8i6yim4 zG`od}Bm)+eN-~9Xqwi>RFR5(YY~AG!-W{dOe!FcM*6d%PyeL3#%m`{rf)Sdz0D3-q z!Cwv=SHREFEI%OcL2#;};!-X`dp1FMTg;inrdE!R-$DLSO+H__nfarW#ndBcgz z-src1iCui}gg~huNhN5J+ITNwYzao|dZ16l?dgZ#L%#m_De}KRxSk1ufrG5HTa>z; zEc14DILf_!+>*y~pSgFdiM4@M4~Z z{lsd{8KQuxVJ8W>I-R+zT@{%T>-ny^b?BFII*6}5RK&^mUL36C{)$cxxpMOuW3XC z>L32-x4!o|GK!!m*aDI`dB@P6%di20lH|D47~*iIv)yrVF=K(z^KR+8b}C4TR~QMN zBm~Q5#mr3uP8=a)2tr|iGEQOm9ZfnI!vJ909q%#ai?~tLU{F?3G~YyeYOWh{S;>UX z)bi5L59lzLBRu-K=1-0eoSYWgn9aw{?O@dCimi0Fk94SMk;6X+N)=?7- zsrO7?I7V7*S^5MtFy(C^y)Qx4INwUu4en*!eg)k1=flI;x1-l$&t%%!;)o1$Yuq}b za@Vdw9ot2_&?xj29wtl1>c|!; zAzilk>clOf{*|VMKkg}5gg^)k$8eOq^BvN09TE*Nd%Q?$*~PpKn8kW^CjA{&w_qNo zuKw;Qa^_Lr7t023K}x{YD!cVFW2?TFMS=Mb0U!DfcMyU?ecxrsu^U`SPTTKxyL=*1 zG`+jX@?3`yC)unKIX$iAn1IQ7D;dodzAYo)E9)i&)G!Q1Fy;>Ne8hD@oKP&Nv7fKb ze{NY{Cd#~d%kxBw04XSflLW@z1tbINCW6~6Jzi}z6_X>21ynL%k^Sz)`|Fnq@kF6@bG#cpjM#Kc0O2@V9>Fmp>*zgMi{d@U5nP^+WD2>Mz_i6TiQJUA)_ zRJoMw6@3)LVp8>smL7&vzg8dq5%*o@#gdb>N*7kOcDXk$YP4>r$}KtZ)r^YkHG5&1 zGw=FkbDGDuhyTK3$^GRIuD4O}X20?%@+9hZT zBfM(>q>gkG(!Hr#<)kqQ5~@ox?LspZBjOyv$0XF3?9MZ>!;U59w>9@yNdpB0WO|ff z2>K3OCb2Z@<0Qt8-ObOfE2oyV!|6oV?1*4(bnDIGLih%_B{7Ii&Zq22_A|43oe4j; zBn}+t)V(XR{3H;RHh4 zHvRC&{F{e=z<>YaTSXM^)Ro2->6`kMiUn;wGqx+8dq$9vS=dBeADrw#3l!>4vmF%-0YECy>t!j%; zoH(J20T1)-W)4yNTa!{&X?PF1p09)*8RS|qf@$c2sZ>LeA9=?lnG3h%W>(KsnxV`I zp)oqXQ)ukgvV8_07r}jp+;SxQcI2UPh2%UxN!^(4#!la%WqCUg32v2yI9BF?pbPTiPrZ2ZV!8 zRRwiqw%^1;UxBjx6B#XdZj%WQY%vZdSlwtt$TdIo3w9(1|O%5u+cF7rb6 zPE|LH!CD3^X~yLEIBylVSizFGwA^}IgH?Usbyg=QbHv`B2WE3#F(MaLU}5)6&b_`y zdPnVzQ}RaVDJWm3#7auwz2|M^;!Ls!e7y<#dR-#j+-Z+2^bRlulkW450L8(0c}p4v zd&uYodg#)|&xz%1PHaraQ#ti~bgyu`(}y}d*z0Jqy@30RXJ)T~M!ayTx}ft;X=x1c z2Wf%2skT_60v}XoaeI+ykzJdi&9|F(7E4c#|JmK?&1lbk1OX}o@J-N+;m941*&fdL z1m@wvjrV*)!|*QAOm;Bl16q~x#WW>}kTmZS;?SD%84>a!P{6@5_%qt?k@FrgFxYl| z_FjPoFNDPD3=uhcfx~pMrTcp;M-G%|73ay|akQxu7^n0gt2~Q4wP45`>jUI~tcI|l zFT#DB^cmN;Y5eJq1}yS5b-(E7Zi*oX7aN>iOJCPK=g$&K+8XBUvYr*Dzoj%#<9Yluvz*)ha1FdvEHD{KL1MN_7)9e@0dIwGO{ZUPjnMv*lh%|vo-itKicG|ZNxt<^O zB{)qZI`-F5n2({`%nNEL+olEp7fO$9WN`< z_rVtG5Kx?q%y$9_3hWp&h%^7@4Npt2d*Z2DTXUu9m6gj%VEy3Mb~-~Bb(i~fxRNLN zdA&*E>m{zM;5dE7@HN_q;o_W!k+rqTC_l z%9vpYPWS2MjpxHHhNHK?GDm`1cDu^Tnx3Lke?&(CJfD(F&k8=UJvk1WpEq-d5C?vYqQ z)94Drj?jk1EB%_XZa;G~uwl7{0dyY9IhcQP1z#!IhA(Evy7f(jPP6UoEQ7j*ZT2Ta zlxQ#)kEy-F8u6no@i%cB*96Co>F~nIZrPrCcES;RYhRg4!e4~lyxD@aobd2d`5(W? zB28PrVyG9sHBHqC#2+xN!pWtpP^`OcR>GFe1}0Yxo<@C1cgN!hrqXx zv{cW@{_27C87?gqI;CV}=FA1sAM$~X@hST%vf48BpDr`O=JjShP#Px{>n&}eAyc_r ztG=3V48IGC6y_E3=*q5{O_-sfK#>_+Zwz!DQPby_b-mi7`2;j8Qwq=4vxQi#G<~mh z=FX*t6S`GQ_^{(a_t?Z?bN`&`Kf9UwtTf(63zWbKl)YnZn{r5>2A9Y9jWn!vkbO?br!};M;D(ccJN`ci8R>&Eg}5l@F-75&&RwXSi3vWk zC5qfpLVX%8d8uBHA~xOgAgsr-Zl~wAr`q@KZaL4t^Wq?Q&f&z{cR&;`@6!3{5W`Gd zQt_E5`;&)vYqXQ7nBziyg^$Xx51y)SJ;_b|eSe@#sbFWQmIgV3s zO$}phd_sG4G7l!4I2S+>8_O8%{3)G-?fT8=3V5lFbFpxMXUJ}`Kz5S~Rn0+KT{Jt# zmYh7`B{|t1_TuerxAy>zcnL{H!>8Sa-fSm?zf=a5)4^S_Wv+5r%plUmGIcZ8u;~VP G-v0#%j3bf& delta 294 zcmV+>0onfR^#QJ$0kAIzliUXuFCb_!I(BL~YADUl<0H z92sDf(;1z!OB{RylPe+`lkXv>ljtIqvmPWc0+SFa2eK#K1+(=oG35K~YVUtu<