From 68ce8e75201b63fc1cc7410bb1683bece3723a34 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sun, 26 Oct 2025 07:56:02 +0000 Subject: [PATCH] feat: finalize swarm service monitoring (#598) --- VERSION | 2 +- cmd/pulse-docker-agent/main.go | 32 + deploy/helm/pulse/Chart.yaml | 4 +- docs/CONFIGURATION.md | 9 +- docs/DOCKER_MONITORING.md | 18 + docs/RELEASE_NOTES.md | 56 +- docs/TEMPERATURE_MONITORING_SECURITY.md | 4 +- docs/development/API_TOKEN_SCOPES.md | 98 --- docs/development/HOME_ASSISTANT_NOTES.md | 15 - docs/development/MOCK_MODE.md | 256 -------- docs/development/PIHOLE_SYNC.md | 11 - docs/development/mock.env.local.example | 27 - docs/frontend-style-guide.md | 82 --- docs/installer-v2-quickref.md | 61 -- .../ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md | 260 -------- docs/operations/ADAPTIVE_POLLING_ROLLOUT.md | 279 --------- docs/operations/audit-log-rotation.md | 163 ----- docs/operations/pulse-sensor-proxy-runbook.md | 105 ---- docs/script-library-guide.md | 208 ------- .../src/components/Alerts/ThresholdsTable.tsx | 159 ++++- .../Alerts/__tests__/ThresholdsTable.test.tsx | 79 ++- .../src/components/Hosts/HostsFilter.tsx | 69 ++- .../src/components/Hosts/HostsOverview.tsx | 12 +- .../src/components/Settings/DockerAgents.tsx | 275 ++++++++- .../src/components/Settings/HostAgents.tsx | 1 - frontend-modern/src/pages/Alerts.tsx | 114 +++- frontend-modern/src/types/alerts.ts | 3 + frontend-modern/src/types/api.ts | 63 ++ internal/alerts/alerts.go | 244 +++++++- internal/alerts/alerts_test.go | 139 +++++ internal/alerts/offline_toggle_test.go | 23 + internal/dockeragent/agent.go | 50 +- internal/dockeragent/agent_internal_test.go | 26 + internal/dockeragent/swarm.go | 563 ++++++++++++++++++ internal/mock/generator.go | 349 +++++++++++ internal/mock/generator_test.go | 99 +++ internal/mock/integration.go | 8 + internal/models/converters.go | 139 +++++ internal/models/models.go | 68 +++ internal/models/models_frontend.go | 68 +++ internal/monitoring/monitor.go | 140 +++++ internal/updates/version.go | 2 +- mock.env | 5 +- pkg/agents/docker/report.go | 88 ++- scripts/toggle-mock.sh | 7 +- 45 files changed, 2823 insertions(+), 1660 deletions(-) delete mode 100644 docs/development/API_TOKEN_SCOPES.md delete mode 100644 docs/development/HOME_ASSISTANT_NOTES.md delete mode 100644 docs/development/MOCK_MODE.md delete mode 100644 docs/development/PIHOLE_SYNC.md delete mode 100644 docs/development/mock.env.local.example delete mode 100644 docs/frontend-style-guide.md delete mode 100644 docs/installer-v2-quickref.md delete mode 100644 docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md delete mode 100644 docs/operations/ADAPTIVE_POLLING_ROLLOUT.md delete mode 100644 docs/operations/audit-log-rotation.md delete mode 100644 docs/operations/pulse-sensor-proxy-runbook.md delete mode 100644 docs/script-library-guide.md create mode 100644 internal/dockeragent/swarm.go diff --git a/VERSION b/VERSION index 4d9fbcf..06edb38 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.25.0 +4.26.0 diff --git a/cmd/pulse-docker-agent/main.go b/cmd/pulse-docker-agent/main.go index cb067ef..b4b02c1 100644 --- a/cmd/pulse-docker-agent/main.go +++ b/cmd/pulse-docker-agent/main.go @@ -66,6 +66,10 @@ func loadConfig() dockeragent.Config { envNoAutoUpdate := strings.TrimSpace(os.Getenv("PULSE_NO_AUTO_UPDATE")) envTargets := strings.TrimSpace(os.Getenv("PULSE_TARGETS")) envContainerStates := strings.TrimSpace(os.Getenv("PULSE_CONTAINER_STATES")) + envSwarmScope := strings.TrimSpace(os.Getenv("PULSE_SWARM_SCOPE")) + envSwarmServices := strings.TrimSpace(os.Getenv("PULSE_SWARM_SERVICES")) + envSwarmTasks := strings.TrimSpace(os.Getenv("PULSE_SWARM_TASKS")) + envIncludeContainers := strings.TrimSpace(os.Getenv("PULSE_INCLUDE_CONTAINERS")) defaultInterval := 30 * time.Second if envInterval != "" { @@ -74,6 +78,26 @@ func loadConfig() dockeragent.Config { } } + swarmScopeDefault := envSwarmScope + if swarmScopeDefault == "" { + swarmScopeDefault = "node" + } + + includeServicesDefault := true + if envSwarmServices != "" { + includeServicesDefault = parseBool(envSwarmServices) + } + + includeTasksDefault := true + if envSwarmTasks != "" { + includeTasksDefault = parseBool(envSwarmTasks) + } + + includeContainersDefault := true + if envIncludeContainers != "" { + includeContainersDefault = parseBool(envIncludeContainers) + } + urlFlag := flag.String("url", envURL, "Pulse server URL (e.g. http://pulse:7655)") tokenFlag := flag.String("token", envToken, "Pulse API token (required)") intervalFlag := flag.Duration("interval", defaultInterval, "Reporting interval (e.g. 30s)") @@ -85,6 +109,10 @@ func loadConfig() dockeragent.Config { flag.Var(&targetFlags, "target", "Pulse target in url|token[|insecure] format. Repeat to send to multiple Pulse instances") var containerStateFlags stringFlagList flag.Var(&containerStateFlags, "container-state", "Only include containers whose status matches this value (repeat to allow multiple). Allowed values: created,running,restarting,removing,paused,exited,dead.") + swarmScopeFlag := flag.String("swarm-scope", strings.ToLower(strings.TrimSpace(swarmScopeDefault)), "Swarm data scope to collect: node, cluster, or auto") + includeServicesFlag := flag.Bool("swarm-services", includeServicesDefault, "Include Swarm service summaries in reports") + includeTasksFlag := flag.Bool("swarm-tasks", includeTasksDefault, "Include Swarm tasks in reports") + includeContainersFlag := flag.Bool("include-containers", includeContainersDefault, "Include per-container metrics in reports") flag.Parse() @@ -146,6 +174,10 @@ func loadConfig() dockeragent.Config { DisableAutoUpdate: *noAutoUpdateFlag, Targets: targets, ContainerStates: containerStates, + SwarmScope: strings.ToLower(strings.TrimSpace(*swarmScopeFlag)), + IncludeServices: *includeServicesFlag, + IncludeTasks: *includeTasksFlag, + IncludeContainers: *includeContainersFlag, } } diff --git a/deploy/helm/pulse/Chart.yaml b/deploy/helm/pulse/Chart.yaml index fc1d7ca..2ff6913 100644 --- a/deploy/helm/pulse/Chart.yaml +++ b/deploy/helm/pulse/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pulse description: Helm chart for deploying the Pulse hub and optional Docker monitoring agent. type: application -version: 4.25.0 -appVersion: "4.25.0" +version: 4.26.0 +appVersion: "4.26.0" icon: https://raw.githubusercontent.com/rcourtman/Pulse/main/docs/images/pulse-logo.svg keywords: - monitoring diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index c58d6a6..f63ab83 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -224,7 +224,11 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout "dockerDefaults": { "cpu": { "trigger": 75, "clear": 60 }, "restartCount": 3, - "restartWindow": 300 + "restartWindow": 300, + "memoryWarnPct": 90, + "memoryCriticalPct": 95, + "serviceWarnGapPercent": 10, + "serviceCriticalGapPercent": 50 }, "dockerIgnoredContainerPrefixes": [ "runner-", @@ -239,7 +243,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout "guest": { "disk": 120, "networkOut": 240 } }, "overrides": { - "delly.lan/qemu/101": { + "example.lan/qemu/101": { "memory": { "trigger": 92, "clear": 80 }, "networkOut": -1, "poweredOffSeverity": "warning" @@ -280,6 +284,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout - `timeThresholds` apply a grace period before an alert fires; `metricTimeThresholds` allow per-metric overrides (e.g., delay network alerts longer than CPU). - `overrides` are indexed by the stable resource ID returned from `/api/state` (VMs: `instance/qemu/vmid`, containers: `instance/lxc/ctid`, nodes: `instance/node`). - `dockerIgnoredContainerPrefixes` lets you silence state/metric/restart alerts for ephemeral containers whose names or IDs share a common, case-insensitive prefix. The Docker tab in the UI keeps this list in sync. +- Swarm service alerts track missing replicas: `serviceWarnGapPercent` defines when a warning fires, and `serviceCriticalGapPercent` must be greater than or equal to the warning gap (Pulse automatically clamps the critical value upward if an older client submits something smaller). - Quiet hours, escalation, deduplication, and restart loop detection are all managed here, and the UI keeps the JSON in sync automatically. > Tip: Back up `alerts.json` alongside `.env` during exports. Restoring it preserves all overrides, quiet-hour schedules, and webhook routing. diff --git a/docs/DOCKER_MONITORING.md b/docs/DOCKER_MONITORING.md index 921808e..ba549b4 100644 --- a/docs/DOCKER_MONITORING.md +++ b/docs/DOCKER_MONITORING.md @@ -88,6 +88,20 @@ The binary reads standard Docker environment variables. If you already use TLS-s High churn environments can flood Pulse with noise from short-lived tasks. Restrict the agent to the container states you care about by repeating `--container-state` (for example, `--container-state running --container-state paused`) or by exporting `PULSE_CONTAINER_STATES=running,paused`. Allowed values match Docker’s status filter: `created`, `running`, `restarting`, `removing`, `paused`, `exited`, and `dead`. If no values are provided the agent reports every container, mirroring the previous behaviour. +### Swarm-aware reporting + +The agent now recognises Docker Swarm roles. Managers query the Swarm control plane for service and task metadata, while workers fall back to the labels present on local containers. The **Settings → Docker Agents** view surfaces role, scope, service counts, and updates per host so you can spot noisy stacks or unhealthy rollouts at a glance. + +Use the new flags to tune the payload: + +- `--swarm-scope` / `PULSE_SWARM_SCOPE` chooses between node-only and cluster-wide aggregation (`auto` switches based on the node’s role). +- `--swarm-services` and `--swarm-tasks` disable service or task blocks if you only need a subset of data. +- `--include-containers` removes per-container metrics when service-level reporting is sufficient (note that workers need container data to derive task info). + +If a manager cannot reach the Swarm API the agent automatically falls back to node scope so updates keep flowing. + +Adjust warning and critical replica gaps (or disable service alerts entirely) under **Alerts → Thresholds → Docker** in the Pulse UI. + ### Multiple Pulse instances A single `pulse-docker-agent` process can now serve any number of Pulse backends. Each target entry keeps its own API token and TLS preference, and Pulse de-duplicates reports using the shared agent ID / machine ID. This avoids running duplicate agents on busy Docker hosts. @@ -139,6 +153,10 @@ docker run -d \ | `--target`, `PULSE_TARGETS` | One or more `url|token[|insecure]` entries to fan-out reports to multiple Pulse servers. Separate entries with `;` or repeat the flag. | — | | `--interval`, `PULSE_INTERVAL` | Reporting cadence (supports `30s`, `1m`, etc.). | `30s` | | `--container-state`, `PULSE_CONTAINER_STATES` | Limit reports to specific Docker statuses (`created`, `running`, `restarting`, `removing`, `paused`, `exited`, `dead`). Separate multiple values with commas/semicolons or repeat the flag. | — | +| `--swarm-scope`, `PULSE_SWARM_SCOPE` | Swarm data scope: `node`, `cluster`, or `auto` (auto picks cluster on managers, node on workers). | `node` | +| `--swarm-services`, `PULSE_SWARM_SERVICES` | Include Swarm service summaries in reports. | `true` | +| `--swarm-tasks`, `PULSE_SWARM_TASKS` | Include individual Swarm tasks in reports. | `true` | +| `--include-containers`, `PULSE_INCLUDE_CONTAINERS` | Include per-container metrics (disable when only Swarm data is needed). | `true` | | `--hostname`, `PULSE_HOSTNAME` | Override host name reported to Pulse. | Docker info / OS hostname | | `--agent-id`, `PULSE_AGENT_ID` | Stable ID for the agent (useful for clustering). | Docker engine ID / machine-id | | `--insecure`, `PULSE_INSECURE_SKIP_VERIFY` | Skip TLS cert validation (unsafe). | `false` | diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index ecf687c..9c52339 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,12 +1,54 @@ -# Upcoming Release Highlights +# Pulse v4.26.0 -## Improvements +## What's Changed +### New +- Standalone host agents now ship with guided Linux, macOS, and Windows installers that stream registration status back to Pulse, generate scoped commands from **Settings → Agents**, and feed host metrics into alerts alongside Proxmox and Docker. +- Alert thresholds gained host-level overrides, connectivity toggles, and snapshot size guardrails so you can tune offline behaviour per host while keeping a global policy for other resources. +- API tokens now support fine-grained scopes with a redesigned manager that previews command templates, highlights unused credentials, and makes revocation a single click. +- Proxmox replication jobs surface in a dedicated **Settings → Hosts → Replication** view with API plumbing to track task health and bubble failures into the monitoring pipeline. +- Docker Swarm environments now receive service/task-aware reporting with configurable scope, plus a Docker settings view that highlights manager/worker roles, stack health, rollout status, and service alert thresholds. -- Host agent installs now report back to Pulse in real time. The **Settings → Agents → Host agents** view surfaces the lookup status, highlights the matching row, and refreshes the connection badge without reloading. -- Host agents participate in the alert system alongside Proxmox and Docker sources. Offline detection, thresholds, and overrides live in **Settings → Alerts** and flow through the existing notification channels. +### Improvements +- Dashboard loads and drawer links respond faster thanks to cached guest metadata, reduced polling allocations, and inline URL editing that no longer flashes on WebSocket updates. +- Settings navigation is reorganized with dedicated Docker and Hosts sections, richer filters, and platform icons that make agent onboarding and discovery workflows clearer. +- LXC guests now report dynamic interface IPs, configuration metadata, and queue metrics so alerting, discovery, and drawers stay accurate even during rapid container churn. +- Notifications consolidate into a consistent toast system, with clearer feedback during agent setup, token generation, and background job state changes. -## Documentation +### Bug Fixes +- Enforced explicit node naming and respected custom Proxmox ports so cluster discovery, overrides, and disk monitoring defaults remain intact after edits. +- Hardened setup-token flows and checksum handling in the installers to prevent stale credentials and guarantee the correct binaries are fetched. +- Treated 501 responses from the Proxmox API as non-fatal during failover, restored FreeBSD disk counter parsing, and stopped guest link icons from re-triggering animations on updates. +- Preserved inline editor state across WebSocket refreshes and ensured Docker host identifiers stay collision-safe in mixed environments. -- Updated the host agent guide with the new lookup workflow, token-free commands, and alert integration notes. +## Installation +- **Install or upgrade with the helper script** + ```bash + curl -sL https://github.com/rcourtman/Pulse/releases/latest/download/install.sh | bash + ``` +- **Binary upgrade on systemd hosts** + ```bash + sudo systemctl stop pulse + curl -fsSL https://github.com/rcourtman/Pulse/releases/download/v4.26.0/pulse-v4.26.0-linux-amd64.tar.gz \ + | sudo tar -xz -C /opt/pulse --strip-components=1 + sudo systemctl start pulse + ``` +- **Docker update** + ```bash + docker pull rcourtman/pulse:v4.26.0 + docker stop pulse || true + docker rm pulse || true + docker run -d --name pulse --restart unless-stopped -p 7655:7655 rcourtman/pulse:v4.26.0 + ``` +- **Helm upgrade** + ```bash + helm upgrade --install pulse oci://ghcr.io/rcourtman/pulse-chart \ + --version 4.26.0 \ + --namespace pulse --create-namespace + ``` -_(Replace this stub with per-version notes when tagging the release.)_ +## Downloads +- Multi-arch Linux tarballs (amd64/arm64/armv7) +- Standalone sensor proxy binaries +- Helm chart archive (pulse-4.26.0-helm.tgz) +- SHA256 checksums (checksums.txt) +- Docker tags: rcourtman/pulse:v4.26.0, :4.26, :4, :latest diff --git a/docs/TEMPERATURE_MONITORING_SECURITY.md b/docs/TEMPERATURE_MONITORING_SECURITY.md index 9f12071..ea2dec4 100644 --- a/docs/TEMPERATURE_MONITORING_SECURITY.md +++ b/docs/TEMPERATURE_MONITORING_SECURITY.md @@ -285,7 +285,7 @@ pulse_proxy_rpc_requests_total{method="get_temperature",result="success"} pulse_proxy_rpc_requests_total{method="ensure_cluster_keys",result="unauthorized"} # SSH request latency -pulse_proxy_ssh_latency_seconds{node="delly"} +pulse_proxy_ssh_latency_seconds{node="example-node"} # Active connections pulse_proxy_queue_depth @@ -417,7 +417,7 @@ curl -s --unix-socket /run/pulse-sensor-proxy/pulse-sensor-proxy.sock \ -X POST -d '{"method":"get_status"}' | jq # 5. Check SSH connectivity -ssh root@delly "sensors -j" +ssh root@example-node "sensors -j" # 6. Inspect adaptive polling for temperature pollers curl -s http://localhost:7655/api/monitoring/scheduler/health \ diff --git a/docs/development/API_TOKEN_SCOPES.md b/docs/development/API_TOKEN_SCOPES.md deleted file mode 100644 index 6949206..0000000 --- a/docs/development/API_TOKEN_SCOPES.md +++ /dev/null @@ -1,98 +0,0 @@ -# API Token Scope Design Brief - -## Objective -Introduce scoped API tokens so administrators can grant the minimum necessary permissions to each integration (Docker agent, host agent, future platform agents, automation scripts, etc.). This replaces today’s “all-or-nothing” tokens and provides safer rotation/revocation paths. - -## Security Rationale -- Agent and automation tokens are often deployed on hosts or third-party services we do not fully trust. If one leaks today, the attacker inherits full administrator powers (issuing new tokens, mutating settings, triggering installs/updates, etc.). -- Constraining tokens to the minimal scope limits the blast radius: a compromised reporting agent can only submit telemetry, not reconfigure Pulse or other integrations. -- Customers operating in regulated or multi-team environments increasingly ask for auditable least-privilege controls. Scopes give us the primitives to surface warnings on over-privileged tokens and eventually add rotation workflows. -- The feature still has to earn adoption, so pair the technical work with UI nudges and reporting that highlight “Full access” tokens and encourage admins to narrow permissions. - -## Requirements Overview - -1. **Token Model Changes** - - Each API token record stores a list of scopes (strings) or a bitmask. Recommended canonical strings: - - `monitoring:read` - - `monitoring:write` - - `docker:report` - - `docker:manage` - - `host-agent:report` - - `settings:read` - - `settings:write` - - `*` (legacy full-access sentinel; backend accepts for migration/edit flows but the UI should not expose it) - - Existing tokens must remain valid. Treat missing scopes as `["*"]` (full access) until the admin edits them. - -2. **Persistence & Migration** - - Extend the token persistence layer (currently BoltDB JSON) to include `scopes: []string`. - - On startup, detect tokens without the new field and default to `["*"]`. - - Expose the complete scope list when returning token metadata (internal API used by Settings UI). - -3. **Middleware Enforcement** - - Add a helper `RequireScope(scope string)` that checks the request’s token record for `scope` or `*`. - - Apply the helper according to the table below: - - | Endpoint (or group) | HTTP verbs | Required scope(s) | Notes | - |-------------------------------------------------|-------------------------------------|-----------------------------|------------------------------------------------------| - | `/api/agents/docker/report` | `POST` | `docker:report` | Docker agent heartbeat payloads | - | `/api/agents/docker/commands/*` | `POST` | `docker:manage` (optional) | If we expose command ack/management over tokens | - | `/api/agents/docker/hosts/*` | `DELETE`, `PUT`, `POST` | `docker:manage` | Admin actions for Docker hosts | - | `/api/agents/host/report` | `POST` | `host-agent:report` | Host agent reporting | - | `/api/state` | `GET` | `monitoring:read` | General state polling (if token-authenticated) | - | `/api/alerts/*` | `GET` | `monitoring:read` | Alerts reading APIs | - | `/api/alerts/*` (mutations) | `POST`, `PUT`, `DELETE` | `monitoring:write` | Acknowledge, silence, etc. | - | `/api/settings/*` | `GET` | `settings:read` | Settings reads via API | - | `/api/settings/*` | `POST`, `PUT`, `DELETE`, `PATCH` | `settings:write` | Any settings mutation | - | `/api/security/tokens*` | all verbs | _n/a (session only)_ | Leave browser-session only; do not allow API tokens yet | - | `/api/install/*`, `/api/updates/*` (mutations) | `POST`, `PUT` | `settings:write` | Sensitive operational endpoints | - - (More endpoints can be added as required; start with the rows above and expand during implementation.) - - Maintain compatibility for admin sessions (browser login) which continue to bypass token checks. - -4. **Token Generation API** - - Update `POST /api/security/tokens` to accept a `scopes` array. - - Validation rules: - - Reject unknown scope identifiers (except the `*` sentinel described above). - - If the array is omitted (legacy callers), default to `['*']` (full access) to preserve backward compatibility. - - If the array is provided but empty, reject with a 400 (“select at least one scope or delete the token”). - - Reject mixed arrays that contain both `'*'` and explicit scopes; if the UI submits such a payload, return a 400 with guidance (“either all scopes or full access”). - - Include the scope list in the response payload so the UI can display it. - -5. **UI/UX Adjustments** - - **Settings → Security** panel: - - When generating or editing a token, show a multi-select with friendly labels (“Docker agent reporting”, “Host agent reporting”, etc.). - - Display the scope summary in the token list (e.g. badges). - - For legacy tokens (implicit `*`), show “Full access” and allow editing to reduce scope. - - **Docker Agents / Host Agents screens:** - - When requesting a token: - - Docker: pre-select both `docker:report` (always) and `docker:manage` if the user needs lifecycle commands (hide manage behind a toggle if desired). - - Host agent: pre-select `host-agent:report`. - - Warn if the stored token lacks the required scope (fallback to showing `` placeholder). - -6. **Testing** - - Unit tests covering: - - Scope parsing/migration - - Middleware checks (token with/without required scope) - - Integration tests for agent endpoints verifying 403 on missing scope. - -7. **Documentation** - - Update `README.md` and relevant docs (e.g. `docs/CONFIGURATION.md`, `docs/HOST_AGENT.md`, Docker docs) to explain scoped tokens. - - Provide an upgrade note for existing installations (“legacy tokens default to full access; edit to restrict scope”). - -## Implementation Notes - -- Use constants for scope strings to avoid typos throughout the codebase. -- Token middleware already retrieves `APITokenRecord`; that struct should grow a `Scopes []string` field with helper methods (`HasScope`). -- For future extensibility, keep the scope checks granular but simple (string equality) rather than regex matching. -- Ensure the Settings UI gracefully handles lack of admin privileges (disable scope selection, show hint). -- Update agent commands (Docker/Host) to mention required scope in their description. -- Guardrails: the backend should never auto-insert `*` once a scoped token exists, and any admin edit that clears all scopes should surface a clear “delete token or assign scopes” decision. - -## Acceptance Criteria - -- Scoped tokens persisted and surfaced via API. -- Middleware rejects tokens missing required scope. -- UI can create, edit, and display scoped tokens; agent panels auto-fill only when valid. -- Documentation updated; existing tokens remain functional without manual migration. - -Once implemented delete this doc. diff --git a/docs/development/HOME_ASSISTANT_NOTES.md b/docs/development/HOME_ASSISTANT_NOTES.md deleted file mode 100644 index 06c35c1..0000000 --- a/docs/development/HOME_ASSISTANT_NOTES.md +++ /dev/null @@ -1,15 +0,0 @@ -# Home Assistant Battery Automation Notes (2025-10-10) - -- Work performed on delly host, LXC VMID 101 (Home Assistant). Pulse codebase unaffected. -- Root cause: malformed `automations.yaml` caused five restarts between 00:23–00:33 BST. -- Fixes applied: - - Restored automation backups. - - Replaced `pyscript.solis_set_charge_current` calls with `number.set_value` targeting `number.solis_rhi_time_charging_charge_current`. - - Removed invalid `source:` attributes from `number.set_value` actions. -- Validation: - - Triggered key automations (`automation.free_electricity_session_maximum_charging`, `automation.intelligent_dispatch_solis_battery_charging`, `automation.emergency_low_soc_protection`, EV guard hold/refresh/release). - - Observed expected charge-current adjustments (100 A car slots/free session, 25 A emergency, 5 A guard) and matching system logs. -- Defaults restored: overnight slot `23:30–05:30`. -- Backups: `/var/lib/docker/volumes/hass_config/_data/automations.yaml.codex_source_cleanup_20251010_090205` (do not delete). -- Testing helpers: API token stored at `/tmp/ha_token.txt` inside VM; use `pct exec 101 -- bash -lc '...'` for commands. -- Final state: charge current reset to 25 A; EV guard sensor `off`. diff --git a/docs/development/MOCK_MODE.md b/docs/development/MOCK_MODE.md deleted file mode 100644 index 8de7b2d..0000000 --- a/docs/development/MOCK_MODE.md +++ /dev/null @@ -1,256 +0,0 @@ -# Mock Mode Development Guide - -Mock mode allows you to develop and test Pulse without requiring real Proxmox infrastructure. It generates realistic mock data including nodes, VMs, containers, storage, backups, and alerts. - -## Quick Start - -### Toggling Mock Mode - -During hot-dev mode (`scripts/hot-dev.sh`), use these npm commands to toggle mock mode: - -```bash -# Enable mock mode -npm run mock:on - -# Disable mock mode (use real infrastructure) -npm run mock:off - -# Check current status -npm run mock:status - -# Edit mock configuration -npm run mock:edit -``` - -The backend will **automatically reload** when you toggle mock mode - no manual restarts needed! - -### Configuration - -Mock mode is configured via `mock.env` in the project root. This file is **tracked in the repository** with sensible defaults, so mock mode works out of the box for all developers. - -**Default configuration (mock.env):** -```bash -PULSE_MOCK_MODE=false # Disabled by default -PULSE_MOCK_NODES=7 -PULSE_MOCK_VMS_PER_NODE=5 -PULSE_MOCK_LXCS_PER_NODE=8 -PULSE_MOCK_DOCKER_HOSTS=3 -PULSE_MOCK_DOCKER_CONTAINERS=12 -PULSE_MOCK_RANDOM_METRICS=true -PULSE_MOCK_STOPPED_PERCENT=20 -``` - -**Local overrides (not tracked):** -Create `mock.env.local` for personal settings that won't be committed: -```bash -# mock.env.local - your personal settings -PULSE_MOCK_MODE=true # Always start in mock mode -PULSE_MOCK_NODES=3 # Fewer nodes for faster startup -``` - -The `.local` file overrides values from `mock.env`, and is gitignored to keep your personal preferences private. - -**Configuration options:** - -- `PULSE_MOCK_MODE`: Enable/disable mock mode (`true`/`false`) -- `PULSE_MOCK_NODES`: Number of nodes to generate (default: 7) -- `PULSE_MOCK_VMS_PER_NODE`: Average VMs per node (default: 5) -- `PULSE_MOCK_LXCS_PER_NODE`: Average containers per node (default: 8) -- `PULSE_MOCK_DOCKER_HOSTS`: Number of Docker hosts to generate (default: 3) -- `PULSE_MOCK_DOCKER_CONTAINERS`: Average containers per Docker host (default: 12) -- `PULSE_MOCK_RANDOM_METRICS`: Enable metric fluctuations (`true`/`false`) -- `PULSE_MOCK_STOPPED_PERCENT`: Percentage of guests in stopped state (default: 20) - -### Data Isolation - -Hot-dev mode now isolates mock data from production credentials automatically: - -- **Mock mode:** data lives in `/opt/pulse/tmp/mock-data` -- **Production mode:** data lives in `/etc/pulse` - -The toggle script exports `PULSE_DATA_DIR` before launching the backend, creates the temporary directory if needed, and cleans up on exit. This guarantees mock credentials never overwrite your real cluster configuration and makes it safe to flip between datasets repeatedly during a session. - -## Hot-Dev Workflow - -Hot reload now runs as a long-lived systemd service so it is always ready when you log in. - -1. **Check the hot-dev service:** - ```bash - systemctl status pulse-hot-dev - ``` - The service is enabled by default; use `sudo systemctl restart pulse-hot-dev` if you need a clean rebuild. - -2. **Toggle mock mode as needed:** - ```bash - npm run mock:on # Backend auto-reloads with mock data - npm run mock:off # Backend auto-reloads with real data - ``` - -3. **Edit mock configuration:** - ```bash - npm run mock:edit # Opens mock.env in your editor - # Save and exit - backend auto-reloads! - ``` - -4. **Frontend changes:** Just save your files - Vite hot-reloads instantly - -**No port changes. No manual restarts. Everything just works!** - -> **Note:** The legacy `pulse-backend.service` is intentionally disabled on this dev box. All backend/API traffic comes from the hot-dev service, so you never need to run the production binary locally. - -### Default credentials - -Authentication is enabled for the dev stack so security-focused features behave exactly like production. Use the shared credentials below when the UI prompts for a login: - -``` -Username: dev -Password: dev -``` - -You can change them at any time by editing `.env` at the repo root (the backend watcher loads it automatically on restart). - - -## Mock Data Generation - -Mock mode generates: - -- **Nodes**: Mix of clustered and standalone nodes -- **Cluster**: First 5 nodes form `mock-cluster`, rest are standalone -- **VMs & Containers**: Realistic distribution with various states -- **Storage**: Local, ZFS, PBS, and shared NFS storage -- **Backups**: Both PVE and PBS backups with realistic metadata -- **Mail Gateway**: PMG instances with mail throughput, spam/virus totals, and quarantine counts -- **Alerts**: CPU, memory, disk, and connectivity alerts -- **Metrics**: Live-updating metrics every 2 seconds - -### Node Characteristics - -- **Clustered nodes** (`pve1`-`pve5`): Part of `mock-cluster` -- **Standalone nodes** (`standalone1`, etc.): Independent instances -- **Offline nodes**: `pve3` is always offline to test error handling -- **Host URLs**: Each node has `Host` field set (e.g., `https://pve1.local:8006`) -- **Cluster fields**: `IsClusterMember` and `ClusterName` properly set - -## API Behavior in Mock Mode - -### Fast, Cached Responses - -In mock mode, `/api/state` returns **cached data instantly** - no locks, no delays, no timeouts. The mock data is stored in memory and updated every 2 seconds with realistic metric fluctuations. - -### WebSocket Updates - -The WebSocket connection receives updates every 2 seconds with changing metrics, just like production. - -### Dashboard Grouping - -Mock nodes include all required fields for proper dashboard grouping: -- `isClusterMember`: Boolean indicating cluster membership -- `clusterName`: Name of the cluster (e.g., "mock-cluster") -- `host`: Full node URL (e.g., "https://pve1.local:8006") - -## Demo Server Usage - -On the demo server, mock mode works the same way: - -### Systemd Service - -If using systemd (`pulse-dev` service): - -```bash -# Toggle mock mode (restarts service) -sudo /opt/pulse/scripts/toggle-mock.sh on -sudo /opt/pulse/scripts/toggle-mock.sh off - -# Check status -/opt/pulse/scripts/toggle-mock.sh status -``` - -### Manual Mode - -If running the backend manually: - -```bash -# Edit mock.env -nano /opt/pulse/mock.env - -# The file watcher will detect changes and auto-reload the backend -# within 5 seconds (or immediately if fsnotify is working) -``` - -## File Watcher Details - -The backend watches `mock.env` using: - -1. **Primary**: `fsnotify` (instant notification on file changes) -2. **Fallback**: Polling every 5 seconds (if fsnotify fails) - -When `mock.env` changes: -- Environment variables are updated -- Monitor is reloaded with new configuration -- New mock data is generated -- WebSocket clients receive updated state - -**No manual process restarts required!** - -## Troubleshooting - -### Mock mode not updating - -1. Check that mock.env exists: `ls -la /opt/pulse/mock.env` -2. Check file watcher logs: Look for "Detected mock.env file change" in logs -3. Verify environment variables: `env | grep PULSE_MOCK` -4. Try touching the file: `touch /opt/pulse/mock.env` - -### Backend not reloading - -1. Ensure the `pulse-hot-dev` systemd service is active -2. Check for errors in backend logs -3. Verify file watcher started successfully -4. Fall back to manual restart if needed - -### Missing cluster grouping - -1. Verify mock data includes `isClusterMember` and `clusterName` -2. Check API response: `curl http://localhost:7656/api/state | jq '.nodes[0]'` -3. Ensure frontend is receiving WebSocket updates - -## Implementation Details - -### Backend Components - -- **Config Watcher** (`internal/config/watcher.go`): Watches both `.env` and `mock.env` -- **Mock Integration** (`internal/mock/integration.go`): Manages mock state and updates -- **Mock Generator** (`internal/mock/generator.go`): Generates realistic mock data -- **Monitor** (`internal/monitoring/monitor.go`): Returns cached mock data when enabled - -### Auto-Reload Flow - -1. User runs `npm run mock:on` (or edits `mock.env`) -2. `toggle-mock.sh` updates `mock.env` and touches the file -3. Config watcher detects file change (via fsnotify or polling) -4. Watcher triggers reload callback -5. ReloadableMonitor reloads with fresh config -6. New monitor instance starts with updated mock mode -7. WebSocket broadcasts new state to connected clients - -### Performance - -- Mock data generation: < 100ms for 7 nodes with 90+ guests -- State snapshot: Instant (returns cached data) -- Memory usage: ~50MB additional for mock data -- Update interval: 2 seconds for metric fluctuations - -## Best Practices - -1. **Use mock mode for frontend development** - Fast, predictable data -2. **Test with real data before PRs** - Ensure real infrastructure works -3. **Adjust mock config to test edge cases** - High load, many nodes, etc. -4. **Use mock.env.local for personal settings** - Your preferences won't be committed -5. **Keep mock.env defaults reasonable** - Other developers will use them -6. **Document any mock data assumptions** - Help other developers - -## See Also - -- [CONFIGURATION.md](../CONFIGURATION.md) - Production configuration -- [TROUBLESHOOTING.md](../TROUBLESHOOTING.md) - Common issues -- [API.md](../API.md) - API documentation diff --git a/docs/development/PIHOLE_SYNC.md b/docs/development/PIHOLE_SYNC.md deleted file mode 100644 index 7f97107..0000000 --- a/docs/development/PIHOLE_SYNC.md +++ /dev/null @@ -1,11 +0,0 @@ -# Pi-hole Nebula Sync Notes - -- Primary Pi-hole: delly CT 114 (192.168.0.102) - Secondary: minipc CT 202 (192.168.0.101) - Virtual IP: 192.168.0.100 -- Runs on the delly host (not inside containers). -- Binary: `/usr/local/bin/nebula-sync`; wrapper script: `/usr/local/bin/pihole-sync.sh`. -- Cron job (`root@delly`): `*/30 * * * *` → logs written to `/var/log/nebula-sync.log`. -- Credentials: `/root/.pihole-sync-credentials` on delly (chmod 600). Request the password from the user if needed. -- Both Pi-holes require `app_sudo = true` inside `/etc/pihole/pihole.toml`. -- When debugging, coordinate with the user before touching production Pi-hole instances. diff --git a/docs/development/mock.env.local.example b/docs/development/mock.env.local.example deleted file mode 100644 index fe33cdf..0000000 --- a/docs/development/mock.env.local.example +++ /dev/null @@ -1,27 +0,0 @@ -# mock.env.local.example -# Copy this to /opt/pulse/mock.env.local for your personal mock mode settings -# The .local file is gitignored and will override values from mock.env -# -# Usage: -# cp docs/development/mock.env.local.example mock.env.local -# # Edit mock.env.local with your preferences -# npm run mock:on - -# Example: Always start in mock mode for frontend development -PULSE_MOCK_MODE=true - -# Example: Use fewer nodes for faster startup -PULSE_MOCK_NODES=3 - -# Example: More VMs for testing high-density scenarios -# PULSE_MOCK_VMS_PER_NODE=10 - -# Example: Adjust Docker coverage -# PULSE_MOCK_DOCKER_HOSTS=2 -# PULSE_MOCK_DOCKER_CONTAINERS=6 - -# Example: Disable metric fluctuations for consistent testing -# PULSE_MOCK_RANDOM_METRICS=false - -# Example: All guests running (no stopped VMs/containers) -# PULSE_MOCK_STOPPED_PERCENT=0 diff --git a/docs/frontend-style-guide.md b/docs/frontend-style-guide.md deleted file mode 100644 index ff58510..0000000 --- a/docs/frontend-style-guide.md +++ /dev/null @@ -1,82 +0,0 @@ -# Frontend UI Style Guide - -This project now ships a handful of shared primitives to keep typography and form layouts consistent. The snippets below show the preferred usage. - -## Section headers - -Use `SectionHeader` for any inline card titles, modal headings, or sub-section titles instead of ad-hoc `

`/`

` elements. - -```tsx -import { SectionHeader } from '@/components/shared/SectionHeader'; - - -``` - -Pass `titleClass`/`descriptionClass` when you need to tweak color or emphasis without rebuilding the layout. - -## Empty states - -Whenever a panel needs to show a loading, error, or "no data" treatment, render `EmptyState` inside a `Card`. - -```tsx -import { Card } from '@/components/shared/Card'; -import { EmptyState } from '@/components/shared/EmptyState'; - - - } - title="No backups yet" - description="Run your first job or adjust the filters to see activity." - actions={( - - )} - /> - -``` - -Icons and actions are optional; omit them when not needed. - -## Form helpers - -Shared form styles live in `@/components/shared/Form`. Import the helpers and apply them to each field container, label, and control for a uniform look. - -```tsx -import { formField, labelClass, controlClass, formHelpText, formCheckbox } from '@/components/shared/Form'; - -
- - -

- Use HTTPS on port 8006 for Proxmox VE and 8007 for PBS. -

-
- - -``` - -Helper summary: - -- `formField`: wraps a label + control stack. -- `labelClass(extra?)`: base typography for labels, with optional extra classes. -- `controlClass(extra?)`: base input styling; append sizing tweaks (`px-2 py-1.5`) as needed. -- `formHelpText`: small secondary text (validation notes, hints). -- `formCheckbox`: shared checkbox styling for toggles inside copy-heavy forms. - -Stick to these helpers when building new settings panels, modals, or detail cards. If a component needs a variant that the helpers do not cover, extend them in `Form.ts` so the convention remains centralized. diff --git a/docs/installer-v2-quickref.md b/docs/installer-v2-quickref.md deleted file mode 100644 index 04d4be5..0000000 --- a/docs/installer-v2-quickref.md +++ /dev/null @@ -1,61 +0,0 @@ -# Installer v2 Quick Reference - -## Opt-In / Opt-Out - -```bash -# Use the new installer -export PULSE_INSTALLER_V2=1 -curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- [flags] - -# Force the legacy installer -export PULSE_INSTALLER_V2=0 -curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- [flags] -``` - -## Common Flags - -- `--url ` — Primary Pulse server URL -- `--token ` — API token for enrollment -- `--target ` — Additional targets (repeatable) -- `--interval ` — Poll interval (default `30s`) -- `--dry-run` — Show actions without applying changes -- `--uninstall` — Remove agent binary, systemd unit, and startup hooks - -## Examples - -```bash -# Preview installation without changes -export PULSE_INSTALLER_V2=1 -curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- \ - --dry-run \ - --url https://pulse.example \ - --token - -# Install with two targets and custom interval -export PULSE_INSTALLER_V2=1 -curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- \ - --url https://pulse-primary \ - --token \ - --target https://pulse-dr| \ - --target https://pulse-edge||true \ - --interval 15s - -# Uninstall -curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- --uninstall -``` - -## Verification - -- Binary path: `/usr/local/bin/pulse-docker-agent` -- Systemd unit: `/etc/systemd/system/pulse-docker-agent.service` -- Logs: `journalctl -u pulse-docker-agent -f` - -## Rollback - -```bash -# Force legacy installer -export PULSE_INSTALLER_V2=0 -curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- ... -``` - -Contact support or the Pulse engineering team if issues arise during rollout. diff --git a/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md b/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md deleted file mode 100644 index 71bbfcc..0000000 --- a/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md +++ /dev/null @@ -1,260 +0,0 @@ -# Adaptive Polling Management Endpoints (Future Enhancement) - -## Status: DEFERRED - -**Decision Date:** 2025-10-20 -**Re-evaluated:** v4.24.0 GA release -**Current Status:** Not implemented in v4.24.0 - -## Overview - -Manual circuit breaker and dead-letter queue (DLQ) management endpoints are **not included in v4.24.0**. The read-only scheduler health API (`/api/monitoring/scheduler/health`) provides full visibility, and automatic recovery mechanisms have proven sufficient during testing and early production rollouts. - ---- - -## What's Available in v4.24.0 - -### Read-Only Scheduler Health API - -**Endpoint:** `GET /api/monitoring/scheduler/health` - -Provides complete visibility into: -- Queue depth and task distribution -- Circuit breaker states per instance -- Dead-letter queue contents and retry schedules -- Per-instance staleness tracking -- Failure streaks and error categorization - -**Documentation:** See [Scheduler Health API](../api/SCHEDULER_HEALTH.md) for complete reference. - -### Existing Management Options - -**v4.24.0 operators can:** - -1. **Toggle adaptive polling** (no restart required) - - Via UI: **Settings → System → Monitoring** - - Via API: Update `system.json` with `adaptivePollingEnabled: false` - -2. **Service restart** (clears transient state) - ```bash - # Systemd - sudo systemctl restart pulse - - # Docker - docker restart pulse - - # LXC - pct restart - ``` - - Clears all circuit breakers - - Resets DLQ (tasks re-queued with fresh state) - - Useful for recovering from stuck states - -3. **Version rollback** (if broader issues) - - Via UI: **Settings → System → Updates → Restore previous version** - - Via CLI: `pulse config rollback` - - Documented in [Operations Runbook](ADAPTIVE_POLLING_ROLLOUT.md) - -4. **Per-instance configuration fixes** - - Update node credentials if authentication failures cause DLQ entries - - Adjust network/firewall if connectivity issues trigger breakers - - Fix underlying infrastructure problems - ---- - -## Why Endpoints Are Deferred - -### Test Results Demonstrate Sufficient Automation - -**Integration testing** (55 seconds, 12 instances): -- Circuit breakers opened and closed automatically -- Transient failures recovered without intervention -- Permanent failures correctly routed to DLQ - -**Soak testing** (2-240 minutes, 80 instances): -- Heap: 2.3MB → 3.1MB (healthy growth) -- Goroutines: 16 → 6 (no leak) -- No scenarios requiring manual intervention - -**Production rollout** (v4.24.0): -- Automatic recovery working as designed -- Service restart sufficient for edge cases -- No operator requests for manual controls - -### Implementation Cost vs. Benefit - -**Would require:** -- Authentication and RBAC integration -- Comprehensive audit logging -- UI integration in Settings → System → Monitoring -- Additional testing and maintenance burden - -**Current workarounds proven effective:** -- Adaptive polling toggle (immediate, no restart) -- Service restart (clears all state in < 30 seconds) -- Version rollback (if systematic issues) - ---- - -## Future Implementation Plan - -### Proposed Endpoints (When Needed) - -If production usage reveals operational gaps, implement: - -#### 1. Reset Circuit Breaker -``` -POST /api/monitoring/breakers/{key}/reset -Authorization: Required (session or API token) -``` - -**Request:** -```json -{ - "reason": "Manual reset after infrastructure fix" -} -``` - -**Response:** -```json -{ - "success": true, - "key": "pve::pve-node1", - "previousState": "open", - "newState": "closed", - "resetBy": "admin", - "resetAt": "2025-10-20T15:30:00Z" -} -``` - -**Use case:** Immediately retry a specific instance after fixing underlying issue (e.g., restored network connectivity) - -#### 2. Retry All DLQ Tasks -``` -POST /api/monitoring/dlq/retry -Authorization: Required (session or API token) -``` - -**Response:** -```json -{ - "success": true, - "tasksRetried": 5, - "keys": ["pve::pve-node1", "pbs::backup-server"] -} -``` - -**Use case:** Bulk retry after fixing widespread issue (e.g., certificate renewal) - -#### 3. Retry Specific DLQ Task -``` -POST /api/monitoring/dlq/{key}/retry -Authorization: Required (session or API token) -``` - -**Response:** -```json -{ - "success": true, - "key": "pve::pve-node1", - "previousRetryCount": 5, - "scheduledFor": "2025-10-20T15:35:00Z" -} -``` - -**Use case:** Targeted retry of single instance - -#### 4. Remove from DLQ -``` -DELETE /api/monitoring/dlq/{key} -Authorization: Required (session or API token) -``` - -**Response:** -```json -{ - "success": true, - "key": "pve::decomissioned-node", - "reason": "Instance permanently decommissioned" -} -``` - -**Use case:** Remove decommissioned instances from DLQ - -### Security Requirements - -All management endpoints would require: -- **Authentication:** Valid session cookie or API token -- **RBAC:** Admin-level permissions -- **Audit logging:** Every action logged with: - - Operator username/IP - - Instance key affected - - Reason provided - - Timestamp - - Previous and new states -- **Rate limiting:** Prevent abuse (e.g., 10 requests/minute) - ---- - -## Re-evaluation Criteria - -**Implement management endpoints if:** - -1. **Operator demand:** >3 requests in first 60 days of v4.24.0 deployment -2. **Service restart frequency:** >5 restarts per week due to stuck breakers/DLQ -3. **Incident impact:** Manual controls would have prevented or accelerated recovery from >1 production incident -4. **Feedback from operations runbook:** [ADAPTIVE_POLLING_ROLLOUT.md](ADAPTIVE_POLLING_ROLLOUT.md) troubleshooting inadequate - -**Don't implement if:** -- Current workarounds remain effective -- Automatic recovery continues to handle 99%+ of scenarios -- No clear operational pain points emerge - ---- - -## Monitoring Current State - -### Check Circuit Breakers -```bash -curl -s http://:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.breaker.state != "closed") | {key, state: .breaker.state, since: .breaker.since}' -``` - -### Check Dead-Letter Queue -```bash -curl -s http://:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.deadLetter.present) | {key, reason: .deadLetter.reason, retryCount: .deadLetter.retryCount, nextRetry: .deadLetter.nextRetry}' -``` - -### Track Recovery Times -```bash -# Monitor breaker state changes -journalctl -u pulse | grep -E "circuit breaker|dead-letter" -``` - ---- - -## Feedback & Requests - -If you encounter scenarios where manual management endpoints would be valuable: - -1. **Document the use case** - - What problem occurred? - - Why wasn't automatic recovery sufficient? - - How would manual control have helped? - -2. **File an issue** - - [GitHub Issues](https://github.com/rcourtman/Pulse/issues) - - Include: scheduler health API output, logs, timeline - -3. **Track frequency** - - If the pattern recurs >3 times, escalate for implementation - ---- - -## Related Documentation - -- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference -- [Operations Runbook](ADAPTIVE_POLLING_ROLLOUT.md) - Steady-state operations and troubleshooting -- [Adaptive Polling Architecture](../monitoring/ADAPTIVE_POLLING.md) - Technical details -- [Configuration Guide](../CONFIGURATION.md) - Adaptive polling settings diff --git a/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md b/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md deleted file mode 100644 index e78335b..0000000 --- a/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md +++ /dev/null @@ -1,279 +0,0 @@ -# Adaptive Polling Operations Runbook - -**GA in v4.24.0 - Enabled by Default** - -This runbook guides operators managing the adaptive polling scheduler in production. Adaptive polling is enabled by default in v4.24.0 and can be toggled via **Settings → System → Monitoring** (no restart) or environment variables (restart required). - -Follow these operational procedures for steady-state monitoring, troubleshooting, and rollback scenarios. - ---- - -## 1. Preparation (Before v4.24.0 Deployment) - -**For new deployments or upgrades to v4.24.0:** - -1. **Monitoring readiness** - - Review available scrape series in [Prometheus Metrics](../monitoring/PROMETHEUS_METRICS.md). - - Set up Grafana dashboard with: - - Instance gauges: `pulse_monitor_poll_queue_depth`, `pulse_monitor_poll_staleness_seconds`, `pulse_monitor_poll_last_success_timestamp` - - **Per-node coverage:** `pulse_monitor_node_poll_staleness_seconds`, `pulse_monitor_node_poll_errors_total`, `pulse_monitor_node_poll_duration_seconds` - - Scheduler health: `pulse_scheduler_queue_depth`, `pulse_scheduler_queue_due_soon`, `pulse_scheduler_dead_letter_depth`, `pulse_scheduler_breaker_state`, and `pulse_scheduler_breaker_failure_count` - - Diagnostics cache sanity: `increase(pulse_diagnostics_cache_misses_total[5m])` vs `pulse_diagnostics_cache_hits_total` - - HTTP SLA: `rate(pulse_http_request_errors_total{status_class="server_error"}[5m])` - - Continue trending `pulse_monitor_poll_total` / `pulse_monitor_poll_errors_total` for throughput - - Configure alerts (see §4) - -2. **Baseline metrics** - - Record pre-upgrade metrics if upgrading from < v4.24.0: - - Typical polling frequency - - Average response times - - Current alert volumes - - These help assess adaptive polling impact - -3. **Rollback readiness** - - Verify update rollback workflow: **Settings → System → Updates → Restore previous version** - - Test rollback in staging environment - - Confirm `/api/monitoring/scheduler/health` accessible - - Document emergency disable procedure (see §5) - -4. **Configuration review** - - Review `system.json` or environment variables for adaptive polling tunables: - - `ADAPTIVE_POLLING_BASE_INTERVAL` (default: 10s) - - `ADAPTIVE_POLLING_MIN_INTERVAL` (default: 5s) - - `ADAPTIVE_POLLING_MAX_INTERVAL` (default: 5m) - - Adjust if needed for your environment (e.g., high-frequency monitoring) - ---- - -## 2. Post-Deployment Verification (v4.24.0+) - -**Adaptive polling is enabled by default. Verify it's working correctly:** - -1. **Check scheduler health** - ```bash - curl -s http://:7655/api/monitoring/scheduler/health | jq - ``` - - **Expected response:** - - `"enabled": true` - - `queue.depth` reasonable (< instances × 1.5) - - `deadLetter.count` = 0 (or only known failing instances) - - `instances[]` array populated with your nodes - - No `breaker.state` stuck in `open` (except known issues) - -2. **Verify UI access** - - Navigate to **Settings → System → Monitoring** - - Confirm "Adaptive Polling" toggle is ON - - Review queue depth and recent poll status - -3. **Check Grafana metrics** (if configured) - - `pulse_monitor_poll_queue_depth` shows reasonable values - - `pulse_monitor_poll_staleness_seconds` < 60s for healthy instances - - `pulse_monitor_poll_errors_total` not rapidly increasing - - `pulse_monitor_poll_last_success_timestamp` updating regularly - -4. **Monitor update history** - - Check **Settings → System → Updates** for update entry - - Verify rollback button is available - - Confirm update status shows "completed" - - **Via API:** - ```bash - curl -s http://:7655/api/updates/history | jq '.entries[0]' - ``` - ---- - -## 3. Steady-State Operations - -**Ongoing monitoring and SLO checks:** - -1. **Daily health checks** - - Review scheduler health dashboard or API endpoint - - Check for: - - Queue depth < 50 (alert if > 50 for 10+ minutes) - - Instance staleness < 60s for healthy instances, < 120s for critical instances - - **Per-node staleness** `pulse_monitor_node_poll_staleness_seconds` < 120s - - DLQ count stable (not growing) - - Circuit breakers mostly `closed` - - Diagnostics cache misses roughly follow hits (`increase(pulse_diagnostics_cache_misses_total[10m])` in line with hits) - - HTTP error rate (`rate(pulse_http_request_errors_total{status_class="server_error"}[5m])`) near zero - -2. **Weekly reviews** - - Analyze trends in Grafana: - - Poll success rates - - Average queue depth over time - - Circuit breaker trip frequency - - Dead-letter queue patterns - - Document any recurring issues - -3. **SLO targets** - - **Queue depth**: < 1.5× instance count (< 50 typical) - - **Staleness**: < 60s for healthy instances, < 120s for critical instances - - **Poll success rate**: > 99% for healthy infrastructure - - **DLQ growth**: < 5% per week (excluding known failures) - - **Circuit breaker recovery**: < 5 minutes for transient failures - -4. **Log correlation** - - Cross-reference scheduler health with update history - - Check `/api/updates/history` for rollback events correlated with scheduler issues - - Review audit logs for adaptive polling configuration changes - ---- - -## 4. Grafana & Alert Configuration - -1. **Dashboard panels** - - **Queue Depth**: `pulse_monitor_poll_queue_depth` - - Use single-stat with alert if > 1.5× active instances for > 10 min - - **Instance & Node Staleness**: combine `pulse_monitor_poll_staleness_seconds` and `pulse_monitor_node_poll_staleness_seconds` - - Alert threshold: > 60 s for > 5 min (excluding known failing instances) - - **Polling Throughput**: rate of `pulse_monitor_poll_total{result="success"}` vs `result="error"` - - **Per-node errors**: table or graph of `pulse_monitor_node_poll_errors_total` to spot noisy nodes - - **Scheduler Health**: panels for `pulse_scheduler_queue_depth`, `pulse_scheduler_queue_due_soon`, `pulse_scheduler_dead_letter_depth`, `pulse_scheduler_breaker_state`, `pulse_scheduler_breaker_failure_count` - - **Diagnostics Cache**: compare `increase(pulse_diagnostics_cache_hits_total[5m])` vs misses so spikes stand out - - **HTTP SLA**: `rate(pulse_http_request_errors_total{status_class="server_error"}[5m])` - - **Last Success Timestamp** (v4.24.0+): `pulse_monitor_poll_last_success_timestamp` to detect polling gaps - -2. **Alerts** - - Queue depth > threshold for >10 min (Warning), >20 min (Critical) - - Instance or node staleness > 60 s for >5 min (Critical) - - Dead-letter count increase > N (based on baseline) triggers Warning - - Any breaker stuck in `open` for >10 min (from `pulse_scheduler_breaker_state`) triggers Critical - - Queue wait > 5 s (95th percentile on `pulse_scheduler_queue_wait_seconds`) triggers Warning - - Permanent failures (`pulse_monitor_poll_errors_total{category="permanent"}`) trigger immediate Critical - - Diagnostics refresh duration > 20 s alongside miss spikes should page engineering (`pulse_diagnostics_refresh_duration_seconds`) - -3. **Notification routing** - - Ensure alerts route to on-call + feature owner - ---- - -## 5. Rollback Procedures - -### Option A: Disable Adaptive Polling (Keep v4.24.0) - -**If adaptive polling causes issues but you want to keep v4.24.0:** - -1. **Via UI (No restart required)** - - Navigate to **Settings → System → Monitoring** - - Toggle "Adaptive Polling" OFF - - Changes apply immediately - -2. **Via environment variables (Restart required)** - ```bash - # Systemd - sudo systemctl edit pulse - # Add: - [Service] - Environment="ADAPTIVE_POLLING_ENABLED=false" - - # Then restart - sudo systemctl restart pulse - ``` - -3. **Verification** - ```bash - curl -s http://:7655/api/monitoring/scheduler/health | jq '.enabled' - # Should return: false - ``` - -### Option B: Full Version Rollback - -**If v4.24.0 causes broader issues:** - -1. **Via UI** - - Navigate to **Settings → System → Updates** - - Click **"Restore previous version"** - - Confirm rollback - - Pulse restarts automatically with previous version - -2. **Via CLI** - ```bash - # Systemd installations - sudo /opt/pulse/pulse config rollback - - # LXC containers - pct exec -- bash -c "cd /opt/pulse && ./pulse config rollback" - ``` - -3. **Verification** - ```bash - # Check version - curl -s http://:7655/api/version | jq '.version' - - # Check update history - curl -s http://:7655/api/updates/history | jq '.entries[0]' - # Should show action="rollback", status="completed" - ``` - -4. **Post-rollback** - - Verify rollback logged in update history - - Check journal: `journalctl -u pulse | grep rollback` - - Monitor for 15-30 minutes to ensure stability - - Document rollback reason and notify stakeholders - ---- - -## 6. Troubleshooting - -| Symptom | Possible Cause | Action | -|---------|----------------|--------| -| Queue depth remains high (> 2× usual) | Insufficient workers, hidden breaker, misconfigured flag | Check scheduler health API for breaker states; consider increasing workers or reverting flag | -| Staleness spikes across many instances | Backend API slowdown or connectivity issues | Inspect backend logs, network health; revert flag if duration > 15 min | -| Dead-letter count climbs rapidly | Downstream API failures | Investigate specific instances via scheduler health API; fix credential/connectivity issues or rollback | -| Circuit breakers stuck half-open/open | Persistent transient failures | Review error logs, ensure backoff/rate limits not starving retries; rollback if unresolved quickly | -| Grafana panels flatline | Metrics exporter or job issue | Ensure Prometheus scraping working; verify service restarted with flag | - -### Accessing Scheduler Health API - -```bash -curl -s http://:7655/api/monitoring/scheduler/health | jq -``` - -Key sections to inspect: - -- `queue.depth`, `queue.perType` -- `instances[].pollStatus` (success/failure streaks and last error) -- `instances[].breaker` (current breaker state, retry windows) -- `instances[].deadLetter` (reason, retry counts, schedules) -- `staleness` (normalized freshness score) - -Common queries: - -**Instances with errors:** -```bash -curl -s http://:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.pollStatus.lastError != null) | {key, lastError: .pollStatus.lastError}' -``` - -**Current dead-letter entries:** -```bash -curl -s http://:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.deadLetter.present) | {key, reason: .deadLetter.reason, retryCount: .deadLetter.retryCount}' -``` - -**Breakers not closed:** -```bash -curl -s http://:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.breaker.state != "closed") | {key, breaker: .breaker}' -``` - -### When to Roll Back - -Rollback immediately if any of the following occurs: -- Queue depth > 3× baseline for > 15 min -- Staleness > 120 s on majority of instances -- Dead-letter count doubles without clear cause -- Customer-facing alerts or latency regressions attributed to adaptive polling - -Document the incident and notify stakeholders after rollback. - ---- - -## 7. Related Documentation - -- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference -- [Adaptive Polling Architecture](../monitoring/ADAPTIVE_POLLING.md) - Technical details -- [Management Endpoints](ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md) - Circuit breaker/DLQ controls -- [Configuration Guide](../CONFIGURATION.md) - Adaptive polling settings diff --git a/docs/operations/audit-log-rotation.md b/docs/operations/audit-log-rotation.md deleted file mode 100644 index 271df60..0000000 --- a/docs/operations/audit-log-rotation.md +++ /dev/null @@ -1,163 +0,0 @@ -# Pulse Sensor Proxy Audit Log Rotation - -The sensor proxy writes a tamper-evident audit trail to -`/var/log/pulse/sensor-proxy/audit.log`. Every entry includes the SHA-256 hash -of the previous entry, so any modification becomes obvious. Because the process -keeps the file open and maintains the running hash in memory, rotation requires -special handling. - -## Rotation Strategy - -Use `logrotate` to rotate the file once it reaches 100 MB. After each rotation, -restart the proxy so it opens a new file and starts a fresh hash chain. - -Create `/etc/logrotate.d/pulse-sensor-proxy` with the following contents: - -```conf -/var/log/pulse/sensor-proxy/audit.log { - daily - size 100M - rotate 90 - compress - delaycompress - missingok - notifempty - create 0640 pulse pulse - sharedscripts - postrotate - systemctl restart pulse-sensor-proxy.service >/dev/null 2>&1 || true - endscript -} -``` - -### Why a Restart Is Mandatory - -`copytruncate` and similar tricks break the chain integrity. Restarting the -service ensures: - -1. The proxy releases the old file descriptor. -2. A new hash chain starts at sequence 1 with an all-zero `prev_hash`. - -If the proxy is not restarted, it will continue writing to the renamed file and -the rotation will have no effect. - -### Chain Continuity Across Rotations - -Each rotated log (`audit.log.1.gz`, `audit.log.2.gz`, …) is self-contained. To -prove continuity between files: - -1. After each rotation, record the final `event_hash` from the rotated file (for - example, store it in the filename or a checksum manifest). -2. When reviewing logs, verify the `prev_hash` of the first entry in the new - file is the zero hash, and reconcile the recorded final hash from the prior - file to show no entries were removed. - -Maintaining this “final hash ledger” allows auditors to stitch the rotated files -together chronologically while preserving the tamper-evident guarantees. - -### Permissions - -Adjust the `create` directive to match the user and group that run the sensor -proxy. The example assumes both user and group are `pulse`. - ---- - -## Post-Rotation Health Checks (v4.24.0+) - -**After rotating audit logs and restarting pulse-sensor-proxy, verify adaptive polling health:** - -### 1. Check Scheduler Health - -```bash -curl -s http://localhost:7655/api/monitoring/scheduler/health | jq -``` - -**Verify:** -- Temperature proxy pollers appear in `instances[]` array -- `pollStatus.lastSuccess` is recent (within last 60 seconds) -- No new entries in `deadLetter` queue for proxy instances -- `breaker.state` is `closed` for proxy nodes - -**Example check for proxy instances:** -```bash -curl -s http://localhost:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.type == "proxy" or .connection | contains("proxy")) | {key, lastSuccess: .pollStatus.lastSuccess, breaker: .breaker.state}' -``` - -### 2. Monitor Metrics (10-15 minutes) - -Watch these metrics to ensure proxy restart didn't cause issues: - -```bash -# Queue depth should remain stable -curl -s http://localhost:7655/api/monitoring/scheduler/health | jq '.queue.depth' - -# Check staleness for proxy instances -curl -s http://localhost:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.type == "proxy") | {key, staleness: .pollStatus.lastSuccess}' -``` - -**Expected behavior:** -- Queue depth: No significant spike (< 10 temporary increase acceptable) -- Staleness: Proxy instances show fresh polls within 30-60 seconds -- No circuit breaker trips for proxy instances -- No new DLQ entries - -### 3. Cross-Reference Audit Logs - -**Link rotation events with scheduler health for security review:** - -```bash -# Check Pulse audit log for rotation timing -journalctl -u pulse-sensor-proxy --since "10 minutes ago" | grep -E "restart|rotation" - -# Check update history for any concurrent events -curl -s http://localhost:7655/api/updates/history?limit=5 | jq '.entries[] | {action, timestamp, status}' -``` - -**Why this matters:** -- Security auditors can correlate proxy restarts with scheduler behavior -- Update rollbacks may be concurrent with log rotations -- Rollback metadata (new in v4.24.0) provides full operational context -- Ensures restart didn't mask polling failures or breaker trips - -### 4. Troubleshooting Rotation Issues - -**If proxy instances don't rejoin queue:** - -1. **Check service status** - ```bash - systemctl status pulse-sensor-proxy - ``` - -2. **Verify scheduler sees the proxy** - ```bash - curl -s http://localhost:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.type == "proxy")' - ``` - -3. **Check for circuit breakers** - ```bash - curl -s http://localhost:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.breaker.state != "closed") | {key, state: .breaker.state, retryAt: .breaker.retryAt}' - ``` - -4. **Review logs for errors** - ```bash - journalctl -u pulse-sensor-proxy -n 50 - journalctl -u pulse | grep -E "proxy|temperature" - ``` - -**Recovery actions:** -- If breakers are stuck: Restart main Pulse service (`systemctl restart pulse`) -- If DLQ entries persist: Check proxy credentials and network connectivity -- If polling doesn't resume: Verify proxy configuration in **Settings → Sensors** - ---- - -## Related Documentation - -- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference -- [Adaptive Polling Operations](ADAPTIVE_POLLING_ROLLOUT.md) - Health monitoring procedures -- [Pulse Sensor Proxy Hardening](../security/pulse-sensor-proxy-hardening.md) - Security configuration -- [Temperature Monitoring Security](../TEMPERATURE_MONITORING_SECURITY.md) - Proxy-specific security notes diff --git a/docs/operations/pulse-sensor-proxy-runbook.md b/docs/operations/pulse-sensor-proxy-runbook.md deleted file mode 100644 index c3035ab..0000000 --- a/docs/operations/pulse-sensor-proxy-runbook.md +++ /dev/null @@ -1,105 +0,0 @@ -# Pulse Sensor Proxy Runbook - -## Quick Reference -- Binary: `/opt/pulse/sensor-proxy/bin/pulse-sensor-proxy` -- Unit: `pulse-sensor-proxy.service` -- Logs: `/var/log/pulse/sensor-proxy/proxy.log` -- Audit trail: `/var/log/pulse/sensor-proxy/audit.log` (hash chained, forwarded via rsyslog) -- Metrics: `http://127.0.0.1:9127/metrics` (set `PULSE_SENSOR_PROXY_METRICS_ADDR` to change/disable) -- Limiters: 1 request/sec per UID (burst 5), per-UID concurrency 2, global concurrency 8, 2 s penalty on validation failures - -## Monitoring Alerts & Response - -```mermaid -sequenceDiagram - participant Backend - participant Proxy - participant Node - - Backend->>Proxy: get_temperature - Proxy->>Proxy: Check rate limit - Proxy->>Node: SSH sensors -j - Node->>Proxy: JSON response - Proxy->>Backend: Temperature data -``` - -### Rate Limit Hits (`pulse_proxy_limiter_rejections_total`) -1. Check audit log entries tagged `limiter.rejection` for offending UID. -2. Confirm workload legitimacy; if expected, consider increasing limits via config override. -3. If malicious, block source process/user and inspect Pulse audit logs. - -### Penalty Events (`pulse_proxy_limiter_penalties_total`) -1. Review corresponding validation failures in audit log (`command.validation_failed`). -2. If repeated invalid JSON/unknown methods, inspect caller code for regressions or intrusion attempts. - -### Audit Log Forwarder Down -1. `journalctl -u rsyslog` to confirm transmission errors. -2. Ensure `/etc/pulse/log-forwarding` certs valid & remote host reachable. -3. Forwarding queue stored locally in `/var/log/pulse/sensor-proxy/forwarding.log`; ship manually if outage exceeds 1 hour. - -### Proxy Health Endpoint Fails -1. `systemctl status pulse-sensor-proxy` -2. Check `/var/log/pulse/sensor-proxy/proxy.log` for panic or limiter exhaustion. -3. Inspect `/var/log/pulse/sensor-proxy/audit.log` for recent privileged method denials. - -## Standard Procedures -### Restart Proxy Safely -```bash -sudo systemctl stop pulse-sensor-proxy -sudo apparmor_parser -r /etc/apparmor.d/pulse-sensor-proxy # if updating policy -sudo systemctl start pulse-sensor-proxy -``` -Verify: -```bash -# Metrics endpoint exposes proxy build/health -curl -s http://127.0.0.1:9127/metrics | grep pulse_proxy_build_info - -# Ensure adaptive polling sees the proxy again -curl -s http://localhost:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.key | contains("temperature")) | {key, pollStatus}' -``` -Temperature instances should show recent `lastSuccess` timestamps with no DLQ entries. - -### Rotate SSH Keys -1. Run `scripts/secure-sensor-files.sh` to regenerate keys (ensure environment locked down). -2. Use RPC `ensure_cluster_keys` to distribute new public key. -3. Confirm nodes accept `ssh` from proxy host. -4. Confirm the scheduler clears any temporary breakers/dlq entries: - ```bash - curl -s http://localhost:7655/api/monitoring/scheduler/health \ - | jq '.instances[] | select(.key | contains("temperature")) | {key, breaker: .breaker.state, deadLetter: .deadLetter.present}' - ``` - Expect `breaker.state=="closed"` and `deadLetter.present==false` for all proxy-driven pollers. - -### Rate Limit Tuning - -| Profile | Nodes | `per_peer_interval_ms` | `per_peer_burst` | Notes | -| --- | --- | --- | --- | --- | -| Default | ≤5 | 1000 | 5 | Shipped with commit 46b8b8d; no action needed for single host clusters. | -| Medium | 6–10 | 500 | 10 | Doubles throughput; monitor `pulse_proxy_limiter_rejects_total`. | -| Large | 11–20 | 250 | 20 | Confirm proxy CPU stays below 70 % and audit logs remain clean. | -| XL | 21–40 | 150 | 30 | Requires high-trust environment; ensure UID filters are locked down. | - -**Procedure:** -1. Edit `/etc/pulse-sensor-proxy/config.yaml` and set the desired profile values under `rate_limit`. -2. Restart the service: - ```bash - sudo systemctl restart pulse-sensor-proxy - ``` -3. Validate: - ```bash - curl -s http://127.0.0.1:9127/metrics \ - | grep pulse_proxy_limiter_rejects_total - ``` - The counter should stop incrementing during steady-state polling. -4. Record the change in the operations log and review audit entries for unexpected callers. - -## Incident Handling -- **Unauthorized Command Attempt:** audit log shows `command.validation_failed` and limiter penalties; capture correlation ID, check Pulse side for compromised container. -- **Excessive Temperature Failures:** refer to `pulse_proxy_ssh_requests_total{result="error"}`; validate network ACLs and node health; escalate to Proxmox team if nodes unreachable. -- **Log Tampering Suspected:** verify audit hash chain by replaying `eventHash` values; compare with remote log store (immutable). Trigger security response if mismatch. - -## Postmortem Checklist -- Timeline: command audit entries, limiter stats, rsyslog queue depth. -- Verify AppArmor/seccomp status (`aa-status`, `systemctl show pulse-sensor-proxy -p AppArmorProfile`). -- Ensure firewall ACLs match `docs/security/pulse-sensor-proxy-network.md`. diff --git a/docs/script-library-guide.md b/docs/script-library-guide.md deleted file mode 100644 index 3eb1d7e..0000000 --- a/docs/script-library-guide.md +++ /dev/null @@ -1,208 +0,0 @@ -# Script Library Guide - -## 1. Introduction - -The script library system standardises helper functions used across Pulse -installers. It reduces duplication, improves testability, and makes it easier to -roll out fixes across the installer fleet. Use the shared libraries when: - -- Multiple scripts need the same functionality (logging, HTTP, systemd, etc.). -- You are refactoring legacy scripts to adopt the v2 pattern. -- New features require reusable helpers (e.g., additional service management). - -## 2. Architecture Overview - -``` -scripts/ -├── lib/ # Shared library modules -│ ├── common.sh # Core utilities -│ ├── systemd.sh # Service management -│ ├── http.sh # HTTP/API operations -│ └── README.md # API documentation -├── tests/ # Test suites -│ ├── run.sh # Test runner -│ ├── test-*.sh # Smoke tests -│ └── integration/ # Integration tests -├── bundle.sh # Bundler tool -├── bundle.manifest # Bundle configuration -└── install-*.sh # Installer scripts - -dist/ # Generated bundled scripts -└── install-*.sh # Ready for distribution -``` - -### Development & Bundling Workflow - -```mermaid -flowchart LR - Code[Write Code
scripts/lib] - Test[Run Tests] - Bundle[Bundle
make bundle-scripts] - Dist[Distribute
dist/*.sh] - - Code --> Test - Test --> Bundle - Bundle --> Dist -``` - -This workflow emphasizes the library's modular design: develop reusable modules in `scripts/lib`, test thoroughly, bundle for distribution, and validate bundled artifacts before release. - -## 3. Using the Library in Your Script - -```bash -#!/usr/bin/env bash -# Example installer using shared libraries - -LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd)/lib" -if [[ -f "$LIB_DIR/common.sh" ]]; then - source "$LIB_DIR/common.sh" - source "$LIB_DIR/systemd.sh" - source "$LIB_DIR/http.sh" -fi - -common::init "$@" - -main() { - common::ensure_root --allow-sudo --args "$@" - common::log_info "Starting installation..." - # ...script logic... -} - -main "$@" -``` - -## 4. Common Migration Patterns - -**Logging** -```bash -# Before -echo "[INFO] Installing..." - -# After -common::log_info "Installing..." -``` - -**Privilege Escalation** -```bash -# Before -if [[ $EUID -ne 0 ]]; then - echo "Must run as root"; exit 1 -fi - -# After -common::ensure_root --allow-sudo --args "$@" -``` - -**Downloads** -```bash -# Before -curl -o file.tar.gz http://example.com/file.tar.gz - -# After -http::download --url http://example.com/file.tar.gz --output file.tar.gz -``` - -**Systemd Unit Creation** -```bash -# Before -cat > /etc/systemd/system/my.service <<'EOF' -... -systemctl daemon-reload -systemctl enable my.service -systemctl start my.service - -# After -systemd::create_service /etc/systemd/system/my.service <<'EOF' -... -EOF -systemd::enable_and_start "my.service" -``` - -## 5. Creating New Library Modules - -Create a new module when functionality is reused across scripts or complex -enough to warrant dedicated helpers. Template: - -```bash -#!/usr/bin/env bash -# Module: mymodule.sh - -mymodule::do_thing() { - local arg="$1" - # implementation -} -``` - -Add the module to `scripts/lib`, document exported functions in -`scripts/lib/README.md`, and update `scripts/bundle.manifest` for any bundles -that need it. - -## 6. Testing Requirements - -Every migrated script must include: - -1. Smoke test (`scripts/tests/test-