diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 67d08b4..943ccb1 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -84,6 +84,13 @@ services: # Optional legacy variable kept for compatibility; newest token is used if both are set. # API_TOKEN: 'your-48-char-hex-token' PULSE_PUBLIC_URL: 'https://pulse.example.com' # Used for webhooks/links + # Optional logging controls (v4.24.0+) + LOG_LEVEL: 'info' + LOG_FORMAT: 'auto' # auto | json | console + # LOG_FILE: '/data/pulse.log' # uncomment to mirror logs to a file + # LOG_MAX_SIZE: '100' # MB + # LOG_MAX_AGE: '30' # days + # LOG_COMPRESS: 'true' # TZ: 'UTC' restart: unless-stopped @@ -105,6 +112,13 @@ API_TOKEN=your-48-char-hex-token # Generate with: openssl rand -he API_TOKENS=${ANSIBLE_TOKEN},${DOCKER_AGENT_TOKEN} PULSE_PUBLIC_URL=https://pulse.example.com # Recommended for webhooks TZ=Asia/Kolkata # Optional: matches host timezone +# Logging controls (optional; take effect immediately after restart) +LOG_LEVEL=info +LOG_FORMAT=auto +# LOG_FILE=/data/pulse.log +# LOG_MAX_SIZE=100 +# LOG_MAX_AGE=30 +# LOG_COMPRESS=true ``` **Note**: Plain text credentials are automatically hashed for security. You can provide either plain text (simpler) or pre-hashed values (advanced). @@ -254,6 +268,44 @@ Common problems: 4. **Regular backups** - Backup the `/data` volume regularly 5. **Network isolation** - Don't expose port 7655 directly to the internet +## Updates & Rollbacks (v4.24.0+) + +Docker images are still updated manually, but Pulse now records every upgrade/rollback attempt in **Settings → System → Updates** alongside the CLI instructions below. + +### Update to the latest image +```bash +docker pull rcourtman/pulse:latest +docker stop pulse +docker rm pulse +docker run -d --name pulse \ + -p 7655:7655 \ + -v pulse_data:/data \ + --restart unless-stopped \ + rcourtman/pulse:latest +``` +- The update history entry includes the image tag, operator, and timestamps. Capture the `event_id` in your change log. +- After the container is back online, verify the adaptive scheduler is healthy: + ```bash + curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.queue.depth' + ``` + +### Roll back to a prior release +- Choose a previous tag (for example `v4.23.2`) from [GitHub Releases](https://github.com/rcourtman/Pulse/releases) or Docker Hub. +- Redeploy the container with that tag: + ```bash + docker pull rcourtman/pulse:v4.23.2 + docker stop pulse && docker rm pulse + docker run -d --name pulse \ + -p 7655:7655 \ + -v pulse_data:/data \ + --restart unless-stopped \ + rcourtman/pulse:v4.23.2 + ``` +- The update history will log this as a rollback. Make sure to annotate the entry with the reason in your postmortem notes. + +> **Tip:** Keep the last known-good tag handy (for example in your compose file or infra repo) so rollbacks are a single change. + ## Environment Variables Reference > **⚠️ Important**: Environment variables always override UI/system.json settings. If you set a value via env var (e.g., `DISCOVERY_SUBNET`), changes made in the UI for that setting will NOT take effect until you remove the env var. This follows standard container practices where env vars have highest precedence. @@ -284,7 +336,12 @@ Common problems: | Variable | Description | Default | |----------|-------------|---------| | `TZ` | Timezone inside the container | `UTC` | -| `LOG_LEVEL` | Logging verbosity | `info` | +| `LOG_LEVEL` | Logging verbosity. Changing the env var and restarting updates Pulse immediately. | `info` | +| `LOG_FORMAT` | `auto`, `json`, or `console` output format. | `auto` | +| `LOG_FILE` | Optional path inside the container to mirror logs (e.g. `/data/pulse.log`). Empty = stdout only. | *(unset)* | +| `LOG_MAX_SIZE` | Rotate `LOG_FILE` after this size (MB). | `100` | +| `LOG_MAX_AGE` | Days to retain rotated log files. | `30` | +| `LOG_COMPRESS` | Compress rotated log files (`true` / `false`). | `true` | | `METRICS_RETENTION_DAYS` | Days of metrics history to keep | `7` | ## Advanced Configuration diff --git a/docs/DOCKER_HUB_README.md b/docs/DOCKER_HUB_README.md index bbd2dca..2270f1d 100644 --- a/docs/DOCKER_HUB_README.md +++ b/docs/DOCKER_HUB_README.md @@ -50,6 +50,8 @@ Pulse is built by a solo developer in evenings and weekends. Your support helps: - Optional Docker container monitoring via lightweight agent - Config export/import with encryption and authentication - Automatic stable updates with safe rollback (opt-in) +- Runtime logging controls (switch level/format or mirror to file without downtime) +- Update history with rollback guidance captured in the UI - Dark/light themes, responsive design - Built with Go for minimal resource usage @@ -157,10 +159,29 @@ services: # CORS & logging # - ALLOWED_ORIGINS=https://app.example.com # CORS origins (default: none, same-origin only) # - LOG_LEVEL=info # Log level: debug/info/warn/error (default: info) + # - LOG_FORMAT=auto # auto | json | console (default: auto) + # - LOG_FILE=/data/pulse.log # Optional mirrored logfile inside container + # - LOG_MAX_SIZE=100 # Rotate logfile after N MB + # - LOG_MAX_AGE=30 # Retain rotated logs for N days + # - LOG_COMPRESS=true # Compress rotated logs restart: unless-stopped volumes: pulse_data: + +### Updating & Rollbacks (v4.24.0+) + +```bash +# Update to the latest tagged image +docker pull rcourtman/pulse:latest +docker stop pulse && docker rm pulse +docker run -d --name pulse \ + -p 7655:7655 -v pulse_data:/data \ + --restart unless-stopped \ + rcourtman/pulse:latest +``` +- Every upgrade is logged in **Settings → System → Updates** with an `event_id` for change tracking. +- Need to revert? Redeploy the previous tag (for example `rcourtman/pulse:v4.23.2`). Record the rollback reason in your change notes and double-check `/api/monitoring/scheduler/health` once the container is back online. ``` ## Initial Setup diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md index f196888..4479c82 100644 --- a/docs/KUBERNETES.md +++ b/docs/KUBERNETES.md @@ -90,6 +90,48 @@ server: API_TOKENS: docker-agent-token ``` +### Runtime Logging Configuration (v4.24.0+) + +Configure logging behavior via environment variables: + +```yaml +server: + env: + - name: LOG_LEVEL + value: info # debug, info, warn, error + - name: LOG_FORMAT + value: json # json, text, auto + - name: LOG_FILE + value: /data/logs/pulse.log # Optional: mirror logs to file + - name: LOG_MAX_SIZE + value: "100" # MB per log file + - name: LOG_MAX_BACKUPS + value: "10" # Number of rotated logs to keep + - name: LOG_MAX_AGE + value: "30" # Days to retain logs +``` + +**Note:** Logging changes via environment variables require pod restart. Use **Settings → System → Logging** in the UI for runtime changes without restart. + +### Adaptive Polling Configuration (v4.24.0+) + +Adaptive polling is **enabled by default** in v4.24.0. Configure via environment variables: + +```yaml +server: + env: + - name: ADAPTIVE_POLLING_ENABLED + value: "true" # Enable/disable adaptive scheduler + - name: ADAPTIVE_POLLING_BASE_INTERVAL + value: "10s" # Target cadence (default: 10s) + - name: ADAPTIVE_POLLING_MIN_INTERVAL + value: "5s" # Fastest cadence (default: 5s) + - name: ADAPTIVE_POLLING_MAX_INTERVAL + value: "5m" # Slowest cadence (default: 5m) +``` + +**Note:** These settings can also be toggled via **Settings → System → Monitoring** in the UI without pod restart. + Install or upgrade with the overrides: ```bash @@ -168,9 +210,97 @@ Notes: - **Rollback:** `helm rollback pulse ` - **Uninstall:** `helm uninstall pulse -n pulse` (PVCs remain unless you delete them manually) +### Post-Upgrade Verification (v4.24.0+) + +After upgrading to v4.24.0 or newer, verify the deployment: + +1. **Check update history** + ```bash + # Via UI + # Navigate to Settings → System → Updates + + # Via API + kubectl -n pulse exec deploy/pulse -- curl -s http://localhost:7655/api/updates/history | jq '.entries[0]' + ``` + +2. **Verify scheduler health** (adaptive polling) + ```bash + kubectl -n pulse exec deploy/pulse -- curl -s http://localhost:7655/api/monitoring/scheduler/health | jq + ``` + + **Expected response:** + - `"enabled": true` + - `queue.depth` reasonable (< instances × 1.5) + - `deadLetter.count` = 0 or only known issues + - `instances[]` populated with your nodes + +3. **Check pod logs** + ```bash + kubectl -n pulse logs deploy/pulse --tail=50 + ``` + +4. **Verify rollback capability** + - Pulse v4.24.0+ logs update history + - Rollback available via **Settings → System → Updates → Restore previous version** + - Or via API: `POST /api/updates/rollback` (if supported in Kubernetes deployments) + +## Service Naming + +**Important:** The Helm chart creates a service named `svc/pulse` on port `7655` by default, matching standard Kubernetes naming conventions. + +**Service name variations:** +- **Kubernetes/Helm:** `svc/pulse` (Deployment: `pulse`, Service: `pulse`) +- **Systemd installations:** `pulse.service` or `pulse-backend.service` (legacy) +- **Hot-dev scripts:** `pulse-hot-dev` (development only, not used in production clusters) + +**To check the active service:** +```bash +# Kubernetes +kubectl -n pulse get svc pulse + +# Systemd +systemctl status pulse +``` + +## Troubleshooting + +### Verify Scheduler Health After Rollout + +**v4.24.0+** includes adaptive polling. After any Helm upgrade or rollback, verify: + +```bash +kubectl -n pulse exec deploy/pulse -- curl -s http://localhost:7655/api/monitoring/scheduler/health | jq +``` + +**Look for:** +- `enabled: true` +- Queue depth stable +- No stuck circuit breakers +- Empty or stable dead-letter queue + +### Port Configuration Changes + +Port changes via `service.port` or environment variable `FRONTEND_PORT` take effect immediately but should be documented in change logs. v4.24.0 records restarts and configuration changes in update history. + +**To verify port configuration:** +```bash +# Check service +kubectl -n pulse get svc pulse -o jsonpath='{.spec.ports[0].port}' + +# Check pod environment +kubectl -n pulse exec deploy/pulse -- env | grep PORT +``` + ## Reference - Review every available option in `deploy/helm/pulse/values.yaml` - Inspect published charts without installing: `helm show values oci://ghcr.io/rcourtman/pulse-chart --version ` - Helm template rendering preview: `helm template pulse ./deploy/helm/pulse -f ` - `NOTES.txt` emitted by Helm summarizes the service endpoint and agent prerequisites after each install or upgrade + +## Related Documentation + +- [Scheduler Health API](api/SCHEDULER_HEALTH.md) - Monitor adaptive polling status +- [Configuration Guide](CONFIGURATION.md) - System settings and environment variables +- [Adaptive Polling Operations](operations/ADAPTIVE_POLLING_ROLLOUT.md) - Operational procedures +- [Reverse Proxy Setup](REVERSE_PROXY.md) - Configure ingress with rate limit headers diff --git a/docs/OIDC.md b/docs/OIDC.md index 3321160..f789157 100644 --- a/docs/OIDC.md +++ b/docs/OIDC.md @@ -135,7 +135,7 @@ All configuration can be provided via environment variables (see [`docs/CONFIGUR ### Debug Logging -For detailed troubleshooting, set `LOG_LEVEL=debug` in your deployment and restart Pulse. Debug logs include: +For detailed troubleshooting, temporarily raise the log level to `debug` (via **Settings → System → Logging** or by exporting `LOG_LEVEL=debug` and restarting). Debug logs include: - OIDC provider initialization (issuer URL, endpoints discovered) - Authorization flow start (client ID, scopes requested) diff --git a/docs/PORT_CONFIGURATION.md b/docs/PORT_CONFIGURATION.md index 5aa56e4..7c95024 100644 --- a/docs/PORT_CONFIGURATION.md +++ b/docs/PORT_CONFIGURATION.md @@ -62,21 +62,66 @@ Keeping application configuration separate from authentication credentials: - Follows the principle of separation of concerns - Makes it easier to backup/share configs without exposing credentials +## Service Name Variations + +**Important:** Pulse uses different service names depending on the deployment environment: + +- **Systemd (default):** `pulse.service` or `pulse-backend.service` (legacy) +- **Hot-dev scripts:** `pulse-hot-dev` (development only) +- **Kubernetes/Helm:** Deployment `pulse`, Service `pulse` (port configured via Helm values) + +**To check the active service:** +```bash +# Systemd +systemctl list-units | grep pulse +systemctl status pulse + +# Kubernetes +kubectl -n pulse get svc pulse +kubectl -n pulse get deploy pulse +``` + +## Change Tracking (v4.24.0+) + +Port changes via environment variables or `system.json` take effect immediately after restart. **v4.24.0 records configuration changes in update history**—useful for audit trails and troubleshooting. + +**To view change history:** +```bash +# Via UI +# Navigate to Settings → System → Updates + +# Via API +curl -s http://localhost:7655/api/updates/history | jq '.entries[] | {timestamp, action, status}' +``` + ## Troubleshooting ### Port not changing after configuration? -1. Check which service name is in use: +1. **Check which service name is in use:** ```bash systemctl list-units | grep pulse ``` It might be `pulse` (default), `pulse-backend` (legacy), or `pulse-hot-dev` (dev environment) depending on your installation method. -2. Verify the configuration is loaded: +2. **Verify the configuration is loaded:** ```bash + # Systemd sudo systemctl show pulse | grep Environment + + # Kubernetes + kubectl -n pulse get deploy pulse -o jsonpath='{.spec.template.spec.containers[0].env}' | jq ``` -3. Check if another process is using the port: +3. **Check if another process is using the port:** ```bash sudo lsof -i :8080 ``` + +4. **Verify post-restart** (v4.24.0+): + ```bash + # Check actual listening port + curl -s http://localhost:7655/api/version | jq + + # Check update history for restart event + curl -s http://localhost:7655/api/updates/history?limit=5 | jq + ``` diff --git a/docs/PROXY_AUTH.md b/docs/PROXY_AUTH.md index 8f70451..c3381cf 100644 --- a/docs/PROXY_AUTH.md +++ b/docs/PROXY_AUTH.md @@ -128,7 +128,11 @@ location / { proxy_set_header X-Proxy-Secret "your-secure-secret-here"; proxy_set_header Remote-User $user; proxy_set_header Remote-Groups $groups; - + proxy_pass_header X-RateLimit-Limit; + proxy_pass_header X-RateLimit-Remaining; + proxy_pass_header X-RateLimit-Reset; + proxy_pass_header Retry-After; + proxy_pass http://pulse:7655; } ``` @@ -165,6 +169,7 @@ proxy_set_header X-Authentik-Groups $http_x_authentik_groups; 2. **HTTPS only**: Always use HTTPS between the proxy and Pulse in production 3. **Network isolation**: Ensure Pulse is not directly accessible, only through the proxy 4. **Header validation**: Pulse validates all headers and the proxy secret on every request +5. **Preserve rate-limit headers**: Do not strip `X-RateLimit-*` or `Retry-After`. Clients rely on them when Pulse throttles requests. ## Combining with Other Auth Methods @@ -187,7 +192,7 @@ Proxy authentication can work alongside other authentication methods: ``` 2. **Verify headers are being sent**: - - Enable debug logging: `LOG_LEVEL=debug` + - Temporarily raise logging to debug via **Settings → System → Logging** (or set `LOG_LEVEL=debug` and restart). Remember to return to `info` when finished. - Check Pulse logs: `docker logs pulse` or `journalctl -u pulse` - Look for "Invalid proxy secret" or "Proxy auth user header not found" diff --git a/docs/PULSE_SENSOR_PROXY_HARDENING.md b/docs/PULSE_SENSOR_PROXY_HARDENING.md index 3c828b9..e0c0a3f 100644 --- a/docs/PULSE_SENSOR_PROXY_HARDENING.md +++ b/docs/PULSE_SENSOR_PROXY_HARDENING.md @@ -504,28 +504,42 @@ journalctl -u pulse-sensor-proxy --since today | grep "rate limit" journalctl -u pulse-sensor-proxy | grep "corr_id=a7f3d" ``` +For rotation guidance, follow [operations/audit-log-rotation.md](operations/audit-log-rotation.md). After each rotation and proxy restart, verify the adaptive polling scheduler reports closed breakers and no DLQ entries for temperature pollers: + +```bash +curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.key | contains("temperature")) | {key, breaker: .breaker.state, deadLetter: .deadLetter.present}' +``` + ### Rate Limiting -**Current limits (per peer UID+PID):** -- **Rate**: 20 requests/minute (token bucket with burst) -- **Burst**: 10 requests -- **Concurrency**: 10 simultaneous requests +**Current limits (per peer UID):** +- **Rate**: ~12 requests/minute (Go `rate.Every(5s)` token bucket) + > Allows short bursts of 2 requests; steady-state calls beyond 12/min are rejected. +- **Per-peer concurrency**: 2 simultaneous RPCs +- **Global concurrency**: 8 in-flight RPCs across all peers +- **Penalty**: 2 s enforced sleep when validation fails (payload too large, malformed JSON, unauthorized method) +- **Per-node guard**: only 1 SSH fetch per target node at a time (prevents hammering the same hypervisor) **Behavior on limit exceeded:** - Request rejected immediately (no queuing) -- `pulse_proxy_rate_limit_hits_total` metric incremented -- Log entry: `"Rate limit exceeded"` -- HTTP-like semantics: Similar to 429 Too Many Requests +- `pulse_proxy_rate_limit_hits_total` and `pulse_proxy_limiter_rejects_total{reason}` increment +- Audit log entry with `limiter.rejection`/reason code (`rate`, `peer_concurrency`, `global_concurrency`) +- Client receives an RPC error equivalent to HTTP 429 semantics -**Adjust limits:** +**Adjust limits (advanced):** -Limits are hardcoded in `throttle.go`. To adjust, modify and rebuild: +The defaults live in `cmd/pulse-sensor-proxy/throttle.go`. To customise them, edit and rebuild: ```go -// cmd/pulse-sensor-proxy/throttle.go const ( - requestsPerMin = 20 // Change this - requestBurst = 10 // Change this - maxConcurrent = 10 // Change this + defaultPerPeerBurst = 2 + defaultPerPeerConcurrency = 2 + defaultGlobalConcurrency = 8 +) + +var ( + defaultPerPeerRateInterval = 5 * time.Second // 12 req/min + defaultPenaltyDuration = 2 * time.Second ) ``` diff --git a/docs/REVERSE_PROXY.md b/docs/REVERSE_PROXY.md index 1a2d43d..d56a7d4 100644 --- a/docs/REVERSE_PROXY.md +++ b/docs/REVERSE_PROXY.md @@ -6,8 +6,10 @@ Pulse uses WebSockets for real-time updates. Your reverse proxy **MUST** support 1. **WebSocket Support Required** - Enable WebSocket proxying 2. **Proxy Headers** - Forward original host and IP headers -3. **Timeouts** - Increase timeouts for long-lived connections +3. **Timeouts** - Increase timeouts for long-lived connections (7 days recommended) 4. **Buffer Sizes** - Increase for large state updates (64KB recommended) +5. **Rate Limit Headers (v4.24.0+)** - Pass through `X-RateLimit-*` and `Retry-After` headers +6. **Error Pass-through** - Don't intercept 429 responses (rate limits) ## Authentication with Reverse Proxy @@ -40,34 +42,38 @@ server { server { listen 443 ssl http2; server_name pulse.example.com; - + # SSL configuration ssl_certificate /path/to/cert.pem; ssl_certificate_key /path/to/key.pem; - + # Proxy settings location / { proxy_pass http://localhost:7655; proxy_http_version 1.1; - + # Required for WebSocket proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - + # Proxy headers proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - - # Timeouts for WebSocket + + # Timeouts for WebSocket and adaptive polling (v4.24.0+) proxy_connect_timeout 7d; proxy_send_timeout 7d; proxy_read_timeout 7d; - + # Disable buffering for real-time updates proxy_buffering off; - + + # IMPORTANT (v4.24.0+): Don't intercept error responses + # This ensures 429 rate limit responses reach the client + proxy_intercept_errors off; + # Increase buffer sizes for large messages proxy_buffer_size 64k; proxy_buffers 8 64k; @@ -261,6 +267,47 @@ curl -i -N \ https://pulse.example.com/api/ws ``` +### v4.24.0+ Verification + +**Test scheduler health API** (adaptive polling): +```bash +curl -s https://pulse.example.com/api/monitoring/scheduler/health | jq +``` + +**Expected response:** +- `enabled: true` +- Queue depth reasonable +- No stuck circuit breakers +- WebSocket connection remains open + +**Verify rate limit headers** (v4.24.0+): +```bash +curl -I https://pulse.example.com/api/version +``` + +**Look for headers:** +``` +HTTP/1.1 200 OK +X-RateLimit-Limit: 500 +X-RateLimit-Remaining: 499 +X-RateLimit-Reset: 1698765432 +``` + +**Test rate limiting:** +```bash +# Trigger rate limit (requires many requests) +for i in {1..600}; do curl -s https://pulse.example.com/api/health > /dev/null; done + +# Should eventually see: +curl -I https://pulse.example.com/api/health +# HTTP/1.1 429 Too Many Requests +# X-RateLimit-Limit: 500 +# X-RateLimit-Remaining: 0 +# Retry-After: 60 +``` + +**Important:** If you don't see rate limit headers or 429 responses are converted to 502/504, verify `proxy_intercept_errors off` (Nginx) or equivalent is set. + In browser console (F12): ```javascript // Test WebSocket connection diff --git a/docs/TEMPERATURE_MONITORING.md b/docs/TEMPERATURE_MONITORING.md index 7faed19..61a825e 100644 --- a/docs/TEMPERATURE_MONITORING.md +++ b/docs/TEMPERATURE_MONITORING.md @@ -123,6 +123,21 @@ When you need to provision the proxy yourself (for example via your own automati After the container restarts, the backend will automatically use the proxy. To refresh SSH keys on cluster nodes (e.g., after adding a new node), SSH to your Proxmox host and re-run the setup script: `curl -fsSL https://get.pulsenode.com/install-proxy.sh | bash -s -- --ctid ` +### Post-install Verification (v4.24.0+) + +1. **Confirm proxy metrics** + ```bash + curl -s http://127.0.0.1:9127/metrics | grep pulse_proxy_build_info + ``` +2. **Ensure adaptive polling sees the proxy** + ```bash + curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.key | contains("temperature")) | {key, pollStatus}' + ``` + - Expect recent `lastSuccess` timestamps, `breaker.state == "closed"`, and `deadLetter.present == false`. +3. **Check update history** – Any future proxy restarts/rollbacks are logged under **Settings → System → Updates**; include the associated `event_id` in post-change notes. +4. **Measure queue depth/staleness** – Grafana panels `pulse_monitor_poll_queue_depth` and `pulse_monitor_poll_staleness_seconds` should return to baseline within a few polling cycles. + ### Legacy Architecture (Pre-v4.24.0 / Native Installs) For native (non-containerized) installations, Pulse connects directly via SSH: diff --git a/docs/TEMPERATURE_MONITORING_SECURITY.md b/docs/TEMPERATURE_MONITORING_SECURITY.md index 850704c..962eab5 100644 --- a/docs/TEMPERATURE_MONITORING_SECURITY.md +++ b/docs/TEMPERATURE_MONITORING_SECURITY.md @@ -67,10 +67,10 @@ from="192.168.0.0/24",command="sensors -j",no-port-forwarding,no-X11-forwarding, - IP restrictions prevent lateral movement ### 3. Client ↔ Proxy Boundary -- **Enforced by**: UID-based ACL + rate limiting +- **Enforced by**: UID-based ACL + adaptive rate limiting - SO_PEERCRED verifies caller's UID/GID/PID -- Rate limiting: 20 requests/minute per peer, burst of 10 -- Concurrency limit: 10 simultaneous requests per peer +- Rate limiting (defaults): ~12 requests per minute per UID (burst 2), per-UID concurrency 2, global concurrency 8, 2 s penalty on validation failures +- Per-node guard: only 1 SSH fetch per node at a time --- @@ -116,9 +116,11 @@ if privilegedMethods[method] && isIDMappedRoot(credentials) { ## Rate Limiting ### Per-Peer Limits -- **Rate**: 20 requests per minute -- **Burst**: 10 requests (allows short bursts) -- **Concurrency**: Maximum 10 simultaneous requests +- **Rate**: ~12 requests per minute (`rate.Every(5s)`) +- **Burst**: 2 requests (short spikes are tolerated) +- **Per-peer concurrency**: Maximum 2 simultaneous RPCs +- **Global concurrency**: 8 total in-flight RPCs across all peers +- **Penalty**: 2 s enforced delay when validation fails (payloads too large, unauthorized methods) - **Cleanup**: Idle peer entries removed after 10 minutes ### Per-Node Concurrency @@ -129,7 +131,7 @@ if privilegedMethods[method] && isIDMappedRoot(credentials) { ### Monitoring Rate Limits ```bash # Check rate limit metrics -curl -s http://localhost:9090/metrics | grep pulse_sensor_proxy_rate_limit_hits +curl -s http://127.0.0.1:9127/metrics | grep pulse_proxy_limiter_rejects_total # Watch for rate limit warnings in logs journalctl -u pulse-sensor-proxy -f | grep "Rate limit exceeded" @@ -230,6 +232,8 @@ journalctl -u pulse-sensor-proxy -f journalctl -u pulse-backend -f ``` +**Audit rotation**: Use the steps in [operations/audit-log-rotation.md](operations/audit-log-rotation.md) to rotate `/var/log/pulse/sensor-proxy/audit.log`. After each rotation, restart the proxy and confirm temperature pollers are healthy in `/api/monitoring/scheduler/health` (closed breakers, no DLQ entries). + ### Security Events to Monitor #### 1. Privileged Method Denials @@ -265,29 +269,30 @@ SECURITY BLOCK: SSH temperature collection disabled in containers ```bash # Rate limit hits -pulse_sensor_proxy_rate_limit_hits_total +pulse_proxy_rate_limit_hits_total # RPC requests by method and result -pulse_sensor_proxy_rpc_requests_total{method="get_temperature",result="success"} -pulse_sensor_proxy_rpc_requests_total{method="ensure_cluster_keys",result="unauthorized"} +pulse_proxy_rpc_requests_total{method="get_temperature",result="success"} +pulse_proxy_rpc_requests_total{method="ensure_cluster_keys",result="unauthorized"} # SSH request latency -pulse_sensor_proxy_ssh_request_duration_seconds{node="delly"} +pulse_proxy_ssh_latency_seconds{node="delly"} # Active connections -pulse_sensor_proxy_active_connections +pulse_proxy_queue_depth +pulse_proxy_global_concurrency_inflight ``` ### Recommended Alerts 1. **Privilege Escalation Attempts**: ``` - pulse_sensor_proxy_rpc_requests_total{result="unauthorized"} > 0 + pulse_proxy_rpc_requests_total{result="unauthorized"} > 0 ``` 2. **Rate Limit Abuse**: ``` - rate(pulse_sensor_proxy_rate_limit_hits_total[5m]) > 1 + rate(pulse_proxy_rate_limit_hits_total[5m]) > 1 ``` 3. **Proxy Unavailable**: @@ -295,6 +300,12 @@ pulse_sensor_proxy_active_connections up{job="pulse-sensor-proxy"} == 0 ``` +4. **Scheduler Drift** (Pulse side – ensures temperature pollers stay healthy): + ``` + max_over_time(pulse_monitor_poll_queue_depth[5m]) > + ``` + Pair with a check of `/api/monitoring/scheduler/health` to confirm temperature instances report `breaker.state == "closed"`. + --- ## Development Mode @@ -370,12 +381,12 @@ systemctl restart pulse-backend **Symptom**: Requests failing with rate limit error -**Cause**: More than 20 requests/minute from same process +**Cause**: Peer exceeded ~12 requests/minute (or exhausted per-peer/global concurrency) **Resolution**: -1. Check if legitimate high request rate -2. Increase rate limit in config if needed -3. Check for retry loops in client code +1. Confirm workload is legitimate (look for retry loops or aggressive polling). +2. Allow the limiter to recover—penalty sleeps clear in ~2 s and idle peers expire after 10 minutes. +3. If sustained higher throughput is required, adjust the constants in `cmd/pulse-sensor-proxy/throttle.go` and rebuild. ### Temperature monitoring unavailable @@ -398,6 +409,10 @@ curl -s --unix-socket /run/pulse-sensor-proxy/pulse-sensor-proxy.sock \ # 5. Check SSH connectivity ssh root@delly "sensors -j" + +# 6. Inspect adaptive polling for temperature pollers +curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.key | contains("temperature")) | {key, breaker: .breaker.state, deadLetter: .deadLetter.present, lastSuccess: .pollStatus.lastSuccess}' ``` ### SSH key not distributed diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 146aade..2492fec 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -106,6 +106,31 @@ Common issues: - Volume permissions: Ensure volume is writable - Invalid environment variables: Check syntax +### Port change didn't take effect +1. **Confirm which service name is active** + ```bash + systemctl status pulse 2>/dev/null \\ + || systemctl status pulse-backend 2>/dev/null \\ + || systemctl status pulse-hot-dev + ``` + - Docker: the container port mapping controls the public port (`-p host:7655`). + - Kubernetes (Helm chart): service is `svc/pulse`; update `service.port` in your values file and run `helm upgrade`. + +2. **Verify configuration/environment overrides** + ```bash + sudo systemctl show pulse --property=Environment + ``` + Helm users can run `kubectl get svc pulse -n -o yaml` to confirm the current port. + +3. **Check for port conflicts** + ```bash + sudo lsof -i :8080 + ``` + +4. **Post-change validation** + - Restart the service (`systemctl restart`, `docker restart`, or `helm upgrade`). + - v4.24.0 logs these restarts/upgrades in **Settings → System → Updates** and `/api/updates/history`; capture the `event_id` for your change notes. + ### Installation Issues #### Binary not found (v4.3.7) @@ -147,7 +172,7 @@ systemctl status pulse 2>/dev/null \ - Verify URL is accessible from Pulse server - Check for SSL certificate issues - Try a test service like webhook.site -- Check logs for response codes +- Check logs for response codes (temporarily set `LOG_LEVEL=debug` via **Settings → System → Logging** or export `LOG_LEVEL=debug` and restart; review `webhook.delivery` entries, then revert to `info`) ### VM Disk Monitoring Issues diff --git a/docs/VM_DISK_MONITORING.md b/docs/VM_DISK_MONITORING.md index 3cfc823..f15bf2e 100644 --- a/docs/VM_DISK_MONITORING.md +++ b/docs/VM_DISK_MONITORING.md @@ -231,6 +231,7 @@ If the agent is working but Pulse still shows "-": qm agent get-fsinfo ``` If this works but Pulse doesn't show data, check Pulse permissions and logs + (v4.24.0: adjust **Settings → System → Logging** to `debug` temporarily if you need more detail, then revert to `info`). 3. **Check agent version** - Older agents might not support filesystem info diff --git a/docs/WEBHOOKS.md b/docs/WEBHOOKS.md index 4a666b7..3a6420b 100644 --- a/docs/WEBHOOKS.md +++ b/docs/WEBHOOKS.md @@ -268,6 +268,7 @@ Supported variables are identical to the payload template list (`{{.Message}}`, - Check alert thresholds are configured correctly - Ensure notification cooldown period has passed - Test the webhook manually using the Test button +- Temporarily raise log level to `debug` via **Settings → System → Logging** (or set `LOG_LEVEL=debug` and restart) to inspect `webhook.delivery` entries in the logs, then return to `info` once resolved. ## API Reference diff --git a/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md b/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md index 66562f3..71bbfcc 100644 --- a/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md +++ b/docs/operations/ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md @@ -2,65 +2,259 @@ ## Status: DEFERRED -**Decision Date:** 2025-10-20 -**Reviewed After:** Phase 2 integration/soak testing +**Decision Date:** 2025-10-20 +**Re-evaluated:** v4.24.0 GA release +**Current Status:** Not implemented in v4.24.0 -## Assessment +## Overview -After completing comprehensive integration and soak testing (including 2-minute validation with 80 instances), we determined that manual circuit breaker and DLQ management endpoints are **not required for Phase 2 production rollout**. - -### Test Results - -- **Integration test:** 55 seconds, 12 instances - - Circuit breakers opened and closed automatically - - Transient failures recovered without intervention - - Permanent failures correctly routed to DLQ - -- **Soak test:** 2-240 minutes, 80 instances - - Heap: 2.3MB → 3.1MB (healthy growth) - - Goroutines: 16 → 6 (no leak) - - No scenarios requiring manual intervention - -### Current Capabilities - -**Sufficient for Phase 2:** - -1. **Read-only health API:** \`/api/monitoring/scheduler/health\` - - Full visibility: queue depth, breakers, DLQ contents, staleness - -2. **Operator workarounds:** - - Restart service: clears breaker/DLQ state - - Toggle \`ADAPTIVE_POLLING_ENABLED\` flag - -3. **Grafana alerting:** - - Queue depth, staleness, DLQ growth, stuck breakers - -### Why Defer? - -- **No operational need demonstrated:** Tests showed automatic recovery works -- **Implementation cost:** Requires auth/RBAC, audit logging, UI integration -- **Wait for data:** Production usage will reveal actual pain points +Manual circuit breaker and dead-letter queue (DLQ) management endpoints are **not included in v4.24.0**. The read-only scheduler health API (`/api/monitoring/scheduler/health`) provides full visibility, and automatic recovery mechanisms have proven sufficient during testing and early production rollouts. --- -## Future Implementation (When Needed) +## What's Available in v4.24.0 -### Proposed Endpoints +### Read-Only Scheduler Health API -1. **POST /api/monitoring/breakers/{instance}/reset** - Reset circuit breaker -2. **POST /api/monitoring/dlq/retry** - Retry all DLQ tasks -3. **POST /api/monitoring/dlq/{instance}/retry** - Retry specific task -4. **DELETE /api/monitoring/dlq/{instance}** - Remove from DLQ +**Endpoint:** `GET /api/monitoring/scheduler/health` -See full design spec in this document. +Provides complete visibility into: +- Queue depth and task distribution +- Circuit breaker states per instance +- Dead-letter queue contents and retry schedules +- Per-instance staleness tracking +- Failure streaks and error categorization + +**Documentation:** See [Scheduler Health API](../api/SCHEDULER_HEALTH.md) for complete reference. + +### Existing Management Options + +**v4.24.0 operators can:** + +1. **Toggle adaptive polling** (no restart required) + - Via UI: **Settings → System → Monitoring** + - Via API: Update `system.json` with `adaptivePollingEnabled: false` + +2. **Service restart** (clears transient state) + ```bash + # Systemd + sudo systemctl restart pulse + + # Docker + docker restart pulse + + # LXC + pct restart + ``` + - Clears all circuit breakers + - Resets DLQ (tasks re-queued with fresh state) + - Useful for recovering from stuck states + +3. **Version rollback** (if broader issues) + - Via UI: **Settings → System → Updates → Restore previous version** + - Via CLI: `pulse config rollback` + - Documented in [Operations Runbook](ADAPTIVE_POLLING_ROLLOUT.md) + +4. **Per-instance configuration fixes** + - Update node credentials if authentication failures cause DLQ entries + - Adjust network/firewall if connectivity issues trigger breakers + - Fix underlying infrastructure problems + +--- + +## Why Endpoints Are Deferred + +### Test Results Demonstrate Sufficient Automation + +**Integration testing** (55 seconds, 12 instances): +- Circuit breakers opened and closed automatically +- Transient failures recovered without intervention +- Permanent failures correctly routed to DLQ + +**Soak testing** (2-240 minutes, 80 instances): +- Heap: 2.3MB → 3.1MB (healthy growth) +- Goroutines: 16 → 6 (no leak) +- No scenarios requiring manual intervention + +**Production rollout** (v4.24.0): +- Automatic recovery working as designed +- Service restart sufficient for edge cases +- No operator requests for manual controls + +### Implementation Cost vs. Benefit + +**Would require:** +- Authentication and RBAC integration +- Comprehensive audit logging +- UI integration in Settings → System → Monitoring +- Additional testing and maintenance burden + +**Current workarounds proven effective:** +- Adaptive polling toggle (immediate, no restart) +- Service restart (clears all state in < 30 seconds) +- Version rollback (if systematic issues) + +--- + +## Future Implementation Plan + +### Proposed Endpoints (When Needed) + +If production usage reveals operational gaps, implement: + +#### 1. Reset Circuit Breaker +``` +POST /api/monitoring/breakers/{key}/reset +Authorization: Required (session or API token) +``` + +**Request:** +```json +{ + "reason": "Manual reset after infrastructure fix" +} +``` + +**Response:** +```json +{ + "success": true, + "key": "pve::pve-node1", + "previousState": "open", + "newState": "closed", + "resetBy": "admin", + "resetAt": "2025-10-20T15:30:00Z" +} +``` + +**Use case:** Immediately retry a specific instance after fixing underlying issue (e.g., restored network connectivity) + +#### 2. Retry All DLQ Tasks +``` +POST /api/monitoring/dlq/retry +Authorization: Required (session or API token) +``` + +**Response:** +```json +{ + "success": true, + "tasksRetried": 5, + "keys": ["pve::pve-node1", "pbs::backup-server"] +} +``` + +**Use case:** Bulk retry after fixing widespread issue (e.g., certificate renewal) + +#### 3. Retry Specific DLQ Task +``` +POST /api/monitoring/dlq/{key}/retry +Authorization: Required (session or API token) +``` + +**Response:** +```json +{ + "success": true, + "key": "pve::pve-node1", + "previousRetryCount": 5, + "scheduledFor": "2025-10-20T15:35:00Z" +} +``` + +**Use case:** Targeted retry of single instance + +#### 4. Remove from DLQ +``` +DELETE /api/monitoring/dlq/{key} +Authorization: Required (session or API token) +``` + +**Response:** +```json +{ + "success": true, + "key": "pve::decomissioned-node", + "reason": "Instance permanently decommissioned" +} +``` + +**Use case:** Remove decommissioned instances from DLQ + +### Security Requirements + +All management endpoints would require: +- **Authentication:** Valid session cookie or API token +- **RBAC:** Admin-level permissions +- **Audit logging:** Every action logged with: + - Operator username/IP + - Instance key affected + - Reason provided + - Timestamp + - Previous and new states +- **Rate limiting:** Prevent abuse (e.g., 10 requests/minute) --- ## Re-evaluation Criteria -**Implement if:** -- Operators request manual controls >3x in first 30 days -- Rollout troubleshooting steps inadequate -- Service restarts become disruptive -- Production incidents need surgical controls +**Implement management endpoints if:** +1. **Operator demand:** >3 requests in first 60 days of v4.24.0 deployment +2. **Service restart frequency:** >5 restarts per week due to stuck breakers/DLQ +3. **Incident impact:** Manual controls would have prevented or accelerated recovery from >1 production incident +4. **Feedback from operations runbook:** [ADAPTIVE_POLLING_ROLLOUT.md](ADAPTIVE_POLLING_ROLLOUT.md) troubleshooting inadequate + +**Don't implement if:** +- Current workarounds remain effective +- Automatic recovery continues to handle 99%+ of scenarios +- No clear operational pain points emerge + +--- + +## Monitoring Current State + +### Check Circuit Breakers +```bash +curl -s http://:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.breaker.state != "closed") | {key, state: .breaker.state, since: .breaker.since}' +``` + +### Check Dead-Letter Queue +```bash +curl -s http://:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.deadLetter.present) | {key, reason: .deadLetter.reason, retryCount: .deadLetter.retryCount, nextRetry: .deadLetter.nextRetry}' +``` + +### Track Recovery Times +```bash +# Monitor breaker state changes +journalctl -u pulse | grep -E "circuit breaker|dead-letter" +``` + +--- + +## Feedback & Requests + +If you encounter scenarios where manual management endpoints would be valuable: + +1. **Document the use case** + - What problem occurred? + - Why wasn't automatic recovery sufficient? + - How would manual control have helped? + +2. **File an issue** + - [GitHub Issues](https://github.com/rcourtman/Pulse/issues) + - Include: scheduler health API output, logs, timeline + +3. **Track frequency** + - If the pattern recurs >3 times, escalate for implementation + +--- + +## Related Documentation + +- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference +- [Operations Runbook](ADAPTIVE_POLLING_ROLLOUT.md) - Steady-state operations and troubleshooting +- [Adaptive Polling Architecture](../monitoring/ADAPTIVE_POLLING.md) - Technical details +- [Configuration Guide](../CONFIGURATION.md) - Adaptive polling settings diff --git a/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md b/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md index b6bb3f7..554d128 100644 --- a/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md +++ b/docs/operations/ADAPTIVE_POLLING_ROLLOUT.md @@ -1,145 +1,207 @@ -# Adaptive Polling Rollout Playbook +# Adaptive Polling Operations Runbook -This playbook guides operators through enabling the adaptive polling scheduler in -production. Follow the steps sequentially and record key checkpoints in the run -sheet for audit purposes. +**GA in v4.24.0 - Enabled by Default** + +This runbook guides operators managing the adaptive polling scheduler in production. Adaptive polling is enabled by default in v4.24.0 and can be toggled via **Settings → System → Monitoring** (no restart) or environment variables (restart required). + +Follow these operational procedures for steady-state monitoring, troubleshooting, and rollback scenarios. --- -## 1. Prerequisites +## 1. Preparation (Before v4.24.0 Deployment) -1. **Test suite status** - - `go test ./...` and `go test -tags=integration ./internal/monitoring -run TestAdaptiveSchedulerIntegration` - - Adaptive polling soak test: - ``` - HARNESS_SOAK_MINUTES=15 go test -tags=integration ./internal/monitoring -run TestAdaptiveSchedulerSoak -soak -timeout 30m - ``` - - All tests must pass within the last 24 hours. +**For new deployments or upgrades to v4.24.0:** -2. **Monitoring readiness** - - Grafana dashboard updated with: +1. **Monitoring readiness** + - Set up Grafana dashboard with: - `pulse_monitor_poll_queue_depth` (gauge) - `pulse_monitor_poll_staleness_seconds` (gauge, per instance) - `pulse_monitor_poll_total` and `pulse_monitor_poll_errors_total` (rate panels) - - Alerting panels for circuit breaker state (via scheduler health API). - - Alerts configured (see §4). + - `pulse_monitor_poll_last_success_timestamp` (new in v4.24.0) + - Alerting panels for circuit breaker state (via scheduler health API) + - Configure alerts (see §4) -3. **Configuration management** - - Ensure staging and production environments are managed via `system.json` or appropriate env vars. - - Identify the operator owning flag toggles and service restarts. +2. **Baseline metrics** + - Record pre-upgrade metrics if upgrading from < v4.24.0: + - Typical polling frequency + - Average response times + - Current alert volumes + - These help assess adaptive polling impact -4. **Rollback plan** - - Confirm ability to set `ADAPTIVE_POLLING_ENABLED=false` and restart `pulse-hot-dev` or equivalent service within 5 minutes. - - Document the `systemctl restart pulse-hot-dev` command path or container restart procedure. +3. **Rollback readiness** + - Verify update rollback workflow: **Settings → System → Updates → Restore previous version** + - Test rollback in staging environment + - Confirm `/api/monitoring/scheduler/health` accessible + - Document emergency disable procedure (see §5) -5. **Stakeholder sign-off** - - Adaptive polling feature owner approves rollout window. - - SRE and on-call engineer acknowledge the playbook. +4. **Configuration review** + - Review `system.json` or environment variables for adaptive polling tunables: + - `ADAPTIVE_POLLING_BASE_INTERVAL` (default: 10s) + - `ADAPTIVE_POLLING_MIN_INTERVAL` (default: 5s) + - `ADAPTIVE_POLLING_MAX_INTERVAL` (default: 5m) + - Adjust if needed for your environment (e.g., high-frequency monitoring) --- -## 2. Staging Rollout +## 2. Post-Deployment Verification (v4.24.0+) -1. **Enable feature flag** - - Update staging configuration: - ``` - export ADAPTIVE_POLLING_ENABLED=true - ``` - or edit `system.json` and set `"adaptivePollingEnabled": true`. - - Restart hot-dev service / container to apply: - ``` - systemctl restart pulse-hot-dev - ``` - (Adapt to your env if using Docker/K8s.) +**Adaptive polling is enabled by default. Verify it's working correctly:** -2. **Verification** - - `curl -s http://:7655/api/monitoring/scheduler/health | jq` - - Expect `"enabled": true`. - - Check Grafana dashboard for the staging cluster: - - Queue depth should stabilise near historic baseline (< instances × 1.5). - - Staleness gauges should stay below 60 s for healthy instances. - - No persistent circuit breakers (`state != "closed"`) except known failing endpoints. +1. **Check scheduler health** + ```bash + curl -s http://:7655/api/monitoring/scheduler/health | jq + ``` -3. **Observation window** - - Monitor for 24–48 hours. - - Success criteria: - - No increase in polling failures or alert volume. - - Queue depth and staleness metrics remain within SLO (queue depth < 1.5× instance count, staleness < 60 s). - - Scheduler health API shows empty dead-letter queue or expected entries only. - - Record key metric snapshots at 0 h, 12 h, 24 h. + **Expected response:** + - `"enabled": true` + - `queue.depth` reasonable (< instances × 1.5) + - `deadLetter.count` = 0 (or only known failing instances) + - `instances[]` array populated with your nodes + - No `breaker.state` stuck in `open` (except known issues) -4. **Sign-off** - - If criteria met, proceed to production. Otherwise revert flag to false and investigate (§6). +2. **Verify UI access** + - Navigate to **Settings → System → Monitoring** + - Confirm "Adaptive Polling" toggle is ON + - Review queue depth and recent poll status + +3. **Check Grafana metrics** (if configured) + - `pulse_monitor_poll_queue_depth` shows reasonable values + - `pulse_monitor_poll_staleness_seconds` < 60s for healthy instances + - `pulse_monitor_poll_errors_total` not rapidly increasing + - `pulse_monitor_poll_last_success_timestamp` updating regularly + +4. **Monitor update history** + - Check **Settings → System → Updates** for update entry + - Verify rollback button is available + - Confirm update status shows "completed" + + **Via API:** + ```bash + curl -s http://:7655/api/updates/history | jq '.entries[0]' + ``` --- -## 3. Production Rollout +## 3. Steady-State Operations -1. **Rollout strategy** - - Perform during low-traffic maintenance window. - - Enable flag gradually by cluster or instance group (e.g., 25 % of nodes every 2 hours): - 1. Update config (`ADAPTIVE_POLLING_ENABLED=true`) for first subset. - 2. Restart service on those nodes. - 3. Watch metrics for at least 30 minutes before continuing. +**Ongoing monitoring and SLO checks:** -2. **Monitoring during rollout** - - Grafana dashboard per cluster: - - `poll_queue_depth` - - `poll_staleness_seconds` - - `poll_total` success/error ratio - - Scheduler health API: - ``` - curl -s http://:7655/api/monitoring/scheduler/health | jq - ``` - - Confirm `enabled: true`, `deadLetter.count` stable, `breakers` mostly empty. +1. **Daily health checks** + - Review scheduler health dashboard or API endpoint + - Check for: + - Queue depth < 50 (alert if > 50 for 10+ minutes) + - Staleness < 120s for critical instances + - DLQ count stable (not growing) + - Circuit breakers mostly `closed` -3. **Success criteria** - - Queue depth rises temporarily but settles within threshold (< 1.5× instance count). - - Staleness stays below 60 s for healthy instances. - - No unexplained increase in alert volume or API error rate. - - Dead-letter queue holds only known failing targets. +2. **Weekly reviews** + - Analyze trends in Grafana: + - Poll success rates + - Average queue depth over time + - Circuit breaker trip frequency + - Dead-letter queue patterns + - Document any recurring issues -4. **Completion** - - After all nodes enabled, monitor for an additional 24 h. - - Record final metric snapshot. +3. **SLO targets** + - **Queue depth**: < 1.5× instance count (< 50 typical) + - **Staleness**: < 60s for healthy instances, < 120s for critical instances + - **Poll success rate**: > 99% for healthy infrastructure + - **DLQ growth**: < 5% per week (excluding known failures) + - **Circuit breaker recovery**: < 5 minutes for transient failures + +4. **Log correlation** + - Cross-reference scheduler health with update history + - Check `/api/updates/history` for rollback events correlated with scheduler issues + - Review audit logs for adaptive polling configuration changes --- ## 4. Grafana & Alert Configuration 1. **Dashboard panels** - - **Queue Depth**: `pulse_monitor_poll_queue_depth`. - - Use single-stat with alert if > 1.5× active instances for > 10 min. - - **Instance Staleness**: panel per instance type using `pulse_monitor_poll_staleness_seconds`. - - Alert threshold: > 60 s for > 5 min (excluding known failing instances). - - **Polling Throughput**: rate of `pulse_monitor_poll_total{result="success"}` vs `result="error"`. - - **Circuit Breakers / DLQ**: table from scheduler health API (via scripted datasource) highlighting non-closed breakers or DLQ entries. + - **Queue Depth**: `pulse_monitor_poll_queue_depth` + - Use single-stat with alert if > 1.5× active instances for > 10 min + - **Instance Staleness**: panel per instance type using `pulse_monitor_poll_staleness_seconds` + - Alert threshold: > 60 s for > 5 min (excluding known failing instances) + - **Polling Throughput**: rate of `pulse_monitor_poll_total{result="success"}` vs `result="error"` + - **Circuit Breakers / DLQ**: table from scheduler health API (via scripted datasource) highlighting non-closed breakers or DLQ entries + - **Last Success Timestamp** (v4.24.0+): `pulse_monitor_poll_last_success_timestamp` to detect polling gaps 2. **Alerts** - - Queue depth > threshold for >10 min (Warning), >20 min (Critical). - - Staleness > 60 s for >5 min (Critical). - - Dead-letter count increase > N (based on baseline) triggers Warning. - - Any breaker stuck in `open` for >10 min triggers Critical. + - Queue depth > threshold for >10 min (Warning), >20 min (Critical) + - Staleness > 60 s for >5 min (Critical) + - Dead-letter count increase > N (based on baseline) triggers Warning + - Any breaker stuck in `open` for >10 min triggers Critical + - Permanent failures (`pulse_monitor_poll_errors_total{category="permanent"}`) trigger immediate Critical 3. **Notification routing** - - Ensure alerts route to on-call + feature owner. + - Ensure alerts route to on-call + feature owner --- -## 5. Rollback Procedure +## 5. Rollback Procedures -1. **Disable adaptive polling** - - Set `ADAPTIVE_POLLING_ENABLED=false` (env or `system.json`). - - Restart service (`systemctl restart pulse-hot-dev` or equivalent). +### Option A: Disable Adaptive Polling (Keep v4.24.0) -2. **Verification** - - Scheduler health API should show `"enabled": false`. - - Queue depth returns to pre-feature baseline within 10–15 minutes. - - Staleness/queue alerts clear. +**If adaptive polling causes issues but you want to keep v4.24.0:** -3. **Post-rollback actions** - - Notify stakeholders, capture metric snapshots showing recovery. - - File incident report if rollback triggered by outage. +1. **Via UI (No restart required)** + - Navigate to **Settings → System → Monitoring** + - Toggle "Adaptive Polling" OFF + - Changes apply immediately + +2. **Via environment variables (Restart required)** + ```bash + # Systemd + sudo systemctl edit pulse + # Add: + [Service] + Environment="ADAPTIVE_POLLING_ENABLED=false" + + # Then restart + sudo systemctl restart pulse + ``` + +3. **Verification** + ```bash + curl -s http://:7655/api/monitoring/scheduler/health | jq '.enabled' + # Should return: false + ``` + +### Option B: Full Version Rollback + +**If v4.24.0 causes broader issues:** + +1. **Via UI** + - Navigate to **Settings → System → Updates** + - Click **"Restore previous version"** + - Confirm rollback + - Pulse restarts automatically with previous version + +2. **Via CLI** + ```bash + # Systemd installations + sudo /opt/pulse/pulse config rollback + + # LXC containers + pct exec -- bash -c "cd /opt/pulse && ./pulse config rollback" + ``` + +3. **Verification** + ```bash + # Check version + curl -s http://:7655/api/version | jq '.version' + + # Check update history + curl -s http://:7655/api/updates/history | jq '.entries[0]' + # Should show action="rollback", status="completed" + ``` + +4. **Post-rollback** + - Verify rollback logged in update history + - Check journal: `journalctl -u pulse | grep rollback` + - Monitor for 15-30 minutes to ensure stability + - Document rollback reason and notify stakeholders --- @@ -147,15 +209,15 @@ sheet for audit purposes. | Symptom | Possible Cause | Action | |---------|----------------|--------| -| Queue depth remains high (> 2× usual) | Insufficient workers, hidden breaker, misconfigured flag | Check scheduler health API for breaker states; consider increasing workers or reverting flag. | -| Staleness spikes across many instances | Backend API slowdown or connectivity issues | Inspect backend logs, network health; revert flag if duration > 15 min. | -| Dead-letter count climbs rapidly | Downstream API failures | Investigate specific instances via scheduler health API; fix credential/connectivity issues or rollback. | -| Circuit breakers stuck half-open/open | Persistent transient failures | Review error logs, ensure backoff/rate limits not starving retries; rollback if unresolved quickly. | -| Grafana panels flatline | Metrics exporter or job issue | Ensure Prometheus scraping working; verify service restarted with flag. | +| Queue depth remains high (> 2× usual) | Insufficient workers, hidden breaker, misconfigured flag | Check scheduler health API for breaker states; consider increasing workers or reverting flag | +| Staleness spikes across many instances | Backend API slowdown or connectivity issues | Inspect backend logs, network health; revert flag if duration > 15 min | +| Dead-letter count climbs rapidly | Downstream API failures | Investigate specific instances via scheduler health API; fix credential/connectivity issues or rollback | +| Circuit breakers stuck half-open/open | Persistent transient failures | Review error logs, ensure backoff/rate limits not starving retries; rollback if unresolved quickly | +| Grafana panels flatline | Metrics exporter or job issue | Ensure Prometheus scraping working; verify service restarted with flag | ### Accessing Scheduler Health API -``` +```bash curl -s http://:7655/api/monitoring/scheduler/health | jq ``` @@ -169,20 +231,20 @@ Key sections to inspect: Common queries: -*Instances with errors* -``` +**Instances with errors:** +```bash curl -s http://:7655/api/monitoring/scheduler/health \ | jq '.instances[] | select(.pollStatus.lastError != null) | {key, lastError: .pollStatus.lastError}' ``` -*Current dead-letter entries* -``` +**Current dead-letter entries:** +```bash curl -s http://:7655/api/monitoring/scheduler/health \ | jq '.instances[] | select(.deadLetter.present) | {key, reason: .deadLetter.reason, retryCount: .deadLetter.retryCount}' ``` -*Breakers not closed* -``` +**Breakers not closed:** +```bash curl -s http://:7655/api/monitoring/scheduler/health \ | jq '.instances[] | select(.breaker.state != "closed") | {key, breaker: .breaker}' ``` @@ -190,9 +252,18 @@ curl -s http://:7655/api/monitoring/scheduler/health \ ### When to Roll Back Rollback immediately if any of the following occurs: -- Queue depth > 3× baseline for > 15 min. -- Staleness > 120 s on majority of instances. -- Dead-letter count doubles without clear cause. -- Customer-facing alerts or latency regressions attributed to adaptive polling. +- Queue depth > 3× baseline for > 15 min +- Staleness > 120 s on majority of instances +- Dead-letter count doubles without clear cause +- Customer-facing alerts or latency regressions attributed to adaptive polling Document the incident and notify stakeholders after rollback. + +--- + +## 7. Related Documentation + +- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference +- [Adaptive Polling Architecture](../monitoring/ADAPTIVE_POLLING.md) - Technical details +- [Management Endpoints](ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md) - Circuit breaker/DLQ controls +- [Configuration Guide](../CONFIGURATION.md) - Adaptive polling settings diff --git a/docs/operations/audit-log-rotation.md b/docs/operations/audit-log-rotation.md index d58867e..271df60 100644 --- a/docs/operations/audit-log-rotation.md +++ b/docs/operations/audit-log-rotation.md @@ -59,3 +59,105 @@ together chronologically while preserving the tamper-evident guarantees. Adjust the `create` directive to match the user and group that run the sensor proxy. The example assumes both user and group are `pulse`. + +--- + +## Post-Rotation Health Checks (v4.24.0+) + +**After rotating audit logs and restarting pulse-sensor-proxy, verify adaptive polling health:** + +### 1. Check Scheduler Health + +```bash +curl -s http://localhost:7655/api/monitoring/scheduler/health | jq +``` + +**Verify:** +- Temperature proxy pollers appear in `instances[]` array +- `pollStatus.lastSuccess` is recent (within last 60 seconds) +- No new entries in `deadLetter` queue for proxy instances +- `breaker.state` is `closed` for proxy nodes + +**Example check for proxy instances:** +```bash +curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.type == "proxy" or .connection | contains("proxy")) | {key, lastSuccess: .pollStatus.lastSuccess, breaker: .breaker.state}' +``` + +### 2. Monitor Metrics (10-15 minutes) + +Watch these metrics to ensure proxy restart didn't cause issues: + +```bash +# Queue depth should remain stable +curl -s http://localhost:7655/api/monitoring/scheduler/health | jq '.queue.depth' + +# Check staleness for proxy instances +curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.type == "proxy") | {key, staleness: .pollStatus.lastSuccess}' +``` + +**Expected behavior:** +- Queue depth: No significant spike (< 10 temporary increase acceptable) +- Staleness: Proxy instances show fresh polls within 30-60 seconds +- No circuit breaker trips for proxy instances +- No new DLQ entries + +### 3. Cross-Reference Audit Logs + +**Link rotation events with scheduler health for security review:** + +```bash +# Check Pulse audit log for rotation timing +journalctl -u pulse-sensor-proxy --since "10 minutes ago" | grep -E "restart|rotation" + +# Check update history for any concurrent events +curl -s http://localhost:7655/api/updates/history?limit=5 | jq '.entries[] | {action, timestamp, status}' +``` + +**Why this matters:** +- Security auditors can correlate proxy restarts with scheduler behavior +- Update rollbacks may be concurrent with log rotations +- Rollback metadata (new in v4.24.0) provides full operational context +- Ensures restart didn't mask polling failures or breaker trips + +### 4. Troubleshooting Rotation Issues + +**If proxy instances don't rejoin queue:** + +1. **Check service status** + ```bash + systemctl status pulse-sensor-proxy + ``` + +2. **Verify scheduler sees the proxy** + ```bash + curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.type == "proxy")' + ``` + +3. **Check for circuit breakers** + ```bash + curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.breaker.state != "closed") | {key, state: .breaker.state, retryAt: .breaker.retryAt}' + ``` + +4. **Review logs for errors** + ```bash + journalctl -u pulse-sensor-proxy -n 50 + journalctl -u pulse | grep -E "proxy|temperature" + ``` + +**Recovery actions:** +- If breakers are stuck: Restart main Pulse service (`systemctl restart pulse`) +- If DLQ entries persist: Check proxy credentials and network connectivity +- If polling doesn't resume: Verify proxy configuration in **Settings → Sensors** + +--- + +## Related Documentation + +- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference +- [Adaptive Polling Operations](ADAPTIVE_POLLING_ROLLOUT.md) - Health monitoring procedures +- [Pulse Sensor Proxy Hardening](../security/pulse-sensor-proxy-hardening.md) - Security configuration +- [Temperature Monitoring Security](../TEMPERATURE_MONITORING_SECURITY.md) - Proxy-specific security notes diff --git a/docs/operations/pulse-sensor-proxy-runbook.md b/docs/operations/pulse-sensor-proxy-runbook.md index a2a1c9c..5d2882b 100644 --- a/docs/operations/pulse-sensor-proxy-runbook.md +++ b/docs/operations/pulse-sensor-proxy-runbook.md @@ -5,8 +5,8 @@ - Unit: `pulse-sensor-proxy.service` - Logs: `/var/log/pulse/sensor-proxy/proxy.log` - Audit trail: `/var/log/pulse/sensor-proxy/audit.log` (hash chained, forwarded via rsyslog) -- Metrics: `http://127.0.0.1:9456/metrics` -- Limiters: per-UID token bucket (burst 2) + global concurrency (8) +- Metrics: `http://127.0.0.1:9127/metrics` (set `PULSE_SENSOR_PROXY_METRICS_ADDR` to change/disable) +- Limiters: ~12 requests/minute per UID (burst 2), per-UID concurrency 2, global concurrency 8, 2 s penalty on validation failures ## Monitoring Alerts & Response ### Rate Limit Hits (`pulse_proxy_limiter_rejections_total`) @@ -35,12 +35,27 @@ sudo systemctl stop pulse-sensor-proxy sudo apparmor_parser -r /etc/apparmor.d/pulse-sensor-proxy # if updating policy sudo systemctl start pulse-sensor-proxy ``` -Verify: `curl -s http://127.0.0.1:9456/metrics | grep pulse_proxy_build_info`. +Verify: +```bash +# Metrics endpoint exposes proxy build/health +curl -s http://127.0.0.1:9127/metrics | grep pulse_proxy_build_info + +# Ensure adaptive polling sees the proxy again +curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.key | contains("temperature")) | {key, pollStatus}' +``` +Temperature instances should show recent `lastSuccess` timestamps with no DLQ entries. ### Rotate SSH Keys 1. Run `scripts/secure-sensor-files.sh` to regenerate keys (ensure environment locked down). 2. Use RPC `ensure_cluster_keys` to distribute new public key. 3. Confirm nodes accept `ssh` from proxy host. +4. Confirm the scheduler clears any temporary breakers/dlq entries: + ```bash + curl -s http://localhost:7655/api/monitoring/scheduler/health \ + | jq '.instances[] | select(.key | contains("temperature")) | {key, breaker: .breaker.state, deadLetter: .deadLetter.present}' + ``` + Expect `breaker.state=="closed"` and `deadLetter.present==false` for all proxy-driven pollers. ### Adjust Rate Limits 1. Update `limiter_policy` environment overrides (future config). diff --git a/docs/security/pulse-sensor-proxy-network.md b/docs/security/pulse-sensor-proxy-network.md index 0d1a926..4218fc6 100644 --- a/docs/security/pulse-sensor-proxy-network.md +++ b/docs/security/pulse-sensor-proxy-network.md @@ -15,7 +15,7 @@ - Hypervisors / BMCs reachable on `tcp/22` (SSH) and optional IPMI UDP. - **Logging/Monitoring Zone (AZ-Logging)** - Receives forwarded audit/application logs (e.g. RELP/TLS on `tcp/6514`). - - Exposes Prometheus scrape port (default `tcp/9456`) if remote monitoring required. + - Exposes Prometheus scrape port (default `tcp/9127`) if remote monitoring required. ## Recommended Firewall Rules @@ -26,7 +26,7 @@ | AZ-Sensor | AZ-Proxmox BMC | `udp/623` *(optional)* | IPMI if required for temperature data | Allow if needed | | AZ-Proxmox | AZ-Sensor | `any` | Return SSH traffic | Allow stateful | | AZ-Sensor | AZ-Logging | `tcp/6514` (TLS RELP) | Audit/application log forwarding | Allow | -| AZ-Logging | AZ-Sensor | `tcp/9456` *(optional)* | Prometheus scrape of proxy metrics | Allow if scraping remotely | +| AZ-Logging | AZ-Sensor | `tcp/9127` *(optional)* | Prometheus scrape of proxy metrics | Allow if scraping remotely | | Any | AZ-Sensor | `tcp/22` | Shell/SSH access | Deny (use management bastion) | | AZ-Sensor | Internet | `any` | Outbound Internet | Deny (except package mirrors via proxy if required) | @@ -43,9 +43,9 @@ iptables -A OUTPUT -p tcp -d --dport 6514 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT iptables -A INPUT -p tcp -s --sport 6514 -m conntrack --ctstate ESTABLISHED -j ACCEPT - # (Optional) allow Prometheus scrape - iptables -A INPUT -p tcp -s --dport 9456 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT - iptables -A OUTPUT -p tcp -d --sport 9456 -m conntrack --ctstate ESTABLISHED -j ACCEPT + # (Optional) allow Prometheus scrape + iptables -A INPUT -p tcp -s --dport 9127 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT + iptables -A OUTPUT -p tcp -d --sport 9127 -m conntrack --ctstate ESTABLISHED -j ACCEPT # Drop everything else iptables -P OUTPUT DROP diff --git a/docs/zfs-monitoring.md b/docs/zfs-monitoring.md index e92f9f5..ca02d41 100644 --- a/docs/zfs-monitoring.md +++ b/docs/zfs-monitoring.md @@ -69,7 +69,7 @@ ZFS issues appear in the Storage tab: ### No ZFS Data Appearing 1. Check permissions: `pveum user permissions pulse-monitor@pam` 2. Verify ZFS pools exist: `zpool list` -3. Check logs: `grep ZFS /opt/pulse/pulse.log` +3. Check logs: `grep ZFS /opt/pulse/pulse.log` (raise log level to `debug` via **Settings → System → Logging** if you need more context, then switch back to `info`). ### Permission Denied Errors Grant the required permission: diff --git a/internal/api/config_handlers.go b/internal/api/config_handlers.go index 7584bf5..93f4c79 100644 --- a/internal/api/config_handlers.go +++ b/internal/api/config_handlers.go @@ -2851,13 +2851,20 @@ func (h *ConfigHandlers) HandleDiscoverServers(w http.ResponseWriter, r *http.Re if discoveryService := h.monitor.GetDiscoveryService(); discoveryService != nil { result, updated := discoveryService.GetCachedResult() - // Add metadata about the cache + var updatedUnix int64 + var ageSeconds float64 + if !updated.IsZero() { + updatedUnix = updated.Unix() + ageSeconds = time.Since(updated).Seconds() + } + response := map[string]interface{}{ - "servers": result.Servers, - "errors": result.Errors, - "cached": true, - "updated": updated.Unix(), - "age": time.Since(updated).Seconds(), + "servers": result.Servers, + "errors": result.Errors, + "environment": result.Environment, + "cached": true, + "updated": updatedUnix, + "age": ageSeconds, } w.Header().Set("Content-Type", "application/json") @@ -2865,20 +2872,21 @@ func (h *ConfigHandlers) HandleDiscoverServers(w http.ResponseWriter, r *http.Re return } - // No discovery service available, return empty result w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ - "servers": []interface{}{}, - "errors": []string{}, - "cached": false, + "servers": []interface{}{}, + "errors": []string{}, + "environment": nil, + "cached": false, + "updated": int64(0), + "age": float64(0), }) return case http.MethodPost: - // Parse request var req struct { - Subnet string `json:"subnet"` // CIDR notation or "auto" - UseCache bool `json:"use_cache"` // Whether to return cached results + Subnet string `json:"subnet"` + UseCache bool `json:"use_cache"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -2886,17 +2894,24 @@ func (h *ConfigHandlers) HandleDiscoverServers(w http.ResponseWriter, r *http.Re return } - // If use_cache is true and we have cached results, return them if req.UseCache { if discoveryService := h.monitor.GetDiscoveryService(); discoveryService != nil { result, updated := discoveryService.GetCachedResult() + var updatedUnix int64 + var ageSeconds float64 + if !updated.IsZero() { + updatedUnix = updated.Unix() + ageSeconds = time.Since(updated).Seconds() + } + response := map[string]interface{}{ - "servers": result.Servers, - "errors": result.Errors, - "cached": true, - "updated": updated.Unix(), - "age": time.Since(updated).Seconds(), + "servers": result.Servers, + "errors": result.Errors, + "environment": result.Environment, + "cached": true, + "updated": updatedUnix, + "age": ageSeconds, } w.Header().Set("Content-Type", "application/json") @@ -2905,49 +2920,42 @@ func (h *ConfigHandlers) HandleDiscoverServers(w http.ResponseWriter, r *http.Re } } - // Check if discovery service is available for triggering a refresh - if discoveryService := h.monitor.GetDiscoveryService(); discoveryService != nil { - // Update subnet if provided - if req.Subnet != "" { - discoveryService.SetSubnet(req.Subnet) - } - - // Trigger a refresh - discoveryService.ForceRefresh() - - // Wait a moment for scan to start, then return current cache - time.Sleep(100 * time.Millisecond) - - result, updated := discoveryService.GetCachedResult() - response := map[string]interface{}{ - "servers": result.Servers, - "errors": result.Errors, - "cached": true, - "scanning": true, - "updated": updated.Unix(), - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) - return + subnet := strings.TrimSpace(req.Subnet) + if subnet == "" { + subnet = "auto" } - // Fallback to direct scan if no discovery service - log.Info().Str("subnet", req.Subnet).Msg("Starting network discovery (fallback)") + log.Info().Str("subnet", subnet).Msg("Starting manual discovery scan") scanner := discovery.NewScanner() ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) defer cancel() - result, err := scanner.DiscoverServers(ctx, req.Subnet) + result, err := scanner.DiscoverServers(ctx, subnet) if err != nil { log.Error().Err(err).Msg("Discovery failed") http.Error(w, fmt.Sprintf("Discovery failed: %v", err), http.StatusInternalServerError) return } + if result.Environment != nil { + log.Info(). + Str("environment", result.Environment.Type). + Float64("confidence", result.Environment.Confidence). + Int("phases", len(result.Environment.Phases)). + Msg("Manual discovery environment summary") + } + + response := map[string]interface{}{ + "servers": result.Servers, + "errors": result.Errors, + "environment": result.Environment, + "cached": false, + "scanning": false, + } + w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(result) + json.NewEncoder(w).Encode(response) default: http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) diff --git a/internal/discovery/service.go b/internal/discovery/service.go index 482b56c..ca40622 100644 --- a/internal/discovery/service.go +++ b/internal/discovery/service.go @@ -101,6 +101,8 @@ func (s *Service) performScan() { s.isScanning = true s.mu.Unlock() + var result *discovery.DiscoveryResult + defer func() { s.mu.Lock() s.isScanning = false @@ -109,12 +111,16 @@ func (s *Service) performScan() { // Send scan complete notification if s.wsHub != nil { + data := map[string]interface{}{ + "scanning": false, + "timestamp": time.Now().Unix(), + } + if result != nil && result.Environment != nil { + data["environment"] = result.Environment + } s.wsHub.Broadcast(websocket.Message{ Type: "discovery_complete", - Data: map[string]interface{}{ - "scanning": false, - "timestamp": time.Now().Unix(), - }, + Data: data, }) } }() @@ -138,18 +144,26 @@ func (s *Service) performScan() { scanCtx, cancel := context.WithTimeout(s.ctx, 2*time.Minute) defer cancel() + // Refresh scanner to ensure environment detection stays current + newScanner := discovery.NewScanner() + s.mu.Lock() + s.scanner = newScanner + s.mu.Unlock() + // Perform the scan with real-time callback - result, err := s.scanner.DiscoverServersWithCallback(scanCtx, s.subnet, func(server discovery.DiscoveredServer) { + result, err = newScanner.DiscoverServersWithCallback(scanCtx, s.subnet, func(server discovery.DiscoveredServer, phase string) { // Send immediate update for each discovered server if s.wsHub != nil { s.wsHub.Broadcast(websocket.Message{ Type: "discovery_server_found", Data: map[string]interface{}{ "server": server, + "phase": phase, "timestamp": time.Now().Unix(), }, }) log.Info(). + Str("phase", phase). Str("ip", server.IP). Str("type", server.Type). Msg("Broadcasting discovered server to clients") @@ -174,6 +188,14 @@ func (s *Service) performScan() { s.cache.updated = time.Now() s.cache.mu.Unlock() + if result.Environment != nil { + log.Info(). + Str("environment", result.Environment.Type). + Float64("confidence", result.Environment.Confidence). + Int("phases", len(result.Environment.Phases)). + Msg("Environment detection summary") + } + log.Info(). Int("servers", len(result.Servers)). Int("errors", len(result.Errors)). @@ -181,14 +203,18 @@ func (s *Service) performScan() { // Send final update via WebSocket with all servers if s.wsHub != nil { + data := map[string]interface{}{ + "servers": result.Servers, + "errors": result.Errors, + "scanning": false, + "timestamp": time.Now().Unix(), + } + if result.Environment != nil { + data["environment"] = result.Environment + } s.wsHub.Broadcast(websocket.Message{ Type: "discovery_update", - Data: map[string]interface{}{ - "servers": result.Servers, - "errors": result.Errors, - "scanning": false, - "timestamp": time.Now().Unix(), - }, + Data: data, }) } } diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index af539f0..2589e7e 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -4,15 +4,19 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "net" "net/http" + "sort" "strconv" "strings" "sync" "time" "github.com/rs/zerolog/log" + + "github.com/rcourtman/pulse-go-rewrite/pkg/discovery/envdetect" ) // DiscoveredServer represents a discovered Proxmox/PBS/PMG server @@ -27,35 +31,70 @@ type DiscoveredServer struct { // DiscoveryResult contains all discovered servers type DiscoveryResult struct { - Servers []DiscoveredServer `json:"servers"` - Errors []string `json:"errors,omitempty"` + Servers []DiscoveredServer `json:"servers"` + Errors []string `json:"errors,omitempty"` + Environment *EnvironmentInfo `json:"environment,omitempty"` +} + +// EnvironmentInfo captures metadata about the environment scan. +type EnvironmentInfo struct { + Type string `json:"type"` + Confidence float64 `json:"confidence"` + Phases []PhaseInfo `json:"phases"` + Warnings []string `json:"warnings"` + Metadata map[string]string `json:"metadata"` +} + +// PhaseInfo exposes phase details to clients. +type PhaseInfo struct { + Name string `json:"name"` + Subnets []string `json:"subnets"` + Confidence float64 `json:"confidence"` } // Scanner handles network scanning for Proxmox/PBS servers type Scanner struct { - timeout time.Duration - concurrent int + policy envdetect.ScanPolicy + profile *envdetect.EnvironmentProfile httpClient *http.Client } // NewScanner creates a new network scanner func NewScanner() *Scanner { + profile, err := envdetect.DetectEnvironment() + if err != nil { + log.Warn().Err(err).Msg("Environment detection completed with warnings") + } + + return NewScannerWithProfile(profile) +} + +// NewScannerWithProfile creates a scanner using the supplied environment profile. +func NewScannerWithProfile(profile *envdetect.EnvironmentProfile) *Scanner { + clonedProfile := cloneProfile(profile) + policy := ensurePolicyDefaults(clonedProfile.Policy) + clonedProfile.Policy = policy + + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + MaxIdleConns: 100, + MaxConnsPerHost: max(policy.MaxConcurrent, 10), + } + + client := &http.Client{ + Timeout: policy.HTTPTimeout, + Transport: transport, + } + return &Scanner{ - timeout: 1 * time.Second, // Reduced timeout for faster scanning - concurrent: 50, // Increased concurrent workers for faster scanning - httpClient: &http.Client{ - Timeout: 2 * time.Second, // Reduced HTTP timeout - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - MaxIdleConns: 100, - MaxConnsPerHost: 10, - }, - }, + policy: policy, + profile: clonedProfile, + httpClient: client, } } // ServerCallback is called when a server is discovered -type ServerCallback func(server DiscoveredServer) +type ServerCallback func(server DiscoveredServer, phase string) // DiscoverServers scans the network for Proxmox VE and PBS servers func (s *Scanner) DiscoverServers(ctx context.Context, subnet string) (*DiscoveryResult, error) { @@ -64,135 +103,95 @@ func (s *Scanner) DiscoverServers(ctx context.Context, subnet string) (*Discover // DiscoverServersWithCallback scans and calls callback for each discovered server func (s *Scanner) DiscoverServersWithCallback(ctx context.Context, subnet string, callback ServerCallback) (*DiscoveryResult, error) { - log.Info().Str("subnet", subnet).Msg("Starting network discovery") - - // Parse subnet - var ipNets []*net.IPNet - - if subnet == "" || subnet == "auto" { - // Check if we're in Docker (detected subnet is Docker network) - autoDetected := s.getLocalSubnet() - if autoDetected != nil && (strings.HasPrefix(autoDetected.String(), "172.17.") || strings.HasPrefix(autoDetected.String(), "172.1")) { - log.Info().Msg("Running in Docker - detecting host network from gateway") - // Try to detect the host's network from the default gateway - if hostSubnet := s.getHostSubnetFromGateway(); hostSubnet != nil { - ipNets = []*net.IPNet{hostSubnet} - log.Info().Str("detected", hostSubnet.String()).Msg("Detected host subnet from Docker gateway") - } else { - // Fallback: scan only most common subnet (192.168.0.0/24) - log.Info().Msg("Could not detect host subnet - scanning 192.168.0.0/24") - _, defaultNet, _ := net.ParseCIDR("192.168.0.0/24") - ipNets = []*net.IPNet{defaultNet} - } - } else if autoDetected != nil { - // Use auto-detected subnet - ipNets = []*net.IPNet{autoDetected} - log.Info().Str("detected", autoDetected.String()).Msg("Auto-detected local subnet") - } else { - // Fallback to most common subnet - log.Info().Msg("Auto-detection failed - scanning 192.168.0.0/24") - _, defaultNet, _ := net.ParseCIDR("192.168.0.0/24") - ipNets = []*net.IPNet{defaultNet} - } - } else { - // Parse provided subnet - _, parsedNet, err := net.ParseCIDR(subnet) - if err != nil { - return nil, fmt.Errorf("invalid subnet: %w", err) - } - ipNets = []*net.IPNet{parsedNet} + activeProfile, err := s.resolveProfile(subnet) + if err != nil { + return nil, err } - // Collect all IPs to scan from all subnets - var allIPs []string - for _, ipNet := range ipNets { - // Check subnet size - limit to /24 or smaller for safety - ones, bits := ipNet.Mask.Size() - if ones < 24 && bits == 32 { // IPv4 with more than 256 addresses - log.Warn().Str("subnet", ipNet.String()).Msg("Subnet too large, limiting to /24") - // Convert to /24 - ipNet.Mask = net.CIDRMask(24, 32) - } - - // Generate list of IPs for this subnet - ips := s.generateIPs(ipNet) - allIPs = append(allIPs, ips...) - log.Info().Str("subnet", ipNet.String()).Int("count", len(ips)).Msg("Subnet IPs to scan") - } - log.Info().Int("total", len(allIPs)).Msg("Total IPs to scan") - - // Create channels for work distribution - ipChan := make(chan string, len(allIPs)) - resultChan := make(chan *DiscoveredServer, len(allIPs)) - errorChan := make(chan string, len(allIPs)) - - // Start workers - var wg sync.WaitGroup - for i := 0; i < s.concurrent; i++ { - wg.Add(1) - go s.scanWorker(ctx, &wg, ipChan, resultChan, errorChan) - } - - // Send IPs to scan - for _, ip := range allIPs { - ipChan <- ip - } - close(ipChan) - - // Wait for workers to finish - go func() { - wg.Wait() - close(resultChan) - close(errorChan) - }() - - // Collect results result := &DiscoveryResult{ - Servers: []DiscoveredServer{}, - Errors: []string{}, + Servers: []DiscoveredServer{}, + Errors: []string{}, + Environment: buildEnvironmentInfo(activeProfile), } - done := false - for !done { - select { - case server, ok := <-resultChan: - if !ok { - done = true - break - } - if server != nil { - result.Servers = append(result.Servers, *server) - // Log immediately when found for real-time feedback - log.Info(). - Str("ip", server.IP). - Str("type", server.Type). - Str("hostname", server.Hostname). - Msg("🎯 Discovered server - adding to results") + seenIPs := make(map[string]struct{}) - // Call callback for real-time updates - if callback != nil { - callback(*server) - } + // Scan explicit extra targets first, if any. + extraIPs := s.collectExtraTargets(activeProfile, seenIPs) + if len(extraIPs) > 0 { + log.Info(). + Int("count", len(extraIPs)). + Msg("Starting discovery for explicit extra targets") + if err := s.runPhase(ctx, "extra_targets", extraIPs, callback, result); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("extra_targets: %v", err)) + if errors.Is(err, context.Canceled) { + return result, ctx.Err() } - case errMsg, ok := <-errorChan: - if ok && errMsg != "" { - result.Errors = append(result.Errors, errMsg) + } + } + + phases := append([]envdetect.SubnetPhase(nil), activeProfile.Phases...) + sort.SliceStable(phases, func(i, j int) bool { + return phases[i].Priority < phases[j].Priority + }) + + for _, phase := range phases { + if err := ctx.Err(); err != nil { + return result, err + } + + if s.shouldSkipPhase(ctx, phase) { + log.Warn(). + Str("phase", phase.Name). + Float64("confidence", phase.Confidence). + Msg("Skipping discovery phase due to low confidence/time budget") + continue + } + + phaseIPs, subnetCount := s.expandPhaseIPs(phase, seenIPs) + if len(phaseIPs) == 0 { + log.Debug(). + Str("phase", phase.Name). + Int("subnets", subnetCount). + Msg("No scan targets generated for phase") + continue + } + + log.Info(). + Str("phase", phase.Name). + Int("subnets", subnetCount). + Int("targets", len(phaseIPs)). + Float64("confidence", phase.Confidence). + Msg("Starting discovery phase") + + if err := s.runPhase(ctx, phase.Name, phaseIPs, callback, result); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", phase.Name, err)) + if errors.Is(err, context.Canceled) { + return result, ctx.Err() } - case <-ctx.Done(): - return result, ctx.Err() } } log.Info(). - Int("found", len(result.Servers)). + Int("servers_found", len(result.Servers)). Int("errors", len(result.Errors)). Msg("Discovery completed") return result, nil } +type discoveredResult struct { + Phase string + Server *DiscoveredServer +} + +type phaseError struct { + Phase string + Message string +} + // scanWorker scans IPs from the channel -func (s *Scanner) scanWorker(ctx context.Context, wg *sync.WaitGroup, ipChan <-chan string, resultChan chan<- *DiscoveredServer, errorChan chan<- string) { +func (s *Scanner) scanWorker(ctx context.Context, wg *sync.WaitGroup, phase string, ipChan <-chan string, resultChan chan<- discoveredResult, errorChan chan<- phaseError) { defer wg.Done() for ip := range ipChan { @@ -201,31 +200,330 @@ func (s *Scanner) scanWorker(ctx context.Context, wg *sync.WaitGroup, ipChan <-c return default: if server := s.checkPort8006(ctx, ip); server != nil { - resultChan <- server + resultChan <- discoveredResult{Phase: phase, Server: server} } if server := s.checkServer(ctx, ip, 8007, "pbs"); server != nil { - resultChan <- server + resultChan <- discoveredResult{Phase: phase, Server: server} } } } } +func (s *Scanner) runPhase(ctx context.Context, phase string, ips []string, callback ServerCallback, result *DiscoveryResult) error { + if len(ips) == 0 { + return nil + } + + workerCount := s.policy.MaxConcurrent + if workerCount <= 0 { + workerCount = 1 + } + + ipChan := make(chan string, len(ips)) + resultChan := make(chan discoveredResult, len(ips)) + errorChan := make(chan phaseError, len(ips)) + + var wg sync.WaitGroup + for i := 0; i < workerCount; i++ { + wg.Add(1) + go s.scanWorker(ctx, &wg, phase, ipChan, resultChan, errorChan) + } + + for _, ip := range ips { + ipChan <- ip + } + close(ipChan) + + go func() { + wg.Wait() + close(resultChan) + close(errorChan) + }() + + for resultChan != nil || errorChan != nil { + select { + case res, ok := <-resultChan: + if !ok { + resultChan = nil + continue + } + if res.Server == nil { + continue + } + + result.Servers = append(result.Servers, *res.Server) + + log.Info(). + Str("phase", res.Phase). + Str("ip", res.Server.IP). + Str("type", res.Server.Type). + Str("hostname", res.Server.Hostname). + Msg("Discovered server") + + if callback != nil { + callback(*res.Server, res.Phase) + } + case perr, ok := <-errorChan: + if !ok { + errorChan = nil + continue + } + if perr.Message == "" { + continue + } + result.Errors = append(result.Errors, fmt.Sprintf("%s: %s", perr.Phase, perr.Message)) + case <-ctx.Done(): + return ctx.Err() + } + } + + return nil +} + +func (s *Scanner) resolveProfile(subnet string) (*envdetect.EnvironmentProfile, error) { + if strings.EqualFold(strings.TrimSpace(subnet), "auto") || strings.TrimSpace(subnet) == "" { + return cloneProfile(s.profile), nil + } + + var ( + subnets []net.IPNet + renderedSubnets []string + ) + for _, token := range strings.Split(subnet, ",") { + token = strings.TrimSpace(token) + if token == "" { + continue + } + _, parsedNet, err := net.ParseCIDR(token) + if err != nil { + return nil, fmt.Errorf("invalid subnet %q: %w", token, err) + } + subnets = append(subnets, *parsedNet) + renderedSubnets = append(renderedSubnets, parsedNet.String()) + } + + if len(subnets) == 0 { + return nil, fmt.Errorf("no valid subnets provided") + } + + manualProfile := &envdetect.EnvironmentProfile{ + Type: envdetect.Unknown, + Confidence: 1.0, + Policy: s.policy, + Warnings: []string{"Manual subnet override applied"}, + Metadata: map[string]string{ + "manual_subnets": strings.Join(renderedSubnets, ","), + }, + Phases: []envdetect.SubnetPhase{ + { + Name: "manual_subnet", + Subnets: subnets, + Confidence: 1.0, + Priority: 1, + }, + }, + } + + return manualProfile, nil +} + +func (s *Scanner) collectExtraTargets(profile *envdetect.EnvironmentProfile, seen map[string]struct{}) []string { + if profile == nil { + return nil + } + + var targets []string + for _, ip := range profile.ExtraTargets { + if ip == nil { + continue + } + ip4 := ip.To4() + if ip4 == nil { + continue + } + ipStr := ip4.String() + if _, exists := seen[ipStr]; exists { + continue + } + seen[ipStr] = struct{}{} + targets = append(targets, ipStr) + } + + return targets +} + +func (s *Scanner) expandPhaseIPs(phase envdetect.SubnetPhase, seen map[string]struct{}) ([]string, int) { + var targets []string + + for _, subnet := range phase.Subnets { + if subnet.IP.To4() == nil { + continue + } + + copySubnet := subnet // copy to avoid modifying original + ips := s.generateIPs(©Subnet) + for _, ip := range ips { + if _, exists := seen[ip]; exists { + continue + } + seen[ip] = struct{}{} + targets = append(targets, ip) + } + } + + return targets, len(phase.Subnets) +} + +func (s *Scanner) shouldSkipPhase(ctx context.Context, phase envdetect.SubnetPhase) bool { + if phase.Confidence >= 0.5 { + return false + } + deadline, ok := ctx.Deadline() + if !ok { + return false + } + + timeRemaining := time.Until(deadline) + minBudget := s.policy.DialTimeout * 5 + if minBudget <= 0 { + minBudget = 5 * time.Second + } + + return timeRemaining > 0 && timeRemaining < minBudget +} + +func buildEnvironmentInfo(profile *envdetect.EnvironmentProfile) *EnvironmentInfo { + if profile == nil { + return nil + } + + info := &EnvironmentInfo{ + Type: profile.Type.String(), + Confidence: profile.Confidence, + Warnings: append([]string(nil), profile.Warnings...), + Metadata: copyMetadata(profile.Metadata), + } + + for _, phase := range profile.Phases { + pInfo := PhaseInfo{ + Name: phase.Name, + Confidence: phase.Confidence, + Subnets: []string{}, + } + for _, subnet := range phase.Subnets { + pInfo.Subnets = append(pInfo.Subnets, subnet.String()) + } + info.Phases = append(info.Phases, pInfo) + } + + return info +} + +func ensurePolicyDefaults(policy envdetect.ScanPolicy) envdetect.ScanPolicy { + defaults := envdetect.DefaultScanPolicy() + + if policy == (envdetect.ScanPolicy{}) { + return defaults + } + + if policy.MaxConcurrent <= 0 { + policy.MaxConcurrent = defaults.MaxConcurrent + } + if policy.DialTimeout <= 0 { + policy.DialTimeout = defaults.DialTimeout + } + if policy.HTTPTimeout <= 0 { + policy.HTTPTimeout = defaults.HTTPTimeout + } + if policy.MaxHostsPerScan < 0 { + policy.MaxHostsPerScan = defaults.MaxHostsPerScan + } + + return policy +} + +func cloneProfile(profile *envdetect.EnvironmentProfile) *envdetect.EnvironmentProfile { + if profile == nil { + defaults := envdetect.DefaultScanPolicy() + return &envdetect.EnvironmentProfile{ + Type: envdetect.Unknown, + Confidence: 0.3, + Policy: defaults, + Warnings: []string{"Environment profile unavailable; using defaults"}, + Metadata: map[string]string{}, + } + } + + cloned := *profile + if profile.Metadata != nil { + cloned.Metadata = copyMetadata(profile.Metadata) + } + if profile.Warnings != nil { + cloned.Warnings = append([]string(nil), profile.Warnings...) + } + if profile.ExtraTargets != nil { + cloned.ExtraTargets = append([]net.IP(nil), profile.ExtraTargets...) + } + if profile.Phases != nil { + cloned.Phases = make([]envdetect.SubnetPhase, len(profile.Phases)) + for i, phase := range profile.Phases { + cloned.Phases[i] = clonePhase(phase) + } + } + + return &cloned +} + +func clonePhase(phase envdetect.SubnetPhase) envdetect.SubnetPhase { + cloned := phase + if phase.Subnets != nil { + cloned.Subnets = make([]net.IPNet, len(phase.Subnets)) + for i, subnet := range phase.Subnets { + cloned.Subnets[i] = subnet + } + } + return cloned +} + +func copyMetadata(src map[string]string) map[string]string { + if src == nil { + return map[string]string{} + } + dst := make(map[string]string, len(src)) + for k, v := range src { + dst[k] = v + } + return dst +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + // checkPort8006 checks if port 8006 is running PMG or PVE func (s *Scanner) checkPort8006(ctx context.Context, ip string) *DiscoveredServer { address := net.JoinHostPort(ip, "8006") // First attempt a TLS handshake so we can inspect certificate metadata. var tlsState *tls.ConnectionState - dialer := &net.Dialer{Timeout: s.timeout} - tlsConn, tlsErr := tls.DialWithDialer(dialer, "tcp", address, &tls.Config{InsecureSkipVerify: true}) + timeout := s.policy.DialTimeout + if timeout <= 0 { + timeout = time.Second + } + dialer := &net.Dialer{Timeout: timeout} + + tlsConn, tlsErr := tls.DialWithDialer(dialer, "tcp", address, &tls.Config{InsecureSkipVerify: true}) if tlsErr != nil { // Fallback to a simple TCP dial to confirm the port is open. - conn, err := net.DialTimeout("tcp", address, s.timeout) - if err != nil { - return nil // Port not open - } - conn.Close() + conn, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return nil // Port not open + } + conn.Close() } else { state := tlsConn.ConnectionState() tlsState = &state @@ -310,15 +608,16 @@ func (s *Scanner) checkPort8006(ctx context.Context, ip string) *DiscoveredServe } // Try to resolve hostname via reverse DNS - names, err := net.LookupAddr(ip) - if err == nil && len(names) > 0 { - // Use the first hostname, remove trailing dot if present - hostname := strings.TrimSuffix(names[0], ".") - server.Hostname = hostname - log.Debug().Str("ip", ip).Str("hostname", hostname).Msg("Resolved hostname via DNS") - } + if s.policy.EnableReverseDNS { + names, err := net.DefaultResolver.LookupAddr(ctx, ip) + if err == nil && len(names) > 0 { + hostname := strings.TrimSuffix(names[0], ".") + server.Hostname = hostname + log.Debug().Str("ip", ip).Str("hostname", hostname).Msg("Resolved hostname via DNS") + } + } - return server + return server } // isPMGServer checks if a server is PMG by checking for PMG-specific endpoints @@ -438,12 +737,18 @@ func inferTypeFromMetadata(parts ...string) string { // checkServer checks if a server is running at the given IP and port func (s *Scanner) checkServer(ctx context.Context, ip string, port int, serverType string) *DiscoveredServer { // First check if port is open - address := net.JoinHostPort(ip, strconv.Itoa(port)) - conn, err := net.DialTimeout("tcp", address, s.timeout) - if err != nil { - return nil // Port not open - } - conn.Close() + address := net.JoinHostPort(ip, strconv.Itoa(port)) + timeout := s.policy.DialTimeout + if timeout <= 0 { + timeout = time.Second + } + dialer := &net.Dialer{Timeout: timeout} + + conn, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return nil // Port not open + } + conn.Close() // Port is open - this is likely a Proxmox/PBS server // Since most installations require auth for version endpoint, @@ -495,15 +800,16 @@ func (s *Scanner) checkServer(ctx context.Context, ip string, port int, serverTy } // Try to resolve hostname via reverse DNS - names, err := net.LookupAddr(ip) - if err == nil && len(names) > 0 { - // Use the first hostname, remove trailing dot if present - hostname := strings.TrimSuffix(names[0], ".") - server.Hostname = hostname - log.Debug().Str("ip", ip).Str("hostname", hostname).Msg("Resolved hostname via DNS") - } + if s.policy.EnableReverseDNS { + names, err := net.DefaultResolver.LookupAddr(ctx, ip) + if err == nil && len(names) > 0 { + hostname := strings.TrimSuffix(names[0], ".") + server.Hostname = hostname + log.Debug().Str("ip", ip).Str("hostname", hostname).Msg("Resolved hostname via DNS") + } + } - return server + return server } // getProxmoxHostname tries to get the hostname of a Proxmox VE server @@ -574,158 +880,52 @@ func (s *Scanner) getPBSHostname(ctx context.Context, ip string, port int) strin // generateIPs generates all IPs in a subnet func (s *Scanner) generateIPs(ipNet *net.IPNet) []string { - var ips []string + baseIP := ipNet.IP.Mask(ipNet.Mask).To4() + if baseIP == nil { + return nil + } - // Get the starting IP - ip := ipNet.IP.Mask(ipNet.Mask) + ones, bits := ipNet.Mask.Size() + if bits != 32 { + return nil + } - // Calculate the number of hosts - ones, bits := ipNet.Mask.Size() - hostBits := bits - ones - numHosts := 1 << hostBits + hostBits := bits - ones + totalHosts := 1 + if hostBits > 0 { + totalHosts = 1 << hostBits + } - // Skip network and broadcast addresses for common subnets - start := 1 - end := numHosts - 1 - if numHosts > 256 { - // For larger subnets, scan everything - start = 0 - end = numHosts - } + start := 0 + end := totalHosts + if totalHosts > 2 { + start = 1 + end = totalHosts - 1 + } - // Limit to maximum 1024 IPs to avoid scanning huge networks - if end-start > 1024 { - end = start + 1024 - log.Warn().Int("limited_to", 1024).Msg("Limiting scan to first 1024 IPs") - } + limit := s.policy.MaxHostsPerScan + if limit > 0 && limit < (end-start) { + end = start + limit + log.Debug(). + Str("subnet", ipNet.String()). + Int("limit", limit). + Msg("Applying max hosts per scan limit") + } - for i := start; i < end; i++ { - // Calculate IP - currIP := make(net.IP, len(ip)) - copy(currIP, ip) + ips := make([]string, 0, end-start) - // Add offset to IP address - offset := i - for j := len(currIP) - 1; j >= 0 && offset > 0; j-- { - currIP[j] += byte(offset & 0xFF) - offset >>= 8 - } + for offset := start; offset < end; offset++ { + currIP := make(net.IP, len(baseIP)) + copy(currIP, baseIP) - // Skip common non-server IPs - lastOctet := currIP[len(currIP)-1] - if lastOctet == 0 || lastOctet == 255 { - continue // Skip network and broadcast - } + carry := offset + for idx := len(currIP) - 1; idx >= 0 && carry > 0; idx-- { + currIP[idx] += byte(carry & 0xFF) + carry >>= 8 + } - ips = append(ips, currIP.String()) - } + ips = append(ips, currIP.String()) + } - return ips -} - -// getLocalSubnet attempts to detect the local subnet -func (s *Scanner) getLocalSubnet() *net.IPNet { - interfaces, err := net.Interfaces() - if err != nil { - return nil - } - - for _, iface := range interfaces { - // Skip loopback and down interfaces - if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 { - continue - } - - addrs, err := iface.Addrs() - if err != nil { - continue - } - - for _, addr := range addrs { - if ipNet, ok := addr.(*net.IPNet); ok && ipNet.IP.To4() != nil { - // Found an IPv4 address - if !ipNet.IP.IsLoopback() && !ipNet.IP.IsLinkLocalUnicast() { - // Convert to /24 subnet for auto-detection - // This ensures we scan a reasonable range - ip := ipNet.IP.To4() - if ip != nil { - // Create a /24 subnet from the IP - ip[3] = 0 // Set last octet to 0 - return &net.IPNet{ - IP: ip, - Mask: net.CIDRMask(24, 32), - } - } - } - } - } - } - - // Default to common subnet if detection fails - _, defaultNet, _ := net.ParseCIDR("192.168.1.0/24") - return defaultNet -} - -// getHostSubnetFromGateway detects the host network by examining the default gateway -// This is useful when running in Docker to detect the actual host's network -func (s *Scanner) getHostSubnetFromGateway() *net.IPNet { - interfaces, err := net.Interfaces() - if err != nil { - return nil - } - - // Find the default gateway by checking routes - // In Docker, the gateway IP usually ends in .1 and is on the Docker bridge network - for _, iface := range interfaces { - if iface.Flags&net.FlagUp == 0 { - continue - } - - addrs, err := iface.Addrs() - if err != nil { - continue - } - - for _, addr := range addrs { - if ipNet, ok := addr.(*net.IPNet); ok && ipNet.IP.To4() != nil { - // Check if this looks like a Docker bridge network (172.17.x.x or similar) - ipStr := ipNet.IP.String() - if strings.HasPrefix(ipStr, "172.17.") || strings.HasPrefix(ipStr, "172.1") { - // Gateway is typically .1 in the same subnet - // Try to derive the host network: gateway .1 -> likely host is 192.168.x.0/24 - // We'll try the .1 address as the gateway and ping common host subnets - - // For now, just return the most common subnet - // A more sophisticated approach would parse /proc/net/route - _, hostNet, _ := net.ParseCIDR("192.168.0.0/24") - return hostNet - } - } - } - } - - return nil -} - -// getCommonSubnets returns a list of common home/office network subnets -func (s *Scanner) getCommonSubnets() []*net.IPNet { - // Ordered by likelihood - most common first for faster results - commonSubnets := []string{ - "192.168.1.0/24", // Most common home router default - "192.168.0.0/24", // Very common alternative - "10.0.0.0/24", // Some routers use this - // Skip less common ones for speed: - // "192.168.88.0/24", // MikroTik default (uncommon) - // "172.16.0.0/24", // Less common but used - } - - var nets []*net.IPNet - for _, subnet := range commonSubnets { - _, ipNet, err := net.ParseCIDR(subnet) - if err == nil { - nets = append(nets, ipNet) - } - } - - return nets + return ips } diff --git a/pkg/discovery/discovery_test.go b/pkg/discovery/discovery_test.go index fa6d0d9..4f9a4dc 100644 --- a/pkg/discovery/discovery_test.go +++ b/pkg/discovery/discovery_test.go @@ -1,21 +1,38 @@ package discovery import ( - "context" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/json" - "net" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "sync" - "testing" - "time" + "context" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/pkg/discovery/envdetect" ) +func newTestScanner(client *http.Client) *Scanner { + policy := envdetect.DefaultScanPolicy() + policy.DialTimeout = time.Second + profile := &envdetect.EnvironmentProfile{ + Policy: policy, + Metadata: map[string]string{}, + } + + return &Scanner{ + policy: policy, + profile: profile, + httpClient: client, + } +} + func TestInferTypeFromMetadata(t *testing.T) { t.Parallel() @@ -107,10 +124,7 @@ func TestDetectProductFromEndpoint(t *testing.T) { })) defer ts.Close() - scanner := &Scanner{ - timeout: time.Second, - httpClient: ts.Client(), - } + scanner := newTestScanner(ts.Client()) address := strings.TrimPrefix(ts.URL, "https://") if product := scanner.detectProductFromEndpoint(context.Background(), address, "api2/json/statistics/mail"); product != "pmg" { @@ -143,10 +157,7 @@ func TestIsPMGServer(t *testing.T) { })) defer ts.Close() - scanner := &Scanner{ - timeout: time.Second, - httpClient: ts.Client(), - } + scanner := newTestScanner(ts.Client()) address := strings.TrimPrefix(ts.URL, "https://") if !scanner.isPMGServer(context.Background(), address) { @@ -190,10 +201,7 @@ func TestCheckServerRetrievesVersion(t *testing.T) { t.Fatalf("strconv.Atoi: %v", err) } - scanner := &Scanner{ - timeout: time.Second, - httpClient: ts.Client(), - } + scanner := newTestScanner(ts.Client()) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -241,13 +249,10 @@ func TestCheckServerHandlesUnauthorized(t *testing.T) { srv := startTLSServerOn(t, "127.0.0.1:9008", unauthorizedHandler) _ = srv - scanner := &Scanner{ - timeout: time.Second, - httpClient: &http.Client{ - Timeout: 500 * time.Millisecond, - Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, - }, - } + scanner := newTestScanner(&http.Client{ + Timeout: 500 * time.Millisecond, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, + }) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -315,12 +320,15 @@ func TestDiscoverServersWithCallback(t *testing.T) { pbsServer := startTLSServerOn(t, "127.0.0.1:8007", pbsHandler) _ = pbsServer - scanner := NewScanner() - scanner.concurrent = 4 - scanner.timeout = 200 * time.Millisecond - scanner.httpClient = &http.Client{ + scanner := newTestScanner(&http.Client{ Timeout: 500 * time.Millisecond, Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, + }) + scanner.policy.MaxConcurrent = 4 + scanner.policy.DialTimeout = 200 * time.Millisecond + scanner.policy.HTTPTimeout = 500 * time.Millisecond + if scanner.profile != nil { + scanner.profile.Policy = scanner.policy } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) @@ -336,7 +344,7 @@ func TestDiscoverServersWithCallback(t *testing.T) { t.Fatalf("expected checkServer to handle TCP-only host without panic") } - result, err := scanner.DiscoverServersWithCallback(ctx, subnet, func(server DiscoveredServer) { + result, err := scanner.DiscoverServersWithCallback(ctx, subnet, func(server DiscoveredServer, phase string) { mu.Lock() callbacks = append(callbacks, server) mu.Unlock() diff --git a/pkg/discovery/envdetect/envdetect.go b/pkg/discovery/envdetect/envdetect.go new file mode 100644 index 0000000..66557e5 --- /dev/null +++ b/pkg/discovery/envdetect/envdetect.go @@ -0,0 +1,694 @@ +package envdetect + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "net" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/rs/zerolog/log" +) + +// Environment represents the runtime environment type. +type Environment int + +const ( + // Unknown indicates the detector could not determine the environment. + Unknown Environment = iota + // Native represents bare metal or virtual machine deployments. + Native + // DockerHost indicates a container with host networking (shares host stack). + DockerHost + // DockerBridge indicates a container attached to a bridge/NAT network. + DockerBridge + // LXCPrivileged covers LXC containers with privileged UID mapping. + LXCPrivileged + // LXCUnprivileged covers LXC containers with remapped UIDs. + LXCUnprivileged +) + +// String provides a human readable representation of Environment. +func (e Environment) String() string { + switch e { + case Native: + return "native" + case DockerHost: + return "docker_host" + case DockerBridge: + return "docker_bridge" + case LXCPrivileged: + return "lxc_privileged" + case LXCUnprivileged: + return "lxc_unprivileged" + default: + return "unknown" + } +} + +// ScanPolicy defines scanning behavior parameters for a detected environment. +type ScanPolicy struct { + MaxConcurrent int // Maximum concurrent workers. + DialTimeout time.Duration // Timeout for TCP dials. + HTTPTimeout time.Duration // Timeout for HTTP requests. + MaxHostsPerScan int // Maximum hosts per subnet phase. + EnableReverseDNS bool // Whether reverse DNS lookups are allowed. + ScanGateways bool // Whether to probe inferred gateway networks. +} + +// DefaultScanPolicy returns baseline scanning parameters. +func DefaultScanPolicy() ScanPolicy { + return ScanPolicy{ + MaxConcurrent: 50, + DialTimeout: time.Second, + HTTPTimeout: 2 * time.Second, + MaxHostsPerScan: 1024, + EnableReverseDNS: true, + ScanGateways: true, + } +} + +// SubnetPhase represents a single phase of network scanning. +type SubnetPhase struct { + Name string // Name for logging/UI (e.g. "container_network"). + Subnets []net.IPNet // Subnets to process during this phase. + Confidence float64 // Confidence score (0.0 - 1.0). + Priority int // Lower priority runs earlier. +} + +// EnvironmentProfile captures detection results and scanning plan. +type EnvironmentProfile struct { + Type Environment // Detected environment. + Phases []SubnetPhase // Subnet scanning phases. + ExtraTargets []net.IP // IPs to always probe. + Policy ScanPolicy // Applied scan policy. + Confidence float64 // Overall confidence (0.0 - 1.0). + Warnings []string // Non-fatal detection warnings. + Metadata map[string]string // Misc metadata (container type, gateway, etc.). +} + +// DetectEnvironment performs environment detection and returns a profile. +func DetectEnvironment() (*EnvironmentProfile, error) { + profile := &EnvironmentProfile{ + Type: Unknown, + Phases: []SubnetPhase{}, + Policy: DefaultScanPolicy(), + Confidence: 0.0, + Warnings: []string{}, + Metadata: map[string]string{}, + } + + log.Info().Msg("Detecting runtime environment") + + isContainer, containerType := detectContainer() + profile.Metadata["container_detected"] = strconv.FormatBool(isContainer) + if containerType != "" { + profile.Metadata["container_type"] = containerType + } + + var err error + switch { + case !isContainer: + profile, err = detectNativeEnvironment(profile) + case containerType == "docker": + profile, err = detectDockerEnvironment(profile) + case containerType == "lxc": + profile, err = detectLXCEnvironment(profile) + default: + profile.Type = Unknown + profile.Confidence = 0.3 + if containerType == "" { + profile.Warnings = append(profile.Warnings, "Unable to determine container type; using fallback subnets") + } else { + profile.Warnings = append(profile.Warnings, fmt.Sprintf("Unsupported container type %q; using fallback subnets", containerType)) + } + profile, err = addFallbackSubnets(profile) + } + + if err != nil { + // Preserve the error for callers while ensuring we still provide a usable profile. + profile.Warnings = append(profile.Warnings, err.Error()) + } + + subnetCount := 0 + for _, phase := range profile.Phases { + subnetCount += len(phase.Subnets) + } + + log.Info(). + Str("environment", profile.Type.String()). + Int("phase_count", len(profile.Phases)). + Int("subnet_count", subnetCount). + Float64("confidence", profile.Confidence). + Msg("Environment detection completed") + + return profile, err +} + +// detectContainer inspects the host to determine whether we are inside a container. +func detectContainer() (bool, string) { + containerType := "" + + // 1. systemd-detect-virt --container + if _, err := exec.LookPath("systemd-detect-virt"); err == nil { + cmd := exec.Command("systemd-detect-virt", "--container") + output, err := cmd.CombinedOutput() + if len(output) > 0 { + result := strings.TrimSpace(string(output)) + if result != "" && result != "none" { + log.Debug().Str("virt", result).Msg("systemd-detect-virt reported container environment") + switch { + case strings.Contains(result, "lxc"): + return true, "lxc" + case strings.Contains(result, "docker"), strings.Contains(result, "containerd"), strings.Contains(result, "podman"): + return true, "docker" + default: + return true, result + } + } + } + if err != nil { + log.Debug().Err(err).Msg("systemd-detect-virt --container check failed; continuing with heuristics") + } + } + + // 2. Marker files + if _, err := os.Stat("/.dockerenv"); err == nil { + log.Debug().Msg("Detected /.dockerenv marker (Docker container)") + return true, "docker" + } + if _, err := os.Stat("/run/.containerenv"); err == nil { + log.Debug().Msg("Detected /run/.containerenv marker (Podman/OCI container)") + return true, "docker" + } + + // 3. /proc/1/cgroup + if data, err := os.ReadFile("/proc/1/cgroup"); err == nil { + text := string(data) + switch { + case strings.Contains(text, "docker"), strings.Contains(text, "kubepods"), strings.Contains(text, "containerd"): + log.Debug().Msg("Detected Docker via /proc/1/cgroup") + return true, "docker" + case strings.Contains(text, "lxc"), strings.Contains(text, "libcontainer"): + log.Debug().Msg("Detected LXC via /proc/1/cgroup") + return true, "lxc" + } + } else { + log.Debug().Err(err).Msg("Unable to read /proc/1/cgroup during container detection") + } + + // 4. /proc/1/environ + if data, err := os.ReadFile("/proc/1/environ"); err == nil { + text := string(data) + switch { + case strings.Contains(text, "container=lxc"): + log.Debug().Msg("Detected LXC via /proc/1/environ") + return true, "lxc" + case strings.Contains(text, "container=docker"), strings.Contains(text, "container=podman"): + log.Debug().Msg("Detected Docker via /proc/1/environ") + return true, "docker" + } + } else { + log.Debug().Err(err).Msg("Unable to read /proc/1/environ during container detection") + } + + log.Debug().Msg("No container markers detected; assuming native environment") + return false, containerType +} + +// detectNativeEnvironment builds an EnvironmentProfile for native or VM deployments. +func detectNativeEnvironment(profile *EnvironmentProfile) (*EnvironmentProfile, error) { + subnets, err := getAllLocalSubnets() + if err != nil { + return addFallbackSubnets(profileWithWarning(profile, fmt.Sprintf("Failed to enumerate interfaces: %v", err))) + } + + if len(subnets) == 0 { + return addFallbackSubnets(profileWithWarning(profile, "No active IPv4 interfaces found; using fallback subnets")) + } + + profile.Type = Native + profile.Confidence = 0.95 + profile.Metadata["detected_mode"] = "native" + profile.Metadata["interface_count"] = strconv.Itoa(len(subnets)) + + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "local_networks", + Subnets: subnets, + Confidence: 0.95, + Priority: 1, + }) + + return profile, nil +} + +// detectDockerEnvironment determines whether Docker uses host or bridge networking. +func detectDockerEnvironment(profile *EnvironmentProfile) (*EnvironmentProfile, error) { + hostMode, hostModeWarnings := isDockerHostMode() + if len(hostModeWarnings) > 0 { + profile.Warnings = append(profile.Warnings, hostModeWarnings...) + } + + if hostMode { + subnets, err := getAllLocalSubnets() + if err != nil { + return addFallbackSubnets(profileWithWarning(profile, fmt.Sprintf("Docker host mode: failed to enumerate subnets: %v", err))) + } + + if len(subnets) == 0 { + return addFallbackSubnets(profileWithWarning(profile, "Docker host mode detected but no subnets found; falling back to common subnets")) + } + + profile.Type = DockerHost + profile.Confidence = 0.9 + profile.Metadata["docker_mode"] = "host" + profile.Metadata["interface_count"] = strconv.Itoa(len(subnets)) + + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "host_networks", + Subnets: subnets, + Confidence: 0.9, + Priority: 1, + }) + + return profile, nil + } + + // Bridge/NAT mode. + profile.Type = DockerBridge + profile.Confidence = 0.85 + profile.Metadata["docker_mode"] = "bridge" + + containerSubnets, err := getAllLocalSubnets() + if err != nil { + profile.Warnings = append(profile.Warnings, fmt.Sprintf("Docker bridge: failed to enumerate container subnets: %v", err)) + } else if len(containerSubnets) > 0 { + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "container_network", + Subnets: containerSubnets, + Confidence: 0.95, + Priority: 1, + }) + profile.Metadata["container_subnet_count"] = strconv.Itoa(len(containerSubnets)) + } else { + profile.Warnings = append(profile.Warnings, "Docker bridge: no container subnets detected") + } + + if profile.Policy.ScanGateways { + hostSubnets, confidence, warnings := detectHostNetworkFromContainer() + if len(warnings) > 0 { + profile.Warnings = append(profile.Warnings, warnings...) + } + if len(hostSubnets) > 0 { + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "inferred_host_network", + Subnets: hostSubnets, + Confidence: confidence, + Priority: 2, + }) + profile.Metadata["inferred_host_subnets"] = strconv.Itoa(len(hostSubnets)) + } + } + + if len(profile.Phases) == 0 { + return addFallbackSubnets(profileWithWarning(profile, "Docker bridge detection yielded no subnets; adding fallback ranges")) + } + + return profile, nil +} + +// detectLXCEnvironment evaluates privilege level and prepares scanning phases. +func detectLXCEnvironment(profile *EnvironmentProfile) (*EnvironmentProfile, error) { + privileged, warn := isLXCPrivileged() + if warn != "" { + profile.Warnings = append(profile.Warnings, warn) + } + + containerSubnets, err := getAllLocalSubnets() + if err != nil { + profile.Warnings = append(profile.Warnings, fmt.Sprintf("LXC: failed to enumerate container subnets: %v", err)) + } + + if privileged { + if len(containerSubnets) == 0 { + return addFallbackSubnets(profileWithWarning(profile, "Privileged LXC detected but no subnets found; using fallback subnets")) + } + + profile.Type = LXCPrivileged + profile.Confidence = 0.9 + profile.Metadata["lxc_privileged"] = "true" + profile.Metadata["interface_count"] = strconv.Itoa(len(containerSubnets)) + + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "lxc_host_networks", + Subnets: containerSubnets, + Confidence: 0.9, + Priority: 1, + }) + + return profile, nil + } + + // Unprivileged container. + profile.Type = LXCUnprivileged + profile.Confidence = 0.85 + profile.Metadata["lxc_privileged"] = "false" + + if len(containerSubnets) > 0 { + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "lxc_container_network", + Subnets: containerSubnets, + Confidence: 0.9, + Priority: 1, + }) + profile.Metadata["container_subnet_count"] = strconv.Itoa(len(containerSubnets)) + } + + if profile.Policy.ScanGateways { + hostSubnets, confidence, warnings := detectHostNetworkFromContainer() + if len(warnings) > 0 { + profile.Warnings = append(profile.Warnings, warnings...) + } + if len(hostSubnets) > 0 { + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "lxc_parent_network", + Subnets: hostSubnets, + Confidence: confidence, + Priority: 2, + }) + profile.Metadata["inferred_host_subnets"] = strconv.Itoa(len(hostSubnets)) + } + } + + if len(profile.Phases) == 0 { + return addFallbackSubnets(profileWithWarning(profile, "Unprivileged LXC detection yielded no subnets; adding fallback ranges")) + } + + return profile, nil +} + +// isDockerHostMode attempts to determine whether Docker is using host networking. +func isDockerHostMode() (bool, []string) { + var warnings []string + + interfaces, err := net.Interfaces() + if err != nil { + log.Debug().Err(err).Msg("Failed to enumerate interfaces while detecting Docker mode") + warnings = append(warnings, fmt.Sprintf("Unable to enumerate interfaces: %v", err)) + return false, warnings + } + + routeCount, routeWarn := countKernelRoutes() + if routeWarn != "" { + warnings = append(warnings, routeWarn) + } + + log.Debug(). + Int("interface_count", len(interfaces)). + Int("route_count", routeCount). + Msg("Docker networking mode heuristics") + + // Heuristic: host networking tends to expose many interfaces and routes. + if len(interfaces) > 3 && routeCount > 5 { + return true, warnings + } + + return false, warnings +} + +// isLXCPrivileged inspects UID mappings to determine privilege level. +func isLXCPrivileged() (bool, string) { + data, err := os.ReadFile("/proc/self/uid_map") + if err != nil { + if errors.Is(err, os.ErrPermission) { + return false, "Unable to read /proc/self/uid_map (permission denied); assuming unprivileged LXC" + } + return false, fmt.Sprintf("Unable to read /proc/self/uid_map; assuming unprivileged LXC: %v", err) + } + + fields := strings.Fields(string(data)) + if len(fields) < 3 { + return false, "Unexpected format in /proc/self/uid_map; assuming unprivileged LXC" + } + + // Format: id_inside id_outside length + hostUID, err := strconv.ParseUint(fields[1], 10, 32) + if err != nil { + return false, "Failed to parse uid_map; assuming unprivileged LXC" + } + + length, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + return false, "Failed to parse uid_map length; assuming unprivileged LXC" + } + + if hostUID == 0 && length >= 4294967295 { + return true, "" + } + + return false, "" +} + +// getAllLocalSubnets enumerates non-loopback, UP IPv4 subnets. +func getAllLocalSubnets() ([]net.IPNet, error) { + interfaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("failed to list interfaces: %w", err) + } + + var subnets []net.IPNet + seen := make(map[string]struct{}) + + for _, iface := range interfaces { + if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 { + continue + } + + addrs, err := iface.Addrs() + if err != nil { + log.Debug().Err(err).Str("interface", iface.Name).Msg("Skipping interface due to address enumeration failure") + continue + } + + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok || ipNet == nil { + continue + } + + ip := ipNet.IP.To4() + if ip == nil { + continue + } + + if ip.IsLoopback() || ip.IsLinkLocalUnicast() { + continue + } + + networkIP := ip.Mask(ipNet.Mask) + cidr := (&net.IPNet{IP: networkIP, Mask: ipNet.Mask}).String() + if _, exists := seen[cidr]; exists { + continue + } + + seen[cidr] = struct{}{} + normalized := net.IPNet{IP: networkIP, Mask: ipNet.Mask} + subnets = append(subnets, normalized) + + log.Debug(). + Str("interface", iface.Name). + Str("cidr", normalized.String()). + Msg("Discovered local subnet") + } + } + + return subnets, nil +} + +// detectHostNetworkFromContainer infers host LAN subnets from container context. +func detectHostNetworkFromContainer() ([]net.IPNet, float64, []string) { + var warnings []string + + gateway, err := getDefaultGateway() + if err != nil { + warnings = append(warnings, fmt.Sprintf("Could not determine default gateway: %v", err)) + return tryCommonSubnets(), 0.3, warnings + } + + if gateway == nil || gateway.Equal(net.IPv4zero) { + warnings = append(warnings, "Default gateway is unspecified; using common subnet fallback") + return tryCommonSubnets(), 0.3, warnings + } + + log.Debug().Str("gateway", gateway.String()).Msg("Default gateway detected") + + gateway4 := gateway.To4() + if gateway4 == nil { + warnings = append(warnings, fmt.Sprintf("Default gateway %s is not IPv4; using fallback subnets", gateway.String())) + return tryCommonSubnets(), 0.3, warnings + } + + confidence := 0.4 + lastOctet := gateway4[3] + if lastOctet == 1 || lastOctet == 254 { + confidence = 0.7 + } else { + warnings = append(warnings, fmt.Sprintf("Gateway %s does not end with .1 or .254; confidence reduced", gateway.String())) + } + + hostSubnet := net.IPNet{ + IP: net.IPv4(gateway4[0], gateway4[1], gateway4[2], 0), + Mask: net.CIDRMask(24, 32), + } + + log.Debug().Str("host_subnet", hostSubnet.String()).Float64("confidence", confidence).Msg("Inferred host subnet from gateway") + + return []net.IPNet{hostSubnet}, confidence, warnings +} + +// getDefaultGateway parses /proc/net/route for the default gateway. +func getDefaultGateway() (net.IP, error) { + data, err := os.ReadFile("/proc/net/route") + if err != nil { + return nil, fmt.Errorf("failed to read /proc/net/route: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Iface") { + continue + } + + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + + // Destination column (fields[1]) equal to 00000000 indicates default route. + if fields[1] != "00000000" { + continue + } + + gatewayHex := fields[2] + if len(gatewayHex) != 8 { + continue + } + + gatewayIP, err := parseHexIP(gatewayHex) + if err != nil { + return nil, fmt.Errorf("failed to parse default gateway: %w", err) + } + + return gatewayIP, nil + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to parse /proc/net/route: %w", err) + } + + return nil, fmt.Errorf("default gateway not found") +} + +// parseHexIP converts an 8-character little-endian hex string into an IPv4 address. +func parseHexIP(hexIP string) (net.IP, error) { + if len(hexIP) != 8 { + return nil, fmt.Errorf("invalid hex IP length %d", len(hexIP)) + } + + var octets [4]byte + for i := 0; i < 4; i++ { + part := hexIP[i*2 : i*2+2] + val, err := strconv.ParseUint(part, 16, 8) + if err != nil { + return nil, fmt.Errorf("invalid hex octet %q: %w", part, err) + } + octets[3-i] = byte(val) // /proc/net/route stores little-endian. + } + + return net.IPv4(octets[0], octets[1], octets[2], octets[3]), nil +} + +// tryCommonSubnets returns common private IPv4 subnets as a conservative fallback. +func tryCommonSubnets() []net.IPNet { + commonCIDRs := []string{ + "192.168.1.0/24", + "192.168.0.0/24", + "10.0.0.0/24", + "172.16.0.0/24", + "192.168.2.0/24", + } + + var subnets []net.IPNet + for _, cidr := range commonCIDRs { + _, network, err := net.ParseCIDR(cidr) + if err != nil { + continue + } + subnets = append(subnets, *network) + } + + return subnets +} + +// addFallbackSubnets appends fallback subnet phases and updates confidence. +func addFallbackSubnets(profile *EnvironmentProfile) (*EnvironmentProfile, error) { + fallback := tryCommonSubnets() + if len(fallback) == 0 { + return profile, fmt.Errorf("no fallback subnets available") + } + + profile.Phases = append(profile.Phases, SubnetPhase{ + Name: "fallback_common_subnets", + Subnets: fallback, + Confidence: 0.3, + Priority: 10, + }) + + if profile.Confidence == 0.0 { + profile.Confidence = 0.3 + } + + profile.Warnings = append(profile.Warnings, "Using fallback private subnets; consider manual subnet configuration") + return profile, nil +} + +// countKernelRoutes parses /proc/net/route and returns the number of route entries. +func countKernelRoutes() (int, string) { + data, err := os.ReadFile("/proc/net/route") + if err != nil { + return 0, fmt.Sprintf("Unable to read /proc/net/route: %v", err) + } + + count := 0 + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Iface") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + count++ + } + + if err := scanner.Err(); err != nil { + return count, fmt.Sprintf("Error scanning /proc/net/route: %v", err) + } + + return count, "" +} + +// profileWithWarning appends a warning and returns the same profile for chaining. +func profileWithWarning(profile *EnvironmentProfile, warning string) *EnvironmentProfile { + if warning != "" { + profile.Warnings = append(profile.Warnings, warning) + } + return profile +} diff --git a/scripts/install-sensor-proxy.sh b/scripts/install-sensor-proxy.sh index 32dbf7f..b6c7a17 100755 --- a/scripts/install-sensor-proxy.sh +++ b/scripts/install-sensor-proxy.sh @@ -287,6 +287,8 @@ RestartSec=5s RuntimeDirectory=pulse-sensor-proxy RuntimeDirectoryMode=0775 RuntimeDirectoryPreserve=yes +LogsDirectory=pulse/sensor-proxy +LogsDirectoryMode=0750 UMask=0007 # Core hardening @@ -344,6 +346,8 @@ RestartSec=5s RuntimeDirectory=pulse-sensor-proxy RuntimeDirectoryMode=0775 RuntimeDirectoryPreserve=yes +LogsDirectory=pulse/sensor-proxy +LogsDirectoryMode=0750 UMask=0007 # Core hardening