diff --git a/docs/DOCKER_HUB_README.md b/docs/DOCKER_HUB_README.md new file mode 100644 index 0000000..50bc714 --- /dev/null +++ b/docs/DOCKER_HUB_README.md @@ -0,0 +1,303 @@ +# Pulse + +[![GitHub release](https://img.shields.io/github/v/release/rcourtman/Pulse)](https://github.com/rcourtman/Pulse/releases/latest) +[![Docker Pulls](https://img.shields.io/docker/pulls/rcourtman/pulse)](https://hub.docker.com/r/rcourtman/pulse) +[![License](https://img.shields.io/github/license/rcourtman/Pulse)](LICENSE) + +**Real-time monitoring for Proxmox VE, Proxmox Mail Gateway, PBS, and Docker infrastructure with alerts and webhooks.** + +Monitor your hybrid Proxmox and Docker estate from a single dashboard. Get instant alerts when nodes go down, containers misbehave, backups fail, or storage fills up. Supports email, Discord, Slack, Telegram, and more. + +**[Try the live demo →](https://demo.pulserelay.pro)** (read-only with mock data) + +## Support Pulse Development + +Pulse is built by a solo developer in evenings and weekends. Your support helps: +- Keep me motivated to add new features +- Prioritize bug fixes and user requests +- Ensure Pulse stays 100% free and open-source forever + +[![GitHub Sponsors](https://img.shields.io/github/sponsors/rcourtman?style=social&label=Sponsor)](https://github.com/sponsors/rcourtman) +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/rcourtman) + +**Not ready to sponsor?** Star the project or share it with your homelab community! + +## Features + +- **Auto-Discovery**: Finds Proxmox nodes on your network, one-liner setup via generated scripts +- **Cluster Support**: Configure one node, monitor entire cluster +- **Enterprise Security**: + - Credentials encrypted at rest, masked in logs, never sent to frontend + - CSRF protection for all state-changing operations + - Rate limiting (500 req/min general, 10 attempts/min for auth) + - Account lockout after failed login attempts + - Secure session management with HttpOnly cookies + - bcrypt password hashing (cost 12) - passwords NEVER stored in plain text + - API tokens stored securely with restricted file permissions + - Security headers (CSP, X-Frame-Options, etc.) + - Comprehensive audit logging +- Live monitoring of VMs, containers, nodes, storage +- **Smart Alerts**: Email and webhooks (Discord, Slack, Telegram, Teams, ntfy.sh, Gotify) + - Example: "VM 'webserver' is down on node 'pve1'" + - Example: "Storage 'local-lvm' at 85% capacity" + - Example: "VM 'database' is back online" +- **Adaptive Thresholds**: Hysteresis-based trigger/clear levels, fractional network thresholds, per-metric search, reset-to-defaults, and Custom overrides with inline audit trail +- **Alert Timeline Analytics**: Rich history explorer with acknowledgement/clear markers, escalation breadcrumbs, and quick filters for noisy resources +- **Ceph Awareness**: Surface Ceph health, pool utilisation, and daemon status automatically when Proxmox exposes Ceph-backed storage +- Unified view of PBS backups, PVE backups, and snapshots +- **Interactive Backup Explorer**: Cross-highlighted bar chart + grid with quick time-range pivots (24h/7d/30d/custom) and contextual tooltips for the busiest jobs +- Proxmox Mail Gateway analytics: mail volume, spam/virus trends, quarantine health, and cluster node status +- Optional Docker container monitoring via lightweight agent +- Config export/import with encryption and authentication +- Automatic stable updates with safe rollback (opt-in) +- Dark/light themes, responsive design +- Built with Go for minimal resource usage + +[View screenshots and full documentation on GitHub →](https://github.com/rcourtman/Pulse) + +## Privacy + +**Pulse respects your privacy:** +- No telemetry or analytics collection +- No phone-home functionality +- No external API calls (except for configured webhooks) +- All data stays on your server +- Open source - verify it yourself + +Your infrastructure data is yours alone. + +## Quick Start with Docker + +### Basic Setup + +```bash +docker run -d \ + --name pulse \ + -p 7655:7655 \ + -v pulse_data:/data \ + --restart unless-stopped \ + rcourtman/pulse:latest +``` + +Then open `http://localhost:7655` and complete the security setup wizard. + +### Network Discovery + +Pulse automatically discovers Proxmox nodes on your network! By default, it scans: +- 192.168.0.0/16 (home networks) +- 10.0.0.0/8 (private networks) +- 172.16.0.0/12 (Docker/internal networks) + +To scan a custom subnet instead: +```bash +docker run -d \ + --name pulse \ + -p 7655:7655 \ + -v pulse_data:/data \ + -e DISCOVERY_SUBNET="192.168.50.0/24" \ + --restart unless-stopped \ + rcourtman/pulse:latest +``` + +### Automated Deployment with Pre-configured Auth + +```bash +# Deploy with authentication pre-configured +docker run -d \ + --name pulse \ + -p 7655:7655 \ + -v pulse_data:/data \ + -e API_TOKEN="your-secure-token" \ + -e PULSE_AUTH_USER="admin" \ + -e PULSE_AUTH_PASS="your-password" \ + --restart unless-stopped \ + rcourtman/pulse:latest + +# Plain text credentials are automatically hashed for security +# No setup required - API works immediately +``` + +### Docker Compose + +```yaml +services: + pulse: + image: rcourtman/pulse:latest + container_name: pulse + ports: + - "7655:7655" + volumes: + - pulse_data:/data + environment: + # NOTE: Env vars override UI settings. Remove env var to allow UI configuration. + + # Network discovery (usually not needed - auto-scans common networks) + # - DISCOVERY_SUBNET=192.168.50.0/24 # Only for non-standard networks + + # Ports + # - PORT=7655 # Backend port (default: 7655) + # - FRONTEND_PORT=7655 # Frontend port (default: 7655) + + # Security (all optional - runs open by default) + # - PULSE_AUTH_USER=admin # Username for web UI login + # - PULSE_AUTH_PASS=your-password # Plain text or bcrypt hash (auto-hashed if plain) + # - API_TOKEN=your-token # Plain text or SHA3-256 hash (auto-hashed if plain) + # - ALLOW_UNPROTECTED_EXPORT=false # Allow export without auth (default: false) + + # Security: Plain text credentials are automatically hashed + # You can provide either: + # 1. Plain text (auto-hashed): PULSE_AUTH_PASS=mypassword + # 2. Pre-hashed (advanced): PULSE_AUTH_PASS='$$2a$$12$$...' + # Note: Escape $ as $$ in docker-compose.yml for pre-hashed values + + # Performance + # - CONNECTION_TIMEOUT=10 # Connection timeout in seconds (default: 10) + + # CORS & logging + # - ALLOWED_ORIGINS=https://app.example.com # CORS origins (default: none, same-origin only) + # - LOG_LEVEL=info # Log level: debug/info/warn/error (default: info) + restart: unless-stopped + +volumes: + pulse_data: +``` + +## Initial Setup + +1. Open `http://:7655` +2. **Complete the mandatory security setup** (first-time only) +3. Create your admin username and password +4. Save the generated API token for automation + +## Configure Proxmox/PBS Nodes + +After logging in: + +1. Go to Settings → Nodes +2. Discovered nodes appear automatically +3. Click "Setup Script" next to any node +4. Click "Generate Setup Code" button (creates a 6-character code valid for 5 minutes) +5. Copy and run the provided one-liner on your Proxmox/PBS host +6. Node is configured and monitoring starts automatically + +**Example setup command:** +```bash +curl -sSL "http://pulse:7655/api/setup-script?type=pve&host=https://pve:8006&auth_token=ABC123" | bash +``` + +## Docker Updates + +```bash +# Latest stable +docker pull rcourtman/pulse:latest + +# Latest RC/pre-release +docker pull rcourtman/pulse:rc + +# Specific version +docker pull rcourtman/pulse:v4.22.0 + +# Then recreate your container +docker stop pulse && docker rm pulse +# Run your docker run or docker-compose command again +``` + +## Security + +- **Authentication required** - Protects your Proxmox infrastructure credentials +- **Quick setup wizard** - Secure your installation in under a minute +- **Multiple auth methods**: Password authentication, API tokens, proxy auth (SSO), or combinations +- **Proxy/SSO support** - Integrate with Authentik, Authelia, and other authentication proxies +- **Enterprise-grade protection**: + - Credentials encrypted at rest (AES-256-GCM) + - CSRF tokens for state-changing operations + - Rate limiting and account lockout protection + - Secure session management with HttpOnly cookies + - bcrypt password hashing (cost 12) - passwords NEVER stored in plain text + - API tokens stored securely with restricted file permissions + - Security headers (CSP, X-Frame-Options, etc.) + - Comprehensive audit logging +- **Security by design**: + - Frontend never receives node credentials + - API tokens visible only to authenticated users + - Export/import requires authentication when configured + +See [Security Documentation](https://github.com/rcourtman/Pulse/blob/main/docs/SECURITY.md) for details. + +## HTTPS/TLS Configuration + +Enable HTTPS by setting these environment variables: + +```bash +docker run -d -p 7655:7655 \ + -e HTTPS_ENABLED=true \ + -e TLS_CERT_FILE=/data/cert.pem \ + -e TLS_KEY_FILE=/data/key.pem \ + -v pulse_data:/data \ + -v /path/to/certs:/data/certs:ro \ + rcourtman/pulse:latest +``` + +## Troubleshooting + +### Authentication Issues + +#### Cannot login after setting up security +- **Docker**: Ensure bcrypt hash is exactly 60 characters and wrapped in single quotes +- **Docker Compose**: MUST escape $ characters as $$ (e.g., `$$2a$$12$$...`) +- **Example (docker run)**: `PULSE_AUTH_PASS='$2a$12$YTZXOCEylj4TaevZ0DCeI.notayQZ..b0OZ97lUZ.Q24fljLiMQHK'` +- **Example (docker-compose.yml)**: `PULSE_AUTH_PASS='$$2a$$12$$YTZXOCEylj4TaevZ0DCeI.notayQZ..b0OZ97lUZ.Q24fljLiMQHK'` +- If hash is truncated or mangled, authentication will fail +- Use Quick Security Setup in the UI to avoid manual configuration errors + +#### .env file not created (Docker) +- **Expected behavior**: When using environment variables, no .env file is created in /data +- The .env file is only created when using Quick Security Setup or password changes +- If you provide credentials via environment variables, they take precedence +- To use Quick Security Setup: Start container WITHOUT auth environment variables + +### VM Disk Stats Show "-" +- VMs require QEMU Guest Agent to report disk usage (Proxmox API returns 0 for VMs) +- Install guest agent in VM: `apt install qemu-guest-agent` (Linux) or virtio-win tools (Windows) +- Enable in VM Options → QEMU Guest Agent, then restart VM +- Container (LXC) disk stats always work (no guest agent needed) + +### Connection Issues +- Check Proxmox API is accessible (port 8006/8007) +- Verify credentials have PVEAuditor role minimum +- For PBS: ensure API token has Datastore.Audit permission + +### Logs +```bash +# View logs +docker logs pulse + +# Follow logs +docker logs -f pulse +``` + +## Documentation + +Full documentation available on GitHub: + +- [Complete Installation Guide](https://github.com/rcourtman/Pulse/blob/main/docs/INSTALL.md) +- [Configuration Guide](https://github.com/rcourtman/Pulse/blob/main/docs/CONFIGURATION.md) +- [VM Disk Monitoring](https://github.com/rcourtman/Pulse/blob/main/docs/VM_DISK_MONITORING.md) - Set up QEMU Guest Agent for accurate VM disk usage +- [Troubleshooting](https://github.com/rcourtman/Pulse/blob/main/docs/TROUBLESHOOTING.md) +- [API Reference](https://github.com/rcourtman/Pulse/blob/main/docs/API.md) +- [Webhook Guide](https://github.com/rcourtman/Pulse/blob/main/docs/WEBHOOKS.md) +- [Proxy Authentication](https://github.com/rcourtman/Pulse/blob/main/docs/PROXY_AUTH.md) - SSO integration with Authentik, Authelia, etc. +- [Reverse Proxy Setup](https://github.com/rcourtman/Pulse/blob/main/docs/REVERSE_PROXY.md) - nginx, Caddy, Apache, Traefik configs +- [Security](https://github.com/rcourtman/Pulse/blob/main/docs/SECURITY.md) +- [FAQ](https://github.com/rcourtman/Pulse/blob/main/docs/FAQ.md) + +## Links + +- [GitHub Repository](https://github.com/rcourtman/Pulse) +- [Releases & Changelog](https://github.com/rcourtman/Pulse/releases) +- [Issues & Feature Requests](https://github.com/rcourtman/Pulse/issues) +- [Live Demo](https://demo.pulserelay.pro) + +## License + +MIT - See [LICENSE](https://github.com/rcourtman/Pulse/blob/main/LICENSE) diff --git a/frontend-modern/src/components/Dashboard/Dashboard.tsx b/frontend-modern/src/components/Dashboard/Dashboard.tsx index 2f0abb4..69e77ac 100644 --- a/frontend-modern/src/components/Dashboard/Dashboard.tsx +++ b/frontend-modern/src/components/Dashboard/Dashboard.tsx @@ -703,7 +703,7 @@ export function Dashboard(props: DashboardProps) { handleSort('name')} onKeyDown={(e) => e.key === 'Enter' && handleSort('name')} tabindex="0" @@ -713,63 +713,63 @@ export function Dashboard(props: DashboardProps) { Name {sortKey() === 'name' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('type')} > Type {sortKey() === 'type' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('vmid')} > VMID {sortKey() === 'vmid' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('uptime')} > Uptime {sortKey() === 'uptime' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('cpu')} > CPU {sortKey() === 'cpu' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('memory')} > Memory {sortKey() === 'memory' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('disk')} > Disk {sortKey() === 'disk' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('diskRead')} > Disk Read{' '} {sortKey() === 'diskRead' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('diskWrite')} > Disk Write{' '} {sortKey() === 'diskWrite' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('networkIn')} > Net In {sortKey() === 'networkIn' && (sortDirection() === 'asc' ? '▲' : '▼')} handleSort('networkOut')} > Net Out{' '} diff --git a/frontend-modern/src/components/Dashboard/GuestRow.tsx b/frontend-modern/src/components/Dashboard/GuestRow.tsx index a32148c..5bffe21 100644 --- a/frontend-modern/src/components/Dashboard/GuestRow.tsx +++ b/frontend-modern/src/components/Dashboard/GuestRow.tsx @@ -218,7 +218,8 @@ export function GuestRow(props: GuestRowProps) { // Get first cell styling const firstCellClass = createMemo(() => { - const base = 'py-0.5 pr-2 whitespace-nowrap relative'; + const base = + 'py-0.5 pr-2 whitespace-nowrap relative w-[170px] lg:w-[190px] xl:w-[210px] 2xl:w-[260px]'; const indent = 'pl-6'; return `${base} ${indent}`; }); @@ -285,7 +286,7 @@ export function GuestRow(props: GuestRowProps) { {/* Type */} - +
{/* VMID */} - + {props.guest.vmid} {/* Uptime */} @@ -316,7 +317,7 @@ export function GuestRow(props: GuestRowProps) { {/* CPU */} - + -}> {/* Memory */} - +
-}> {/* Disk – surface usage even if guest is currently stopped so users can see last reported values */} - + {/* Disk I/O */} - +
- +
{/* Network I/O */} - +
- +
diff --git a/frontend-modern/src/components/Settings/Settings.tsx b/frontend-modern/src/components/Settings/Settings.tsx index 585a834..4931b19 100644 --- a/frontend-modern/src/components/Settings/Settings.tsx +++ b/frontend-modern/src/components/Settings/Settings.tsx @@ -3805,12 +3805,25 @@ const Settings: Component = (props) => { {(() => { const sanitizeForGitHub = (data: Record) => { // Deep clone the data - const sanitized = JSON.parse(JSON.stringify(data)) as Record< - string, - unknown - >; + const sanitized = JSON.parse(JSON.stringify(data)) as Record< + string, + unknown + >; - const connectionKeyMap = new Map(); + const connectionKeyMap = new Map(); + const instanceKeyMap = new Map(); + + const getSanitizedInstance = (instance: string) => { + if (!instance) return instance; + if (instanceKeyMap.has(instance)) { + return instanceKeyMap.get(instance)!; + } + const suffixMatch = instance.match(/\.(lan|local|home|internal)$/); + const suffix = suffixMatch ? suffixMatch[0] : ''; + const label = `instance-REDACTED${suffix}`; + instanceKeyMap.set(instance, label); + return label; + }; // Sanitize IP addresses (keep first octet for network type identification) const sanitizeIP = (ip: string) => { @@ -3838,6 +3851,90 @@ const Settings: Component = (props) => { .replace(/\b\d{1,3}(?:\.\d{1,3}){3}\b/g, 'xxx.xxx.xxx.xxx'); }; + const sanitizeNotesArray = (notes: unknown) => { + if (!Array.isArray(notes)) return notes; + return notes.map((note) => { + if (typeof note !== 'string') return note; + const sanitizedNote = sanitizeText(note); + return sanitizedNote ?? note; + }); + }; + + const sanitizeNodeSnapshots = ( + snapshots: Array>, + ) => + snapshots.map((snapshot, index: number) => { + const sanitizedSnapshot = { ...snapshot }; + const originalNode = + typeof sanitizedSnapshot.node === 'string' + ? (sanitizedSnapshot.node as string) + : ''; + if (originalNode) { + sanitizedSnapshot.node = + connectionKeyMap.get(originalNode) || + sanitizeHostname(originalNode); + } + + const originalInstance = + typeof sanitizedSnapshot.instance === 'string' + ? (sanitizedSnapshot.instance as string) + : ''; + if (originalInstance) { + sanitizedSnapshot.instance = getSanitizedInstance(originalInstance); + } + + if ( + typeof sanitizedSnapshot.id === 'string' && + sanitizedSnapshot.id + ) { + sanitizedSnapshot.id = `node-snapshot-${index}`; + } + + return sanitizedSnapshot; + }); + + const sanitizeGuestSnapshots = ( + snapshots: Array>, + ) => + snapshots.map((snapshot, index: number) => { + const sanitizedSnapshot = { ...snapshot }; + const originalNode = + typeof sanitizedSnapshot.node === 'string' + ? (sanitizedSnapshot.node as string) + : ''; + if (originalNode) { + sanitizedSnapshot.node = + connectionKeyMap.get(originalNode) || + sanitizeHostname(originalNode); + } + + const originalInstance = + typeof sanitizedSnapshot.instance === 'string' + ? (sanitizedSnapshot.instance as string) + : ''; + if (originalInstance) { + sanitizedSnapshot.instance = getSanitizedInstance(originalInstance); + } + + if (typeof sanitizedSnapshot.name === 'string') { + sanitizedSnapshot.name = 'vm-REDACTED'; + } + + if (typeof sanitizedSnapshot.vmid === 'number') { + sanitizedSnapshot.vmid = index + 1; + } else if (typeof sanitizedSnapshot.vmid === 'string') { + sanitizedSnapshot.vmid = `vm-${index + 1}`; + } + + if (Array.isArray(sanitizedSnapshot.notes)) { + sanitizedSnapshot.notes = sanitizeNotesArray( + sanitizedSnapshot.notes, + ); + } + + return sanitizedSnapshot; + }); + // Sanitize nodes if (sanitized.nodes) { sanitized.nodes = ( @@ -3944,6 +4041,18 @@ const Settings: Component = (props) => { }); } + if (Array.isArray(sanitized.nodeSnapshots)) { + sanitized.nodeSnapshots = sanitizeNodeSnapshots( + sanitized.nodeSnapshots as Array>, + ); + } + + if (Array.isArray(sanitized.guestSnapshots)) { + sanitized.guestSnapshots = sanitizeGuestSnapshots( + sanitized.guestSnapshots as Array>, + ); + } + // Sanitize backend diagnostics (if present) if ( sanitized.backendDiagnostics && @@ -3951,6 +4060,18 @@ const Settings: Component = (props) => { ) { const backend = sanitized.backendDiagnostics as Record; + if (Array.isArray(backend.nodeSnapshots)) { + backend.nodeSnapshots = sanitizeNodeSnapshots( + backend.nodeSnapshots as Array>, + ); + } + + if (Array.isArray(backend.guestSnapshots)) { + backend.guestSnapshots = sanitizeGuestSnapshots( + backend.guestSnapshots as Array>, + ); + } + if (Array.isArray(backend.nodes)) { backend.nodes = backend.nodes.map((rawNode, index: number) => { const node = rawNode as Record; diff --git a/internal/api/diagnostics.go b/internal/api/diagnostics.go index 34f440a..43f4182 100644 --- a/internal/api/diagnostics.go +++ b/internal/api/diagnostics.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" "github.com/rcourtman/pulse-go-rewrite/internal/updates" "github.com/rcourtman/pulse-go-rewrite/pkg/pbs" "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" @@ -25,6 +26,10 @@ type DiagnosticsInfo struct { PBS []PBSDiagnostic `json:"pbs"` System SystemDiagnostic `json:"system"` Errors []string `json:"errors"` + // NodeSnapshots captures the raw memory payload and derived usage Pulse last observed per node. + NodeSnapshots []monitoring.NodeMemorySnapshot `json:"nodeSnapshots,omitempty"` + // GuestSnapshots captures recent per-guest memory breakdowns (VM/LXC) with the raw Proxmox fields. + GuestSnapshots []monitoring.GuestMemorySnapshot `json:"guestSnapshots,omitempty"` } // NodeDiagnostic contains diagnostic info for a Proxmox node @@ -280,6 +285,17 @@ func (r *Router) handleDiagnostics(w http.ResponseWriter, req *http.Request) { diag.PBS = append(diag.PBS, pbsDiag) } + // Include cached monitor snapshots for memory diagnostics if available + if r.monitor != nil { + snapshots := r.monitor.GetDiagnosticSnapshots() + if len(snapshots.Nodes) > 0 { + diag.NodeSnapshots = snapshots.Nodes + } + if len(snapshots.Guests) > 0 { + diag.GuestSnapshots = snapshots.Guests + } + } + // Add any recent errors from logs (this would need a log collector) // For now, just check basic connectivity diff --git a/internal/monitoring/diagnostic_snapshots.go b/internal/monitoring/diagnostic_snapshots.go new file mode 100644 index 0000000..c88e97f --- /dev/null +++ b/internal/monitoring/diagnostic_snapshots.go @@ -0,0 +1,172 @@ +package monitoring + +import ( + "fmt" + "sort" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +// NodeMemoryRaw captures the raw memory fields returned by Proxmox for a node. +type NodeMemoryRaw struct { + Total uint64 `json:"total"` + Used uint64 `json:"used"` + Free uint64 `json:"free"` + Available uint64 `json:"available"` + Avail uint64 `json:"avail"` + Buffers uint64 `json:"buffers"` + Cached uint64 `json:"cached"` + Shared uint64 `json:"shared"` + EffectiveAvailable uint64 `json:"effectiveAvailable"` + FallbackTotal uint64 `json:"fallbackTotal,omitempty"` + FallbackUsed uint64 `json:"fallbackUsed,omitempty"` + FallbackFree uint64 `json:"fallbackFree,omitempty"` + FallbackCalculated bool `json:"fallbackCalculated,omitempty"` + ProxmoxMemorySource string `json:"proxmoxMemorySource,omitempty"` +} + +// NodeMemorySnapshot records the final node memory calculation alongside the raw data. +type NodeMemorySnapshot struct { + Instance string `json:"instance"` + Node string `json:"node"` + RetrievedAt time.Time `json:"retrievedAt"` + MemorySource string `json:"memorySource"` + FallbackReason string `json:"fallbackReason,omitempty"` + Memory models.Memory `json:"memory"` + Raw NodeMemoryRaw `json:"raw"` +} + +// VMMemoryRaw captures both the listing and detailed status memory fields for a VM/CT. +type VMMemoryRaw struct { + ListingMem uint64 `json:"listingMem"` + ListingMaxMem uint64 `json:"listingMaxmem"` + StatusMem uint64 `json:"statusMem,omitempty"` + StatusFreeMem uint64 `json:"statusFreemem,omitempty"` + StatusMaxMem uint64 `json:"statusMaxmem,omitempty"` + Balloon uint64 `json:"balloon,omitempty"` + BalloonMin uint64 `json:"balloonMin,omitempty"` + MemInfoUsed uint64 `json:"meminfoUsed,omitempty"` + MemInfoFree uint64 `json:"meminfoFree,omitempty"` + MemInfoTotal uint64 `json:"meminfoTotal,omitempty"` + Agent int `json:"agent,omitempty"` + DerivedFromBall bool `json:"derivedFromBalloon,omitempty"` +} + +// GuestMemorySnapshot records the memory calculation for a guest (VM/LXC). +type GuestMemorySnapshot struct { + GuestType string `json:"guestType"` + Instance string `json:"instance"` + Node string `json:"node"` + Name string `json:"name"` + VMID int `json:"vmid"` + Status string `json:"status"` + RetrievedAt time.Time `json:"retrievedAt"` + MemorySource string `json:"memorySource"` + Memory models.Memory `json:"memory"` + Raw VMMemoryRaw `json:"raw"` + Notes []string `json:"notes,omitempty"` +} + +// DiagnosticSnapshotSet aggregates the latest node and guest snapshots. +type DiagnosticSnapshotSet struct { + Nodes []NodeMemorySnapshot `json:"nodes"` + Guests []GuestMemorySnapshot `json:"guests"` +} + +func makeNodeSnapshotKey(instance, node string) string { + return fmt.Sprintf("%s|%s", instance, node) +} + +func makeGuestSnapshotKey(instance, guestType, node string, vmid int) string { + return fmt.Sprintf("%s|%s|%s|%d", instance, guestType, node, vmid) +} + +func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemorySnapshot) { + if m == nil { + return + } + + snapshot.Instance = instance + snapshot.Node = node + if snapshot.RetrievedAt.IsZero() { + snapshot.RetrievedAt = time.Now() + } + + m.diagMu.Lock() + defer m.diagMu.Unlock() + if m.nodeSnapshots == nil { + m.nodeSnapshots = make(map[string]NodeMemorySnapshot) + } + m.nodeSnapshots[makeNodeSnapshotKey(instance, node)] = snapshot +} + +func (m *Monitor) recordGuestSnapshot(instance, guestType, node string, vmid int, snapshot GuestMemorySnapshot) { + if m == nil { + return + } + + snapshot.Instance = instance + snapshot.GuestType = guestType + snapshot.Node = node + snapshot.VMID = vmid + if snapshot.RetrievedAt.IsZero() { + snapshot.RetrievedAt = time.Now() + } + + m.diagMu.Lock() + defer m.diagMu.Unlock() + if m.guestSnapshots == nil { + m.guestSnapshots = make(map[string]GuestMemorySnapshot) + } + m.guestSnapshots[makeGuestSnapshotKey(instance, guestType, node, vmid)] = snapshot +} + +// GetDiagnosticSnapshots returns copies of the latest node and guest memory snapshots. +func (m *Monitor) GetDiagnosticSnapshots() DiagnosticSnapshotSet { + result := DiagnosticSnapshotSet{ + Nodes: []NodeMemorySnapshot{}, + Guests: []GuestMemorySnapshot{}, + } + + if m == nil { + return result + } + + m.diagMu.RLock() + defer m.diagMu.RUnlock() + + if len(m.nodeSnapshots) > 0 { + result.Nodes = make([]NodeMemorySnapshot, 0, len(m.nodeSnapshots)) + for _, snapshot := range m.nodeSnapshots { + result.Nodes = append(result.Nodes, snapshot) + } + sort.Slice(result.Nodes, func(i, j int) bool { + if result.Nodes[i].Instance == result.Nodes[j].Instance { + return result.Nodes[i].Node < result.Nodes[j].Node + } + return result.Nodes[i].Instance < result.Nodes[j].Instance + }) + } + + if len(m.guestSnapshots) > 0 { + result.Guests = make([]GuestMemorySnapshot, 0, len(m.guestSnapshots)) + for _, snapshot := range m.guestSnapshots { + result.Guests = append(result.Guests, snapshot) + } + sort.Slice(result.Guests, func(i, j int) bool { + if result.Guests[i].Instance == result.Guests[j].Instance { + if result.Guests[i].Node == result.Guests[j].Node { + if result.Guests[i].GuestType == result.Guests[j].GuestType { + return result.Guests[i].VMID < result.Guests[j].VMID + } + return result.Guests[i].GuestType < result.Guests[j].GuestType + } + return result.Guests[i].Node < result.Guests[j].Node + } + return result.Guests[i].Instance < result.Guests[j].Instance + }) + } + + return result +} diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 8e7ee86..f2b8999 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -203,6 +203,9 @@ type Monitor struct { pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance runtimeCtx context.Context // Context used while monitor is running wsHub *websocket.Hub // Hub used for broadcasting state + diagMu sync.RWMutex // Protects diagnostic snapshot maps + nodeSnapshots map[string]NodeMemorySnapshot + guestSnapshots map[string]GuestMemorySnapshot } // safePercentage calculates percentage safely, returning 0 if divisor is 0 @@ -763,6 +766,8 @@ func New(cfg *config.Config) (*Monitor, error) { lastPhysicalDiskPoll: make(map[string]time.Time), persistence: config.NewConfigPersistence(cfg.DataPath), pbsBackupPollers: make(map[string]bool), + nodeSnapshots: make(map[string]NodeMemorySnapshot), + guestSnapshots: make(map[string]GuestMemorySnapshot), } // Load saved configurations @@ -1698,6 +1703,19 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie ClusterName: instanceCfg.ClusterName, } + nodeSnapshotRaw := NodeMemoryRaw{ + Total: node.MaxMem, + Used: node.Mem, + Free: node.MaxMem - node.Mem, + FallbackTotal: node.MaxMem, + FallbackUsed: node.Mem, + FallbackFree: node.MaxMem - node.Mem, + FallbackCalculated: true, + ProxmoxMemorySource: "nodes-endpoint", + } + nodeMemorySource := "nodes-endpoint" + var nodeFallbackReason string + // Debug logging for disk metrics - note that these values can fluctuate // due to thin provisioning and dynamic allocation if node.Disk > 0 && node.MaxDisk > 0 { @@ -1716,6 +1734,7 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie if node.Status == "online" { nodeInfo, nodeErr := client.GetNodeStatus(ctx, node.Node) if nodeErr != nil { + nodeFallbackReason = "node-status-unavailable" // If we can't get node status, log but continue with data from /nodes endpoint if node.Disk > 0 && node.MaxDisk > 0 { log.Warn(). @@ -1735,6 +1754,20 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie Msg("Could not get node status - no fallback metrics available (memory will include cache/buffers)") } } else if nodeInfo != nil { + if nodeInfo.Memory != nil { + nodeSnapshotRaw.Total = nodeInfo.Memory.Total + nodeSnapshotRaw.Used = nodeInfo.Memory.Used + nodeSnapshotRaw.Free = nodeInfo.Memory.Free + nodeSnapshotRaw.Available = nodeInfo.Memory.Available + nodeSnapshotRaw.Avail = nodeInfo.Memory.Avail + nodeSnapshotRaw.Buffers = nodeInfo.Memory.Buffers + nodeSnapshotRaw.Cached = nodeInfo.Memory.Cached + nodeSnapshotRaw.Shared = nodeInfo.Memory.Shared + nodeSnapshotRaw.EffectiveAvailable = nodeInfo.Memory.EffectiveAvailable() + nodeSnapshotRaw.ProxmoxMemorySource = "node-status" + nodeSnapshotRaw.FallbackCalculated = false + } + // Convert LoadAvg from interface{} to float64 loadAvg := make([]float64, 0, len(nodeInfo.LoadAvg)) for _, val := range nodeInfo.LoadAvg { @@ -1807,14 +1840,17 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie switch { case nodeInfo.Memory.Available > 0: logCtx.Msg("Node memory: using available field (excludes reclaimable cache)") + nodeMemorySource = "available-field" case nodeInfo.Memory.Avail > 0: logCtx.Msg("Node memory: using avail field (excludes reclaimable cache)") + nodeMemorySource = "avail-field" default: logCtx. Uint64("free", nodeInfo.Memory.Free). Uint64("buffers", nodeInfo.Memory.Buffers). Uint64("cached", nodeInfo.Memory.Cached). Msg("Node memory: derived available from free+buffers+cached (excludes reclaimable cache)") + nodeMemorySource = "derived-free-buffers-cached" } default: // Fallback to traditional used memory if no cache-aware data is exposed @@ -1824,6 +1860,7 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie Uint64("total", nodeInfo.Memory.Total). Uint64("used", nodeInfo.Memory.Used). Msg("Node memory: no cache-aware metrics - using traditional calculation (includes cache)") + nodeMemorySource = "node-status-used" } free := int64(nodeInfo.Memory.Total - actualUsed) @@ -1894,9 +1931,25 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie Str("instance", instanceName). Str("node", node.Node). Msg("Preserving previous memory metrics - node status unavailable this cycle") + + if nodeFallbackReason == "" { + nodeFallbackReason = "preserved-previous-snapshot" + } + nodeMemorySource = "previous-snapshot" + if nodeSnapshotRaw.ProxmoxMemorySource == "node-status" && nodeSnapshotRaw.Total == 0 { + nodeSnapshotRaw.ProxmoxMemorySource = "previous-snapshot" + } } } + m.recordNodeSnapshot(instanceName, node.Node, NodeMemorySnapshot{ + RetrievedAt: time.Now(), + MemorySource: nodeMemorySource, + FallbackReason: nodeFallbackReason, + Memory: modelNode.Memory, + Raw: nodeSnapshotRaw, + }) + // Collect temperature data via SSH (non-blocking, best effort) // Only attempt for online nodes if node.Status == "online" && m.tempCollector != nil { @@ -2432,6 +2485,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam continue } + memTotal := res.MaxMem + memUsed := res.Mem + memorySource := "cluster-resources" + guestRaw := VMMemoryRaw{ + ListingMem: res.Mem, + ListingMaxMem: res.MaxMem, + } + var detailedStatus *proxmox.VMStatus + // Try to get actual disk usage from guest agent if VM is running diskUsed := res.Disk diskTotal := res.MaxDisk @@ -2449,7 +2511,7 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam // We should prefer guest agent data when available for accurate metrics if res.Status == "running" && res.Type == "qemu" { // First check if agent is enabled by getting VM status - vmStatus, err := client.GetVMStatus(ctx, res.Node, res.VMID) + status, err := client.GetVMStatus(ctx, res.Node, res.VMID) if err != nil { log.Debug(). Err(err). @@ -2457,15 +2519,47 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam Str("vm", res.Name). Int("vmid", res.VMID). Msg("Could not get VM status to check guest agent availability") - } else if vmStatus != nil { + } else if status != nil { + detailedStatus = status + guestRaw.StatusMaxMem = detailedStatus.MaxMem + guestRaw.StatusMem = detailedStatus.Mem + guestRaw.StatusFreeMem = detailedStatus.FreeMem + guestRaw.Balloon = detailedStatus.Balloon + guestRaw.BalloonMin = detailedStatus.BalloonMin + guestRaw.Agent = detailedStatus.Agent + if detailedStatus.MemInfo != nil { + guestRaw.MemInfoUsed = detailedStatus.MemInfo.Used + guestRaw.MemInfoFree = detailedStatus.MemInfo.Free + guestRaw.MemInfoTotal = detailedStatus.MemInfo.Total + } + // Use actual disk I/O values from detailed status - diskReadBytes = int64(vmStatus.DiskRead) - diskWriteBytes = int64(vmStatus.DiskWrite) - networkInBytes = int64(vmStatus.NetIn) - networkOutBytes = int64(vmStatus.NetOut) + diskReadBytes = int64(detailedStatus.DiskRead) + diskWriteBytes = int64(detailedStatus.DiskWrite) + networkInBytes = int64(detailedStatus.NetIn) + networkOutBytes = int64(detailedStatus.NetOut) + + if detailedStatus.Balloon > 0 && detailedStatus.Balloon < detailedStatus.MaxMem { + memTotal = detailedStatus.Balloon + guestRaw.DerivedFromBall = true + } else if detailedStatus.MaxMem > 0 { + memTotal = detailedStatus.MaxMem + guestRaw.DerivedFromBall = false + } + + if detailedStatus.FreeMem > 0 && memTotal >= detailedStatus.FreeMem { + memUsed = memTotal - detailedStatus.FreeMem + memorySource = "status-freemem" + } else if detailedStatus.Mem > 0 { + memUsed = detailedStatus.Mem + memorySource = "status-mem" + } + if memUsed > memTotal { + memUsed = memTotal + } // Gather guest metadata from the agent when available - guestIPs, guestIfaces, guestOSName, guestOSVersion := fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, vmStatus) + guestIPs, guestIfaces, guestOSName, guestOSVersion := fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, detailedStatus) if len(guestIPs) > 0 { ipAddresses = guestIPs } @@ -2481,12 +2575,12 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam // Always try to get filesystem info if agent is enabled // Prefer guest agent data over cluster/resources data for accuracy - if vmStatus.Agent > 0 { + if detailedStatus.Agent > 0 { log.Debug(). Str("instance", instanceName). Str("vm", res.Name). Int("vmid", res.VMID). - Int("agent", vmStatus.Agent). + Int("agent", detailedStatus.Agent). Uint64("current_disk", diskUsed). Uint64("current_maxdisk", diskTotal). Msg("Guest agent enabled, querying filesystem info for accurate disk usage") @@ -2748,7 +2842,7 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam Str("instance", instanceName). Str("vm", res.Name). Int("vmid", res.VMID). - Int("agent", vmStatus.Agent). + Int("agent", detailedStatus.Agent). Msg("VM does not have guest agent enabled in config") } } else { @@ -2761,16 +2855,43 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam } } - // Calculate I/O rates after we have the actual values + if res.Status != "running" { + memorySource = "powered-off" + memUsed = 0 + } + + memFree := uint64(0) + if memTotal >= memUsed { + memFree = memTotal - memUsed + } + + sampleTime := time.Now() currentMetrics := IOMetrics{ DiskRead: diskReadBytes, DiskWrite: diskWriteBytes, NetworkIn: networkInBytes, NetworkOut: networkOutBytes, - Timestamp: time.Now(), + Timestamp: sampleTime, } diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics) + memoryUsage := safePercentage(float64(memUsed), float64(memTotal)) + memory := models.Memory{ + Total: int64(memTotal), + Used: int64(memUsed), + Free: int64(memFree), + Usage: memoryUsage, + } + if memory.Free < 0 { + memory.Free = 0 + } + if memory.Used > memory.Total { + memory.Used = memory.Total + } + if detailedStatus != nil && detailedStatus.Balloon > 0 { + memory.Balloon = int64(detailedStatus.Balloon) + } + vm := models.VM{ ID: guestID, VMID: res.VMID, @@ -2781,12 +2902,7 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam Type: "qemu", CPU: safeFloat(res.CPU), CPUs: res.MaxCPU, - Memory: models.Memory{ - Total: int64(res.MaxMem), - Used: int64(res.Mem), - Free: int64(res.MaxMem - res.Mem), - Usage: safePercentage(float64(res.Mem), float64(res.MaxMem)), - }, + Memory: memory, Disk: models.Disk{ Total: int64(diskTotal), Used: int64(diskUsed), @@ -2804,7 +2920,7 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam DiskWrite: maxInt64(0, int64(diskWriteRate)), Uptime: int64(res.Uptime), Template: res.Template == 1, - LastSeen: time.Now(), + LastSeen: sampleTime, } // Parse tags @@ -2826,6 +2942,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam allVMs = append(allVMs, vm) + m.recordGuestSnapshot(instanceName, vm.Type, res.Node, res.VMID, GuestMemorySnapshot{ + Name: vm.Name, + Status: vm.Status, + RetrievedAt: sampleTime, + MemorySource: memorySource, + Memory: vm.Memory, + Raw: guestRaw, + }) + // For non-running VMs, zero out resource usage metrics to prevent false alerts // Proxmox may report stale or residual metrics for stopped VMs if vm.Status != "running" { @@ -3026,6 +3151,13 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli guestID = fmt.Sprintf("%s-%s-%d", instanceName, node.Node, vm.VMID) } + guestRaw := VMMemoryRaw{ + ListingMem: vm.Mem, + ListingMaxMem: vm.MaxMem, + Agent: vm.Agent, + } + memorySource := "listing-mem" + // Initialize I/O metrics from VM listing (may be 0 for disk I/O) diskReadBytes := int64(vm.DiskRead) diskWriteBytes := int64(vm.DiskWrite) @@ -3042,6 +3174,17 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli if vm.Status == "running" { // Try to get detailed VM status for more accurate memory reporting and disk I/O if status, err := client.GetVMStatus(ctx, node.Node, vm.VMID); err == nil && status != nil { + guestRaw.StatusMaxMem = status.MaxMem + guestRaw.StatusMem = status.Mem + guestRaw.StatusFreeMem = status.FreeMem + guestRaw.Balloon = status.Balloon + guestRaw.BalloonMin = status.BalloonMin + guestRaw.Agent = status.Agent + if status.MemInfo != nil { + guestRaw.MemInfoUsed = status.MemInfo.Used + guestRaw.MemInfoFree = status.MemInfo.Free + guestRaw.MemInfoTotal = status.MemInfo.Total + } // Use actual disk I/O values from detailed status diskReadBytes = int64(status.DiskRead) @@ -3052,16 +3195,21 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli // If balloon is enabled, use balloon as the total available memory if status.Balloon > 0 && status.Balloon < status.MaxMem { memTotal = status.Balloon + guestRaw.DerivedFromBall = true + } else { + guestRaw.DerivedFromBall = false } // If we have free memory from guest agent, calculate actual usage if status.FreeMem > 0 { // Guest agent reports free memory, so calculate used memUsed = memTotal - status.FreeMem + memorySource = "status-freemem" } else if status.Mem > 0 { // No guest agent free memory data, but we have actual memory usage // Use the reported memory usage from Proxmox memUsed = status.Mem + memorySource = "status-mem" } else { // No memory data available at all - show 0% usage memUsed = 0 @@ -3083,10 +3231,12 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli } else { // Failed to get detailed status - show 0% usage memUsed = 0 + memorySource = "status-unavailable" } } else { // VM is not running, show 0 usage memUsed = 0 + memorySource = "powered-off" } // Set CPU to 0 for non-running VMs to avoid false alerts @@ -3296,12 +3446,13 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli } // Calculate I/O rates after we have the actual values + sampleTime := time.Now() currentMetrics := IOMetrics{ DiskRead: diskReadBytes, DiskWrite: diskWriteBytes, NetworkIn: networkInBytes, NetworkOut: networkOutBytes, - Timestamp: time.Now(), + Timestamp: sampleTime, } diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics) @@ -3341,19 +3492,27 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli Template: vm.Template == 1, Tags: tags, Lock: vm.Lock, - LastSeen: time.Now(), + LastSeen: sampleTime, } allVMs = append(allVMs, modelVM) + m.recordGuestSnapshot(instanceName, modelVM.Type, node.Node, vm.VMID, GuestMemorySnapshot{ + Name: vm.Name, + Status: vm.Status, + RetrievedAt: sampleTime, + MemorySource: memorySource, + Memory: modelVM.Memory, + Raw: guestRaw, + }) + // Record metrics history - now := time.Now() - m.metricsHistory.AddGuestMetric(modelVM.ID, "cpu", modelVM.CPU*100, now) - m.metricsHistory.AddGuestMetric(modelVM.ID, "memory", modelVM.Memory.Usage, now) - m.metricsHistory.AddGuestMetric(modelVM.ID, "disk", modelVM.Disk.Usage, now) - m.metricsHistory.AddGuestMetric(modelVM.ID, "diskread", float64(modelVM.DiskRead), now) - m.metricsHistory.AddGuestMetric(modelVM.ID, "diskwrite", float64(modelVM.DiskWrite), now) - m.metricsHistory.AddGuestMetric(modelVM.ID, "netin", float64(modelVM.NetworkIn), now) - m.metricsHistory.AddGuestMetric(modelVM.ID, "netout", float64(modelVM.NetworkOut), now) + m.metricsHistory.AddGuestMetric(modelVM.ID, "cpu", modelVM.CPU*100, sampleTime) + m.metricsHistory.AddGuestMetric(modelVM.ID, "memory", modelVM.Memory.Usage, sampleTime) + m.metricsHistory.AddGuestMetric(modelVM.ID, "disk", modelVM.Disk.Usage, sampleTime) + m.metricsHistory.AddGuestMetric(modelVM.ID, "diskread", float64(modelVM.DiskRead), sampleTime) + m.metricsHistory.AddGuestMetric(modelVM.ID, "diskwrite", float64(modelVM.DiskWrite), sampleTime) + m.metricsHistory.AddGuestMetric(modelVM.ID, "netin", float64(modelVM.NetworkIn), sampleTime) + m.metricsHistory.AddGuestMetric(modelVM.ID, "netout", float64(modelVM.NetworkOut), sampleTime) // Check thresholds for alerts m.alertManager.CheckGuest(modelVM, instanceName) diff --git a/internal/monitoring/monitor_optimized.go b/internal/monitoring/monitor_optimized.go index 2ea34c9..0c7125d 100644 --- a/internal/monitoring/monitor_optimized.go +++ b/internal/monitoring/monitor_optimized.go @@ -131,6 +131,13 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st guestID = fmt.Sprintf("%s-%s-%d", instanceName, n.Node, vm.VMID) } + guestRaw := VMMemoryRaw{ + ListingMem: vm.Mem, + ListingMaxMem: vm.MaxMem, + Agent: vm.Agent, + } + memorySource := "listing-mem" + // Initialize metrics from VM listing (may be 0 for disk I/O) diskReadBytes := int64(vm.DiskRead) diskWriteBytes := int64(vm.DiskWrite) @@ -150,13 +157,27 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st statusCtx, cancel := context.WithTimeout(ctx, 2*time.Second) if status, err := client.GetVMStatus(statusCtx, n.Node, vm.VMID); err == nil { vmStatus = status + guestRaw.StatusMaxMem = status.MaxMem + guestRaw.StatusMem = status.Mem + guestRaw.StatusFreeMem = status.FreeMem + guestRaw.Balloon = status.Balloon + guestRaw.BalloonMin = status.BalloonMin + guestRaw.Agent = status.Agent + if status.MemInfo != nil { + guestRaw.MemInfoUsed = status.MemInfo.Used + guestRaw.MemInfoFree = status.MemInfo.Free + guestRaw.MemInfoTotal = status.MemInfo.Total + } if vmStatus.Balloon > 0 && vmStatus.Balloon < vmStatus.MaxMem { memTotal = vmStatus.Balloon + guestRaw.DerivedFromBall = true } if vmStatus.FreeMem > 0 { memUsed = memTotal - vmStatus.FreeMem + memorySource = "status-freemem" } else if vmStatus.Mem > 0 { memUsed = vmStatus.Mem + memorySource = "status-mem" } // Use actual disk I/O values from detailed status diskReadBytes = int64(vmStatus.DiskRead) @@ -167,6 +188,12 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st cancel() } + if vm.Status != "running" { + memorySource = "powered-off" + } else if vmStatus == nil { + memorySource = "status-unavailable" + } + if vm.Status == "running" && vmStatus != nil { guestIPs, guestIfaces, guestOSName, guestOSVersion := fetchGuestAgentMetadata(ctx, client, instanceName, n.Node, vm.Name, vm.VMID, vmStatus) if len(guestIPs) > 0 { @@ -184,12 +211,13 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st } // Calculate I/O rates after we have the actual values + sampleTime := time.Now() currentMetrics := IOMetrics{ DiskRead: diskReadBytes, DiskWrite: diskWriteBytes, NetworkIn: networkInBytes, NetworkOut: networkOutBytes, - Timestamp: time.Now(), + Timestamp: sampleTime, } diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics) @@ -469,7 +497,7 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st DiskWrite: maxInt64(0, int64(diskWriteRate)), Uptime: int64(vm.Uptime), Template: vm.Template == 1, - LastSeen: time.Now(), + LastSeen: sampleTime, Tags: tags, IPAddresses: ipAddresses, OSName: osName, @@ -490,6 +518,15 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st nodeVMs = append(nodeVMs, modelVM) + m.recordGuestSnapshot(instanceName, modelVM.Type, n.Node, vm.VMID, GuestMemorySnapshot{ + Name: vm.Name, + Status: vm.Status, + RetrievedAt: sampleTime, + MemorySource: memorySource, + Memory: modelVM.Memory, + Raw: guestRaw, + }) + // Check alerts m.alertManager.CheckGuest(modelVM, instanceName) } diff --git a/scripts/toggle-mock.sh b/scripts/toggle-mock.sh index 2da6eb0..edd02c0 100755 --- a/scripts/toggle-mock.sh +++ b/scripts/toggle-mock.sh @@ -31,6 +31,21 @@ fi restart_backend() { echo -e "${YELLOW}Restarting backend...${NC}" + # Check if running under systemd + if systemctl is-active --quiet pulse-hot-dev; then + echo "Detected systemd hot-dev service, restarting it..." + sudo systemctl restart pulse-hot-dev + sleep 3 + if systemctl is-active --quiet pulse-hot-dev; then + echo -e "${GREEN}✓ Hot-dev service restarted successfully${NC}" + return 0 + else + echo -e "${RED}✗ Hot-dev service failed to start. Check: sudo journalctl -u pulse-hot-dev -n 50${NC}" + return 1 + fi + fi + + # Fallback: manual restart (for standalone hot-dev.sh) # Kill existing pulse backend pkill -x pulse 2>/dev/null || true sleep 1