diff --git a/internal/api/diagnostics.go b/internal/api/diagnostics.go index 17700ca..336da97 100644 --- a/internal/api/diagnostics.go +++ b/internal/api/diagnostics.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "runtime" + "sort" "strings" "time" @@ -32,6 +33,26 @@ type DiagnosticsInfo struct { NodeSnapshots []monitoring.NodeMemorySnapshot `json:"nodeSnapshots,omitempty"` // GuestSnapshots captures recent per-guest memory breakdowns (VM/LXC) with the raw Proxmox fields. GuestSnapshots []monitoring.GuestMemorySnapshot `json:"guestSnapshots,omitempty"` + // MemorySources summarizes how many nodes currently rely on each memory source per instance. + MemorySources []MemorySourceStat `json:"memorySources,omitempty"` +} + +// MemorySourceStat aggregates memory-source usage per instance. +type MemorySourceStat struct { + Instance string `json:"instance"` + Source string `json:"source"` + NodeCount int `json:"nodeCount"` + LastUpdated string `json:"lastUpdated"` + Fallback bool `json:"fallback"` +} + +func isFallbackMemorySource(source string) bool { + switch strings.ToLower(source) { + case "", "unknown", "nodes-endpoint", "node-status-used", "previous-snapshot": + return true + default: + return false + } } // NodeDiagnostic contains diagnostic info for a Proxmox node @@ -338,6 +359,54 @@ func (r *Router) handleDiagnostics(w http.ResponseWriter, req *http.Request) { snapshots := r.monitor.GetDiagnosticSnapshots() if len(snapshots.Nodes) > 0 { diag.NodeSnapshots = snapshots.Nodes + + type memorySourceAgg struct { + stat MemorySourceStat + latest time.Time + } + + sourceAverages := make(map[string]*memorySourceAgg) + for _, snap := range snapshots.Nodes { + source := snap.MemorySource + if source == "" { + source = "unknown" + } + + key := fmt.Sprintf("%s|%s", snap.Instance, source) + entry, ok := sourceAverages[key] + if !ok { + entry = &memorySourceAgg{ + stat: MemorySourceStat{ + Instance: snap.Instance, + Source: source, + Fallback: isFallbackMemorySource(source), + }, + } + sourceAverages[key] = entry + } + + entry.stat.NodeCount++ + if snap.RetrievedAt.After(entry.latest) { + entry.latest = snap.RetrievedAt + } + } + + if len(sourceAverages) > 0 { + diag.MemorySources = make([]MemorySourceStat, 0, len(sourceAverages)) + for _, entry := range sourceAverages { + if !entry.latest.IsZero() { + entry.stat.LastUpdated = entry.latest.UTC().Format(time.RFC3339) + } + diag.MemorySources = append(diag.MemorySources, entry.stat) + } + + sort.Slice(diag.MemorySources, func(i, j int) bool { + if diag.MemorySources[i].Instance == diag.MemorySources[j].Instance { + return diag.MemorySources[i].Source < diag.MemorySources[j].Source + } + return diag.MemorySources[i].Instance < diag.MemorySources[j].Instance + }) + } } if len(snapshots.Guests) > 0 { diag.GuestSnapshots = snapshots.Guests diff --git a/internal/monitoring/diagnostic_snapshots.go b/internal/monitoring/diagnostic_snapshots.go index 184a6e1..9f02aa2 100644 --- a/internal/monitoring/diagnostic_snapshots.go +++ b/internal/monitoring/diagnostic_snapshots.go @@ -6,6 +6,8 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) // NodeMemoryRaw captures the raw memory fields returned by Proxmox for a node. @@ -88,6 +90,70 @@ func makeGuestSnapshotKey(instance, guestType, node string, vmid int) string { return fmt.Sprintf("%s|%s|%s|%d", instance, guestType, node, vmid) } +func (m *Monitor) logNodeMemorySource(instance, node string, snapshot NodeMemorySnapshot) { + if m == nil { + return + } + + key := makeNodeSnapshotKey(instance, node) + + source := snapshot.MemorySource + var prevSource string + m.diagMu.RLock() + if existing, ok := m.nodeSnapshots[key]; ok { + prevSource = existing.MemorySource + } + m.diagMu.RUnlock() + + if prevSource == snapshot.MemorySource { + return + } + + var evt *zerolog.Event + switch source { + case "", "nodes-endpoint", "node-status-used", "previous-snapshot": + evt = log.Warn() + default: + evt = log.Debug() + } + + evt = evt. + Str("instance", instance). + Str("node", node). + Str("memorySource", source). + Str("proxmoxSource", snapshot.Raw.ProxmoxMemorySource) + + if snapshot.FallbackReason != "" { + evt = evt.Str("fallbackReason", snapshot.FallbackReason) + } + if snapshot.Raw.Available > 0 { + evt = evt.Uint64("rawAvailable", snapshot.Raw.Available) + } + if snapshot.Raw.Buffers > 0 { + evt = evt.Uint64("rawBuffers", snapshot.Raw.Buffers) + } + if snapshot.Raw.Cached > 0 { + evt = evt.Uint64("rawCached", snapshot.Raw.Cached) + } + if snapshot.Raw.TotalMinusUsed > 0 { + evt = evt.Uint64("rawTotalMinusUsed", snapshot.Raw.TotalMinusUsed) + } + if snapshot.Memory.Total > 0 { + evt = evt.Int64("total", snapshot.Memory.Total) + } + if snapshot.Memory.Used > 0 { + evt = evt.Int64("used", snapshot.Memory.Used) + } + if snapshot.Memory.Free > 0 { + evt = evt.Int64("free", snapshot.Memory.Free) + } + if snapshot.Memory.Usage > 0 { + evt = evt.Float64("usage", snapshot.Memory.Usage) + } + + evt.Msg("Node memory source updated") +} + func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemorySnapshot) { if m == nil { return @@ -99,6 +165,8 @@ func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemoryS snapshot.RetrievedAt = time.Now() } + m.logNodeMemorySource(instance, node, snapshot) + m.diagMu.Lock() defer m.diagMu.Unlock() if m.nodeSnapshots == nil { diff --git a/pkg/proxmox/client_test.go b/pkg/proxmox/client_test.go index fe106fe..01a36ed 100644 --- a/pkg/proxmox/client_test.go +++ b/pkg/proxmox/client_test.go @@ -101,6 +101,17 @@ func TestMemoryStatusEffectiveAvailable(t *testing.T) { status: MemoryStatus{Total: 32 * 1024, Free: 4 * 1024, Buffers: 2 * 1024, Cached: 6 * 1024}, want: 12 * 1024, }, + { + name: "available zero but buffers and cache present", + status: MemoryStatus{ + Total: 64 * 1024, + Used: 40 * 1024, + Free: 6 * 1024, + Buffers: 8 * 1024, + Cached: 10 * 1024, + }, + want: 24 * 1024, + }, { name: "caps derived value at total", status: MemoryStatus{Total: 8 * 1024, Free: 4 * 1024, Buffers: 4 * 1024, Cached: 4 * 1024}, @@ -196,6 +207,19 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) { wantUsedPct: 42.4, // Should be ~42%, not 86%! description: "Real user report: 86% shown when actual usage is 42%", }, + { + name: "missing available but buffers/cached still present (issue 553 guard)", + status: MemoryStatus{ + Total: 34359738368, // 32GB + Used: 27917287424, // ~26GB reported used (includes cache) + Free: 2147483648, // 2GB + Buffers: 3221225472, // 3GB + Cached: 8053063680, // 7.5GB + }, + wantAvailable: 13421772800, // Free + Buffers + Cached + wantUsedPct: 60.94, + description: "When available=0 but buffers/cached exist, derive reclaimable memory instead of alerting", + }, { name: "missing all cache-aware fields - fallback to zero", status: MemoryStatus{