diff --git a/internal/monitoring/diagnostic_snapshots.go b/internal/monitoring/diagnostic_snapshots.go index 5667c88..184a6e1 100644 --- a/internal/monitoring/diagnostic_snapshots.go +++ b/internal/monitoring/diagnostic_snapshots.go @@ -19,6 +19,7 @@ type NodeMemoryRaw struct { Cached uint64 `json:"cached"` Shared uint64 `json:"shared"` EffectiveAvailable uint64 `json:"effectiveAvailable"` + TotalMinusUsed uint64 `json:"totalMinusUsed,omitempty"` FallbackTotal uint64 `json:"fallbackTotal,omitempty"` FallbackUsed uint64 `json:"fallbackUsed,omitempty"` FallbackFree uint64 `json:"fallbackFree,omitempty"` @@ -39,22 +40,23 @@ type NodeMemorySnapshot struct { // VMMemoryRaw captures both the listing and detailed status memory fields for a VM/CT. type VMMemoryRaw struct { - ListingMem uint64 `json:"listingMem"` - ListingMaxMem uint64 `json:"listingMaxmem"` - StatusMem uint64 `json:"statusMem,omitempty"` - StatusFreeMem uint64 `json:"statusFreemem,omitempty"` - StatusMaxMem uint64 `json:"statusMaxmem,omitempty"` - Balloon uint64 `json:"balloon,omitempty"` - BalloonMin uint64 `json:"balloonMin,omitempty"` - MemInfoUsed uint64 `json:"meminfoUsed,omitempty"` - MemInfoFree uint64 `json:"meminfoFree,omitempty"` - MemInfoTotal uint64 `json:"meminfoTotal,omitempty"` - MemInfoAvailable uint64 `json:"meminfoAvailable,omitempty"` - MemInfoBuffers uint64 `json:"meminfoBuffers,omitempty"` - MemInfoCached uint64 `json:"meminfoCached,omitempty"` - MemInfoShared uint64 `json:"meminfoShared,omitempty"` - Agent int `json:"agent,omitempty"` - DerivedFromBall bool `json:"derivedFromBalloon,omitempty"` + ListingMem uint64 `json:"listingMem"` + ListingMaxMem uint64 `json:"listingMaxmem"` + StatusMem uint64 `json:"statusMem,omitempty"` + StatusFreeMem uint64 `json:"statusFreemem,omitempty"` + StatusMaxMem uint64 `json:"statusMaxmem,omitempty"` + Balloon uint64 `json:"balloon,omitempty"` + BalloonMin uint64 `json:"balloonMin,omitempty"` + MemInfoUsed uint64 `json:"meminfoUsed,omitempty"` + MemInfoFree uint64 `json:"meminfoFree,omitempty"` + MemInfoTotal uint64 `json:"meminfoTotal,omitempty"` + MemInfoAvailable uint64 `json:"meminfoAvailable,omitempty"` + MemInfoBuffers uint64 `json:"meminfoBuffers,omitempty"` + MemInfoCached uint64 `json:"meminfoCached,omitempty"` + MemInfoShared uint64 `json:"meminfoShared,omitempty"` + MemInfoTotalMinusUsed uint64 `json:"meminfoTotalMinusUsed,omitempty"` + Agent int `json:"agent,omitempty"` + DerivedFromBall bool `json:"derivedFromBalloon,omitempty"` } // GuestMemorySnapshot records the memory calculation for a guest (VM/LXC). diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 432ac75..26a15a2 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -2055,12 +2055,37 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie var actualUsed uint64 effectiveAvailable := nodeInfo.Memory.EffectiveAvailable() usedRRDFallback := false + componentAvailable := nodeInfo.Memory.Free + if nodeInfo.Memory.Buffers > 0 { + if math.MaxUint64-componentAvailable < nodeInfo.Memory.Buffers { + componentAvailable = math.MaxUint64 + } else { + componentAvailable += nodeInfo.Memory.Buffers + } + } + if nodeInfo.Memory.Cached > 0 { + if math.MaxUint64-componentAvailable < nodeInfo.Memory.Cached { + componentAvailable = math.MaxUint64 + } else { + componentAvailable += nodeInfo.Memory.Cached + } + } + if nodeInfo.Memory.Total > 0 && componentAvailable > nodeInfo.Memory.Total { + componentAvailable = nodeInfo.Memory.Total + } - if effectiveAvailable == 0 && - nodeInfo.Memory.Available == 0 && + availableFromUsed := uint64(0) + if nodeInfo.Memory.Total > 0 && nodeInfo.Memory.Used > 0 && nodeInfo.Memory.Total >= nodeInfo.Memory.Used { + availableFromUsed = nodeInfo.Memory.Total - nodeInfo.Memory.Used + } + nodeSnapshotRaw.TotalMinusUsed = availableFromUsed + + missingCacheMetrics := nodeInfo.Memory.Available == 0 && nodeInfo.Memory.Avail == 0 && nodeInfo.Memory.Buffers == 0 && - nodeInfo.Memory.Cached == 0 { + nodeInfo.Memory.Cached == 0 + + if effectiveAvailable == 0 && missingCacheMetrics { if memAvail, err := m.getNodeRRDMemAvailable(ctx, client, node.Node); err == nil && memAvail > 0 { effectiveAvailable = memAvail usedRRDFallback = true @@ -2073,6 +2098,21 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie } } + const totalMinusUsedGapTolerance uint64 = 16 * 1024 * 1024 + gapGreaterThanComponents := false + if availableFromUsed > componentAvailable { + gap := availableFromUsed - componentAvailable + if componentAvailable == 0 || gap >= totalMinusUsedGapTolerance { + gapGreaterThanComponents = true + } + } + + derivedFromTotalMinusUsed := !usedRRDFallback && + missingCacheMetrics && + availableFromUsed > 0 && + gapGreaterThanComponents && + effectiveAvailable == availableFromUsed + switch { case effectiveAvailable > 0 && effectiveAvailable <= nodeInfo.Memory.Total: // Prefer available/avail fields or derived buffers+cache values when present. @@ -2087,29 +2127,36 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie Uint64("effectiveAvailable", effectiveAvailable). Uint64("actualUsed", actualUsed). Float64("usage", safePercentage(float64(actualUsed), float64(nodeInfo.Memory.Total))) - if usedRRDFallback { logCtx.Msg("Node memory: using RRD memavailable fallback (excludes reclaimable cache)") nodeMemorySource = "rrd-memavailable" nodeFallbackReason = "rrd-memavailable" nodeSnapshotRaw.FallbackCalculated = true nodeSnapshotRaw.ProxmoxMemorySource = "rrd-memavailable" - } else { - switch { - case nodeInfo.Memory.Available > 0: - logCtx.Msg("Node memory: using available field (excludes reclaimable cache)") - nodeMemorySource = "available-field" - case nodeInfo.Memory.Avail > 0: - logCtx.Msg("Node memory: using avail field (excludes reclaimable cache)") - nodeMemorySource = "avail-field" - default: - logCtx. - Uint64("free", nodeInfo.Memory.Free). - Uint64("buffers", nodeInfo.Memory.Buffers). - Uint64("cached", nodeInfo.Memory.Cached). - Msg("Node memory: derived available from free+buffers+cached (excludes reclaimable cache)") - nodeMemorySource = "derived-free-buffers-cached" + } else if nodeInfo.Memory.Available > 0 { + logCtx.Msg("Node memory: using available field (excludes reclaimable cache)") + nodeMemorySource = "available-field" + } else if nodeInfo.Memory.Avail > 0 { + logCtx.Msg("Node memory: using avail field (excludes reclaimable cache)") + nodeMemorySource = "avail-field" + } else if derivedFromTotalMinusUsed { + logCtx. + Uint64("availableFromUsed", availableFromUsed). + Uint64("reportedFree", nodeInfo.Memory.Free). + Msg("Node memory: derived available from total-used gap (cache fields missing)") + nodeMemorySource = "derived-total-minus-used" + if nodeFallbackReason == "" { + nodeFallbackReason = "node-status-total-minus-used" } + nodeSnapshotRaw.FallbackCalculated = true + nodeSnapshotRaw.ProxmoxMemorySource = "node-status-total-minus-used" + } else { + logCtx. + Uint64("free", nodeInfo.Memory.Free). + Uint64("buffers", nodeInfo.Memory.Buffers). + Uint64("cached", nodeInfo.Memory.Cached). + Msg("Node memory: derived available from free+buffers+cached (excludes reclaimable cache)") + nodeMemorySource = "derived-free-buffers-cached" } default: // Fallback to traditional used memory if no cache-aware data is exposed diff --git a/internal/monitoring/monitor_optimized.go b/internal/monitoring/monitor_optimized.go index c89d583..42d9221 100644 --- a/internal/monitoring/monitor_optimized.go +++ b/internal/monitoring/monitor_optimized.go @@ -3,6 +3,7 @@ package monitoring import ( "context" "fmt" + "math" "os" "sort" "strings" @@ -173,6 +174,34 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st guestRaw.MemInfoBuffers = status.MemInfo.Buffers guestRaw.MemInfoCached = status.MemInfo.Cached guestRaw.MemInfoShared = status.MemInfo.Shared + componentAvailable := status.MemInfo.Free + if status.MemInfo.Buffers > 0 { + if math.MaxUint64-componentAvailable < status.MemInfo.Buffers { + componentAvailable = math.MaxUint64 + } else { + componentAvailable += status.MemInfo.Buffers + } + } + if status.MemInfo.Cached > 0 { + if math.MaxUint64-componentAvailable < status.MemInfo.Cached { + componentAvailable = math.MaxUint64 + } else { + componentAvailable += status.MemInfo.Cached + } + } + if status.MemInfo.Total > 0 && componentAvailable > status.MemInfo.Total { + componentAvailable = status.MemInfo.Total + } + + availableFromUsed := uint64(0) + if status.MemInfo.Total > 0 && status.MemInfo.Used > 0 && status.MemInfo.Total >= status.MemInfo.Used { + availableFromUsed = status.MemInfo.Total - status.MemInfo.Used + guestRaw.MemInfoTotalMinusUsed = availableFromUsed + } + + missingCacheMetrics := status.MemInfo.Available == 0 && + status.MemInfo.Buffers == 0 && + status.MemInfo.Cached == 0 switch { case status.MemInfo.Available > 0: @@ -186,6 +215,17 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st status.MemInfo.Cached memorySource = "meminfo-derived" } + + if memAvailable == 0 && availableFromUsed > 0 && missingCacheMetrics { + const vmTotalMinusUsedGapTolerance uint64 = 4 * 1024 * 1024 + if availableFromUsed > componentAvailable { + gap := availableFromUsed - componentAvailable + if componentAvailable == 0 || gap >= vmTotalMinusUsedGapTolerance { + memAvailable = availableFromUsed + memorySource = "meminfo-total-minus-used" + } + } + } } if vmStatus.Balloon > 0 && vmStatus.Balloon < vmStatus.MaxMem { memTotal = vmStatus.Balloon diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go index aca2627..d704e1d 100644 --- a/pkg/proxmox/client.go +++ b/pkg/proxmox/client.go @@ -447,6 +447,13 @@ func (m *MemoryStatus) EffectiveAvailable() uint64 { } derived := m.Free + m.Buffers + m.Cached + if m.Total > 0 && m.Used > 0 && m.Total >= m.Used { + availableFromUsed := m.Total - m.Used + if availableFromUsed > derived { + derived = availableFromUsed + } + } + if derived == 0 { return 0 } diff --git a/pkg/proxmox/client_test.go b/pkg/proxmox/client_test.go index b4f569c..fe106fe 100644 --- a/pkg/proxmox/client_test.go +++ b/pkg/proxmox/client_test.go @@ -168,9 +168,22 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) { Cached: 4294967296, // 4GB }, wantAvailable: 6979321856, // Free + Buffers + Cached = ~6.5GB - wantUsedPct: 58.4, // Should be ~58%, not ~80% + wantUsedPct: 58.4, // Should be ~58%, not ~80% description: "When available/avail missing, derive from free+buffers+cached", }, + { + name: "proxmox 8.4 hides cache fields - derive from total-minus-used gap", + status: MemoryStatus{ + Total: 134794743808, // ~125.6GB + Used: 107351023616, // ~100GB actual usage + Free: 6471057408, // ~6GB bare free reported + Buffers: 0, + Cached: 0, + }, + wantAvailable: 27443720192, // total - used => ~25.6GB reclaimable (free + cache) + wantUsedPct: 79.6, // Matches Proxmox node dashboard + description: "Proxmox 8.4 stops reporting buffers/cached; use total-used gap to recover cache-aware metric", + }, { name: "issue #435 specific case - 86% vs 42% real usage", status: MemoryStatus{ @@ -190,9 +203,9 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) { Used: 6871947674, Free: 0, // All fields missing }, - wantAvailable: 0, - wantUsedPct: 80.0, // Falls back to cache-inclusive calculation - description: "When all cache fields missing, EffectiveAvailable returns 0", + wantAvailable: 1717986918, // Derived from total - used + wantUsedPct: 80.0, // Still aligns with cache-inclusive calculation when nothing else reported + description: "When all cache fields missing, fall back to total-used gap instead of zero", }, }