danvics/Pulse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refs #533: add total-minus-used memory fallback

Browse source
This commit is contained in:
rcourtman 2025-10-15 18:19:54 +00:00
parent 5ce47a72ec
commit 32421b36b8
5 changed files with 148 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
internal/monitoring/diagnostic_snapshots.go
View file

@ -19,6 +19,7 @@ type NodeMemoryRaw struct {
Cached uint64 `json:"cached"`
Shared uint64 `json:"shared"`
EffectiveAvailable uint64 `json:"effectiveAvailable"`
TotalMinusUsed uint64 `json:"totalMinusUsed,omitempty"`
FallbackTotal uint64 `json:"fallbackTotal,omitempty"`
FallbackUsed uint64 `json:"fallbackUsed,omitempty"`
FallbackFree uint64 `json:"fallbackFree,omitempty"`
@ -39,22 +40,23 @@ type NodeMemorySnapshot struct {
// VMMemoryRaw captures both the listing and detailed status memory fields for a VM/CT.
type VMMemoryRaw struct {
ListingMem uint64 `json:"listingMem"`
ListingMaxMem uint64 `json:"listingMaxmem"`
StatusMem uint64 `json:"statusMem,omitempty"`
StatusFreeMem uint64 `json:"statusFreemem,omitempty"`
StatusMaxMem uint64 `json:"statusMaxmem,omitempty"`
Balloon uint64 `json:"balloon,omitempty"`
BalloonMin uint64 `json:"balloonMin,omitempty"`
MemInfoUsed uint64 `json:"meminfoUsed,omitempty"`
MemInfoFree uint64 `json:"meminfoFree,omitempty"`
MemInfoTotal uint64 `json:"meminfoTotal,omitempty"`
MemInfoAvailable uint64 `json:"meminfoAvailable,omitempty"`
MemInfoBuffers uint64 `json:"meminfoBuffers,omitempty"`
MemInfoCached uint64 `json:"meminfoCached,omitempty"`
MemInfoShared uint64 `json:"meminfoShared,omitempty"`
Agent int `json:"agent,omitempty"`
DerivedFromBall bool `json:"derivedFromBalloon,omitempty"`
ListingMem uint64 `json:"listingMem"`
ListingMaxMem uint64 `json:"listingMaxmem"`
StatusMem uint64 `json:"statusMem,omitempty"`
StatusFreeMem uint64 `json:"statusFreemem,omitempty"`
StatusMaxMem uint64 `json:"statusMaxmem,omitempty"`
Balloon uint64 `json:"balloon,omitempty"`
BalloonMin uint64 `json:"balloonMin,omitempty"`
MemInfoUsed uint64 `json:"meminfoUsed,omitempty"`
MemInfoFree uint64 `json:"meminfoFree,omitempty"`
MemInfoTotal uint64 `json:"meminfoTotal,omitempty"`
MemInfoAvailable uint64 `json:"meminfoAvailable,omitempty"`
MemInfoBuffers uint64 `json:"meminfoBuffers,omitempty"`
MemInfoCached uint64 `json:"meminfoCached,omitempty"`
MemInfoShared uint64 `json:"meminfoShared,omitempty"`
MemInfoTotalMinusUsed uint64 `json:"meminfoTotalMinusUsed,omitempty"`
Agent int `json:"agent,omitempty"`
DerivedFromBall bool `json:"derivedFromBalloon,omitempty"`
}
// GuestMemorySnapshot records the memory calculation for a guest (VM/LXC).

85
internal/monitoring/monitor.go
View file

@ -2055,12 +2055,37 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
var actualUsed uint64
effectiveAvailable := nodeInfo.Memory.EffectiveAvailable()
usedRRDFallback := false
componentAvailable := nodeInfo.Memory.Free
if nodeInfo.Memory.Buffers > 0 {
if math.MaxUint64-componentAvailable < nodeInfo.Memory.Buffers {
componentAvailable = math.MaxUint64
} else {
componentAvailable += nodeInfo.Memory.Buffers
}
}
if nodeInfo.Memory.Cached > 0 {
if math.MaxUint64-componentAvailable < nodeInfo.Memory.Cached {
componentAvailable = math.MaxUint64
} else {
componentAvailable += nodeInfo.Memory.Cached
}
}
if nodeInfo.Memory.Total > 0 && componentAvailable > nodeInfo.Memory.Total {
componentAvailable = nodeInfo.Memory.Total
}
if effectiveAvailable == 0 &&
nodeInfo.Memory.Available == 0 &&
availableFromUsed := uint64(0)
if nodeInfo.Memory.Total > 0 && nodeInfo.Memory.Used > 0 && nodeInfo.Memory.Total >= nodeInfo.Memory.Used {
availableFromUsed = nodeInfo.Memory.Total - nodeInfo.Memory.Used
}
nodeSnapshotRaw.TotalMinusUsed = availableFromUsed
missingCacheMetrics := nodeInfo.Memory.Available == 0 &&
nodeInfo.Memory.Avail == 0 &&
nodeInfo.Memory.Buffers == 0 &&
nodeInfo.Memory.Cached == 0 {
nodeInfo.Memory.Cached == 0
if effectiveAvailable == 0 && missingCacheMetrics {
if memAvail, err := m.getNodeRRDMemAvailable(ctx, client, node.Node); err == nil && memAvail > 0 {
effectiveAvailable = memAvail
usedRRDFallback = true
@ -2073,6 +2098,21 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
}
}
const totalMinusUsedGapTolerance uint64 = 16 * 1024 * 1024
gapGreaterThanComponents := false
if availableFromUsed > componentAvailable {
gap := availableFromUsed - componentAvailable
if componentAvailable == 0 || gap >= totalMinusUsedGapTolerance {
gapGreaterThanComponents = true
}
}
derivedFromTotalMinusUsed := !usedRRDFallback &&
missingCacheMetrics &&
availableFromUsed > 0 &&
gapGreaterThanComponents &&
effectiveAvailable == availableFromUsed
switch {
case effectiveAvailable > 0 && effectiveAvailable <= nodeInfo.Memory.Total:
// Prefer available/avail fields or derived buffers+cache values when present.
@ -2087,29 +2127,36 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
Uint64("effectiveAvailable", effectiveAvailable).
Uint64("actualUsed", actualUsed).
Float64("usage", safePercentage(float64(actualUsed), float64(nodeInfo.Memory.Total)))
if usedRRDFallback {
logCtx.Msg("Node memory: using RRD memavailable fallback (excludes reclaimable cache)")
nodeMemorySource = "rrd-memavailable"
nodeFallbackReason = "rrd-memavailable"
nodeSnapshotRaw.FallbackCalculated = true
nodeSnapshotRaw.ProxmoxMemorySource = "rrd-memavailable"
} else {
switch {
case nodeInfo.Memory.Available > 0:
logCtx.Msg("Node memory: using available field (excludes reclaimable cache)")
nodeMemorySource = "available-field"
case nodeInfo.Memory.Avail > 0:
logCtx.Msg("Node memory: using avail field (excludes reclaimable cache)")
nodeMemorySource = "avail-field"
default:
logCtx.
Uint64("free", nodeInfo.Memory.Free).
Uint64("buffers", nodeInfo.Memory.Buffers).
Uint64("cached", nodeInfo.Memory.Cached).
Msg("Node memory: derived available from free+buffers+cached (excludes reclaimable cache)")
nodeMemorySource = "derived-free-buffers-cached"
} else if nodeInfo.Memory.Available > 0 {
logCtx.Msg("Node memory: using available field (excludes reclaimable cache)")
nodeMemorySource = "available-field"
} else if nodeInfo.Memory.Avail > 0 {
logCtx.Msg("Node memory: using avail field (excludes reclaimable cache)")
nodeMemorySource = "avail-field"
} else if derivedFromTotalMinusUsed {
logCtx.
Uint64("availableFromUsed", availableFromUsed).
Uint64("reportedFree", nodeInfo.Memory.Free).
Msg("Node memory: derived available from total-used gap (cache fields missing)")
nodeMemorySource = "derived-total-minus-used"
if nodeFallbackReason == "" {
nodeFallbackReason = "node-status-total-minus-used"
}
nodeSnapshotRaw.FallbackCalculated = true
nodeSnapshotRaw.ProxmoxMemorySource = "node-status-total-minus-used"
} else {
logCtx.
Uint64("free", nodeInfo.Memory.Free).
Uint64("buffers", nodeInfo.Memory.Buffers).
Uint64("cached", nodeInfo.Memory.Cached).
Msg("Node memory: derived available from free+buffers+cached (excludes reclaimable cache)")
nodeMemorySource = "derived-free-buffers-cached"
}
default:
// Fallback to traditional used memory if no cache-aware data is exposed

40
internal/monitoring/monitor_optimized.go
View file

@ -3,6 +3,7 @@ package monitoring
import (
"context"
"fmt"
"math"
"os"
"sort"
"strings"
@ -173,6 +174,34 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st
guestRaw.MemInfoBuffers = status.MemInfo.Buffers
guestRaw.MemInfoCached = status.MemInfo.Cached
guestRaw.MemInfoShared = status.MemInfo.Shared
componentAvailable := status.MemInfo.Free
if status.MemInfo.Buffers > 0 {
if math.MaxUint64-componentAvailable < status.MemInfo.Buffers {
componentAvailable = math.MaxUint64
} else {
componentAvailable += status.MemInfo.Buffers
}
}
if status.MemInfo.Cached > 0 {
if math.MaxUint64-componentAvailable < status.MemInfo.Cached {
componentAvailable = math.MaxUint64
} else {
componentAvailable += status.MemInfo.Cached
}
}
if status.MemInfo.Total > 0 && componentAvailable > status.MemInfo.Total {
componentAvailable = status.MemInfo.Total
}
availableFromUsed := uint64(0)
if status.MemInfo.Total > 0 && status.MemInfo.Used > 0 && status.MemInfo.Total >= status.MemInfo.Used {
availableFromUsed = status.MemInfo.Total - status.MemInfo.Used
guestRaw.MemInfoTotalMinusUsed = availableFromUsed
}
missingCacheMetrics := status.MemInfo.Available == 0 &&
status.MemInfo.Buffers == 0 &&
status.MemInfo.Cached == 0
switch {
case status.MemInfo.Available > 0:
@ -186,6 +215,17 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st
status.MemInfo.Cached
memorySource = "meminfo-derived"
}
if memAvailable == 0 && availableFromUsed > 0 && missingCacheMetrics {
const vmTotalMinusUsedGapTolerance uint64 = 4 * 1024 * 1024
if availableFromUsed > componentAvailable {
gap := availableFromUsed - componentAvailable
if componentAvailable == 0 || gap >= vmTotalMinusUsedGapTolerance {
memAvailable = availableFromUsed
memorySource = "meminfo-total-minus-used"
}
}
}
}
if vmStatus.Balloon > 0 && vmStatus.Balloon < vmStatus.MaxMem {
memTotal = vmStatus.Balloon

7
pkg/proxmox/client.go
View file

@ -447,6 +447,13 @@ func (m *MemoryStatus) EffectiveAvailable() uint64 {
}
derived := m.Free + m.Buffers + m.Cached
if m.Total > 0 && m.Used > 0 && m.Total >= m.Used {
availableFromUsed := m.Total - m.Used
if availableFromUsed > derived {
derived = availableFromUsed
}
}
if derived == 0 {
return 0
}

21
pkg/proxmox/client_test.go
View file

@ -168,9 +168,22 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) {
Cached: 4294967296, // 4GB
},
wantAvailable: 6979321856, // Free + Buffers + Cached = ~6.5GB
wantUsedPct: 58.4, // Should be ~58%, not ~80%
wantUsedPct: 58.4, // Should be ~58%, not ~80%
description: "When available/avail missing, derive from free+buffers+cached",
},
{
name: "proxmox 8.4 hides cache fields - derive from total-minus-used gap",
status: MemoryStatus{
Total: 134794743808, // ~125.6GB
Used: 107351023616, // ~100GB actual usage
Free: 6471057408, // ~6GB bare free reported
Buffers: 0,
Cached: 0,
},
wantAvailable: 27443720192, // total - used => ~25.6GB reclaimable (free + cache)
wantUsedPct: 79.6, // Matches Proxmox node dashboard
description: "Proxmox 8.4 stops reporting buffers/cached; use total-used gap to recover cache-aware metric",
},
{
name: "issue #435 specific case - 86% vs 42% real usage",
status: MemoryStatus{
@ -190,9 +203,9 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) {
Used: 6871947674,
Free: 0, // All fields missing
},
wantAvailable: 0,
wantUsedPct: 80.0, // Falls back to cache-inclusive calculation
description: "When all cache fields missing, EffectiveAvailable returns 0",
wantAvailable: 1717986918, // Derived from total - used
wantUsedPct: 80.0, // Still aligns with cache-inclusive calculation when nothing else reported
description: "When all cache fields missing, fall back to total-used gap instead of zero",
},
}