Log memory source transitions for diagnostics (#553)
This commit is contained in:
rcourtman
parent
665fe7902c
commit
aaae27dc11
3 changed files with 161 additions and 0 deletions
|
|
@ -8,6 +8,7 @@ import (
|
|||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -32,6 +33,26 @@ type DiagnosticsInfo struct {
|
|||
NodeSnapshots []monitoring.NodeMemorySnapshot `json:"nodeSnapshots,omitempty"`
|
||||
// GuestSnapshots captures recent per-guest memory breakdowns (VM/LXC) with the raw Proxmox fields.
|
||||
GuestSnapshots []monitoring.GuestMemorySnapshot `json:"guestSnapshots,omitempty"`
|
||||
// MemorySources summarizes how many nodes currently rely on each memory source per instance.
|
||||
MemorySources []MemorySourceStat `json:"memorySources,omitempty"`
|
||||
}
|
||||
|
||||
// MemorySourceStat aggregates memory-source usage per instance.
|
||||
type MemorySourceStat struct {
|
||||
Instance string `json:"instance"`
|
||||
Source string `json:"source"`
|
||||
NodeCount int `json:"nodeCount"`
|
||||
LastUpdated string `json:"lastUpdated"`
|
||||
Fallback bool `json:"fallback"`
|
||||
}
|
||||
|
||||
func isFallbackMemorySource(source string) bool {
|
||||
switch strings.ToLower(source) {
|
||||
case "", "unknown", "nodes-endpoint", "node-status-used", "previous-snapshot":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// NodeDiagnostic contains diagnostic info for a Proxmox node
|
||||
|
|
@ -338,6 +359,54 @@ func (r *Router) handleDiagnostics(w http.ResponseWriter, req *http.Request) {
|
|||
snapshots := r.monitor.GetDiagnosticSnapshots()
|
||||
if len(snapshots.Nodes) > 0 {
|
||||
diag.NodeSnapshots = snapshots.Nodes
|
||||
|
||||
type memorySourceAgg struct {
|
||||
stat MemorySourceStat
|
||||
latest time.Time
|
||||
}
|
||||
|
||||
sourceAverages := make(map[string]*memorySourceAgg)
|
||||
for _, snap := range snapshots.Nodes {
|
||||
source := snap.MemorySource
|
||||
if source == "" {
|
||||
source = "unknown"
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s|%s", snap.Instance, source)
|
||||
entry, ok := sourceAverages[key]
|
||||
if !ok {
|
||||
entry = &memorySourceAgg{
|
||||
stat: MemorySourceStat{
|
||||
Instance: snap.Instance,
|
||||
Source: source,
|
||||
Fallback: isFallbackMemorySource(source),
|
||||
},
|
||||
}
|
||||
sourceAverages[key] = entry
|
||||
}
|
||||
|
||||
entry.stat.NodeCount++
|
||||
if snap.RetrievedAt.After(entry.latest) {
|
||||
entry.latest = snap.RetrievedAt
|
||||
}
|
||||
}
|
||||
|
||||
if len(sourceAverages) > 0 {
|
||||
diag.MemorySources = make([]MemorySourceStat, 0, len(sourceAverages))
|
||||
for _, entry := range sourceAverages {
|
||||
if !entry.latest.IsZero() {
|
||||
entry.stat.LastUpdated = entry.latest.UTC().Format(time.RFC3339)
|
||||
}
|
||||
diag.MemorySources = append(diag.MemorySources, entry.stat)
|
||||
}
|
||||
|
||||
sort.Slice(diag.MemorySources, func(i, j int) bool {
|
||||
if diag.MemorySources[i].Instance == diag.MemorySources[j].Instance {
|
||||
return diag.MemorySources[i].Source < diag.MemorySources[j].Source
|
||||
}
|
||||
return diag.MemorySources[i].Instance < diag.MemorySources[j].Instance
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(snapshots.Guests) > 0 {
|
||||
diag.GuestSnapshots = snapshots.Guests
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// NodeMemoryRaw captures the raw memory fields returned by Proxmox for a node.
|
||||
|
|
@ -88,6 +90,70 @@ func makeGuestSnapshotKey(instance, guestType, node string, vmid int) string {
|
|||
return fmt.Sprintf("%s|%s|%s|%d", instance, guestType, node, vmid)
|
||||
}
|
||||
|
||||
func (m *Monitor) logNodeMemorySource(instance, node string, snapshot NodeMemorySnapshot) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
|
||||
key := makeNodeSnapshotKey(instance, node)
|
||||
|
||||
source := snapshot.MemorySource
|
||||
var prevSource string
|
||||
m.diagMu.RLock()
|
||||
if existing, ok := m.nodeSnapshots[key]; ok {
|
||||
prevSource = existing.MemorySource
|
||||
}
|
||||
m.diagMu.RUnlock()
|
||||
|
||||
if prevSource == snapshot.MemorySource {
|
||||
return
|
||||
}
|
||||
|
||||
var evt *zerolog.Event
|
||||
switch source {
|
||||
case "", "nodes-endpoint", "node-status-used", "previous-snapshot":
|
||||
evt = log.Warn()
|
||||
default:
|
||||
evt = log.Debug()
|
||||
}
|
||||
|
||||
evt = evt.
|
||||
Str("instance", instance).
|
||||
Str("node", node).
|
||||
Str("memorySource", source).
|
||||
Str("proxmoxSource", snapshot.Raw.ProxmoxMemorySource)
|
||||
|
||||
if snapshot.FallbackReason != "" {
|
||||
evt = evt.Str("fallbackReason", snapshot.FallbackReason)
|
||||
}
|
||||
if snapshot.Raw.Available > 0 {
|
||||
evt = evt.Uint64("rawAvailable", snapshot.Raw.Available)
|
||||
}
|
||||
if snapshot.Raw.Buffers > 0 {
|
||||
evt = evt.Uint64("rawBuffers", snapshot.Raw.Buffers)
|
||||
}
|
||||
if snapshot.Raw.Cached > 0 {
|
||||
evt = evt.Uint64("rawCached", snapshot.Raw.Cached)
|
||||
}
|
||||
if snapshot.Raw.TotalMinusUsed > 0 {
|
||||
evt = evt.Uint64("rawTotalMinusUsed", snapshot.Raw.TotalMinusUsed)
|
||||
}
|
||||
if snapshot.Memory.Total > 0 {
|
||||
evt = evt.Int64("total", snapshot.Memory.Total)
|
||||
}
|
||||
if snapshot.Memory.Used > 0 {
|
||||
evt = evt.Int64("used", snapshot.Memory.Used)
|
||||
}
|
||||
if snapshot.Memory.Free > 0 {
|
||||
evt = evt.Int64("free", snapshot.Memory.Free)
|
||||
}
|
||||
if snapshot.Memory.Usage > 0 {
|
||||
evt = evt.Float64("usage", snapshot.Memory.Usage)
|
||||
}
|
||||
|
||||
evt.Msg("Node memory source updated")
|
||||
}
|
||||
|
||||
func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemorySnapshot) {
|
||||
if m == nil {
|
||||
return
|
||||
|
|
@ -99,6 +165,8 @@ func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemoryS
|
|||
snapshot.RetrievedAt = time.Now()
|
||||
}
|
||||
|
||||
m.logNodeMemorySource(instance, node, snapshot)
|
||||
|
||||
m.diagMu.Lock()
|
||||
defer m.diagMu.Unlock()
|
||||
if m.nodeSnapshots == nil {
|
||||
|
|
|
|||
|
|
@ -101,6 +101,17 @@ func TestMemoryStatusEffectiveAvailable(t *testing.T) {
|
|||
status: MemoryStatus{Total: 32 * 1024, Free: 4 * 1024, Buffers: 2 * 1024, Cached: 6 * 1024},
|
||||
want: 12 * 1024,
|
||||
},
|
||||
{
|
||||
name: "available zero but buffers and cache present",
|
||||
status: MemoryStatus{
|
||||
Total: 64 * 1024,
|
||||
Used: 40 * 1024,
|
||||
Free: 6 * 1024,
|
||||
Buffers: 8 * 1024,
|
||||
Cached: 10 * 1024,
|
||||
},
|
||||
want: 24 * 1024,
|
||||
},
|
||||
{
|
||||
name: "caps derived value at total",
|
||||
status: MemoryStatus{Total: 8 * 1024, Free: 4 * 1024, Buffers: 4 * 1024, Cached: 4 * 1024},
|
||||
|
|
@ -196,6 +207,19 @@ func TestMemoryStatusEffectiveAvailable_RegressionIssue435(t *testing.T) {
|
|||
wantUsedPct: 42.4, // Should be ~42%, not 86%!
|
||||
description: "Real user report: 86% shown when actual usage is 42%",
|
||||
},
|
||||
{
|
||||
name: "missing available but buffers/cached still present (issue 553 guard)",
|
||||
status: MemoryStatus{
|
||||
Total: 34359738368, // 32GB
|
||||
Used: 27917287424, // ~26GB reported used (includes cache)
|
||||
Free: 2147483648, // 2GB
|
||||
Buffers: 3221225472, // 3GB
|
||||
Cached: 8053063680, // 7.5GB
|
||||
},
|
||||
wantAvailable: 13421772800, // Free + Buffers + Cached
|
||||
wantUsedPct: 60.94,
|
||||
description: "When available=0 but buffers/cached exist, derive reclaimable memory instead of alerting",
|
||||
},
|
||||
{
|
||||
name: "missing all cache-aware fields - fallback to zero",
|
||||
status: MemoryStatus{
|
||||
|
|
|
|||
Loading…
Reference in a new issue