Fix inflated RAM usage reporting for LXC containers

Related to #553

## Problem

LXC containers showed inflated memory usage (e.g., 90%+ when actual usage was 50-60%,
96% when actual was 61%) because the code used the raw `mem` value from Proxmox's
`/cluster/resources` API endpoint. This value comes from cgroup `memory.current` which
includes reclaimable cache and buffers, making memory appear nearly full even when
plenty is available.

## Root Cause

- **Nodes**: Had sophisticated cache-aware memory calculation with RRD fallbacks
- **VMs (qemu)**: Had detailed memory calculation using guest agent meminfo
- **LXCs**: Naively used `res.Mem` directly without any cache-aware correction

The Proxmox cluster resources API's `mem` field for LXCs includes cache/buffers
(from cgroup memory accounting), which should be excluded for accurate "used" memory.

## Solution

Implement cache-aware memory calculation for LXC containers by:

1. Adding `GetLXCRRDData()` method to fetch RRD metrics for LXC containers from
   `/nodes/{node}/lxc/{vmid}/rrddata`
2. Using RRD `memavailable` to calculate actual used memory (total - available)
3. Falling back to RRD `memused` if `memavailable` is not available
4. Only using cluster resources `mem` value as last resort

This matches the approach already used for nodes and VMs, providing consistent
cache-aware memory reporting across all resource types.

## Changes

- Added `GuestRRDPoint` type and `GetLXCRRDData()` method to pkg/proxmox
- Added `GetLXCRRDData()` to ClusterClient for cluster-aware operations
- Modified LXC memory calculation in `pollPVEInstance()` to use RRD data when available
- Added guest memory snapshot recording for LXC containers
- Updated test stubs to implement the new interface method

## Testing

- Code compiles successfully
- Follows the same proven pattern used for nodes and VMs
- Includes diagnostic snapshot recording for troubleshooting
This commit is contained in:
rcourtman 2025-11-06 00:16:18 +00:00
parent 88ad986877
commit af55362009
6 changed files with 152 additions and 5 deletions

View file

@ -46,6 +46,7 @@ type PVEClientInterface interface {
GetNodes(ctx context.Context) ([]proxmox.Node, error)
GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error)
GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error)
GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error)
GetVMs(ctx context.Context, node string) ([]proxmox.VM, error)
GetContainers(ctx context.Context, node string) ([]proxmox.Container, error)
GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error)
@ -6562,15 +6563,82 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
}
// Calculate I/O rates for container
sampleTime := time.Now()
currentMetrics := IOMetrics{
DiskRead: int64(res.DiskRead),
DiskWrite: int64(res.DiskWrite),
NetworkIn: int64(res.NetIn),
NetworkOut: int64(res.NetOut),
Timestamp: time.Now(),
Timestamp: sampleTime,
}
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
// Calculate cache-aware memory for LXC containers
// The cluster resources API returns mem from cgroup which includes cache/buffers (inflated).
// Try to get more accurate memory metrics from RRD data.
memTotal := res.MaxMem
memUsed := res.Mem
memorySource := "cluster-resources"
guestRaw := VMMemoryRaw{
ListingMem: res.Mem,
ListingMaxMem: res.MaxMem,
}
// For running containers, try to get RRD data for cache-aware memory calculation
if res.Status == "running" {
rrdCtx, rrdCancel := context.WithTimeout(ctx, 5*time.Second)
rrdPoints, err := client.GetLXCRRDData(rrdCtx, res.Node, res.VMID, "hour", "AVERAGE", []string{"memavailable", "memused", "maxmem"})
rrdCancel()
if err == nil && len(rrdPoints) > 0 {
// Use the most recent RRD point
point := rrdPoints[len(rrdPoints)-1]
if point.MaxMem != nil && *point.MaxMem > 0 {
guestRaw.StatusMaxMem = uint64(*point.MaxMem)
}
// Prefer memavailable-based calculation (excludes cache/buffers)
if point.MemAvailable != nil && *point.MemAvailable > 0 {
memAvailable := uint64(*point.MemAvailable)
if memAvailable <= memTotal {
memUsed = memTotal - memAvailable
memorySource = "rrd-memavailable"
guestRaw.MemInfoAvailable = memAvailable
log.Debug().
Str("container", res.Name).
Str("node", res.Node).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Uint64("used", memUsed).
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
Msg("LXC memory: using RRD memavailable (excludes reclaimable cache)")
}
} else if point.MemUsed != nil && *point.MemUsed > 0 {
// Fall back to memused from RRD if available
memUsed = uint64(*point.MemUsed)
if memUsed <= memTotal {
memorySource = "rrd-memused"
guestRaw.MemInfoUsed = memUsed
log.Debug().
Str("container", res.Name).
Str("node", res.Node).
Uint64("total", memTotal).
Uint64("used", memUsed).
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
Msg("LXC memory: using RRD memused (excludes reclaimable cache)")
}
}
} else if err != nil {
log.Debug().
Err(err).
Str("instance", instanceName).
Str("container", res.Name).
Int("vmid", res.VMID).
Msg("RRD memory data unavailable for LXC, using cluster resources value")
}
}
container := models.Container{
ID: guestID,
VMID: res.VMID,
@ -6582,10 +6650,10 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
CPU: safeFloat(res.CPU),
CPUs: int(res.MaxCPU),
Memory: models.Memory{
Total: int64(res.MaxMem),
Used: int64(res.Mem),
Free: int64(res.MaxMem - res.Mem),
Usage: safePercentage(float64(res.Mem), float64(res.MaxMem)),
Total: int64(memTotal),
Used: int64(memUsed),
Free: int64(memTotal - memUsed),
Usage: safePercentage(float64(memUsed), float64(memTotal)),
},
Disk: models.Disk{
Total: int64(res.MaxDisk),
@ -6623,6 +6691,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
allContainers = append(allContainers, container)
m.recordGuestSnapshot(instanceName, container.Type, res.Node, res.VMID, GuestMemorySnapshot{
Name: container.Name,
Status: container.Status,
RetrievedAt: sampleTime,
MemorySource: memorySource,
Memory: container.Memory,
Raw: guestRaw,
})
// For non-running containers, zero out resource usage metrics to prevent false alerts
// Proxmox may report stale or residual metrics for stopped containers
if container.Status != "running" {

View file

@ -33,6 +33,10 @@ func (s *stubPVEClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf
return s.rrdPoints, nil
}
func (s *stubPVEClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
return nil, nil
}
func (s *stubPVEClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
return nil, nil
}

View file

@ -20,6 +20,9 @@ func (f fakeSnapshotClient) GetNodeStatus(ctx context.Context, node string) (*pr
func (f fakeSnapshotClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
return nil, nil
}

View file

@ -30,6 +30,10 @@ func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, tim
return nil, nil
}
func (f *fakeStorageClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
return nil, nil
}
func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
return nil, nil
}

View file

@ -467,6 +467,14 @@ type NodeRRDPoint struct {
MemAvailable *float64 `json:"memavailable,omitempty"`
}
// GuestRRDPoint represents a single RRD datapoint for a VM or LXC container.
type GuestRRDPoint struct {
Time int64 `json:"time"`
MaxMem *float64 `json:"maxmem,omitempty"`
MemUsed *float64 `json:"memused,omitempty"`
MemAvailable *float64 `json:"memavailable,omitempty"`
}
// NodeStatus represents detailed node status from /nodes/{node}/status endpoint
// This endpoint provides real-time metrics that update every second
type NodeStatus struct {
@ -709,6 +717,44 @@ func (c *Client) GetNodeRRDData(ctx context.Context, node, timeframe, cf string,
return result.Data, nil
}
// GetLXCRRDData retrieves RRD metrics for an LXC container.
func (c *Client) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
if timeframe == "" {
timeframe = "hour"
}
if cf == "" {
cf = "AVERAGE"
}
params := url.Values{}
params.Set("timeframe", timeframe)
params.Set("cf", cf)
if len(ds) > 0 {
params.Set("ds", strings.Join(ds, ","))
}
path := fmt.Sprintf("/nodes/%s/lxc/%d/rrddata", url.PathEscape(node), vmid)
if query := params.Encode(); query != "" {
path = fmt.Sprintf("%s?%s", path, query)
}
resp, err := c.get(ctx, path)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result struct {
Data []GuestRRDPoint `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return result.Data, nil
}
// VM represents a Proxmox VE virtual machine
type VM struct {
VMID int `json:"vmid"`

View file

@ -792,6 +792,19 @@ func (cc *ClusterClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf
return result, err
}
func (cc *ClusterClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
var result []GuestRRDPoint
err := cc.executeWithFailover(ctx, func(client *Client) error {
points, err := client.GetLXCRRDData(ctx, node, vmid, timeframe, cf, ds)
if err != nil {
return err
}
result = points
return nil
})
return result, err
}
func (cc *ClusterClient) GetVMs(ctx context.Context, node string) ([]VM, error) {
var result []VM
err := cc.executeWithFailover(ctx, func(client *Client) error {