Fix inflated RAM usage reporting for LXC containers
Related to #553 ## Problem LXC containers showed inflated memory usage (e.g., 90%+ when actual usage was 50-60%, 96% when actual was 61%) because the code used the raw `mem` value from Proxmox's `/cluster/resources` API endpoint. This value comes from cgroup `memory.current` which includes reclaimable cache and buffers, making memory appear nearly full even when plenty is available. ## Root Cause - **Nodes**: Had sophisticated cache-aware memory calculation with RRD fallbacks - **VMs (qemu)**: Had detailed memory calculation using guest agent meminfo - **LXCs**: Naively used `res.Mem` directly without any cache-aware correction The Proxmox cluster resources API's `mem` field for LXCs includes cache/buffers (from cgroup memory accounting), which should be excluded for accurate "used" memory. ## Solution Implement cache-aware memory calculation for LXC containers by: 1. Adding `GetLXCRRDData()` method to fetch RRD metrics for LXC containers from `/nodes/{node}/lxc/{vmid}/rrddata` 2. Using RRD `memavailable` to calculate actual used memory (total - available) 3. Falling back to RRD `memused` if `memavailable` is not available 4. Only using cluster resources `mem` value as last resort This matches the approach already used for nodes and VMs, providing consistent cache-aware memory reporting across all resource types. ## Changes - Added `GuestRRDPoint` type and `GetLXCRRDData()` method to pkg/proxmox - Added `GetLXCRRDData()` to ClusterClient for cluster-aware operations - Modified LXC memory calculation in `pollPVEInstance()` to use RRD data when available - Added guest memory snapshot recording for LXC containers - Updated test stubs to implement the new interface method ## Testing - Code compiles successfully - Follows the same proven pattern used for nodes and VMs - Includes diagnostic snapshot recording for troubleshooting
This commit is contained in:
parent
88ad986877
commit
af55362009
6 changed files with 152 additions and 5 deletions
|
|
@ -46,6 +46,7 @@ type PVEClientInterface interface {
|
|||
GetNodes(ctx context.Context) ([]proxmox.Node, error)
|
||||
GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error)
|
||||
GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error)
|
||||
GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error)
|
||||
GetVMs(ctx context.Context, node string) ([]proxmox.VM, error)
|
||||
GetContainers(ctx context.Context, node string) ([]proxmox.Container, error)
|
||||
GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error)
|
||||
|
|
@ -6562,15 +6563,82 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
|||
}
|
||||
|
||||
// Calculate I/O rates for container
|
||||
sampleTime := time.Now()
|
||||
currentMetrics := IOMetrics{
|
||||
DiskRead: int64(res.DiskRead),
|
||||
DiskWrite: int64(res.DiskWrite),
|
||||
NetworkIn: int64(res.NetIn),
|
||||
NetworkOut: int64(res.NetOut),
|
||||
Timestamp: time.Now(),
|
||||
Timestamp: sampleTime,
|
||||
}
|
||||
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
|
||||
|
||||
// Calculate cache-aware memory for LXC containers
|
||||
// The cluster resources API returns mem from cgroup which includes cache/buffers (inflated).
|
||||
// Try to get more accurate memory metrics from RRD data.
|
||||
memTotal := res.MaxMem
|
||||
memUsed := res.Mem
|
||||
memorySource := "cluster-resources"
|
||||
guestRaw := VMMemoryRaw{
|
||||
ListingMem: res.Mem,
|
||||
ListingMaxMem: res.MaxMem,
|
||||
}
|
||||
|
||||
// For running containers, try to get RRD data for cache-aware memory calculation
|
||||
if res.Status == "running" {
|
||||
rrdCtx, rrdCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
rrdPoints, err := client.GetLXCRRDData(rrdCtx, res.Node, res.VMID, "hour", "AVERAGE", []string{"memavailable", "memused", "maxmem"})
|
||||
rrdCancel()
|
||||
|
||||
if err == nil && len(rrdPoints) > 0 {
|
||||
// Use the most recent RRD point
|
||||
point := rrdPoints[len(rrdPoints)-1]
|
||||
|
||||
if point.MaxMem != nil && *point.MaxMem > 0 {
|
||||
guestRaw.StatusMaxMem = uint64(*point.MaxMem)
|
||||
}
|
||||
|
||||
// Prefer memavailable-based calculation (excludes cache/buffers)
|
||||
if point.MemAvailable != nil && *point.MemAvailable > 0 {
|
||||
memAvailable := uint64(*point.MemAvailable)
|
||||
if memAvailable <= memTotal {
|
||||
memUsed = memTotal - memAvailable
|
||||
memorySource = "rrd-memavailable"
|
||||
guestRaw.MemInfoAvailable = memAvailable
|
||||
log.Debug().
|
||||
Str("container", res.Name).
|
||||
Str("node", res.Node).
|
||||
Uint64("total", memTotal).
|
||||
Uint64("available", memAvailable).
|
||||
Uint64("used", memUsed).
|
||||
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
|
||||
Msg("LXC memory: using RRD memavailable (excludes reclaimable cache)")
|
||||
}
|
||||
} else if point.MemUsed != nil && *point.MemUsed > 0 {
|
||||
// Fall back to memused from RRD if available
|
||||
memUsed = uint64(*point.MemUsed)
|
||||
if memUsed <= memTotal {
|
||||
memorySource = "rrd-memused"
|
||||
guestRaw.MemInfoUsed = memUsed
|
||||
log.Debug().
|
||||
Str("container", res.Name).
|
||||
Str("node", res.Node).
|
||||
Uint64("total", memTotal).
|
||||
Uint64("used", memUsed).
|
||||
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
|
||||
Msg("LXC memory: using RRD memused (excludes reclaimable cache)")
|
||||
}
|
||||
}
|
||||
} else if err != nil {
|
||||
log.Debug().
|
||||
Err(err).
|
||||
Str("instance", instanceName).
|
||||
Str("container", res.Name).
|
||||
Int("vmid", res.VMID).
|
||||
Msg("RRD memory data unavailable for LXC, using cluster resources value")
|
||||
}
|
||||
}
|
||||
|
||||
container := models.Container{
|
||||
ID: guestID,
|
||||
VMID: res.VMID,
|
||||
|
|
@ -6582,10 +6650,10 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
|||
CPU: safeFloat(res.CPU),
|
||||
CPUs: int(res.MaxCPU),
|
||||
Memory: models.Memory{
|
||||
Total: int64(res.MaxMem),
|
||||
Used: int64(res.Mem),
|
||||
Free: int64(res.MaxMem - res.Mem),
|
||||
Usage: safePercentage(float64(res.Mem), float64(res.MaxMem)),
|
||||
Total: int64(memTotal),
|
||||
Used: int64(memUsed),
|
||||
Free: int64(memTotal - memUsed),
|
||||
Usage: safePercentage(float64(memUsed), float64(memTotal)),
|
||||
},
|
||||
Disk: models.Disk{
|
||||
Total: int64(res.MaxDisk),
|
||||
|
|
@ -6623,6 +6691,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
|||
|
||||
allContainers = append(allContainers, container)
|
||||
|
||||
m.recordGuestSnapshot(instanceName, container.Type, res.Node, res.VMID, GuestMemorySnapshot{
|
||||
Name: container.Name,
|
||||
Status: container.Status,
|
||||
RetrievedAt: sampleTime,
|
||||
MemorySource: memorySource,
|
||||
Memory: container.Memory,
|
||||
Raw: guestRaw,
|
||||
})
|
||||
|
||||
// For non-running containers, zero out resource usage metrics to prevent false alerts
|
||||
// Proxmox may report stale or residual metrics for stopped containers
|
||||
if container.Status != "running" {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,10 @@ func (s *stubPVEClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf
|
|||
return s.rrdPoints, nil
|
||||
}
|
||||
|
||||
func (s *stubPVEClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *stubPVEClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ func (f fakeSnapshotClient) GetNodeStatus(ctx context.Context, node string) (*pr
|
|||
func (f fakeSnapshotClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeSnapshotClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeSnapshotClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, tim
|
|||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -467,6 +467,14 @@ type NodeRRDPoint struct {
|
|||
MemAvailable *float64 `json:"memavailable,omitempty"`
|
||||
}
|
||||
|
||||
// GuestRRDPoint represents a single RRD datapoint for a VM or LXC container.
|
||||
type GuestRRDPoint struct {
|
||||
Time int64 `json:"time"`
|
||||
MaxMem *float64 `json:"maxmem,omitempty"`
|
||||
MemUsed *float64 `json:"memused,omitempty"`
|
||||
MemAvailable *float64 `json:"memavailable,omitempty"`
|
||||
}
|
||||
|
||||
// NodeStatus represents detailed node status from /nodes/{node}/status endpoint
|
||||
// This endpoint provides real-time metrics that update every second
|
||||
type NodeStatus struct {
|
||||
|
|
@ -709,6 +717,44 @@ func (c *Client) GetNodeRRDData(ctx context.Context, node, timeframe, cf string,
|
|||
return result.Data, nil
|
||||
}
|
||||
|
||||
// GetLXCRRDData retrieves RRD metrics for an LXC container.
|
||||
func (c *Client) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
|
||||
if timeframe == "" {
|
||||
timeframe = "hour"
|
||||
}
|
||||
if cf == "" {
|
||||
cf = "AVERAGE"
|
||||
}
|
||||
|
||||
params := url.Values{}
|
||||
params.Set("timeframe", timeframe)
|
||||
params.Set("cf", cf)
|
||||
if len(ds) > 0 {
|
||||
params.Set("ds", strings.Join(ds, ","))
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("/nodes/%s/lxc/%d/rrddata", url.PathEscape(node), vmid)
|
||||
if query := params.Encode(); query != "" {
|
||||
path = fmt.Sprintf("%s?%s", path, query)
|
||||
}
|
||||
|
||||
resp, err := c.get(ctx, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Data []GuestRRDPoint `json:"data"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Data, nil
|
||||
}
|
||||
|
||||
// VM represents a Proxmox VE virtual machine
|
||||
type VM struct {
|
||||
VMID int `json:"vmid"`
|
||||
|
|
|
|||
|
|
@ -792,6 +792,19 @@ func (cc *ClusterClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf
|
|||
return result, err
|
||||
}
|
||||
|
||||
func (cc *ClusterClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
|
||||
var result []GuestRRDPoint
|
||||
err := cc.executeWithFailover(ctx, func(client *Client) error {
|
||||
points, err := client.GetLXCRRDData(ctx, node, vmid, timeframe, cf, ds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result = points
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (cc *ClusterClient) GetVMs(ctx context.Context, node string) ([]VM, error) {
|
||||
var result []VM
|
||||
err := cc.executeWithFailover(ctx, func(client *Client) error {
|
||||
|
|
|
|||
Loading…
Reference in a new issue