Fix inflated RAM usage reporting for LXC containers
Related to #553 ## Problem LXC containers showed inflated memory usage (e.g., 90%+ when actual usage was 50-60%, 96% when actual was 61%) because the code used the raw `mem` value from Proxmox's `/cluster/resources` API endpoint. This value comes from cgroup `memory.current` which includes reclaimable cache and buffers, making memory appear nearly full even when plenty is available. ## Root Cause - **Nodes**: Had sophisticated cache-aware memory calculation with RRD fallbacks - **VMs (qemu)**: Had detailed memory calculation using guest agent meminfo - **LXCs**: Naively used `res.Mem` directly without any cache-aware correction The Proxmox cluster resources API's `mem` field for LXCs includes cache/buffers (from cgroup memory accounting), which should be excluded for accurate "used" memory. ## Solution Implement cache-aware memory calculation for LXC containers by: 1. Adding `GetLXCRRDData()` method to fetch RRD metrics for LXC containers from `/nodes/{node}/lxc/{vmid}/rrddata` 2. Using RRD `memavailable` to calculate actual used memory (total - available) 3. Falling back to RRD `memused` if `memavailable` is not available 4. Only using cluster resources `mem` value as last resort This matches the approach already used for nodes and VMs, providing consistent cache-aware memory reporting across all resource types. ## Changes - Added `GuestRRDPoint` type and `GetLXCRRDData()` method to pkg/proxmox - Added `GetLXCRRDData()` to ClusterClient for cluster-aware operations - Modified LXC memory calculation in `pollPVEInstance()` to use RRD data when available - Added guest memory snapshot recording for LXC containers - Updated test stubs to implement the new interface method ## Testing - Code compiles successfully - Follows the same proven pattern used for nodes and VMs - Includes diagnostic snapshot recording for troubleshooting
This commit is contained in:
parent
88ad986877
commit
af55362009
6 changed files with 152 additions and 5 deletions
|
|
@ -46,6 +46,7 @@ type PVEClientInterface interface {
|
||||||
GetNodes(ctx context.Context) ([]proxmox.Node, error)
|
GetNodes(ctx context.Context) ([]proxmox.Node, error)
|
||||||
GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error)
|
GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error)
|
||||||
GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error)
|
GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error)
|
||||||
|
GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error)
|
||||||
GetVMs(ctx context.Context, node string) ([]proxmox.VM, error)
|
GetVMs(ctx context.Context, node string) ([]proxmox.VM, error)
|
||||||
GetContainers(ctx context.Context, node string) ([]proxmox.Container, error)
|
GetContainers(ctx context.Context, node string) ([]proxmox.Container, error)
|
||||||
GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error)
|
GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error)
|
||||||
|
|
@ -6562,15 +6563,82 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate I/O rates for container
|
// Calculate I/O rates for container
|
||||||
|
sampleTime := time.Now()
|
||||||
currentMetrics := IOMetrics{
|
currentMetrics := IOMetrics{
|
||||||
DiskRead: int64(res.DiskRead),
|
DiskRead: int64(res.DiskRead),
|
||||||
DiskWrite: int64(res.DiskWrite),
|
DiskWrite: int64(res.DiskWrite),
|
||||||
NetworkIn: int64(res.NetIn),
|
NetworkIn: int64(res.NetIn),
|
||||||
NetworkOut: int64(res.NetOut),
|
NetworkOut: int64(res.NetOut),
|
||||||
Timestamp: time.Now(),
|
Timestamp: sampleTime,
|
||||||
}
|
}
|
||||||
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
|
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
|
||||||
|
|
||||||
|
// Calculate cache-aware memory for LXC containers
|
||||||
|
// The cluster resources API returns mem from cgroup which includes cache/buffers (inflated).
|
||||||
|
// Try to get more accurate memory metrics from RRD data.
|
||||||
|
memTotal := res.MaxMem
|
||||||
|
memUsed := res.Mem
|
||||||
|
memorySource := "cluster-resources"
|
||||||
|
guestRaw := VMMemoryRaw{
|
||||||
|
ListingMem: res.Mem,
|
||||||
|
ListingMaxMem: res.MaxMem,
|
||||||
|
}
|
||||||
|
|
||||||
|
// For running containers, try to get RRD data for cache-aware memory calculation
|
||||||
|
if res.Status == "running" {
|
||||||
|
rrdCtx, rrdCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
rrdPoints, err := client.GetLXCRRDData(rrdCtx, res.Node, res.VMID, "hour", "AVERAGE", []string{"memavailable", "memused", "maxmem"})
|
||||||
|
rrdCancel()
|
||||||
|
|
||||||
|
if err == nil && len(rrdPoints) > 0 {
|
||||||
|
// Use the most recent RRD point
|
||||||
|
point := rrdPoints[len(rrdPoints)-1]
|
||||||
|
|
||||||
|
if point.MaxMem != nil && *point.MaxMem > 0 {
|
||||||
|
guestRaw.StatusMaxMem = uint64(*point.MaxMem)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prefer memavailable-based calculation (excludes cache/buffers)
|
||||||
|
if point.MemAvailable != nil && *point.MemAvailable > 0 {
|
||||||
|
memAvailable := uint64(*point.MemAvailable)
|
||||||
|
if memAvailable <= memTotal {
|
||||||
|
memUsed = memTotal - memAvailable
|
||||||
|
memorySource = "rrd-memavailable"
|
||||||
|
guestRaw.MemInfoAvailable = memAvailable
|
||||||
|
log.Debug().
|
||||||
|
Str("container", res.Name).
|
||||||
|
Str("node", res.Node).
|
||||||
|
Uint64("total", memTotal).
|
||||||
|
Uint64("available", memAvailable).
|
||||||
|
Uint64("used", memUsed).
|
||||||
|
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
|
||||||
|
Msg("LXC memory: using RRD memavailable (excludes reclaimable cache)")
|
||||||
|
}
|
||||||
|
} else if point.MemUsed != nil && *point.MemUsed > 0 {
|
||||||
|
// Fall back to memused from RRD if available
|
||||||
|
memUsed = uint64(*point.MemUsed)
|
||||||
|
if memUsed <= memTotal {
|
||||||
|
memorySource = "rrd-memused"
|
||||||
|
guestRaw.MemInfoUsed = memUsed
|
||||||
|
log.Debug().
|
||||||
|
Str("container", res.Name).
|
||||||
|
Str("node", res.Node).
|
||||||
|
Uint64("total", memTotal).
|
||||||
|
Uint64("used", memUsed).
|
||||||
|
Float64("usage", safePercentage(float64(memUsed), float64(memTotal))).
|
||||||
|
Msg("LXC memory: using RRD memused (excludes reclaimable cache)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
log.Debug().
|
||||||
|
Err(err).
|
||||||
|
Str("instance", instanceName).
|
||||||
|
Str("container", res.Name).
|
||||||
|
Int("vmid", res.VMID).
|
||||||
|
Msg("RRD memory data unavailable for LXC, using cluster resources value")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
container := models.Container{
|
container := models.Container{
|
||||||
ID: guestID,
|
ID: guestID,
|
||||||
VMID: res.VMID,
|
VMID: res.VMID,
|
||||||
|
|
@ -6582,10 +6650,10 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
||||||
CPU: safeFloat(res.CPU),
|
CPU: safeFloat(res.CPU),
|
||||||
CPUs: int(res.MaxCPU),
|
CPUs: int(res.MaxCPU),
|
||||||
Memory: models.Memory{
|
Memory: models.Memory{
|
||||||
Total: int64(res.MaxMem),
|
Total: int64(memTotal),
|
||||||
Used: int64(res.Mem),
|
Used: int64(memUsed),
|
||||||
Free: int64(res.MaxMem - res.Mem),
|
Free: int64(memTotal - memUsed),
|
||||||
Usage: safePercentage(float64(res.Mem), float64(res.MaxMem)),
|
Usage: safePercentage(float64(memUsed), float64(memTotal)),
|
||||||
},
|
},
|
||||||
Disk: models.Disk{
|
Disk: models.Disk{
|
||||||
Total: int64(res.MaxDisk),
|
Total: int64(res.MaxDisk),
|
||||||
|
|
@ -6623,6 +6691,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
||||||
|
|
||||||
allContainers = append(allContainers, container)
|
allContainers = append(allContainers, container)
|
||||||
|
|
||||||
|
m.recordGuestSnapshot(instanceName, container.Type, res.Node, res.VMID, GuestMemorySnapshot{
|
||||||
|
Name: container.Name,
|
||||||
|
Status: container.Status,
|
||||||
|
RetrievedAt: sampleTime,
|
||||||
|
MemorySource: memorySource,
|
||||||
|
Memory: container.Memory,
|
||||||
|
Raw: guestRaw,
|
||||||
|
})
|
||||||
|
|
||||||
// For non-running containers, zero out resource usage metrics to prevent false alerts
|
// For non-running containers, zero out resource usage metrics to prevent false alerts
|
||||||
// Proxmox may report stale or residual metrics for stopped containers
|
// Proxmox may report stale or residual metrics for stopped containers
|
||||||
if container.Status != "running" {
|
if container.Status != "running" {
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,10 @@ func (s *stubPVEClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf
|
||||||
return s.rrdPoints, nil
|
return s.rrdPoints, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *stubPVEClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *stubPVEClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
func (s *stubPVEClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,9 @@ func (f fakeSnapshotClient) GetNodeStatus(ctx context.Context, node string) (*pr
|
||||||
func (f fakeSnapshotClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
|
func (f fakeSnapshotClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
func (f fakeSnapshotClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
func (f fakeSnapshotClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
func (f fakeSnapshotClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,10 @@ func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, tim
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakeStorageClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -467,6 +467,14 @@ type NodeRRDPoint struct {
|
||||||
MemAvailable *float64 `json:"memavailable,omitempty"`
|
MemAvailable *float64 `json:"memavailable,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GuestRRDPoint represents a single RRD datapoint for a VM or LXC container.
|
||||||
|
type GuestRRDPoint struct {
|
||||||
|
Time int64 `json:"time"`
|
||||||
|
MaxMem *float64 `json:"maxmem,omitempty"`
|
||||||
|
MemUsed *float64 `json:"memused,omitempty"`
|
||||||
|
MemAvailable *float64 `json:"memavailable,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// NodeStatus represents detailed node status from /nodes/{node}/status endpoint
|
// NodeStatus represents detailed node status from /nodes/{node}/status endpoint
|
||||||
// This endpoint provides real-time metrics that update every second
|
// This endpoint provides real-time metrics that update every second
|
||||||
type NodeStatus struct {
|
type NodeStatus struct {
|
||||||
|
|
@ -709,6 +717,44 @@ func (c *Client) GetNodeRRDData(ctx context.Context, node, timeframe, cf string,
|
||||||
return result.Data, nil
|
return result.Data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetLXCRRDData retrieves RRD metrics for an LXC container.
|
||||||
|
func (c *Client) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
|
||||||
|
if timeframe == "" {
|
||||||
|
timeframe = "hour"
|
||||||
|
}
|
||||||
|
if cf == "" {
|
||||||
|
cf = "AVERAGE"
|
||||||
|
}
|
||||||
|
|
||||||
|
params := url.Values{}
|
||||||
|
params.Set("timeframe", timeframe)
|
||||||
|
params.Set("cf", cf)
|
||||||
|
if len(ds) > 0 {
|
||||||
|
params.Set("ds", strings.Join(ds, ","))
|
||||||
|
}
|
||||||
|
|
||||||
|
path := fmt.Sprintf("/nodes/%s/lxc/%d/rrddata", url.PathEscape(node), vmid)
|
||||||
|
if query := params.Encode(); query != "" {
|
||||||
|
path = fmt.Sprintf("%s?%s", path, query)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := c.get(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Data []GuestRRDPoint `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.Data, nil
|
||||||
|
}
|
||||||
|
|
||||||
// VM represents a Proxmox VE virtual machine
|
// VM represents a Proxmox VE virtual machine
|
||||||
type VM struct {
|
type VM struct {
|
||||||
VMID int `json:"vmid"`
|
VMID int `json:"vmid"`
|
||||||
|
|
|
||||||
|
|
@ -792,6 +792,19 @@ func (cc *ClusterClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cc *ClusterClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) {
|
||||||
|
var result []GuestRRDPoint
|
||||||
|
err := cc.executeWithFailover(ctx, func(client *Client) error {
|
||||||
|
points, err := client.GetLXCRRDData(ctx, node, vmid, timeframe, cf, ds)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result = points
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
func (cc *ClusterClient) GetVMs(ctx context.Context, node string) ([]VM, error) {
|
func (cc *ClusterClient) GetVMs(ctx context.Context, node string) ([]VM, error) {
|
||||||
var result []VM
|
var result []VM
|
||||||
err := cc.executeWithFailover(ctx, func(client *Client) error {
|
err := cc.executeWithFailover(ctx, func(client *Client) error {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue