diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 7f10e63..05b23f4 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -46,6 +46,7 @@ type PVEClientInterface interface { GetNodes(ctx context.Context) ([]proxmox.Node, error) GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) + GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) @@ -6562,15 +6563,82 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam } // Calculate I/O rates for container + sampleTime := time.Now() currentMetrics := IOMetrics{ DiskRead: int64(res.DiskRead), DiskWrite: int64(res.DiskWrite), NetworkIn: int64(res.NetIn), NetworkOut: int64(res.NetOut), - Timestamp: time.Now(), + Timestamp: sampleTime, } diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics) + // Calculate cache-aware memory for LXC containers + // The cluster resources API returns mem from cgroup which includes cache/buffers (inflated). + // Try to get more accurate memory metrics from RRD data. + memTotal := res.MaxMem + memUsed := res.Mem + memorySource := "cluster-resources" + guestRaw := VMMemoryRaw{ + ListingMem: res.Mem, + ListingMaxMem: res.MaxMem, + } + + // For running containers, try to get RRD data for cache-aware memory calculation + if res.Status == "running" { + rrdCtx, rrdCancel := context.WithTimeout(ctx, 5*time.Second) + rrdPoints, err := client.GetLXCRRDData(rrdCtx, res.Node, res.VMID, "hour", "AVERAGE", []string{"memavailable", "memused", "maxmem"}) + rrdCancel() + + if err == nil && len(rrdPoints) > 0 { + // Use the most recent RRD point + point := rrdPoints[len(rrdPoints)-1] + + if point.MaxMem != nil && *point.MaxMem > 0 { + guestRaw.StatusMaxMem = uint64(*point.MaxMem) + } + + // Prefer memavailable-based calculation (excludes cache/buffers) + if point.MemAvailable != nil && *point.MemAvailable > 0 { + memAvailable := uint64(*point.MemAvailable) + if memAvailable <= memTotal { + memUsed = memTotal - memAvailable + memorySource = "rrd-memavailable" + guestRaw.MemInfoAvailable = memAvailable + log.Debug(). + Str("container", res.Name). + Str("node", res.Node). + Uint64("total", memTotal). + Uint64("available", memAvailable). + Uint64("used", memUsed). + Float64("usage", safePercentage(float64(memUsed), float64(memTotal))). + Msg("LXC memory: using RRD memavailable (excludes reclaimable cache)") + } + } else if point.MemUsed != nil && *point.MemUsed > 0 { + // Fall back to memused from RRD if available + memUsed = uint64(*point.MemUsed) + if memUsed <= memTotal { + memorySource = "rrd-memused" + guestRaw.MemInfoUsed = memUsed + log.Debug(). + Str("container", res.Name). + Str("node", res.Node). + Uint64("total", memTotal). + Uint64("used", memUsed). + Float64("usage", safePercentage(float64(memUsed), float64(memTotal))). + Msg("LXC memory: using RRD memused (excludes reclaimable cache)") + } + } + } else if err != nil { + log.Debug(). + Err(err). + Str("instance", instanceName). + Str("container", res.Name). + Int("vmid", res.VMID). + Msg("RRD memory data unavailable for LXC, using cluster resources value") + } + } + container := models.Container{ ID: guestID, VMID: res.VMID, @@ -6582,10 +6650,10 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam CPU: safeFloat(res.CPU), CPUs: int(res.MaxCPU), Memory: models.Memory{ - Total: int64(res.MaxMem), - Used: int64(res.Mem), - Free: int64(res.MaxMem - res.Mem), - Usage: safePercentage(float64(res.Mem), float64(res.MaxMem)), + Total: int64(memTotal), + Used: int64(memUsed), + Free: int64(memTotal - memUsed), + Usage: safePercentage(float64(memUsed), float64(memTotal)), }, Disk: models.Disk{ Total: int64(res.MaxDisk), @@ -6623,6 +6691,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam allContainers = append(allContainers, container) + m.recordGuestSnapshot(instanceName, container.Type, res.Node, res.VMID, GuestMemorySnapshot{ + Name: container.Name, + Status: container.Status, + RetrievedAt: sampleTime, + MemorySource: memorySource, + Memory: container.Memory, + Raw: guestRaw, + }) + // For non-running containers, zero out resource usage metrics to prevent false alerts // Proxmox may report stale or residual metrics for stopped containers if container.Status != "running" { diff --git a/internal/monitoring/monitor_memory_test.go b/internal/monitoring/monitor_memory_test.go index bcf5741..d0cdeed 100644 --- a/internal/monitoring/monitor_memory_test.go +++ b/internal/monitoring/monitor_memory_test.go @@ -33,6 +33,10 @@ func (s *stubPVEClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf return s.rrdPoints, nil } +func (s *stubPVEClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) { + return nil, nil +} + func (s *stubPVEClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) { return nil, nil } diff --git a/internal/monitoring/monitor_snapshots_test.go b/internal/monitoring/monitor_snapshots_test.go index d82473e..efa71e4 100644 --- a/internal/monitoring/monitor_snapshots_test.go +++ b/internal/monitoring/monitor_snapshots_test.go @@ -20,6 +20,9 @@ func (f fakeSnapshotClient) GetNodeStatus(ctx context.Context, node string) (*pr func (f fakeSnapshotClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) { return nil, nil } +func (f fakeSnapshotClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) { + return nil, nil +} func (f fakeSnapshotClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) { return nil, nil } diff --git a/internal/monitoring/monitor_storage_test.go b/internal/monitoring/monitor_storage_test.go index ba7c84c..20cb68c 100644 --- a/internal/monitoring/monitor_storage_test.go +++ b/internal/monitoring/monitor_storage_test.go @@ -30,6 +30,10 @@ func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, tim return nil, nil } +func (f *fakeStorageClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) { + return nil, nil +} + func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) { return nil, nil } diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go index b70f57b..306e9bf 100644 --- a/pkg/proxmox/client.go +++ b/pkg/proxmox/client.go @@ -467,6 +467,14 @@ type NodeRRDPoint struct { MemAvailable *float64 `json:"memavailable,omitempty"` } +// GuestRRDPoint represents a single RRD datapoint for a VM or LXC container. +type GuestRRDPoint struct { + Time int64 `json:"time"` + MaxMem *float64 `json:"maxmem,omitempty"` + MemUsed *float64 `json:"memused,omitempty"` + MemAvailable *float64 `json:"memavailable,omitempty"` +} + // NodeStatus represents detailed node status from /nodes/{node}/status endpoint // This endpoint provides real-time metrics that update every second type NodeStatus struct { @@ -709,6 +717,44 @@ func (c *Client) GetNodeRRDData(ctx context.Context, node, timeframe, cf string, return result.Data, nil } +// GetLXCRRDData retrieves RRD metrics for an LXC container. +func (c *Client) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) { + if timeframe == "" { + timeframe = "hour" + } + if cf == "" { + cf = "AVERAGE" + } + + params := url.Values{} + params.Set("timeframe", timeframe) + params.Set("cf", cf) + if len(ds) > 0 { + params.Set("ds", strings.Join(ds, ",")) + } + + path := fmt.Sprintf("/nodes/%s/lxc/%d/rrddata", url.PathEscape(node), vmid) + if query := params.Encode(); query != "" { + path = fmt.Sprintf("%s?%s", path, query) + } + + resp, err := c.get(ctx, path) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result struct { + Data []GuestRRDPoint `json:"data"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + return result.Data, nil +} + // VM represents a Proxmox VE virtual machine type VM struct { VMID int `json:"vmid"` diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go index daf50ad..f04a12f 100644 --- a/pkg/proxmox/cluster_client.go +++ b/pkg/proxmox/cluster_client.go @@ -792,6 +792,19 @@ func (cc *ClusterClient) GetNodeRRDData(ctx context.Context, node, timeframe, cf return result, err } +func (cc *ClusterClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]GuestRRDPoint, error) { + var result []GuestRRDPoint + err := cc.executeWithFailover(ctx, func(client *Client) error { + points, err := client.GetLXCRRDData(ctx, node, vmid, timeframe, cf, ds) + if err != nil { + return err + } + result = points + return nil + }) + return result, err +} + func (cc *ClusterClient) GetVMs(ctx context.Context, node string) ([]VM, error) { var result []VM err := cc.executeWithFailover(ctx, func(client *Client) error {