Improve LXC guest metrics visibility (#596)
This commit is contained in:
rcourtman
parent
e1fe8354e9
commit
aac3dacd63
6 changed files with 747 additions and 66 deletions
|
|
@ -181,6 +181,15 @@ func (c Container) ToFrontend() ContainerFrontend {
|
|||
ct.Disks = c.Disks
|
||||
}
|
||||
|
||||
if len(c.IPAddresses) > 0 {
|
||||
ct.IPAddresses = append([]string(nil), c.IPAddresses...)
|
||||
}
|
||||
|
||||
if len(c.NetworkInterfaces) > 0 {
|
||||
ct.NetworkInterfaces = make([]GuestNetworkInterface, len(c.NetworkInterfaces))
|
||||
copy(ct.NetworkInterfaces, c.NetworkInterfaces)
|
||||
}
|
||||
|
||||
return ct
|
||||
}
|
||||
|
||||
|
|
@ -233,6 +242,68 @@ func (d DockerHost) ToFrontend() DockerHostFrontend {
|
|||
return h
|
||||
}
|
||||
|
||||
// ToFrontend converts a Host to HostFrontend.
|
||||
func (h Host) ToFrontend() HostFrontend {
|
||||
host := HostFrontend{
|
||||
ID: h.ID,
|
||||
Hostname: h.Hostname,
|
||||
DisplayName: h.DisplayName,
|
||||
Platform: h.Platform,
|
||||
OSName: h.OSName,
|
||||
OSVersion: h.OSVersion,
|
||||
KernelVersion: h.KernelVersion,
|
||||
Architecture: h.Architecture,
|
||||
CPUCount: h.CPUCount,
|
||||
CPUUsage: h.CPUUsage,
|
||||
Status: h.Status,
|
||||
UptimeSeconds: h.UptimeSeconds,
|
||||
IntervalSeconds: h.IntervalSeconds,
|
||||
AgentVersion: h.AgentVersion,
|
||||
TokenID: h.TokenID,
|
||||
TokenName: h.TokenName,
|
||||
TokenHint: h.TokenHint,
|
||||
Tags: append([]string(nil), h.Tags...),
|
||||
LastSeen: h.LastSeen.Unix() * 1000,
|
||||
}
|
||||
|
||||
if host.DisplayName == "" {
|
||||
if h.DisplayName != "" {
|
||||
host.DisplayName = h.DisplayName
|
||||
} else if h.Hostname != "" {
|
||||
host.DisplayName = h.Hostname
|
||||
}
|
||||
}
|
||||
|
||||
if len(h.LoadAverage) > 0 {
|
||||
host.LoadAverage = append([]float64(nil), h.LoadAverage...)
|
||||
}
|
||||
|
||||
if (h.Memory != Memory{}) {
|
||||
mem := h.Memory
|
||||
host.Memory = &mem
|
||||
}
|
||||
|
||||
if len(h.Disks) > 0 {
|
||||
host.Disks = append([]Disk(nil), h.Disks...)
|
||||
}
|
||||
|
||||
if len(h.NetworkInterfaces) > 0 {
|
||||
host.NetworkInterfaces = make([]HostNetworkInterface, len(h.NetworkInterfaces))
|
||||
copy(host.NetworkInterfaces, h.NetworkInterfaces)
|
||||
}
|
||||
|
||||
if s := hostSensorSummaryToFrontend(h.Sensors); s != nil {
|
||||
host.Sensors = s
|
||||
}
|
||||
|
||||
if h.TokenLastUsedAt != nil && !h.TokenLastUsedAt.IsZero() {
|
||||
ts := h.TokenLastUsedAt.Unix() * 1000
|
||||
host.TokenLastUsedAt = &ts
|
||||
}
|
||||
|
||||
return host
|
||||
}
|
||||
|
||||
// ToFrontend converts a DockerContainer to DockerContainerFrontend
|
||||
func (c DockerContainer) ToFrontend() DockerContainerFrontend {
|
||||
container := DockerContainerFrontend{
|
||||
|
|
@ -291,6 +362,35 @@ func (c DockerContainer) ToFrontend() DockerContainerFrontend {
|
|||
return container
|
||||
}
|
||||
|
||||
func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFrontend {
|
||||
if len(src.TemperatureCelsius) == 0 && len(src.FanRPM) == 0 && len(src.Additional) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
dest := &HostSensorSummaryFrontend{}
|
||||
if len(src.TemperatureCelsius) > 0 {
|
||||
dest.TemperatureCelsius = copyStringFloatMap(src.TemperatureCelsius)
|
||||
}
|
||||
if len(src.FanRPM) > 0 {
|
||||
dest.FanRPM = copyStringFloatMap(src.FanRPM)
|
||||
}
|
||||
if len(src.Additional) > 0 {
|
||||
dest.Additional = copyStringFloatMap(src.Additional)
|
||||
}
|
||||
return dest
|
||||
}
|
||||
|
||||
func copyStringFloatMap(src map[string]float64) map[string]float64 {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
dest := make(map[string]float64, len(src))
|
||||
for k, v := range src {
|
||||
dest[k] = v
|
||||
}
|
||||
return dest
|
||||
}
|
||||
|
||||
func toDockerHostCommandFrontend(cmd DockerHostCommandStatus) *DockerHostCommandFrontend {
|
||||
result := &DockerHostCommandFrontend{
|
||||
ID: cmd.ID,
|
||||
|
|
|
|||
|
|
@ -116,28 +116,30 @@ type VM struct {
|
|||
|
||||
// Container represents an LXC container
|
||||
type Container struct {
|
||||
ID string `json:"id"`
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"`
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUs int `json:"cpus"`
|
||||
Memory Memory `json:"memory"`
|
||||
Disk Disk `json:"disk"`
|
||||
Disks []Disk `json:"disks,omitempty"`
|
||||
NetworkIn int64 `json:"networkIn"`
|
||||
NetworkOut int64 `json:"networkOut"`
|
||||
DiskRead int64 `json:"diskRead"`
|
||||
DiskWrite int64 `json:"diskWrite"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Template bool `json:"template"`
|
||||
LastBackup time.Time `json:"lastBackup,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Lock string `json:"lock,omitempty"`
|
||||
LastSeen time.Time `json:"lastSeen"`
|
||||
ID string `json:"id"`
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"`
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUs int `json:"cpus"`
|
||||
Memory Memory `json:"memory"`
|
||||
Disk Disk `json:"disk"`
|
||||
Disks []Disk `json:"disks,omitempty"`
|
||||
NetworkIn int64 `json:"networkIn"`
|
||||
NetworkOut int64 `json:"networkOut"`
|
||||
DiskRead int64 `json:"diskRead"`
|
||||
DiskWrite int64 `json:"diskWrite"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Template bool `json:"template"`
|
||||
LastBackup time.Time `json:"lastBackup,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Lock string `json:"lock,omitempty"`
|
||||
LastSeen time.Time `json:"lastSeen"`
|
||||
IPAddresses []string `json:"ipAddresses,omitempty"`
|
||||
NetworkInterfaces []GuestNetworkInterface `json:"networkInterfaces,omitempty"`
|
||||
}
|
||||
|
||||
// Host represents a generic infrastructure host reporting via external agents.
|
||||
|
|
|
|||
|
|
@ -67,30 +67,32 @@ type VMFrontend struct {
|
|||
|
||||
// ContainerFrontend represents a Container with frontend-friendly field names
|
||||
type ContainerFrontend struct {
|
||||
ID string `json:"id"`
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"`
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUs int `json:"cpus"`
|
||||
Memory *Memory `json:"memory,omitempty"` // Full memory object
|
||||
Mem int64 `json:"mem"` // Maps to Memory.Used
|
||||
MaxMem int64 `json:"maxmem"` // Maps to Memory.Total
|
||||
DiskObj *Disk `json:"disk,omitempty"` // Full disk object
|
||||
Disks []Disk `json:"disks,omitempty"` // Individual filesystem/disk usage
|
||||
NetIn int64 `json:"networkIn"` // Maps to NetworkIn (camelCase for frontend)
|
||||
NetOut int64 `json:"networkOut"` // Maps to NetworkOut (camelCase for frontend)
|
||||
DiskRead int64 `json:"diskRead"` // Maps to DiskRead (camelCase for frontend)
|
||||
DiskWrite int64 `json:"diskWrite"` // Maps to DiskWrite (camelCase for frontend)
|
||||
Uptime int64 `json:"uptime"`
|
||||
Template bool `json:"template"`
|
||||
LastBackup int64 `json:"lastBackup,omitempty"` // Unix timestamp
|
||||
Tags string `json:"tags,omitempty"` // Joined string
|
||||
Lock string `json:"lock,omitempty"`
|
||||
LastSeen int64 `json:"lastSeen"` // Unix timestamp
|
||||
ID string `json:"id"`
|
||||
VMID int `json:"vmid"`
|
||||
Name string `json:"name"`
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"`
|
||||
Status string `json:"status"`
|
||||
Type string `json:"type"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUs int `json:"cpus"`
|
||||
Memory *Memory `json:"memory,omitempty"` // Full memory object
|
||||
Mem int64 `json:"mem"` // Maps to Memory.Used
|
||||
MaxMem int64 `json:"maxmem"` // Maps to Memory.Total
|
||||
DiskObj *Disk `json:"disk,omitempty"` // Full disk object
|
||||
Disks []Disk `json:"disks,omitempty"` // Individual filesystem/disk usage
|
||||
NetworkInterfaces []GuestNetworkInterface `json:"networkInterfaces,omitempty"`
|
||||
IPAddresses []string `json:"ipAddresses,omitempty"`
|
||||
NetIn int64 `json:"networkIn"` // Maps to NetworkIn (camelCase for frontend)
|
||||
NetOut int64 `json:"networkOut"` // Maps to NetworkOut (camelCase for frontend)
|
||||
DiskRead int64 `json:"diskRead"` // Maps to DiskRead (camelCase for frontend)
|
||||
DiskWrite int64 `json:"diskWrite"` // Maps to DiskWrite (camelCase for frontend)
|
||||
Uptime int64 `json:"uptime"`
|
||||
Template bool `json:"template"`
|
||||
LastBackup int64 `json:"lastBackup,omitempty"` // Unix timestamp
|
||||
Tags string `json:"tags,omitempty"` // Joined string
|
||||
Lock string `json:"lock,omitempty"`
|
||||
LastSeen int64 `json:"lastSeen"` // Unix timestamp
|
||||
}
|
||||
|
||||
// DockerHostFrontend represents a Docker host with frontend-friendly fields
|
||||
|
|
@ -174,6 +176,42 @@ type DockerHostCommandFrontend struct {
|
|||
ExpiresAt *int64 `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
// HostFrontend represents a generic infrastructure host exposed to the UI.
|
||||
type HostFrontend struct {
|
||||
ID string `json:"id"`
|
||||
Hostname string `json:"hostname"`
|
||||
DisplayName string `json:"displayName"`
|
||||
Platform string `json:"platform,omitempty"`
|
||||
OSName string `json:"osName,omitempty"`
|
||||
OSVersion string `json:"osVersion,omitempty"`
|
||||
KernelVersion string `json:"kernelVersion,omitempty"`
|
||||
Architecture string `json:"architecture,omitempty"`
|
||||
CPUCount int `json:"cpuCount,omitempty"`
|
||||
CPUUsage float64 `json:"cpuUsage,omitempty"`
|
||||
LoadAverage []float64 `json:"loadAverage,omitempty"`
|
||||
Memory *Memory `json:"memory,omitempty"`
|
||||
Disks []Disk `json:"disks,omitempty"`
|
||||
NetworkInterfaces []HostNetworkInterface `json:"networkInterfaces,omitempty"`
|
||||
Sensors *HostSensorSummaryFrontend `json:"sensors,omitempty"`
|
||||
Status string `json:"status"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
|
||||
LastSeen int64 `json:"lastSeen"`
|
||||
IntervalSeconds int `json:"intervalSeconds,omitempty"`
|
||||
AgentVersion string `json:"agentVersion,omitempty"`
|
||||
TokenID string `json:"tokenId,omitempty"`
|
||||
TokenName string `json:"tokenName,omitempty"`
|
||||
TokenHint string `json:"tokenHint,omitempty"`
|
||||
TokenLastUsedAt *int64 `json:"tokenLastUsedAt,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// HostSensorSummaryFrontend mirrors HostSensorSummary with primitives for the frontend.
|
||||
type HostSensorSummaryFrontend struct {
|
||||
TemperatureCelsius map[string]float64 `json:"temperatureCelsius,omitempty"`
|
||||
FanRPM map[string]float64 `json:"fanRpm,omitempty"`
|
||||
Additional map[string]float64 `json:"additional,omitempty"`
|
||||
}
|
||||
|
||||
// StorageFrontend represents Storage with frontend-friendly field names
|
||||
type StorageFrontend struct {
|
||||
ID string `json:"id"`
|
||||
|
|
@ -263,6 +301,7 @@ type StateFrontend struct {
|
|||
VMs []VMFrontend `json:"vms"`
|
||||
Containers []ContainerFrontend `json:"containers"`
|
||||
DockerHosts []DockerHostFrontend `json:"dockerHosts"`
|
||||
Hosts []HostFrontend `json:"hosts"`
|
||||
Storage []StorageFrontend `json:"storage"`
|
||||
CephClusters []CephClusterFrontend `json:"cephClusters"`
|
||||
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import (
|
|||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
|
||||
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
|
||||
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
|
|
@ -509,6 +510,17 @@ func safeFloat(val float64) float64 {
|
|||
return val
|
||||
}
|
||||
|
||||
func cloneStringFloatMap(src map[string]float64) map[string]float64 {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]float64, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// shouldRunBackupPoll determines whether a backup polling cycle should execute.
|
||||
// Returns whether polling should run, a human-readable skip reason, and the timestamp to record.
|
||||
func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, string, time.Time) {
|
||||
|
|
@ -547,6 +559,7 @@ func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, stri
|
|||
|
||||
const (
|
||||
dockerConnectionPrefix = "docker-"
|
||||
hostConnectionPrefix = "host-"
|
||||
dockerOfflineGraceMultiplier = 4
|
||||
dockerMinimumHealthWindow = 30 * time.Second
|
||||
dockerMaximumHealthWindow = 10 * time.Minute
|
||||
|
|
@ -1266,6 +1279,170 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
|
|||
return host, nil
|
||||
}
|
||||
|
||||
// ApplyHostReport ingests a host agent report into the shared state.
|
||||
func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.APITokenRecord) (models.Host, error) {
|
||||
hostname := strings.TrimSpace(report.Host.Hostname)
|
||||
if hostname == "" {
|
||||
return models.Host{}, fmt.Errorf("host report missing hostname")
|
||||
}
|
||||
|
||||
identifier := strings.TrimSpace(report.Host.ID)
|
||||
if identifier != "" {
|
||||
identifier = sanitizeDockerHostSuffix(identifier)
|
||||
}
|
||||
if identifier == "" {
|
||||
if machine := sanitizeDockerHostSuffix(report.Host.MachineID); machine != "" {
|
||||
identifier = machine
|
||||
}
|
||||
}
|
||||
if identifier == "" {
|
||||
if agentID := sanitizeDockerHostSuffix(report.Agent.ID); agentID != "" {
|
||||
identifier = agentID
|
||||
}
|
||||
}
|
||||
if identifier == "" {
|
||||
if hostName := sanitizeDockerHostSuffix(hostname); hostName != "" {
|
||||
identifier = hostName
|
||||
}
|
||||
}
|
||||
if identifier == "" {
|
||||
seedParts := uniqueNonEmptyStrings(
|
||||
report.Host.MachineID,
|
||||
report.Agent.ID,
|
||||
report.Host.Hostname,
|
||||
)
|
||||
if len(seedParts) == 0 {
|
||||
seedParts = []string{hostname}
|
||||
}
|
||||
seed := strings.Join(seedParts, "|")
|
||||
sum := sha1.Sum([]byte(seed))
|
||||
identifier = fmt.Sprintf("host-%s", hex.EncodeToString(sum[:6]))
|
||||
}
|
||||
|
||||
existingHosts := m.state.GetHosts()
|
||||
var previous models.Host
|
||||
var hasPrevious bool
|
||||
for _, candidate := range existingHosts {
|
||||
if candidate.ID == identifier {
|
||||
previous = candidate
|
||||
hasPrevious = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
displayName := strings.TrimSpace(report.Host.DisplayName)
|
||||
if displayName == "" {
|
||||
displayName = hostname
|
||||
}
|
||||
|
||||
timestamp := report.Timestamp
|
||||
if timestamp.IsZero() {
|
||||
timestamp = time.Now().UTC()
|
||||
}
|
||||
|
||||
memory := models.Memory{
|
||||
Total: report.Metrics.Memory.TotalBytes,
|
||||
Used: report.Metrics.Memory.UsedBytes,
|
||||
Free: report.Metrics.Memory.FreeBytes,
|
||||
Usage: safeFloat(report.Metrics.Memory.Usage),
|
||||
SwapTotal: report.Metrics.Memory.SwapTotal,
|
||||
SwapUsed: report.Metrics.Memory.SwapUsed,
|
||||
}
|
||||
if memory.Usage <= 0 && memory.Total > 0 {
|
||||
memory.Usage = safePercentage(float64(memory.Used), float64(memory.Total))
|
||||
}
|
||||
|
||||
disks := make([]models.Disk, 0, len(report.Disks))
|
||||
for _, disk := range report.Disks {
|
||||
usage := safeFloat(disk.Usage)
|
||||
if usage <= 0 && disk.TotalBytes > 0 {
|
||||
usage = safePercentage(float64(disk.UsedBytes), float64(disk.TotalBytes))
|
||||
}
|
||||
disks = append(disks, models.Disk{
|
||||
Total: disk.TotalBytes,
|
||||
Used: disk.UsedBytes,
|
||||
Free: disk.FreeBytes,
|
||||
Usage: usage,
|
||||
Mountpoint: disk.Mountpoint,
|
||||
Type: disk.Type,
|
||||
Device: disk.Device,
|
||||
})
|
||||
}
|
||||
|
||||
network := make([]models.HostNetworkInterface, 0, len(report.Network))
|
||||
for _, nic := range report.Network {
|
||||
network = append(network, models.HostNetworkInterface{
|
||||
Name: nic.Name,
|
||||
MAC: nic.MAC,
|
||||
Addresses: append([]string(nil), nic.Addresses...),
|
||||
RXBytes: nic.RXBytes,
|
||||
TXBytes: nic.TXBytes,
|
||||
SpeedMbps: nic.SpeedMbps,
|
||||
})
|
||||
}
|
||||
|
||||
host := models.Host{
|
||||
ID: identifier,
|
||||
Hostname: hostname,
|
||||
DisplayName: displayName,
|
||||
Platform: strings.TrimSpace(strings.ToLower(report.Host.Platform)),
|
||||
OSName: strings.TrimSpace(report.Host.OSName),
|
||||
OSVersion: strings.TrimSpace(report.Host.OSVersion),
|
||||
KernelVersion: strings.TrimSpace(report.Host.KernelVersion),
|
||||
Architecture: strings.TrimSpace(report.Host.Architecture),
|
||||
CPUCount: report.Host.CPUCount,
|
||||
CPUUsage: safeFloat(report.Metrics.CPUUsagePercent),
|
||||
LoadAverage: append([]float64(nil), report.Host.LoadAverage...),
|
||||
Memory: memory,
|
||||
Disks: disks,
|
||||
NetworkInterfaces: network,
|
||||
Sensors: models.HostSensorSummary{
|
||||
TemperatureCelsius: cloneStringFloatMap(report.Sensors.TemperatureCelsius),
|
||||
FanRPM: cloneStringFloatMap(report.Sensors.FanRPM),
|
||||
Additional: cloneStringFloatMap(report.Sensors.Additional),
|
||||
},
|
||||
Status: "online",
|
||||
UptimeSeconds: report.Host.UptimeSeconds,
|
||||
IntervalSeconds: report.Agent.IntervalSeconds,
|
||||
LastSeen: timestamp,
|
||||
AgentVersion: strings.TrimSpace(report.Agent.Version),
|
||||
Tags: append([]string(nil), report.Tags...),
|
||||
}
|
||||
|
||||
if len(host.LoadAverage) == 0 {
|
||||
host.LoadAverage = nil
|
||||
}
|
||||
if len(host.Disks) == 0 {
|
||||
host.Disks = nil
|
||||
}
|
||||
if len(host.NetworkInterfaces) == 0 {
|
||||
host.NetworkInterfaces = nil
|
||||
}
|
||||
|
||||
if tokenRecord != nil {
|
||||
host.TokenID = tokenRecord.ID
|
||||
host.TokenName = tokenRecord.Name
|
||||
host.TokenHint = tokenHintFromRecord(tokenRecord)
|
||||
if tokenRecord.LastUsedAt != nil {
|
||||
t := tokenRecord.LastUsedAt.UTC()
|
||||
host.TokenLastUsedAt = &t
|
||||
} else {
|
||||
now := time.Now().UTC()
|
||||
host.TokenLastUsedAt = &now
|
||||
}
|
||||
} else if hasPrevious {
|
||||
host.TokenID = previous.TokenID
|
||||
host.TokenName = previous.TokenName
|
||||
host.TokenHint = previous.TokenHint
|
||||
host.TokenLastUsedAt = previous.TokenLastUsedAt
|
||||
}
|
||||
|
||||
m.state.UpsertHost(host)
|
||||
m.state.SetConnectionHealth(hostConnectionPrefix+host.ID, true)
|
||||
|
||||
return host, nil
|
||||
}
|
||||
|
||||
const (
|
||||
removedDockerHostsTTL = 24 * time.Hour // Clean up removed hosts tracking after 24 hours
|
||||
)
|
||||
|
|
@ -1659,6 +1836,336 @@ func anyToInt64(val interface{}) int64 {
|
|||
return 0
|
||||
}
|
||||
|
||||
func (m *Monitor) enrichContainerMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName string, container *models.Container) {
|
||||
if container == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ensureContainerRootDiskEntry(container)
|
||||
|
||||
if client == nil || container.Status != "running" {
|
||||
return
|
||||
}
|
||||
|
||||
statusCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
status, err := client.GetContainerStatus(statusCtx, nodeName, container.VMID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Debug().
|
||||
Err(err).
|
||||
Str("instance", instanceName).
|
||||
Str("node", nodeName).
|
||||
Str("container", container.Name).
|
||||
Int("vmid", container.VMID).
|
||||
Msg("Container status metadata unavailable")
|
||||
return
|
||||
}
|
||||
if status == nil {
|
||||
return
|
||||
}
|
||||
|
||||
addressSet := make(map[string]struct{})
|
||||
addressOrder := make([]string, 0, 4)
|
||||
|
||||
addAddress := func(addr string) {
|
||||
addr = strings.TrimSpace(addr)
|
||||
if addr == "" {
|
||||
return
|
||||
}
|
||||
if _, exists := addressSet[addr]; exists {
|
||||
return
|
||||
}
|
||||
addressSet[addr] = struct{}{}
|
||||
addressOrder = append(addressOrder, addr)
|
||||
}
|
||||
|
||||
for _, addr := range sanitizeGuestAddressStrings(status.IP) {
|
||||
addAddress(addr)
|
||||
}
|
||||
for _, addr := range sanitizeGuestAddressStrings(status.IP6) {
|
||||
addAddress(addr)
|
||||
}
|
||||
for _, addr := range parseContainerRawIPs(status.IPv4) {
|
||||
addAddress(addr)
|
||||
}
|
||||
for _, addr := range parseContainerRawIPs(status.IPv6) {
|
||||
addAddress(addr)
|
||||
}
|
||||
|
||||
networkIfaces := make([]models.GuestNetworkInterface, 0, len(status.Network))
|
||||
for rawName, cfg := range status.Network {
|
||||
if cfg == (proxmox.ContainerNetworkConfig{}) {
|
||||
continue
|
||||
}
|
||||
|
||||
iface := models.GuestNetworkInterface{}
|
||||
name := strings.TrimSpace(cfg.Name)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(rawName)
|
||||
}
|
||||
if name != "" {
|
||||
iface.Name = name
|
||||
}
|
||||
if mac := strings.TrimSpace(cfg.HWAddr); mac != "" {
|
||||
iface.MAC = mac
|
||||
}
|
||||
|
||||
addrCandidates := make([]string, 0, 4)
|
||||
addrCandidates = append(addrCandidates, collectIPsFromInterface(cfg.IP)...)
|
||||
addrCandidates = append(addrCandidates, collectIPsFromInterface(cfg.IP6)...)
|
||||
addrCandidates = append(addrCandidates, collectIPsFromInterface(cfg.IPv4)...)
|
||||
addrCandidates = append(addrCandidates, collectIPsFromInterface(cfg.IPv6)...)
|
||||
|
||||
if len(addrCandidates) > 0 {
|
||||
deduped := dedupeStringsPreserveOrder(addrCandidates)
|
||||
if len(deduped) > 0 {
|
||||
iface.Addresses = deduped
|
||||
for _, addr := range deduped {
|
||||
addAddress(addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if iface.Name != "" || iface.MAC != "" || len(iface.Addresses) > 0 {
|
||||
networkIfaces = append(networkIfaces, iface)
|
||||
}
|
||||
}
|
||||
|
||||
if len(networkIfaces) > 1 {
|
||||
sort.SliceStable(networkIfaces, func(i, j int) bool {
|
||||
left := strings.TrimSpace(networkIfaces[i].Name)
|
||||
right := strings.TrimSpace(networkIfaces[j].Name)
|
||||
return left < right
|
||||
})
|
||||
}
|
||||
|
||||
if len(addressOrder) > 1 {
|
||||
sort.Strings(addressOrder)
|
||||
}
|
||||
|
||||
if len(addressOrder) > 0 {
|
||||
container.IPAddresses = addressOrder
|
||||
}
|
||||
|
||||
if len(networkIfaces) > 0 {
|
||||
container.NetworkInterfaces = networkIfaces
|
||||
}
|
||||
|
||||
if disks := convertContainerDiskInfo(status); len(disks) > 0 {
|
||||
container.Disks = disks
|
||||
}
|
||||
|
||||
ensureContainerRootDiskEntry(container)
|
||||
}
|
||||
|
||||
func ensureContainerRootDiskEntry(container *models.Container) {
|
||||
if container == nil || len(container.Disks) > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
total := container.Disk.Total
|
||||
used := container.Disk.Used
|
||||
if total <= 0 && used <= 0 {
|
||||
return
|
||||
}
|
||||
if total > 0 && used > total {
|
||||
used = total
|
||||
}
|
||||
|
||||
free := total - used
|
||||
if free < 0 {
|
||||
free = 0
|
||||
}
|
||||
|
||||
usage := container.Disk.Usage
|
||||
if total > 0 && usage <= 0 {
|
||||
usage = safePercentage(float64(used), float64(total))
|
||||
}
|
||||
|
||||
container.Disks = []models.Disk{
|
||||
{
|
||||
Total: total,
|
||||
Used: used,
|
||||
Free: free,
|
||||
Usage: usage,
|
||||
Mountpoint: "/",
|
||||
Type: "rootfs",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func convertContainerDiskInfo(status *proxmox.Container) []models.Disk {
|
||||
if status == nil || len(status.DiskInfo) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
disks := make([]models.Disk, 0, len(status.DiskInfo))
|
||||
for name, info := range status.DiskInfo {
|
||||
total := clampToInt64(info.Total)
|
||||
used := clampToInt64(info.Used)
|
||||
if total <= 0 && used <= 0 {
|
||||
continue
|
||||
}
|
||||
if total > 0 && used > total {
|
||||
used = total
|
||||
}
|
||||
free := total - used
|
||||
if free < 0 {
|
||||
free = 0
|
||||
}
|
||||
|
||||
disk := models.Disk{
|
||||
Total: total,
|
||||
Used: used,
|
||||
Free: free,
|
||||
}
|
||||
|
||||
if total > 0 {
|
||||
disk.Usage = safePercentage(float64(used), float64(total))
|
||||
}
|
||||
|
||||
label := strings.TrimSpace(name)
|
||||
if strings.EqualFold(label, "rootfs") || label == "" {
|
||||
disk.Mountpoint = "/"
|
||||
disk.Type = "rootfs"
|
||||
if device := sanitizeRootFSDevice(status.RootFS); device != "" {
|
||||
disk.Device = device
|
||||
}
|
||||
} else {
|
||||
disk.Mountpoint = label
|
||||
disk.Type = strings.ToLower(label)
|
||||
}
|
||||
|
||||
disks = append(disks, disk)
|
||||
}
|
||||
|
||||
if len(disks) > 1 {
|
||||
sort.SliceStable(disks, func(i, j int) bool {
|
||||
return disks[i].Mountpoint < disks[j].Mountpoint
|
||||
})
|
||||
}
|
||||
|
||||
return disks
|
||||
}
|
||||
|
||||
func sanitizeRootFSDevice(root string) string {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return ""
|
||||
}
|
||||
if idx := strings.Index(root, ","); idx != -1 {
|
||||
root = root[:idx]
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func parseContainerRawIPs(raw json.RawMessage) []string {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
var data interface{}
|
||||
if err := json.Unmarshal(raw, &data); err != nil {
|
||||
return nil
|
||||
}
|
||||
return collectIPsFromInterface(data)
|
||||
}
|
||||
|
||||
func collectIPsFromInterface(value interface{}) []string {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return nil
|
||||
case string:
|
||||
return sanitizeGuestAddressStrings(v)
|
||||
case []interface{}:
|
||||
results := make([]string, 0, len(v))
|
||||
for _, item := range v {
|
||||
results = append(results, collectIPsFromInterface(item)...)
|
||||
}
|
||||
return results
|
||||
case []string:
|
||||
results := make([]string, 0, len(v))
|
||||
for _, item := range v {
|
||||
results = append(results, sanitizeGuestAddressStrings(item)...)
|
||||
}
|
||||
return results
|
||||
case map[string]interface{}:
|
||||
results := make([]string, 0)
|
||||
for _, key := range []string{"ip", "ip6", "ipv4", "ipv6", "address", "value"} {
|
||||
if val, ok := v[key]; ok {
|
||||
results = append(results, collectIPsFromInterface(val)...)
|
||||
}
|
||||
}
|
||||
return results
|
||||
case json.Number:
|
||||
return sanitizeGuestAddressStrings(v.String())
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeGuestAddressStrings(value string) []string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
lower := strings.ToLower(value)
|
||||
switch lower {
|
||||
case "dhcp", "manual", "static", "auto", "none", "n/a", "unknown", "0.0.0.0", "::", "::1":
|
||||
return nil
|
||||
}
|
||||
|
||||
parts := strings.FieldsFunc(value, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || r == ',' || r == ';'
|
||||
})
|
||||
|
||||
if len(parts) > 1 {
|
||||
results := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
results = append(results, sanitizeGuestAddressStrings(part)...)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
if strings.HasPrefix(value, "127.") || strings.HasPrefix(strings.ToLower(value), "0.0.0.0") {
|
||||
return nil
|
||||
}
|
||||
|
||||
lower = strings.ToLower(value)
|
||||
if strings.HasPrefix(lower, "fe80") {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(lower, "::1") {
|
||||
return nil
|
||||
}
|
||||
|
||||
return []string{value}
|
||||
}
|
||||
|
||||
func dedupeStringsPreserveOrder(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
result := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v] = struct{}{}
|
||||
result = append(result, v)
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return nil
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetConnectionStatuses returns the current connection status for all nodes
|
||||
func (m *Monitor) GetConnectionStatuses() map[string]bool {
|
||||
if mock.IsMockEnabled() {
|
||||
|
|
@ -5185,6 +5692,8 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
|
|||
}
|
||||
}
|
||||
|
||||
m.enrichContainerMetadata(ctx, client, instanceName, res.Node, &container)
|
||||
|
||||
allContainers = append(allContainers, container)
|
||||
|
||||
// For non-running containers, zero out resource usage metrics to prevent false alerts
|
||||
|
|
|
|||
|
|
@ -975,6 +975,8 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
|
|||
modelContainer.Disk.Usage = safePercentage(float64(modelContainer.Disk.Used), float64(modelContainer.Disk.Total))
|
||||
}
|
||||
|
||||
m.enrichContainerMetadata(ctx, client, instanceName, n.Node, &modelContainer)
|
||||
|
||||
// Zero out metrics for non-running containers
|
||||
if container.Status != "running" {
|
||||
modelContainer.CPU = 0
|
||||
|
|
|
|||
|
|
@ -734,26 +734,55 @@ type VM struct {
|
|||
|
||||
// Container represents a Proxmox VE LXC container
|
||||
type Container struct {
|
||||
VMID FlexInt `json:"vmid"` // Changed to FlexInt to handle string VMIDs from some Proxmox versions
|
||||
Name string `json:"name"`
|
||||
Node string `json:"node"`
|
||||
Status string `json:"status"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUs FlexInt `json:"cpus"`
|
||||
Mem uint64 `json:"mem"`
|
||||
MaxMem uint64 `json:"maxmem"`
|
||||
Swap uint64 `json:"swap"`
|
||||
MaxSwap uint64 `json:"maxswap"`
|
||||
Disk uint64 `json:"disk"`
|
||||
MaxDisk uint64 `json:"maxdisk"`
|
||||
NetIn uint64 `json:"netin"`
|
||||
NetOut uint64 `json:"netout"`
|
||||
DiskRead uint64 `json:"diskread"`
|
||||
DiskWrite uint64 `json:"diskwrite"`
|
||||
Uptime uint64 `json:"uptime"`
|
||||
Template int `json:"template"`
|
||||
Tags string `json:"tags"`
|
||||
Lock string `json:"lock"`
|
||||
VMID FlexInt `json:"vmid"` // Changed to FlexInt to handle string VMIDs from some Proxmox versions
|
||||
Name string `json:"name"`
|
||||
Node string `json:"node"`
|
||||
Status string `json:"status"`
|
||||
CPU float64 `json:"cpu"`
|
||||
CPUs FlexInt `json:"cpus"`
|
||||
Mem uint64 `json:"mem"`
|
||||
MaxMem uint64 `json:"maxmem"`
|
||||
Swap uint64 `json:"swap"`
|
||||
MaxSwap uint64 `json:"maxswap"`
|
||||
Disk uint64 `json:"disk"`
|
||||
MaxDisk uint64 `json:"maxdisk"`
|
||||
NetIn uint64 `json:"netin"`
|
||||
NetOut uint64 `json:"netout"`
|
||||
DiskRead uint64 `json:"diskread"`
|
||||
DiskWrite uint64 `json:"diskwrite"`
|
||||
Uptime uint64 `json:"uptime"`
|
||||
Template int `json:"template"`
|
||||
Tags string `json:"tags"`
|
||||
Lock string `json:"lock"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
IP string `json:"ip,omitempty"`
|
||||
IP6 string `json:"ip6,omitempty"`
|
||||
IPv4 json.RawMessage `json:"ipv4,omitempty"`
|
||||
IPv6 json.RawMessage `json:"ipv6,omitempty"`
|
||||
Network map[string]ContainerNetworkConfig `json:"network,omitempty"`
|
||||
DiskInfo map[string]ContainerDiskUsage `json:"diskinfo,omitempty"`
|
||||
RootFS string `json:"rootfs,omitempty"`
|
||||
}
|
||||
|
||||
// ContainerNetworkConfig captures basic container network status information.
|
||||
type ContainerNetworkConfig struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
HWAddr string `json:"hwaddr,omitempty"`
|
||||
Bridge string `json:"bridge,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
IP interface{} `json:"ip,omitempty"`
|
||||
IP6 interface{} `json:"ip6,omitempty"`
|
||||
IPv4 interface{} `json:"ipv4,omitempty"`
|
||||
IPv6 interface{} `json:"ipv6,omitempty"`
|
||||
Firewall interface{} `json:"firewall,omitempty"`
|
||||
Tag interface{} `json:"tag,omitempty"`
|
||||
}
|
||||
|
||||
// ContainerDiskUsage captures disk usage details returned by the LXC status API.
|
||||
type ContainerDiskUsage struct {
|
||||
Total uint64 `json:"total,omitempty"`
|
||||
Used uint64 `json:"used,omitempty"`
|
||||
}
|
||||
|
||||
// Storage represents a Proxmox VE storage
|
||||
|
|
|
|||
Loading…
Reference in a new issue