danvics/Pulse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Related to #692: Skip unsupported guest OS info calls

Browse source
This commit is contained in:
rcourtman 2025-11-12 19:17:09 +00:00
parent b829d86d2c
commit 923293fbf7
2 changed files with 119 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
internal/monitoring/helpers_test.go
View file

@ -1,6 +1,7 @@
package monitoring package monitoring
import ( import (
"errors"
"fmt" "fmt"
"math" "math"
"testing" "testing"
@ -278,3 +279,41 @@ func TestConvertPoolInfoToModelNil(t *testing.T) {
t.Fatalf("expected nil result for nil input") t.Fatalf("expected nil result for nil input")
} }
} }
func TestIsGuestAgentOSInfoUnsupportedError(t *testing.T) {
t.Parallel()
cases := []struct {
name string
err error
want bool
}{
{name: "nil error", err: nil, want: false},
{name: "unrelated error", err: errors.New("guest agent timeout"), want: false},
{
name: "missing os-release path",
err: errors.New(`API error 500: {"errors":{"message":"guest agent command failed: Failed to open file '/etc/os-release': No such file or directory"}}`),
want: true,
},
{
name: "missing usr lib os-release",
err: errors.New("API error 500: guest agent command failed: Failed to open file '/usr/lib/os-release': No such file or directory"),
want: true,
},
{
name: "unsupported command",
err: errors.New("API error 500: unsupported command: guest-get-osinfo"),
want: true,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if got := isGuestAgentOSInfoUnsupportedError(tc.err); got != tc.want {
t.Fatalf("isGuestAgentOSInfoUnsupportedError(%v) = %v, want %v", tc.err, got, tc.want)
}
})
}
}

128
internal/monitoring/monitor.go
View file

@ -569,19 +569,19 @@ type Monitor struct {
guestMetadataRetryBackoff time.Duration guestMetadataRetryBackoff time.Duration
guestMetadataHoldDuration time.Duration guestMetadataHoldDuration time.Duration
// Configurable guest agent timeouts (refs #592) // Configurable guest agent timeouts (refs #592)
guestAgentFSInfoTimeout time.Duration guestAgentFSInfoTimeout time.Duration
guestAgentNetworkTimeout time.Duration guestAgentNetworkTimeout time.Duration
guestAgentOSInfoTimeout time.Duration guestAgentOSInfoTimeout time.Duration
guestAgentVersionTimeout time.Duration guestAgentVersionTimeout time.Duration
guestAgentRetries int guestAgentRetries int
executor PollExecutor executor PollExecutor
breakerBaseRetry time.Duration breakerBaseRetry time.Duration
breakerMaxDelay time.Duration breakerMaxDelay time.Duration
breakerHalfOpenWindow time.Duration breakerHalfOpenWindow time.Duration
instanceInfoCache map[string]*instanceInfo instanceInfoCache map[string]*instanceInfo
pollStatusMap map[string]*pollStatus pollStatusMap map[string]*pollStatus
dlqInsightMap map[string]*dlqInsight dlqInsightMap map[string]*dlqInsight
nodeLastOnline map[string]time.Time // Track last time each node was seen online (for grace period) nodeLastOnline map[string]time.Time // Track last time each node was seen online (for grace period)
} }
type rrdMemCacheEntry struct { type rrdMemCacheEntry struct {
@ -873,26 +873,26 @@ const (
// Guest agent timeout defaults (configurable via environment variables) // Guest agent timeout defaults (configurable via environment variables)
// Increased from 3-5s to 10-15s to handle high-load environments better (refs #592) // Increased from 3-5s to 10-15s to handle high-load environments better (refs #592)
defaultGuestAgentFSInfoTimeout = 15 * time.Second // GUEST_AGENT_FSINFO_TIMEOUT defaultGuestAgentFSInfoTimeout = 15 * time.Second // GUEST_AGENT_FSINFO_TIMEOUT
defaultGuestAgentNetworkTimeout = 10 * time.Second // GUEST_AGENT_NETWORK_TIMEOUT defaultGuestAgentNetworkTimeout = 10 * time.Second // GUEST_AGENT_NETWORK_TIMEOUT
defaultGuestAgentOSInfoTimeout = 10 * time.Second // GUEST_AGENT_OSINFO_TIMEOUT defaultGuestAgentOSInfoTimeout = 10 * time.Second // GUEST_AGENT_OSINFO_TIMEOUT
defaultGuestAgentVersionTimeout = 10 * time.Second // GUEST_AGENT_VERSION_TIMEOUT defaultGuestAgentVersionTimeout = 10 * time.Second // GUEST_AGENT_VERSION_TIMEOUT
defaultGuestAgentRetries = 1 // GUEST_AGENT_RETRIES (0 = no retry, 1 = one retry) defaultGuestAgentRetries = 1 // GUEST_AGENT_RETRIES (0 = no retry, 1 = one retry)
defaultGuestAgentRetryDelay = 500 * time.Millisecond defaultGuestAgentRetryDelay = 500 * time.Millisecond
// Skip OS info calls after this many consecutive failures to avoid triggering buggy guest agents (refs #692) // Skip OS info calls after this many consecutive failures to avoid triggering buggy guest agents (refs #692)
guestAgentOSInfoFailureThreshold = 3 guestAgentOSInfoFailureThreshold = 3
) )
type guestMetadataCacheEntry struct { type guestMetadataCacheEntry struct {
ipAddresses []string ipAddresses []string
networkInterfaces []models.GuestNetworkInterface networkInterfaces []models.GuestNetworkInterface
osName string osName string
osVersion string osVersion string
agentVersion string agentVersion string
fetchedAt time.Time fetchedAt time.Time
osInfoFailureCount int // Track consecutive OS info failures osInfoFailureCount int // Track consecutive OS info failures
osInfoSkip bool // Skip OS info calls after repeated failures (refs #692) osInfoSkip bool // Skip OS info calls after repeated failures (refs #692)
} }
type taskOutcome struct { type taskOutcome struct {
@ -2485,23 +2485,34 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
return client.GetVMAgentInfo(ctx, nodeName, vmid) return client.GetVMAgentInfo(ctx, nodeName, vmid)
}) })
if err != nil { if err != nil {
osInfoFailureCount++ if isGuestAgentOSInfoUnsupportedError(err) {
if osInfoFailureCount >= guestAgentOSInfoFailureThreshold {
osInfoSkip = true osInfoSkip = true
log.Info(). osInfoFailureCount = guestAgentOSInfoFailureThreshold
log.Warn().
Str("instance", instanceName). Str("instance", instanceName).
Str("vm", vmName). Str("vm", vmName).
Int("vmid", vmid). Int("vmid", vmid).
Int("failureCount", osInfoFailureCount).
Msg("Guest agent OS info consistently fails, skipping future calls to avoid triggering buggy guest agents")
} else {
log.Debug().
Str("instance", instanceName).
Str("vm", vmName).
Int("vmid", vmid).
Int("failureCount", osInfoFailureCount).
Err(err). Err(err).
Msg("Guest agent OS info unavailable") Msg("Guest agent OS info unsupported (missing os-release). Skipping future calls to avoid qemu-ga issues (refs #692)")
} else {
osInfoFailureCount++
if osInfoFailureCount >= guestAgentOSInfoFailureThreshold {
osInfoSkip = true
log.Info().
Str("instance", instanceName).
Str("vm", vmName).
Int("vmid", vmid).
Int("failureCount", osInfoFailureCount).
Msg("Guest agent OS info consistently fails, skipping future calls to avoid triggering buggy guest agents")
} else {
log.Debug().
Str("instance", instanceName).
Str("vm", vmName).
Int("vmid", vmid).
Int("failureCount", osInfoFailureCount).
Err(err).
Msg("Guest agent OS info unavailable")
}
} }
} else if agentInfo, ok := agentInfoRaw.(map[string]interface{}); ok && len(agentInfo) > 0 { } else if agentInfo, ok := agentInfoRaw.(map[string]interface{}); ok && len(agentInfo) > 0 {
osName, osVersion = extractGuestOSInfo(agentInfo) osName, osVersion = extractGuestOSInfo(agentInfo)
@ -2720,6 +2731,27 @@ func extractGuestOSInfo(data map[string]interface{}) (string, string) {
return osName, osVersion return osName, osVersion
} }
func isGuestAgentOSInfoUnsupportedError(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
// OpenBSD qemu-ga emits "Failed to open file '/etc/os-release'" (refs #692)
if strings.Contains(msg, "os-release") &&
(strings.Contains(msg, "failed to open file") || strings.Contains(msg, "no such file or directory")) {
return true
}
// Some Proxmox builds bubble up "unsupported command: guest-get-osinfo"
if strings.Contains(msg, "guest-get-osinfo") && strings.Contains(msg, "unsupported") {
return true
}
return false
}
func stringValue(val interface{}) string { func stringValue(val interface{}) string {
switch v := val.(type) { switch v := val.(type) {
case string: case string:
@ -5325,10 +5357,10 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
Free: int64(node.MaxDisk - node.Disk), Free: int64(node.MaxDisk - node.Disk),
Usage: safePercentage(float64(node.Disk), float64(node.MaxDisk)), Usage: safePercentage(float64(node.Disk), float64(node.MaxDisk)),
}, },
Uptime: int64(node.Uptime), Uptime: int64(node.Uptime),
LoadAverage: []float64{}, LoadAverage: []float64{},
LastSeen: time.Now(), LastSeen: time.Now(),
ConnectionHealth: connectionHealthStr, // Use the determined health status ConnectionHealth: connectionHealthStr, // Use the determined health status
IsClusterMember: instanceCfg.IsCluster, IsClusterMember: instanceCfg.IsCluster,
ClusterName: instanceCfg.ClusterName, ClusterName: instanceCfg.ClusterName,
TemperatureMonitoringEnabled: instanceCfg.TemperatureMonitoringEnabled, TemperatureMonitoringEnabled: instanceCfg.TemperatureMonitoringEnabled,
@ -7129,8 +7161,8 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
// If we got ZERO resources but had resources before, and we have no node data, // If we got ZERO resources but had resources before, and we have no node data,
// this likely means the cluster health check failed. Preserve everything. // this likely means the cluster health check failed. Preserve everything.
if len(allVMs) == 0 && len(allContainers) == 0 && if len(allVMs) == 0 && len(allContainers) == 0 &&
(prevVMCount > 0 || prevContainerCount > 0) && (prevVMCount > 0 || prevContainerCount > 0) &&
len(nodeEffectiveStatus) == 0 { len(nodeEffectiveStatus) == 0 {
log.Warn(). log.Warn().
Str("instance", instanceName). Str("instance", instanceName).
Int("prevVMs", prevVMCount). Int("prevVMs", prevVMCount).
@ -9096,9 +9128,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
log.Debug().Str("instance", instanceName).Msg("Polling PBS backups") log.Debug().Str("instance", instanceName).Msg("Polling PBS backups")
var allBackups []models.PBSBackup var allBackups []models.PBSBackup
datastoreCount := len(datastores) // Number of datastores to query datastoreCount := len(datastores) // Number of datastores to query
datastoreFetches := 0 // Number of successful datastore fetches datastoreFetches := 0 // Number of successful datastore fetches
datastoreErrors := 0 // Number of failed datastore fetches datastoreErrors := 0 // Number of failed datastore fetches
// Process each datastore // Process each datastore
for _, ds := range datastores { for _, ds := range datastores {