danvics/Pulse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix guest agent OS info calls causing OpenBSD VM crashes (related to #692)

Browse source 
Add defensive mitigation to prevent repeated guest-get-osinfo calls that
trigger buggy behavior in QEMU guest agent 9.0.2 on OpenBSD 7.6.

The issue: OpenBSD doesn't have /etc/os-release (Linux convention), and
qemu-ga 9.0.2 appears to spawn excessive helper processes trying to read
this file whenever guest-get-osinfo is called. These helpers don't clean
up properly, eventually exhausting the process table and crashing the VM.

The fix: Track consecutive OS info failures per VM. After 3 failures,
automatically skip future guest-get-osinfo calls for that VM while
continuing to fetch other guest agent data (network interfaces, version).
This prevents triggering the buggy code path while maintaining most guest
agent functionality.

The counter resets on success, so if the guest agent is upgraded or the
issue is resolved, Pulse will automatically resume OS info collection.

Related to #692
This commit is contained in:
rcourtman 2025-11-11 22:27:22 +00:00
parent 135b378820
commit 3e90737448
1 changed files with 57 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
internal/monitoring/monitor.go
View file

@ -879,15 +879,20 @@ const (
defaultGuestAgentVersionTimeout = 10 * time.Second // GUEST_AGENT_VERSION_TIMEOUT defaultGuestAgentVersionTimeout = 10 * time.Second // GUEST_AGENT_VERSION_TIMEOUT
defaultGuestAgentRetries = 1 // GUEST_AGENT_RETRIES (0 = no retry, 1 = one retry) defaultGuestAgentRetries = 1 // GUEST_AGENT_RETRIES (0 = no retry, 1 = one retry)
defaultGuestAgentRetryDelay = 500 * time.Millisecond defaultGuestAgentRetryDelay = 500 * time.Millisecond
// Skip OS info calls after this many consecutive failures to avoid triggering buggy guest agents (refs #692)
guestAgentOSInfoFailureThreshold = 3
) )
type guestMetadataCacheEntry struct { type guestMetadataCacheEntry struct {
ipAddresses []string ipAddresses []string
networkInterfaces []models.GuestNetworkInterface networkInterfaces []models.GuestNetworkInterface
osName string osName string
osVersion string osVersion string
agentVersion string agentVersion string
fetchedAt time.Time fetchedAt time.Time
osInfoFailureCount int // Track consecutive OS info failures
osInfoSkip bool // Skip OS info calls after repeated failures (refs #692)
} }
type taskOutcome struct { type taskOutcome struct {
@ -2464,21 +2469,48 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
} }
// OS info with configurable timeout and retry (refs #592) // OS info with configurable timeout and retry (refs #592)
agentInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentOSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) { // Skip OS info calls if we've seen repeated failures (refs #692 - OpenBSD qemu-ga issue)
return client.GetVMAgentInfo(ctx, nodeName, vmid) osInfoFailureCount := cached.osInfoFailureCount
}) osInfoSkip := cached.osInfoSkip
if err != nil {
if !osInfoSkip {
agentInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentOSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
return client.GetVMAgentInfo(ctx, nodeName, vmid)
})
if err != nil {
osInfoFailureCount++
if osInfoFailureCount >= guestAgentOSInfoFailureThreshold {
osInfoSkip = true
log.Info().
Str("instance", instanceName).
Str("vm", vmName).
Int("vmid", vmid).
Int("failureCount", osInfoFailureCount).
Msg("Guest agent OS info consistently fails, skipping future calls to avoid triggering buggy guest agents")
} else {
log.Debug().
Str("instance", instanceName).
Str("vm", vmName).
Int("vmid", vmid).
Int("failureCount", osInfoFailureCount).
Err(err).
Msg("Guest agent OS info unavailable")
}
} else if agentInfo, ok := agentInfoRaw.(map[string]interface{}); ok && len(agentInfo) > 0 {
osName, osVersion = extractGuestOSInfo(agentInfo)
osInfoFailureCount = 0 // Reset on success
osInfoSkip = false
} else {
osName = ""
osVersion = ""
}
} else {
// Skipping OS info call due to repeated failures
log.Debug(). log.Debug().
Str("instance", instanceName). Str("instance", instanceName).
Str("vm", vmName). Str("vm", vmName).
Int("vmid", vmid). Int("vmid", vmid).
Err(err). Msg("Skipping guest agent OS info call (disabled after repeated failures)")
Msg("Guest agent OS info unavailable")
} else if agentInfo, ok := agentInfoRaw.(map[string]interface{}); ok && len(agentInfo) > 0 {
osName, osVersion = extractGuestOSInfo(agentInfo)
} else {
osName = ""
osVersion = ""
} }
// Agent version with configurable timeout and retry (refs #592) // Agent version with configurable timeout and retry (refs #592)
@ -2499,12 +2531,14 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
} }
entry := guestMetadataCacheEntry{ entry := guestMetadataCacheEntry{
ipAddresses: cloneStringSlice(ipAddresses), ipAddresses: cloneStringSlice(ipAddresses),
networkInterfaces: cloneGuestNetworkInterfaces(networkIfaces), networkInterfaces: cloneGuestNetworkInterfaces(networkIfaces),
osName: osName, osName: osName,
osVersion: osVersion, osVersion: osVersion,
agentVersion: agentVersion, agentVersion: agentVersion,
fetchedAt: time.Now(), fetchedAt: time.Now(),
osInfoFailureCount: osInfoFailureCount,
osInfoSkip: osInfoSkip,
} }
m.guestMetadataMu.Lock() m.guestMetadataMu.Lock()