Add direct node fallback for storage polling

This commit is contained in:
rcourtman 2025-11-18 19:58:38 +00:00
parent 936894a5fe
commit 274c45799f
2 changed files with 84 additions and 0 deletions

View file

@ -123,6 +123,18 @@ func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string {
return baseName
}
func (m *Monitor) getInstanceConfig(instanceName string) *config.PVEInstance {
if m == nil || m.config == nil {
return nil
}
for i := range m.config.PVEInstances {
if strings.EqualFold(m.config.PVEInstances[i].Name, instanceName) {
return &m.config.PVEInstances[i]
}
}
return nil
}
func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk {
if len(disks) == 0 || len(nodes) == 0 {
return disks

View file

@ -10,6 +10,7 @@ import (
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/errors"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
@ -1122,6 +1123,8 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
startTime := time.Now()
instanceCfg := m.getInstanceConfig(instanceName)
// Get cluster storage configuration first (single call)
clusterStorages, err := client.GetAllStorage(ctx)
clusterStorageAvailable := err == nil
@ -1207,6 +1210,25 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
// Fetch storage for this node
nodeStorage, err := client.GetStorage(ctx, n.Node)
if err != nil {
if shouldAttemptFallback(err) {
if fallbackStorage, ferr := m.fetchNodeStorageFallback(ctx, instanceCfg, n.Node); ferr == nil {
log.Warn().
Str("instance", instanceName).
Str("node", n.Node).
Err(err).
Msg("Primary storage query failed; using direct node fallback")
nodeStorage = fallbackStorage
err = nil
} else {
log.Warn().
Str("instance", instanceName).
Str("node", n.Node).
Err(ferr).
Msg("Storage fallback to direct node query failed")
}
}
}
if err != nil {
// Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this
if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") {
@ -1545,3 +1567,53 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
Msg("Parallel storage polling completed")
}
}
func shouldAttemptFallback(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context canceled")
}
func (m *Monitor) fetchNodeStorageFallback(ctx context.Context, instanceCfg *config.PVEInstance, nodeName string) ([]proxmox.Storage, error) {
if m == nil || instanceCfg == nil || !instanceCfg.IsCluster || len(instanceCfg.ClusterEndpoints) == 0 {
return nil, fmt.Errorf("fallback unavailable")
}
var target string
hasFingerprint := strings.TrimSpace(instanceCfg.Fingerprint) != ""
for _, ep := range instanceCfg.ClusterEndpoints {
if !strings.EqualFold(ep.NodeName, nodeName) {
continue
}
target = clusterEndpointEffectiveURL(ep, instanceCfg.VerifySSL, hasFingerprint)
if target != "" {
break
}
}
if strings.TrimSpace(target) == "" {
return nil, fmt.Errorf("fallback endpoint missing for node %s", nodeName)
}
cfg := proxmox.ClientConfig{
Host: target,
VerifySSL: instanceCfg.VerifySSL,
Fingerprint: instanceCfg.Fingerprint,
Timeout: m.pollTimeout,
}
if instanceCfg.TokenName != "" && instanceCfg.TokenValue != "" {
cfg.TokenName = instanceCfg.TokenName
cfg.TokenValue = instanceCfg.TokenValue
} else {
cfg.User = instanceCfg.User
cfg.Password = instanceCfg.Password
}
directClient, err := proxmox.NewClient(cfg)
if err != nil {
return nil, err
}
return directClient.GetStorage(ctx, nodeName)
}