Add direct node fallback for storage polling
This commit is contained in:
parent
936894a5fe
commit
274c45799f
2 changed files with 84 additions and 0 deletions
|
|
@ -123,6 +123,18 @@ func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string {
|
|||
return baseName
|
||||
}
|
||||
|
||||
func (m *Monitor) getInstanceConfig(instanceName string) *config.PVEInstance {
|
||||
if m == nil || m.config == nil {
|
||||
return nil
|
||||
}
|
||||
for i := range m.config.PVEInstances {
|
||||
if strings.EqualFold(m.config.PVEInstances[i].Name, instanceName) {
|
||||
return &m.config.PVEInstances[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk {
|
||||
if len(disks) == 0 || len(nodes) == 0 {
|
||||
return disks
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/errors"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
|
|
@ -1122,6 +1123,8 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
|
|||
func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
startTime := time.Now()
|
||||
|
||||
instanceCfg := m.getInstanceConfig(instanceName)
|
||||
|
||||
// Get cluster storage configuration first (single call)
|
||||
clusterStorages, err := client.GetAllStorage(ctx)
|
||||
clusterStorageAvailable := err == nil
|
||||
|
|
@ -1207,6 +1210,25 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
|
|||
|
||||
// Fetch storage for this node
|
||||
nodeStorage, err := client.GetStorage(ctx, n.Node)
|
||||
if err != nil {
|
||||
if shouldAttemptFallback(err) {
|
||||
if fallbackStorage, ferr := m.fetchNodeStorageFallback(ctx, instanceCfg, n.Node); ferr == nil {
|
||||
log.Warn().
|
||||
Str("instance", instanceName).
|
||||
Str("node", n.Node).
|
||||
Err(err).
|
||||
Msg("Primary storage query failed; using direct node fallback")
|
||||
nodeStorage = fallbackStorage
|
||||
err = nil
|
||||
} else {
|
||||
log.Warn().
|
||||
Str("instance", instanceName).
|
||||
Str("node", n.Node).
|
||||
Err(ferr).
|
||||
Msg("Storage fallback to direct node query failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this
|
||||
if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") {
|
||||
|
|
@ -1545,3 +1567,53 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
|
|||
Msg("Parallel storage polling completed")
|
||||
}
|
||||
}
|
||||
|
||||
func shouldAttemptFallback(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context canceled")
|
||||
}
|
||||
|
||||
func (m *Monitor) fetchNodeStorageFallback(ctx context.Context, instanceCfg *config.PVEInstance, nodeName string) ([]proxmox.Storage, error) {
|
||||
if m == nil || instanceCfg == nil || !instanceCfg.IsCluster || len(instanceCfg.ClusterEndpoints) == 0 {
|
||||
return nil, fmt.Errorf("fallback unavailable")
|
||||
}
|
||||
|
||||
var target string
|
||||
hasFingerprint := strings.TrimSpace(instanceCfg.Fingerprint) != ""
|
||||
for _, ep := range instanceCfg.ClusterEndpoints {
|
||||
if !strings.EqualFold(ep.NodeName, nodeName) {
|
||||
continue
|
||||
}
|
||||
target = clusterEndpointEffectiveURL(ep, instanceCfg.VerifySSL, hasFingerprint)
|
||||
if target != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(target) == "" {
|
||||
return nil, fmt.Errorf("fallback endpoint missing for node %s", nodeName)
|
||||
}
|
||||
|
||||
cfg := proxmox.ClientConfig{
|
||||
Host: target,
|
||||
VerifySSL: instanceCfg.VerifySSL,
|
||||
Fingerprint: instanceCfg.Fingerprint,
|
||||
Timeout: m.pollTimeout,
|
||||
}
|
||||
if instanceCfg.TokenName != "" && instanceCfg.TokenValue != "" {
|
||||
cfg.TokenName = instanceCfg.TokenName
|
||||
cfg.TokenValue = instanceCfg.TokenValue
|
||||
} else {
|
||||
cfg.User = instanceCfg.User
|
||||
cfg.Password = instanceCfg.Password
|
||||
}
|
||||
|
||||
directClient, err := proxmox.NewClient(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return directClient.GetStorage(ctx, nodeName)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue