Add direct node fallback for storage polling
This commit is contained in:
parent
936894a5fe
commit
274c45799f
2 changed files with 84 additions and 0 deletions
|
|
@ -123,6 +123,18 @@ func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string {
|
||||||
return baseName
|
return baseName
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) getInstanceConfig(instanceName string) *config.PVEInstance {
|
||||||
|
if m == nil || m.config == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i := range m.config.PVEInstances {
|
||||||
|
if strings.EqualFold(m.config.PVEInstances[i].Name, instanceName) {
|
||||||
|
return &m.config.PVEInstances[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk {
|
func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk {
|
||||||
if len(disks) == 0 || len(nodes) == 0 {
|
if len(disks) == 0 || len(nodes) == 0 {
|
||||||
return disks
|
return disks
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/errors"
|
"github.com/rcourtman/pulse-go-rewrite/internal/errors"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
|
|
@ -1122,6 +1123,8 @@ func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName stri
|
||||||
func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
|
instanceCfg := m.getInstanceConfig(instanceName)
|
||||||
|
|
||||||
// Get cluster storage configuration first (single call)
|
// Get cluster storage configuration first (single call)
|
||||||
clusterStorages, err := client.GetAllStorage(ctx)
|
clusterStorages, err := client.GetAllStorage(ctx)
|
||||||
clusterStorageAvailable := err == nil
|
clusterStorageAvailable := err == nil
|
||||||
|
|
@ -1207,6 +1210,25 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
|
||||||
|
|
||||||
// Fetch storage for this node
|
// Fetch storage for this node
|
||||||
nodeStorage, err := client.GetStorage(ctx, n.Node)
|
nodeStorage, err := client.GetStorage(ctx, n.Node)
|
||||||
|
if err != nil {
|
||||||
|
if shouldAttemptFallback(err) {
|
||||||
|
if fallbackStorage, ferr := m.fetchNodeStorageFallback(ctx, instanceCfg, n.Node); ferr == nil {
|
||||||
|
log.Warn().
|
||||||
|
Str("instance", instanceName).
|
||||||
|
Str("node", n.Node).
|
||||||
|
Err(err).
|
||||||
|
Msg("Primary storage query failed; using direct node fallback")
|
||||||
|
nodeStorage = fallbackStorage
|
||||||
|
err = nil
|
||||||
|
} else {
|
||||||
|
log.Warn().
|
||||||
|
Str("instance", instanceName).
|
||||||
|
Str("node", n.Node).
|
||||||
|
Err(ferr).
|
||||||
|
Msg("Storage fallback to direct node query failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this
|
// Handle timeout gracefully - unavailable storage (e.g., NFS mounts) can cause this
|
||||||
if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") {
|
if strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "deadline exceeded") {
|
||||||
|
|
@ -1545,3 +1567,53 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
|
||||||
Msg("Parallel storage polling completed")
|
Msg("Parallel storage polling completed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func shouldAttemptFallback(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
msg := strings.ToLower(err.Error())
|
||||||
|
return strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context canceled")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Monitor) fetchNodeStorageFallback(ctx context.Context, instanceCfg *config.PVEInstance, nodeName string) ([]proxmox.Storage, error) {
|
||||||
|
if m == nil || instanceCfg == nil || !instanceCfg.IsCluster || len(instanceCfg.ClusterEndpoints) == 0 {
|
||||||
|
return nil, fmt.Errorf("fallback unavailable")
|
||||||
|
}
|
||||||
|
|
||||||
|
var target string
|
||||||
|
hasFingerprint := strings.TrimSpace(instanceCfg.Fingerprint) != ""
|
||||||
|
for _, ep := range instanceCfg.ClusterEndpoints {
|
||||||
|
if !strings.EqualFold(ep.NodeName, nodeName) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target = clusterEndpointEffectiveURL(ep, instanceCfg.VerifySSL, hasFingerprint)
|
||||||
|
if target != "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(target) == "" {
|
||||||
|
return nil, fmt.Errorf("fallback endpoint missing for node %s", nodeName)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := proxmox.ClientConfig{
|
||||||
|
Host: target,
|
||||||
|
VerifySSL: instanceCfg.VerifySSL,
|
||||||
|
Fingerprint: instanceCfg.Fingerprint,
|
||||||
|
Timeout: m.pollTimeout,
|
||||||
|
}
|
||||||
|
if instanceCfg.TokenName != "" && instanceCfg.TokenValue != "" {
|
||||||
|
cfg.TokenName = instanceCfg.TokenName
|
||||||
|
cfg.TokenValue = instanceCfg.TokenValue
|
||||||
|
} else {
|
||||||
|
cfg.User = instanceCfg.User
|
||||||
|
cfg.Password = instanceCfg.Password
|
||||||
|
}
|
||||||
|
|
||||||
|
directClient, err := proxmox.NewClient(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return directClient.GetStorage(ctx, nodeName)
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue