Handle PVE portless fallback when default port fails
This commit is contained in:
parent
5dbaf7c596
commit
8606bb47cf
2 changed files with 126 additions and 9 deletions
|
|
@ -15,6 +15,15 @@ const (
|
||||||
defaultPBSPort = "8007"
|
defaultPBSPort = "8007"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// DefaultPVEPort is the standard API port for Proxmox VE.
|
||||||
|
DefaultPVEPort = defaultPVEPort
|
||||||
|
// DefaultPBSPort is the standard API port for Proxmox Backup Server.
|
||||||
|
DefaultPBSPort = defaultPBSPort
|
||||||
|
// DefaultPMGPort reuses the PVE API port.
|
||||||
|
DefaultPMGPort = defaultPVEPort
|
||||||
|
)
|
||||||
|
|
||||||
// normalizeHostPort ensures we always have a scheme and explicit port when talking to
|
// normalizeHostPort ensures we always have a scheme and explicit port when talking to
|
||||||
// Proxmox APIs. It preserves existing ports/schemes and strips any path/query segments.
|
// Proxmox APIs. It preserves existing ports/schemes and strips any path/query segments.
|
||||||
func normalizeHostPort(host, defaultPort string) string {
|
func normalizeHostPort(host, defaultPort string) string {
|
||||||
|
|
@ -48,13 +57,28 @@ func normalizeHostPort(host, defaultPort string) string {
|
||||||
|
|
||||||
// CreateProxmoxConfig creates a proxmox.ClientConfig from a PVEInstance
|
// CreateProxmoxConfig creates a proxmox.ClientConfig from a PVEInstance
|
||||||
func CreateProxmoxConfig(node *PVEInstance) proxmox.ClientConfig {
|
func CreateProxmoxConfig(node *PVEInstance) proxmox.ClientConfig {
|
||||||
|
return createProxmoxConfigWithHost(node, node.Host, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateProxmoxConfigWithHost builds a proxmox.ClientConfig using an explicit host.
|
||||||
|
// When normalizeHost is true, the host will be normalized (scheme + default port);
|
||||||
|
// when false, the host is used verbatim (useful for fallback attempts).
|
||||||
|
func CreateProxmoxConfigWithHost(node *PVEInstance, host string, normalizeHost bool) proxmox.ClientConfig {
|
||||||
|
return createProxmoxConfigWithHost(node, host, normalizeHost)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createProxmoxConfigWithHost(node *PVEInstance, host string, normalizeHost bool) proxmox.ClientConfig {
|
||||||
user := node.User
|
user := node.User
|
||||||
if node.TokenName == "" && node.TokenValue == "" && user != "" && !strings.Contains(user, "@") {
|
if node.TokenName == "" && node.TokenValue == "" && user != "" && !strings.Contains(user, "@") {
|
||||||
user = user + "@pam"
|
user = user + "@pam"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if normalizeHost {
|
||||||
|
host = normalizeHostPort(host, defaultPVEPort)
|
||||||
|
}
|
||||||
|
|
||||||
return proxmox.ClientConfig{
|
return proxmox.ClientConfig{
|
||||||
Host: normalizeHostPort(node.Host, defaultPVEPort),
|
Host: host,
|
||||||
User: user,
|
User: user,
|
||||||
Password: node.Password,
|
Password: node.Password,
|
||||||
TokenName: node.TokenName,
|
TokenName: node.TokenName,
|
||||||
|
|
@ -132,3 +156,32 @@ func CreatePMGConfigFromFields(host, user, password, tokenName, tokenValue, fing
|
||||||
Fingerprint: fingerprint,
|
Fingerprint: fingerprint,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StripDefaultPort removes the supplied defaultPort from a host URL if it matches.
|
||||||
|
// Useful for retrying portless endpoints (e.g., reverse proxies on 443) when the
|
||||||
|
// normalized :8006/:8007 host is unreachable.
|
||||||
|
func StripDefaultPort(host, defaultPort string) string {
|
||||||
|
trimmed := strings.TrimSpace(host)
|
||||||
|
if trimmed == "" || defaultPort == "" {
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
candidate := trimmed
|
||||||
|
if !strings.HasPrefix(candidate, "http://") && !strings.HasPrefix(candidate, "https://") {
|
||||||
|
candidate = "https://" + candidate
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(candidate)
|
||||||
|
if err != nil || parsed.Host == "" {
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
if parsed.Port() != defaultPort {
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed.Host = parsed.Hostname()
|
||||||
|
parsed.RawQuery = ""
|
||||||
|
parsed.Fragment = ""
|
||||||
|
return parsed.Scheme + "://" + parsed.Host + strings.TrimSuffix(parsed.Path, "/")
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -5478,6 +5478,65 @@ func isTransientError(err error) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func shouldTryPortlessFallback(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
msg := strings.ToLower(err.Error())
|
||||||
|
if strings.Contains(msg, "connection refused") ||
|
||||||
|
strings.Contains(msg, "connection reset") ||
|
||||||
|
strings.Contains(msg, "no such host") ||
|
||||||
|
strings.Contains(msg, "client.timeout exceeded") ||
|
||||||
|
strings.Contains(msg, "i/o timeout") ||
|
||||||
|
strings.Contains(msg, "context deadline exceeded") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryPVEPortFallback handles the case where a normalized :8006 host is unreachable
|
||||||
|
// because the actual endpoint is fronted by a reverse proxy on 443. If the initial
|
||||||
|
// GetNodes call fails with a connection error and the host has the default PVE port,
|
||||||
|
// retry without the default port to hit the proxy. On success, swap the client so
|
||||||
|
// subsequent polls reuse the working endpoint.
|
||||||
|
func (m *Monitor) retryPVEPortFallback(ctx context.Context, instanceName string, instanceCfg *config.PVEInstance, currentClient PVEClientInterface, cause error) ([]proxmox.Node, PVEClientInterface, error) {
|
||||||
|
if instanceCfg == nil || !shouldTryPortlessFallback(cause) {
|
||||||
|
return nil, currentClient, cause
|
||||||
|
}
|
||||||
|
|
||||||
|
fallbackHost := config.StripDefaultPort(instanceCfg.Host, config.DefaultPVEPort)
|
||||||
|
if fallbackHost == "" || fallbackHost == instanceCfg.Host {
|
||||||
|
return nil, currentClient, cause
|
||||||
|
}
|
||||||
|
|
||||||
|
clientCfg := config.CreateProxmoxConfigWithHost(instanceCfg, fallbackHost, false)
|
||||||
|
if clientCfg.Timeout <= 0 {
|
||||||
|
clientCfg.Timeout = m.config.ConnectionTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
fallbackClient, err := proxmox.NewClient(clientCfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, currentClient, cause
|
||||||
|
}
|
||||||
|
|
||||||
|
fallbackNodes, err := fallbackClient.GetNodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, currentClient, cause
|
||||||
|
}
|
||||||
|
|
||||||
|
// Switch to the working host for the remainder of the poll (and future polls)
|
||||||
|
primaryHost := instanceCfg.Host
|
||||||
|
instanceCfg.Host = fallbackHost
|
||||||
|
m.pveClients[instanceName] = fallbackClient
|
||||||
|
log.Warn().
|
||||||
|
Str("instance", instanceName).
|
||||||
|
Str("primary", primaryHost).
|
||||||
|
Str("fallback", fallbackHost).
|
||||||
|
Msg("Primary PVE host failed; using fallback without default port")
|
||||||
|
|
||||||
|
return fallbackNodes, fallbackClient, nil
|
||||||
|
}
|
||||||
|
|
||||||
// pollPVEInstance polls a single PVE instance
|
// pollPVEInstance polls a single PVE instance
|
||||||
func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, client PVEClientInterface) {
|
func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, client PVEClientInterface) {
|
||||||
defer recoverFromPanic(fmt.Sprintf("pollPVEInstance-%s", instanceName))
|
defer recoverFromPanic(fmt.Sprintf("pollPVEInstance-%s", instanceName))
|
||||||
|
|
@ -5541,16 +5600,21 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
|
||||||
// Poll nodes
|
// Poll nodes
|
||||||
nodes, err := client.GetNodes(ctx)
|
nodes, err := client.GetNodes(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
monErr := errors.WrapConnectionError("poll_nodes", instanceName, err)
|
if fallbackNodes, fallbackClient, fallbackErr := m.retryPVEPortFallback(ctx, instanceName, instanceCfg, client, err); fallbackErr == nil {
|
||||||
pollErr = monErr
|
client = fallbackClient
|
||||||
log.Error().Err(monErr).Str("instance", instanceName).Msg("Failed to get nodes")
|
nodes = fallbackNodes
|
||||||
m.state.SetConnectionHealth(instanceName, false)
|
} else {
|
||||||
|
monErr := errors.WrapConnectionError("poll_nodes", instanceName, err)
|
||||||
|
pollErr = monErr
|
||||||
|
log.Error().Err(monErr).Str("instance", instanceName).Msg("Failed to get nodes")
|
||||||
|
m.state.SetConnectionHealth(instanceName, false)
|
||||||
|
|
||||||
// Track auth failure if it's an authentication error
|
// Track auth failure if it's an authentication error
|
||||||
if errors.IsAuthError(err) {
|
if errors.IsAuthError(err) {
|
||||||
m.recordAuthFailure(instanceName, "pve")
|
m.recordAuthFailure(instanceName, "pve")
|
||||||
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset auth failures on successful connection
|
// Reset auth failures on successful connection
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue