Add HTTP mode to pulse-sensor-proxy for multi-instance temperature monitoring

This implements HTTP/HTTPS support for pulse-sensor-proxy to enable
temperature monitoring across multiple separate Proxmox instances.

Architecture changes:
- Dual-mode operation: Unix socket (local) + HTTPS (remote)
- Unix socket remains default for security/performance (no breaking change)
- HTTP mode enables temps from external PVE hosts

Backend implementation:
- Add HTTPS server with TLS + Bearer token authentication to sensor-proxy
- Add TemperatureProxyURL and TemperatureProxyToken fields to PVEInstance
- Add HTTP client (internal/tempproxy/http_client.go) for remote proxy calls
- Update temperature collector to prefer HTTP proxy when configured
- Fallback logic: HTTP proxy → Unix socket → direct SSH (if not containerized)

Configuration:
- pulse-sensor-proxy config: http_enabled, http_listen_addr, http_tls_cert/key, http_auth_token
- PVEInstance config: temperature_proxy_url, temperature_proxy_token
- Environment variables: PULSE_SENSOR_PROXY_HTTP_* for all HTTP settings

Security:
- TLS 1.2+ with modern cipher suites
- Constant-time token comparison (timing attack prevention)
- Rate limiting applied to HTTP requests (shared with socket mode)
- Audit logging for all HTTP requests

Next steps:
- Update installer script to support HTTP mode + auto-registration
- Add Pulse API endpoint for proxy registration
- Generate TLS certificates during installation
- Test multi-instance temperature collection

Related to #571 (multi-instance architecture)
This commit is contained in:
rcourtman 2025-11-13 16:13:53 +00:00
parent ea80a6153c
commit 22f092f941
9 changed files with 619 additions and 2 deletions

View file

@ -316,3 +316,21 @@ func int64Ptr(v int64) *int64 {
value := v
return &value
}
// LogHTTPRequest logs HTTP requests for audit trail
func (a *auditLogger) LogHTTPRequest(remoteAddr, method, path string, statusCode int, reason string) {
if a == nil {
return
}
event := AuditEvent{
EventType: "http_request",
RemoteAddr: remoteAddr,
Command: method,
Target: path,
ExitCode: &statusCode,
Reason: reason,
}
a.log(&event)
}

View file

@ -37,6 +37,13 @@ type Config struct {
AllowedIDMapUsers []string `yaml:"allowed_idmap_users"`
RateLimit *RateLimitConfig `yaml:"rate_limit,omitempty"`
// HTTP mode configuration
HTTPEnabled bool `yaml:"http_enabled"` // Enable HTTP server mode
HTTPListenAddr string `yaml:"http_listen_addr"` // Address to listen on (e.g., ":8443")
HTTPTLSCertFile string `yaml:"http_tls_cert"` // Path to TLS certificate
HTTPTLSKeyFile string `yaml:"http_tls_key"` // Path to TLS private key
HTTPAuthToken string `yaml:"http_auth_token"` // Bearer token for authentication
}
// PeerConfig represents a peer entry with capabilities.
@ -272,6 +279,50 @@ func loadConfig(configPath string) (*Config, error) {
log.Info().Str("log_level", cfg.LogLevel).Msg("Log level set from environment")
}
// HTTP mode configuration from environment variables
if envHTTPEnabled := os.Getenv("PULSE_SENSOR_PROXY_HTTP_ENABLED"); envHTTPEnabled != "" {
if parsed, err := parseBool(envHTTPEnabled); err != nil {
log.Warn().Str("value", envHTTPEnabled).Err(err).Msg("Invalid PULSE_SENSOR_PROXY_HTTP_ENABLED value, ignoring")
} else {
cfg.HTTPEnabled = parsed
log.Info().Bool("http_enabled", parsed).Msg("HTTP mode enabled from environment")
}
}
if envHTTPAddr := os.Getenv("PULSE_SENSOR_PROXY_HTTP_ADDR"); envHTTPAddr != "" {
cfg.HTTPListenAddr = strings.TrimSpace(envHTTPAddr)
log.Info().Str("http_addr", cfg.HTTPListenAddr).Msg("HTTP listen address set from environment")
}
if envHTTPCert := os.Getenv("PULSE_SENSOR_PROXY_HTTP_TLS_CERT"); envHTTPCert != "" {
cfg.HTTPTLSCertFile = strings.TrimSpace(envHTTPCert)
log.Info().Str("tls_cert", cfg.HTTPTLSCertFile).Msg("HTTP TLS cert path set from environment")
}
if envHTTPKey := os.Getenv("PULSE_SENSOR_PROXY_HTTP_TLS_KEY"); envHTTPKey != "" {
cfg.HTTPTLSKeyFile = strings.TrimSpace(envHTTPKey)
log.Info().Str("tls_key", cfg.HTTPTLSKeyFile).Msg("HTTP TLS key path set from environment")
}
if envHTTPToken := os.Getenv("PULSE_SENSOR_PROXY_HTTP_AUTH_TOKEN"); envHTTPToken != "" {
cfg.HTTPAuthToken = strings.TrimSpace(envHTTPToken)
log.Info().Msg("HTTP auth token set from environment")
}
// Validate HTTP configuration if enabled
if cfg.HTTPEnabled {
if cfg.HTTPListenAddr == "" {
cfg.HTTPListenAddr = ":8443" // Default port
log.Info().Str("http_addr", cfg.HTTPListenAddr).Msg("Using default HTTP listen address")
}
if cfg.HTTPAuthToken == "" {
return nil, fmt.Errorf("http_enabled=true requires http_auth_token to be configured")
}
if cfg.HTTPTLSCertFile == "" || cfg.HTTPTLSKeyFile == "" {
return nil, fmt.Errorf("http_enabled=true requires both http_tls_cert and http_tls_key")
}
}
return cfg, nil
}

View file

@ -0,0 +1,308 @@
package main
import (
"context"
"crypto/subtle"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"strings"
"time"
"github.com/rs/zerolog/log"
)
// HTTPServer provides HTTP/HTTPS access to temperature data
type HTTPServer struct {
proxy *Proxy
server *http.Server
config *Config
}
// NewHTTPServer creates a new HTTP server for the proxy
func NewHTTPServer(proxy *Proxy, config *Config) *HTTPServer {
return &HTTPServer{
proxy: proxy,
config: config,
}
}
// Start starts the HTTP server with TLS
func (h *HTTPServer) Start() error {
if !h.config.HTTPEnabled {
return nil
}
// Validate TLS certificate and key exist
if h.config.HTTPTLSCertFile == "" || h.config.HTTPTLSKeyFile == "" {
return fmt.Errorf("TLS cert and key required for HTTP mode")
}
mux := http.NewServeMux()
// Register endpoints
mux.HandleFunc("/temps", h.handleTemperature)
mux.HandleFunc("/health", h.handleHealth)
// Create TLS config with modern security settings
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
PreferServerCipherSuites: true,
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
},
}
h.server = &http.Server{
Addr: h.config.HTTPListenAddr,
Handler: h.rateLimitMiddleware(h.authMiddleware(mux)),
TLSConfig: tlsConfig,
ReadTimeout: h.config.ReadTimeout,
WriteTimeout: h.config.WriteTimeout,
IdleTimeout: 120 * time.Second,
MaxHeaderBytes: 1 << 20, // 1 MB
}
log.Info().
Str("addr", h.config.HTTPListenAddr).
Str("cert", h.config.HTTPTLSCertFile).
Msg("Starting HTTPS server")
go func() {
if err := h.server.ListenAndServeTLS(h.config.HTTPTLSCertFile, h.config.HTTPTLSKeyFile); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("HTTPS server failed")
}
}()
return nil
}
// Stop gracefully shuts down the HTTP server
func (h *HTTPServer) Stop(ctx context.Context) error {
if h.server == nil {
return nil
}
log.Info().Msg("Shutting down HTTPS server")
return h.server.Shutdown(ctx)
}
// authMiddleware validates Bearer token authentication
func (h *HTTPServer) authMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Extract Authorization header
authHeader := r.Header.Get("Authorization")
if authHeader == "" {
h.sendJSONError(w, http.StatusUnauthorized, "missing authorization header")
if h.proxy.audit != nil {
h.proxy.audit.LogHTTPRequest(r.RemoteAddr, r.Method, r.URL.Path, http.StatusUnauthorized, "missing_auth_header")
}
return
}
// Check Bearer token format
parts := strings.SplitN(authHeader, " ", 2)
if len(parts) != 2 || parts[0] != "Bearer" {
h.sendJSONError(w, http.StatusUnauthorized, "invalid authorization format")
if h.proxy.audit != nil {
h.proxy.audit.LogHTTPRequest(r.RemoteAddr, r.Method, r.URL.Path, http.StatusUnauthorized, "invalid_auth_format")
}
return
}
// Constant-time token comparison to prevent timing attacks
providedToken := parts[1]
if subtle.ConstantTimeCompare([]byte(providedToken), []byte(h.config.HTTPAuthToken)) != 1 {
h.sendJSONError(w, http.StatusUnauthorized, "invalid token")
if h.proxy.audit != nil {
h.proxy.audit.LogHTTPRequest(r.RemoteAddr, r.Method, r.URL.Path, http.StatusUnauthorized, "invalid_token")
}
return
}
// Token valid, proceed to next handler
next.ServeHTTP(w, r)
})
}
// rateLimitMiddleware applies rate limiting per client IP
func (h *HTTPServer) rateLimitMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Extract client IP
clientIP, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
clientIP = r.RemoteAddr
}
// Create synthetic peer credentials for rate limiting
// Use IP hash as UID for HTTP clients
peerCred := &peerCredentials{
uid: hashIPToUID(clientIP),
gid: 0,
pid: 0,
}
if h.proxy.rateLimiter == nil {
h.sendJSONError(w, http.StatusServiceUnavailable, "rate limiter not available")
return
}
// Check rate limit
peer := h.proxy.rateLimiter.identifyPeer(peerCred)
peerLabel := peer.String()
releaseLimiter, limitReason, allowed := h.proxy.rateLimiter.allow(peer)
if !allowed {
log.Warn().
Str("client_ip", clientIP).
Str("reason", limitReason).
Msg("HTTP rate limit exceeded")
if h.proxy.audit != nil {
h.proxy.audit.LogHTTPRequest(r.RemoteAddr, r.Method, r.URL.Path, http.StatusTooManyRequests, "rate_limit_"+limitReason)
}
h.sendJSONError(w, http.StatusTooManyRequests, "rate limit exceeded")
return
}
defer func() {
if releaseLimiter != nil {
releaseLimiter()
}
}()
// Apply penalty if handler returns error
releaseFn := releaseLimiter
applyPenalty := func(reason string) {
if releaseFn != nil {
releaseFn()
releaseFn = nil
}
h.proxy.rateLimiter.penalize(peerLabel, reason)
}
// Wrap response writer to detect errors
wrappedWriter := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
next.ServeHTTP(wrappedWriter, r)
// Apply penalty for errors
if wrappedWriter.statusCode >= 400 && wrappedWriter.statusCode != http.StatusTooManyRequests {
applyPenalty("http_error")
}
})
}
// handleTemperature handles GET /temps?node=<nodename>
func (h *HTTPServer) handleTemperature(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
h.sendJSONError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
// Extract node parameter
nodeName := r.URL.Query().Get("node")
if nodeName == "" {
h.sendJSONError(w, http.StatusBadRequest, "missing 'node' query parameter")
return
}
// Validate node name
nodeName = strings.TrimSpace(nodeName)
if err := validateNodeName(nodeName); err != nil {
h.sendJSONError(w, http.StatusBadRequest, "invalid node name format")
return
}
// Validate node against allowlist
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
if h.proxy.nodeValidator != nil {
if err := h.proxy.nodeValidator.Validate(ctx, nodeName); err != nil {
log.Warn().Err(err).Str("node", nodeName).Msg("Node validation failed")
h.sendJSONError(w, http.StatusForbidden, "node not allowed")
return
}
}
// Acquire per-node concurrency lock
releaseNode := h.proxy.nodeGate.acquire(nodeName)
defer releaseNode()
// Fetch temperature data via SSH
log.Debug().Str("node", nodeName).Msg("Fetching temperature via SSH (HTTP request)")
tempData, err := h.proxy.getTemperatureViaSSH(nodeName)
if err != nil {
log.Warn().Err(err).Str("node", nodeName).Msg("Failed to get temperatures via SSH")
h.sendJSONError(w, http.StatusInternalServerError, fmt.Sprintf("failed to get temperatures: %v", err))
return
}
// Return temperature data as JSON
response := map[string]interface{}{
"node": nodeName,
"temperature": tempData,
}
log.Info().Str("node", nodeName).Msg("Temperature data fetched successfully via HTTP")
h.sendJSON(w, http.StatusOK, response)
if h.proxy.audit != nil {
h.proxy.audit.LogHTTPRequest(r.RemoteAddr, r.Method, r.URL.Path, http.StatusOK, "temperature_success")
}
}
// handleHealth handles GET /health
func (h *HTTPServer) handleHealth(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
h.sendJSONError(w, http.StatusMethodNotAllowed, "method not allowed")
return
}
response := map[string]interface{}{
"status": "ok",
"version": Version,
}
h.sendJSON(w, http.StatusOK, response)
}
// sendJSON sends a JSON response
func (h *HTTPServer) sendJSON(w http.ResponseWriter, statusCode int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(statusCode)
if err := json.NewEncoder(w).Encode(data); err != nil {
log.Error().Err(err).Msg("Failed to encode JSON response")
}
}
// sendJSONError sends a JSON error response
func (h *HTTPServer) sendJSONError(w http.ResponseWriter, statusCode int, message string) {
h.sendJSON(w, statusCode, map[string]interface{}{
"error": message,
})
}
// hashIPToUID creates a deterministic UID from an IP address for rate limiting
func hashIPToUID(ip string) uint32 {
// Simple hash function: sum of byte values
var hash uint32
for i := 0; i < len(ip); i++ {
hash = hash*31 + uint32(ip[i])
}
// Ensure it's in a reasonable range for UID
return 100000 + (hash % 900000)
}
// responseWriter wraps http.ResponseWriter to capture status code
type responseWriter struct {
http.ResponseWriter
statusCode int
}
func (rw *responseWriter) WriteHeader(code int) {
rw.statusCode = code
rw.ResponseWriter.WriteHeader(code)
}

View file

@ -432,6 +432,15 @@ func runProxy() {
log.Fatal().Err(err).Msg("Failed to start proxy")
}
// Start HTTP server if enabled
var httpServer *HTTPServer
if cfg.HTTPEnabled {
httpServer = NewHTTPServer(proxy, cfg)
if err := httpServer.Start(); err != nil {
log.Fatal().Err(err).Msg("Failed to start HTTP server")
}
}
// Start metrics server
if err := metrics.Start(cfg.MetricsAddress); err != nil {
log.Fatal().Err(err).Msg("Failed to start metrics server")
@ -443,6 +452,16 @@ func runProxy() {
<-sigChan
log.Info().Msg("Shutting down proxy...")
// Shutdown HTTP server if running
if httpServer != nil {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := httpServer.Stop(shutdownCtx); err != nil {
log.Error().Err(err).Msg("Error shutting down HTTP server")
}
}
proxy.Stop()
if proxy.rateLimiter != nil {
proxy.rateLimiter.shutdown()

View file

@ -66,6 +66,8 @@ export interface NodeInstance {
*/
export interface PVENodeConfig extends NodeInstance {
realm?: string; // Authentication realm (pam, pve, etc.)
temperatureProxyURL?: string; // Optional HTTPS URL to pulse-sensor-proxy
temperatureProxyToken?: string; // Bearer token for proxy authentication
}
/**

View file

@ -406,6 +406,10 @@ type PVEInstance struct {
TemperatureMonitoringEnabled *bool // Monitor temperature via SSH (nil = use global setting, true/false = override)
SSHPort int // SSH port for temperature monitoring (0 = use global default)
// Temperature proxy configuration (for external PVE hosts)
TemperatureProxyURL string // Optional HTTPS URL to pulse-sensor-proxy (e.g., https://pve1.lan:8443)
TemperatureProxyToken string // Bearer token for proxy authentication
// Cluster support
IsCluster bool // True if this is a cluster
ClusterName string // Cluster name if applicable

View file

@ -5764,7 +5764,8 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie
sshHost = node.Node
}
temp, err := m.tempCollector.CollectTemperature(tempCtx, sshHost, node.Node)
// Use HTTP proxy if configured for this instance, otherwise fall back to socket/SSH
temp, err := m.tempCollector.CollectTemperatureWithProxy(tempCtx, sshHost, node.Node, instanceCfg.TemperatureProxyURL, instanceCfg.TemperatureProxyToken)
tempCancel()
if err == nil && temp != nil && temp.Available {

View file

@ -91,13 +91,39 @@ func NewTemperatureCollectorWithPort(sshUser, sshKeyPath string, sshPort int) *T
// CollectTemperature collects temperature data from a node via SSH
func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost, nodeName string) (*models.Temperature, error) {
return tc.CollectTemperatureWithProxy(ctx, nodeHost, nodeName, "", "")
}
// CollectTemperatureWithProxy collects temperature data with optional HTTP proxy configuration
func (tc *TemperatureCollector) CollectTemperatureWithProxy(ctx context.Context, nodeHost, nodeName, proxyURL, proxyToken string) (*models.Temperature, error) {
// Extract hostname/IP from the host URL (might be https://hostname:8006)
host := extractHostname(nodeHost)
var output string
var err error
// Use proxy if available, otherwise fall back to direct SSH
// Try HTTP proxy first if configured for this instance
if proxyURL != "" && proxyToken != "" {
httpClient := tempproxy.NewHTTPClient(proxyURL, proxyToken)
if httpClient.IsAvailable() {
output, err = httpClient.GetTemperature(host)
if err != nil {
log.Debug().
Str("node", nodeName).
Str("host", host).
Str("proxy_url", proxyURL).
Err(err).
Msg("Failed to collect temperature data via HTTP proxy")
// Don't fall back to socket/SSH for HTTP proxy failures
// If HTTP proxy is configured, it's the intended method
return &models.Temperature{Available: false}, nil
}
// HTTP proxy succeeded
goto parseOutput
}
}
// Use Unix socket proxy if available (local deployment)
if tc.isProxyEnabled() {
output, err = tc.proxyClient.GetTemperature(host)
if err != nil {
@ -176,6 +202,7 @@ func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost
}
}
parseOutput:
// Parse sensors JSON output
temp, err := tc.parseSensorsJSON(output)
if err != nil {

View file

@ -0,0 +1,187 @@
package tempproxy
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// HTTPClient communicates with pulse-sensor-proxy via HTTPS
type HTTPClient struct {
baseURL string
authToken string
httpClient *http.Client
timeout time.Duration
}
// NewHTTPClient creates a new HTTP-based proxy client
func NewHTTPClient(baseURL, authToken string) *HTTPClient {
// Create HTTP client with TLS and reasonable timeouts
httpClient := &http.Client{
Timeout: defaultTimeout,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
DisableCompression: false,
DisableKeepAlives: false,
MaxIdleConnsPerHost: 2,
},
}
return &HTTPClient{
baseURL: strings.TrimSuffix(baseURL, "/"),
authToken: authToken,
httpClient: httpClient,
timeout: defaultTimeout,
}
}
// IsAvailable checks if the HTTP proxy is accessible
// For HTTP mode, we consider it available if URL and token are configured
func (c *HTTPClient) IsAvailable() bool {
return c.baseURL != "" && c.authToken != ""
}
// GetTemperature fetches temperature data from a node via HTTP
func (c *HTTPClient) GetTemperature(nodeHost string) (string, error) {
if !c.IsAvailable() {
return "", &ProxyError{
Type: ErrorTypeTransport,
Message: "HTTP proxy not configured",
Retryable: false,
}
}
// Build request URL
reqURL := fmt.Sprintf("%s/temps?node=%s", c.baseURL, url.QueryEscape(nodeHost))
// Create request
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
if err != nil {
return "", &ProxyError{
Type: ErrorTypeTransport,
Message: "failed to create HTTP request",
Retryable: false,
Wrapped: err,
}
}
// Add authorization header
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.authToken))
req.Header.Set("Accept", "application/json")
// Execute request with retries
var lastErr error
for attempt := 0; attempt < maxRetries; attempt++ {
if attempt > 0 {
backoff := calculateBackoff(attempt)
time.Sleep(backoff)
}
resp, err := c.httpClient.Do(req)
if err != nil {
lastErr = &ProxyError{
Type: ErrorTypeTransport,
Message: fmt.Sprintf("HTTP request failed (attempt %d/%d)", attempt+1, maxRetries),
Retryable: true,
Wrapped: err,
}
continue
}
// Read response body
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
lastErr = &ProxyError{
Type: ErrorTypeTransport,
Message: "failed to read response body",
Retryable: true,
Wrapped: err,
}
continue
}
// Check HTTP status
if resp.StatusCode == http.StatusUnauthorized {
return "", &ProxyError{
Type: ErrorTypeAuth,
Message: "authentication failed - invalid token",
Retryable: false,
}
}
if resp.StatusCode == http.StatusForbidden {
return "", &ProxyError{
Type: ErrorTypeAuth,
Message: "node not allowed by proxy",
Retryable: false,
}
}
if resp.StatusCode == http.StatusTooManyRequests {
lastErr = &ProxyError{
Type: ErrorTypeTransport,
Message: "rate limit exceeded",
Retryable: true,
}
continue
}
if resp.StatusCode != http.StatusOK {
return "", &ProxyError{
Type: ErrorTypeTransport,
Message: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, string(body)),
Retryable: resp.StatusCode >= 500,
}
}
// Parse JSON response
var jsonResp struct {
Node string `json:"node"`
Temperature interface{} `json:"temperature"`
}
if err := json.Unmarshal(body, &jsonResp); err != nil {
return "", &ProxyError{
Type: ErrorTypeTransport,
Message: "failed to parse response JSON",
Retryable: false,
Wrapped: err,
}
}
// Convert temperature data back to JSON string (matching socket interface)
tempJSON, err := json.Marshal(jsonResp.Temperature)
if err != nil {
return "", &ProxyError{
Type: ErrorTypeTransport,
Message: "failed to marshal temperature data",
Retryable: false,
Wrapped: err,
}
}
return string(tempJSON), nil
}
// All retries exhausted
if lastErr != nil {
return "", lastErr
}
return "", &ProxyError{
Type: ErrorTypeUnknown,
Message: "all retry attempts failed",
Retryable: false,
}
}