diff --git a/cmd/pulse-sensor-proxy/http_server.go b/cmd/pulse-sensor-proxy/http_server.go index 495b236..834ef89 100644 --- a/cmd/pulse-sensor-proxy/http_server.go +++ b/cmd/pulse-sensor-proxy/http_server.go @@ -281,9 +281,13 @@ func (h *HTTPServer) handleTemperature(w http.ResponseWriter, r *http.Request) { releaseNode := h.proxy.nodeGate.acquire(nodeName) defer releaseNode() - // Fetch temperature data via SSH + // Fetch temperature data via SSH with context timeout + // Use a shorter timeout than the HTTP client to ensure we respond before client timeout + sshCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + log.Debug().Str("node", nodeName).Msg("Fetching temperature via SSH (HTTP request)") - tempData, err := h.proxy.getTemperatureViaSSH(nodeName) + tempData, err := h.proxy.getTemperatureViaSSH(sshCtx, nodeName) if err != nil { log.Warn().Err(err).Str("node", nodeName).Msg("Failed to get temperatures via SSH") h.sendJSONError(w, http.StatusInternalServerError, fmt.Sprintf("failed to get temperatures: %v", err)) diff --git a/cmd/pulse-sensor-proxy/main.go b/cmd/pulse-sensor-proxy/main.go index 2fbb816..368df74 100644 --- a/cmd/pulse-sensor-proxy/main.go +++ b/cmd/pulse-sensor-proxy/main.go @@ -1010,8 +1010,10 @@ func (p *Proxy) handleGetTemperature(req RPCRequest) RPCResponse { } } - // Fetch temperature data - tempData, err := p.getTemperatureViaSSH(node) + // Fetch temperature data with timeout + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + tempData, err := p.getTemperatureViaSSH(ctx, node) if err != nil { return RPCResponse{ Success: false, @@ -1185,8 +1187,10 @@ func (p *Proxy) handleGetTemperatureV2(ctx context.Context, req *RPCRequest, log logger.Debug().Str("node", node).Msg("Fetching temperature via SSH") - // Fetch temperature data - tempData, err := p.getTemperatureViaSSH(node) + // Fetch temperature data with timeout + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + tempData, err := p.getTemperatureViaSSH(ctx, node) if err != nil { logger.Warn().Err(err).Str("node", node).Msg("Failed to get temperatures") return nil, fmt.Errorf("failed to get temperatures: %w", err) diff --git a/cmd/pulse-sensor-proxy/ssh.go b/cmd/pulse-sensor-proxy/ssh.go index 18f0b74..72eee94 100644 --- a/cmd/pulse-sensor-proxy/ssh.go +++ b/cmd/pulse-sensor-proxy/ssh.go @@ -56,6 +56,74 @@ func execCommand(cmd string) (string, error) { return string(out), err } +// execCommandWithLimitsContext runs a shell command with output limits and context cancellation +func execCommandWithLimitsContext(ctx context.Context, cmd string, stdoutLimit, stderrLimit int64) (string, string, bool, bool, error) { + command := exec.CommandContext(ctx, "sh", "-c", cmd) + + stdoutPipe, err := command.StdoutPipe() + if err != nil { + return "", "", false, false, fmt.Errorf("stdout pipe: %w", err) + } + stderrPipe, err := command.StderrPipe() + if err != nil { + return "", "", false, false, fmt.Errorf("stderr pipe: %w", err) + } + + if err := command.Start(); err != nil { + return "", "", false, false, err + } + + type pipeResult struct { + data []byte + exceeded bool + err error + } + + var wg sync.WaitGroup + wg.Add(2) + + stdoutCh := make(chan pipeResult, 1) + stderrCh := make(chan pipeResult, 1) + + go func() { + defer wg.Done() + data, exceeded, readErr := readAllWithLimit(stdoutPipe, stdoutLimit) + stdoutCh <- pipeResult{data: data, exceeded: exceeded, err: readErr} + }() + + go func() { + defer wg.Done() + data, exceeded, readErr := readAllWithLimit(stderrPipe, stderrLimit) + stderrCh <- pipeResult{data: data, exceeded: exceeded, err: readErr} + }() + + var stdoutRes, stderrRes pipeResult + wgDone := make(chan struct{}) + go func() { + wg.Wait() + close(wgDone) + }() + + <-wgDone + stdoutRes = <-stdoutCh + stderrRes = <-stderrCh + + waitErr := command.Wait() + + if stdoutRes.err != nil { + return "", "", stdoutRes.exceeded, stderrRes.exceeded, fmt.Errorf("stdout read: %w", stdoutRes.err) + } + if stderrRes.err != nil { + return "", "", stdoutRes.exceeded, stderrRes.exceeded, fmt.Errorf("stderr read: %w", stderrRes.err) + } + + if waitErr != nil { + return string(stdoutRes.data), string(stderrRes.data), stdoutRes.exceeded, stderrRes.exceeded, waitErr + } + + return string(stdoutRes.data), string(stderrRes.data), stdoutRes.exceeded, stderrRes.exceeded, nil +} + func execCommandWithLimits(cmd string, stdoutLimit, stderrLimit int64) (string, string, bool, bool, error) { command := exec.Command("sh", "-c", cmd) @@ -504,7 +572,7 @@ func (p *Proxy) testSSHConnection(nodeHost string) error { } // getTemperatureViaSSH fetches temperature data from a node -func (p *Proxy) getTemperatureViaSSH(nodeHost string) (string, error) { +func (p *Proxy) getTemperatureViaSSH(ctx context.Context, nodeHost string) (string, error) { startTime := time.Now() nodeLabel := sanitizeNodeLabel(nodeHost) @@ -533,7 +601,7 @@ func (p *Proxy) getTemperatureViaSSH(nodeHost string) (string, error) { nodeHost, ) - stdout, stderr, stdoutExceeded, stderrExceeded, err := execCommandWithLimits(cmd, p.maxSSHOutputBytes, p.maxSSHOutputBytes) + stdout, stderr, stdoutExceeded, stderrExceeded, err := execCommandWithLimitsContext(ctx, cmd, p.maxSSHOutputBytes, p.maxSSHOutputBytes) if stdoutExceeded { log.Warn().Str("node", nodeHost).Int64("limit_bytes", p.maxSSHOutputBytes).Msg("SSH temperature output exceeded limit") if p.metrics != nil { diff --git a/internal/tempproxy/http_client.go b/internal/tempproxy/http_client.go index 27e051b..0799a72 100644 --- a/internal/tempproxy/http_client.go +++ b/internal/tempproxy/http_client.go @@ -149,7 +149,7 @@ func (c *HTTPClient) GetTemperature(nodeHost string) (string, error) { // Parse JSON response var jsonResp struct { Node string `json:"node"` - Temperature string `json:"temperature"` + Temperature string `json:"temperature"` // This is a JSON-encoded string } if err := json.Unmarshal(body, &jsonResp); err != nil { @@ -161,7 +161,8 @@ func (c *HTTPClient) GetTemperature(nodeHost string) (string, error) { } } - // Temperature field is already a JSON string, return as-is + // The temperature field contains JSON-encoded sensor data as a string + // Return it as-is since the caller expects raw JSON return jsonResp.Temperature, nil }