From 823508dc4868d7f3c6e262dcb0a240b8e0c28c28 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 20 Nov 2025 15:45:21 +0000 Subject: [PATCH] Related to #712: auto-restore host agent binaries for download --- cmd/pulse/main.go | 5 +- .../agentbinaries/host_agent.go | 201 ++++++++++-------- internal/api/router.go | 145 +++++++++---- internal/api/router_download_test.go | 23 ++ internal/api/router_integration_test.go | 7 +- 5 files changed, 251 insertions(+), 130 deletions(-) rename cmd/pulse/agent_binaries.go => internal/agentbinaries/host_agent.go (68%) diff --git a/cmd/pulse/main.go b/cmd/pulse/main.go index fb27b9a..993d0e7 100644 --- a/cmd/pulse/main.go +++ b/cmd/pulse/main.go @@ -12,6 +12,7 @@ import ( "syscall" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/agentbinaries" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/api" "github.com/rcourtman/pulse-go-rewrite/internal/config" @@ -104,7 +105,7 @@ func runServer() { log.Info().Msg("Starting Pulse monitoring server") // Validate agent binaries are available for download - validateAgentBinaries() + agentbinaries.EnsureHostAgentBinaries(Version) // Create context that cancels on interrupt ctx, cancel := context.WithCancel(context.Background()) @@ -170,7 +171,7 @@ func runServer() { } return nil } - router = api.NewRouter(cfg, reloadableMonitor.GetMonitor(), wsHub, reloadFunc) + router = api.NewRouter(cfg, reloadableMonitor.GetMonitor(), wsHub, reloadFunc, Version) // Create HTTP server with unified configuration // In production, serve everything (frontend + API) on the frontend port diff --git a/cmd/pulse/agent_binaries.go b/internal/agentbinaries/host_agent.go similarity index 68% rename from cmd/pulse/agent_binaries.go rename to internal/agentbinaries/host_agent.go index 7b25c21..448f2af 100644 --- a/cmd/pulse/agent_binaries.go +++ b/internal/agentbinaries/host_agent.go @@ -1,4 +1,4 @@ -package main +package agentbinaries import ( "archive/tar" @@ -12,84 +12,47 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" "github.com/rs/zerolog/log" ) -type hostAgentBinary struct { - platform string - arch string - filenames []string +type HostAgentBinary struct { + Platform string + Arch string + Filenames []string } -var requiredHostAgentBinaries = []hostAgentBinary{ - {platform: "linux", arch: "amd64", filenames: []string{"pulse-host-agent-linux-amd64"}}, - {platform: "linux", arch: "arm64", filenames: []string{"pulse-host-agent-linux-arm64"}}, - {platform: "linux", arch: "armv7", filenames: []string{"pulse-host-agent-linux-armv7"}}, - {platform: "darwin", arch: "amd64", filenames: []string{"pulse-host-agent-darwin-amd64"}}, - {platform: "darwin", arch: "arm64", filenames: []string{"pulse-host-agent-darwin-arm64"}}, +var requiredHostAgentBinaries = []HostAgentBinary{ + {Platform: "linux", Arch: "amd64", Filenames: []string{"pulse-host-agent-linux-amd64"}}, + {Platform: "linux", Arch: "arm64", Filenames: []string{"pulse-host-agent-linux-arm64"}}, + {Platform: "linux", Arch: "armv7", Filenames: []string{"pulse-host-agent-linux-armv7"}}, + {Platform: "linux", Arch: "armv6", Filenames: []string{"pulse-host-agent-linux-armv6"}}, + {Platform: "linux", Arch: "386", Filenames: []string{"pulse-host-agent-linux-386"}}, + {Platform: "darwin", Arch: "amd64", Filenames: []string{"pulse-host-agent-darwin-amd64"}}, + {Platform: "darwin", Arch: "arm64", Filenames: []string{"pulse-host-agent-darwin-arm64"}}, { - platform: "windows", - arch: "amd64", - filenames: []string{"pulse-host-agent-windows-amd64", "pulse-host-agent-windows-amd64.exe"}, + Platform: "windows", + Arch: "amd64", + Filenames: []string{"pulse-host-agent-windows-amd64", "pulse-host-agent-windows-amd64.exe"}, }, { - platform: "windows", - arch: "arm64", - filenames: []string{"pulse-host-agent-windows-arm64", "pulse-host-agent-windows-arm64.exe"}, + Platform: "windows", + Arch: "arm64", + Filenames: []string{"pulse-host-agent-windows-arm64", "pulse-host-agent-windows-arm64.exe"}, }, { - platform: "windows", - arch: "386", - filenames: []string{"pulse-host-agent-windows-386", "pulse-host-agent-windows-386.exe"}, + Platform: "windows", + Arch: "386", + Filenames: []string{"pulse-host-agent-windows-386", "pulse-host-agent-windows-386.exe"}, }, } -func validateAgentBinaries() { - binDirs := hostAgentSearchPaths() - missing := findMissingHostAgentBinaries(binDirs) - if len(missing) == 0 { - log.Info().Msg("All host agent binaries available for download") - return - } +var downloadMu sync.Mutex - missingPlatforms := make([]string, 0, len(missing)) - for key := range missing { - missingPlatforms = append(missingPlatforms, key) - } - sort.Strings(missingPlatforms) - - log.Warn(). - Strs("missing_platforms", missingPlatforms). - Msg("Host agent binaries missing - attempting to download bundle from GitHub release") - - if err := downloadAndInstallHostAgentBinaries(binDirs[0]); err != nil { - log.Error(). - Err(err). - Str("target_dir", binDirs[0]). - Strs("missing_platforms", missingPlatforms). - Msg("Failed to automatically install host agent binaries; install script downloads will fail") - return - } - - remaining := findMissingHostAgentBinaries(binDirs) - if len(remaining) == 0 { - log.Info().Msg("Host agent binaries restored from GitHub release bundle") - return - } - - stillMissing := make([]string, 0, len(remaining)) - for key := range remaining { - stillMissing = append(stillMissing, key) - } - sort.Strings(stillMissing) - log.Warn(). - Strs("missing_platforms", stillMissing). - Msg("Host agent binaries still missing after automatic restoration attempt") -} - -func hostAgentSearchPaths() []string { +// HostAgentSearchPaths returns the directories to search for host agent binaries. +func HostAgentSearchPaths() []string { primary := strings.TrimSpace(os.Getenv("PULSE_BIN_DIR")) if primary == "" { primary = "/opt/pulse/bin" @@ -109,32 +72,64 @@ func hostAgentSearchPaths() []string { return result } -func findMissingHostAgentBinaries(binDirs []string) map[string]hostAgentBinary { - missing := make(map[string]hostAgentBinary) - for _, binary := range requiredHostAgentBinaries { - if !hostAgentBinaryExists(binDirs, binary.filenames) { - key := fmt.Sprintf("%s-%s", binary.platform, binary.arch) - missing[key] = binary - } +// EnsureHostAgentBinaries verifies that all host agent binaries are present locally. +// If any are missing, it attempts to restore them from the matching GitHub release. +// The returned map contains any binaries that remain missing after the attempt. +func EnsureHostAgentBinaries(version string) map[string]HostAgentBinary { + binDirs := HostAgentSearchPaths() + missing := findMissingHostAgentBinaries(binDirs) + if len(missing) == 0 { + return nil } - return missing + + downloadMu.Lock() + defer downloadMu.Unlock() + + // Re-check after acquiring the lock in case another goroutine restored them. + missing = findMissingHostAgentBinaries(binDirs) + if len(missing) == 0 { + return nil + } + + missingPlatforms := make([]string, 0, len(missing)) + for key := range missing { + missingPlatforms = append(missingPlatforms, key) + } + sort.Strings(missingPlatforms) + + log.Warn(). + Strs("missing_platforms", missingPlatforms). + Msg("Host agent binaries missing - attempting to download bundle from GitHub release") + + if err := DownloadAndInstallHostAgentBinaries(version, binDirs[0]); err != nil { + log.Error(). + Err(err). + Str("target_dir", binDirs[0]). + Strs("missing_platforms", missingPlatforms). + Msg("Failed to automatically install host agent binaries; download endpoints will return 404s") + return missing + } + + if remaining := findMissingHostAgentBinaries(binDirs); len(remaining) > 0 { + stillMissing := make([]string, 0, len(remaining)) + for key := range remaining { + stillMissing = append(stillMissing, key) + } + sort.Strings(stillMissing) + log.Warn(). + Strs("missing_platforms", stillMissing). + Msg("Host agent binaries still missing after automatic restoration attempt") + return remaining + } + + log.Info().Msg("Host agent binaries restored from GitHub release bundle") + return nil } -func hostAgentBinaryExists(binDirs, filenames []string) bool { - for _, dir := range binDirs { - for _, name := range filenames { - path := filepath.Join(dir, name) - if info, err := os.Stat(path); err == nil && !info.IsDir() { - return true - } - } - } - return false -} - -func downloadAndInstallHostAgentBinaries(targetDir string) error { - version := strings.TrimSpace(Version) - if version == "" || strings.EqualFold(version, "dev") { +// DownloadAndInstallHostAgentBinaries fetches the universal host agent bundle for the given version and installs it. +func DownloadAndInstallHostAgentBinaries(version string, targetDir string) error { + normalizedVersion := normalizeVersionTag(version) + if normalizedVersion == "" || strings.EqualFold(normalizedVersion, "vdev") { return fmt.Errorf("cannot download host agent bundle for non-release version %q", version) } @@ -142,7 +137,7 @@ func downloadAndInstallHostAgentBinaries(targetDir string) error { return fmt.Errorf("failed to ensure bin directory %s: %w", targetDir, err) } - url := fmt.Sprintf("https://github.com/rcourtman/Pulse/releases/download/%[1]s/pulse-%[1]s.tar.gz", version) + url := fmt.Sprintf("https://github.com/rcourtman/Pulse/releases/download/%[1]s/pulse-%[1]s.tar.gz", normalizedVersion) tempFile, err := os.CreateTemp("", "pulse-host-agent-*.tar.gz") if err != nil { return fmt.Errorf("failed to create temporary archive file: %w", err) @@ -176,6 +171,38 @@ func downloadAndInstallHostAgentBinaries(targetDir string) error { return nil } +func findMissingHostAgentBinaries(binDirs []string) map[string]HostAgentBinary { + missing := make(map[string]HostAgentBinary) + for _, binary := range requiredHostAgentBinaries { + if !hostAgentBinaryExists(binDirs, binary.Filenames) { + key := fmt.Sprintf("%s-%s", binary.Platform, binary.Arch) + missing[key] = binary + } + } + return missing +} + +func hostAgentBinaryExists(binDirs, filenames []string) bool { + for _, dir := range binDirs { + for _, name := range filenames { + path := filepath.Join(dir, name) + if info, err := os.Stat(path); err == nil && !info.IsDir() { + return true + } + } + } + return false +} + +func normalizeVersionTag(version string) string { + v := strings.TrimSpace(version) + if v == "" { + return "" + } + v = strings.TrimPrefix(v, "v") + return "v" + v +} + func extractHostAgentBinaries(archivePath, targetDir string) error { file, err := os.Open(archivePath) if err != nil { diff --git a/internal/api/router.go b/internal/api/router.go index 35cb646..e45b574 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -18,11 +18,13 @@ import ( "path/filepath" "regexp" "runtime" + "sort" "strconv" "strings" "sync" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/agentbinaries" "github.com/rcourtman/pulse-go-rewrite/internal/auth" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/dockeragent" @@ -58,6 +60,7 @@ type Router struct { oidcMu sync.Mutex oidcService *OIDCService wrapped http.Handler + serverVersion string projectRoot string // Cached system settings to avoid loading from disk on every request settingsMu sync.RWMutex @@ -97,7 +100,7 @@ func isDirectLoopbackRequest(req *http.Request) bool { } // NewRouter creates a new router instance -func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, wsHub *websocket.Hub, reloadFunc func() error) *Router { +func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, wsHub *websocket.Hub, reloadFunc func() error, serverVersion string) *Router { // Initialize persistent session and CSRF stores InitSessionStore(cfg.DataPath) InitCSRFStore(cfg.DataPath) @@ -125,6 +128,7 @@ func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, wsHub *websocket updateHistory: updateHistory, exportLimiter: NewRateLimiter(5, 1*time.Minute), // 5 attempts per minute persistence: config.NewConfigPersistence(cfg.DataPath), + serverVersion: strings.TrimSpace(serverVersion), projectRoot: projectRoot, } @@ -3394,36 +3398,53 @@ func (r *Router) handleDownloadHostAgent(w http.ResponseWriter, req *http.Reques return } - searchPaths := make([]string, 0, 12) - strictMode := platformParam != "" && archParam != "" - - // Try platform-specific binary first - if strictMode { - searchPaths = append(searchPaths, - filepath.Join(pulseBinDir(), fmt.Sprintf("pulse-host-agent-%s-%s", platformParam, archParam)), - filepath.Join("/opt/pulse", fmt.Sprintf("pulse-host-agent-%s-%s", platformParam, archParam)), - filepath.Join("/app", fmt.Sprintf("pulse-host-agent-%s-%s", platformParam, archParam)), - ) + checkedPaths, served := r.tryServeHostAgentBinary(w, req, platformParam, archParam) + if served { + return } - if platformParam != "" && !strictMode { - searchPaths = append(searchPaths, - filepath.Join(pulseBinDir(), "pulse-host-agent-"+platformParam), - filepath.Join("/opt/pulse", "pulse-host-agent-"+platformParam), - filepath.Join("/app", "pulse-host-agent-"+platformParam), - ) + remainingMissing := agentbinaries.EnsureHostAgentBinaries(r.serverVersion) + + afterRestorePaths, served := r.tryServeHostAgentBinary(w, req, platformParam, archParam) + checkedPaths = append(checkedPaths, afterRestorePaths...) + if served { + return } - // Default locations (host architecture) - only when no platform/arch specified - if !strictMode && platformParam == "" { - searchPaths = append(searchPaths, - filepath.Join(pulseBinDir(), "pulse-host-agent"), - "/opt/pulse/pulse-host-agent", - filepath.Join("/app", "pulse-host-agent"), - ) + // Build detailed error message with troubleshooting guidance + var errorMsg strings.Builder + errorMsg.WriteString(fmt.Sprintf("Host agent binary not found for %s/%s\n\n", platformParam, archParam)) + errorMsg.WriteString("Troubleshooting:\n") + errorMsg.WriteString("1. If running in Docker: Rebuild the Docker image to include all platform binaries\n") + errorMsg.WriteString("2. If running bare metal: Run 'scripts/build-release.sh' to build all platform binaries\n") + errorMsg.WriteString("3. Build from source:\n") + errorMsg.WriteString(fmt.Sprintf(" GOOS=%s GOARCH=%s go build -o pulse-host-agent-%s-%s ./cmd/pulse-host-agent\n", platformParam, archParam, platformParam, archParam)) + errorMsg.WriteString(fmt.Sprintf(" sudo mv pulse-host-agent-%s-%s /opt/pulse/bin/\n\n", platformParam, archParam)) + + if len(remainingMissing) > 0 { + errorMsg.WriteString("Automatic repair attempted but the following binaries are still missing:\n") + for _, key := range sortedHostAgentKeys(remainingMissing) { + errorMsg.WriteString(fmt.Sprintf(" - %s\n", key)) + } + if r.serverVersion != "" { + errorMsg.WriteString(fmt.Sprintf("Release bundle used: %s\n\n", strings.TrimSpace(r.serverVersion))) + } else { + errorMsg.WriteString("\n") + } } + errorMsg.WriteString("Searched locations:\n") + for _, path := range dedupeStrings(checkedPaths) { + errorMsg.WriteString(fmt.Sprintf(" - %s\n", path)) + } + + http.Error(w, errorMsg.String(), http.StatusNotFound) +} + +func (r *Router) tryServeHostAgentBinary(w http.ResponseWriter, req *http.Request, platformParam, archParam string) ([]string, bool) { + searchPaths := hostAgentSearchCandidates(platformParam, archParam) checkedPaths := make([]string, 0, len(searchPaths)*2) + shouldCheckWindowsExe := func(path string) bool { base := strings.ToLower(filepath.Base(path)) return strings.Contains(base, "windows") && !strings.HasSuffix(base, ".exe") @@ -3441,32 +3462,76 @@ func (r *Router) handleDownloadHostAgent(w http.ResponseWriter, req *http.Reques for _, path := range pathsToCheck { checkedPaths = append(checkedPaths, path) if info, err := os.Stat(path); err == nil && !info.IsDir() { - // Check if this is a checksum request if strings.HasSuffix(req.URL.Path, ".sha256") { r.serveChecksum(w, req, path) - return + return checkedPaths, true } http.ServeFile(w, req, path) - return + return checkedPaths, true } } } - // Build detailed error message with troubleshooting guidance - var errorMsg strings.Builder - errorMsg.WriteString(fmt.Sprintf("Host agent binary not found for %s/%s\n\n", platformParam, archParam)) - errorMsg.WriteString("Troubleshooting:\n") - errorMsg.WriteString("1. If running in Docker: Rebuild the Docker image to include all platform binaries\n") - errorMsg.WriteString("2. If running bare metal: Run 'scripts/build-release.sh' to build all platform binaries\n") - errorMsg.WriteString("3. Build from source:\n") - errorMsg.WriteString(fmt.Sprintf(" GOOS=%s GOARCH=%s go build -o pulse-host-agent-%s-%s ./cmd/pulse-host-agent\n", platformParam, archParam, platformParam, archParam)) - errorMsg.WriteString(fmt.Sprintf(" sudo mv pulse-host-agent-%s-%s /opt/pulse/bin/\n\n", platformParam, archParam)) - errorMsg.WriteString("Searched locations:\n") - for _, path := range checkedPaths { - errorMsg.WriteString(fmt.Sprintf(" - %s\n", path)) + return checkedPaths, false +} + +func hostAgentSearchCandidates(platformParam, archParam string) []string { + searchPaths := make([]string, 0, 12) + strictMode := platformParam != "" && archParam != "" + + if strictMode { + searchPaths = append(searchPaths, + filepath.Join(pulseBinDir(), fmt.Sprintf("pulse-host-agent-%s-%s", platformParam, archParam)), + filepath.Join("/opt/pulse", fmt.Sprintf("pulse-host-agent-%s-%s", platformParam, archParam)), + filepath.Join("/app", fmt.Sprintf("pulse-host-agent-%s-%s", platformParam, archParam)), + ) } - http.Error(w, errorMsg.String(), http.StatusNotFound) + if platformParam != "" && !strictMode { + searchPaths = append(searchPaths, + filepath.Join(pulseBinDir(), "pulse-host-agent-"+platformParam), + filepath.Join("/opt/pulse", "pulse-host-agent-"+platformParam), + filepath.Join("/app", "pulse-host-agent-"+platformParam), + ) + } + + if !strictMode && platformParam == "" { + searchPaths = append(searchPaths, + filepath.Join(pulseBinDir(), "pulse-host-agent"), + "/opt/pulse/pulse-host-agent", + filepath.Join("/app", "pulse-host-agent"), + ) + } + + return searchPaths +} + +func dedupeStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + for _, value := range values { + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +func sortedHostAgentKeys(missing map[string]agentbinaries.HostAgentBinary) []string { + if len(missing) == 0 { + return nil + } + keys := make([]string, 0, len(missing)) + for key := range missing { + keys = append(keys, key) + } + sort.Strings(keys) + return keys } // serveChecksum computes and serves the SHA256 checksum of a file diff --git a/internal/api/router_download_test.go b/internal/api/router_download_test.go index 9c71802..d9b5f44 100644 --- a/internal/api/router_download_test.go +++ b/internal/api/router_download_test.go @@ -40,6 +40,29 @@ func TestHandleDownloadHostAgentServesWindowsExe(t *testing.T) { } } +func TestHandleDownloadHostAgentServesLinuxArm64(t *testing.T) { + binDir := setupTempPulseBin(t) + filePath := filepath.Join(binDir, "pulse-host-agent-linux-arm64") + payload := []byte("arm64-binary") + if err := os.WriteFile(filePath, payload, 0o755); err != nil { + t.Fatalf("failed to write test binary: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/download/pulse-host-agent?platform=linux&arch=arm64", nil) + rr := httptest.NewRecorder() + + router := &Router{} + router.handleDownloadHostAgent(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200 OK, got %d", rr.Code) + } + + if got := rr.Body.String(); got != string(payload) { + t.Fatalf("unexpected response body: %q", got) + } +} + func TestHandleDownloadHostAgentServesChecksumForWindowsExe(t *testing.T) { const ( arch = "unit-sha" diff --git a/internal/api/router_integration_test.go b/internal/api/router_integration_test.go index 5b1b680..624f178 100644 --- a/internal/api/router_integration_test.go +++ b/internal/api/router_integration_test.go @@ -79,10 +79,15 @@ func newIntegrationServerWithConfig(t *testing.T, customize func(*config.Config) return monitor.GetState().ToFrontend() }) + version := readRuntimeVersion(t) + if version == "" { + version = "dev" + } + router := api.NewRouter(cfg, monitor, hub, func() error { monitor.SyncAlertState() return nil - }) + }, version) srv := httptest.NewServer(router.Handler()) t.Cleanup(func() {