From e29fda5e5a8282b6a55c74c57c7b78b087585010 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sat, 29 Nov 2025 11:09:22 +0000 Subject: [PATCH] Rebuild agent token bindings on API token config reload When api_tokens.json is modified on disk, the ConfigWatcher reloads the tokens into memory. However, the Monitor's dockerTokenBindings and hostTokenBindings maps were not synchronized with the new token set, causing orphaned bindings when agents reconnect after reinstall. Add SetAPITokenReloadCallback to ConfigWatcher that triggers Monitor's new RebuildTokenBindings method after token reload. This method reconstructs the binding maps from current Docker host and host agent state, keeping only bindings for tokens that still exist in config. Related to #773 --- cmd/pulse/main.go | 8 ++++ internal/config/watcher.go | 31 +++++++++++++- internal/monitoring/monitor.go | 75 ++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) diff --git a/cmd/pulse/main.go b/cmd/pulse/main.go index 2797da5..afb0c32 100644 --- a/cmd/pulse/main.go +++ b/cmd/pulse/main.go @@ -201,6 +201,14 @@ func runServer() { } }) + // Set callback to rebuild token bindings when API tokens are reloaded from disk. + // This fixes issue #773 where agent token bindings become orphaned after config reload. + configWatcher.SetAPITokenReloadCallback(func() { + if monitor := reloadableMonitor.GetMonitor(); monitor != nil { + monitor.RebuildTokenBindings() + } + }) + if err := configWatcher.Start(); err != nil { log.Warn().Err(err).Msg("Failed to start config watcher") } diff --git a/internal/config/watcher.go b/internal/config/watcher.go index 6439288..0e7f1fd 100644 --- a/internal/config/watcher.go +++ b/internal/config/watcher.go @@ -32,6 +32,7 @@ type ConfigWatcher struct { apiTokensLastModTime time.Time mu sync.RWMutex onMockReload func() // Callback to trigger backend restart + onAPITokenReload func() // Callback when API tokens are reloaded from disk } // NewConfigWatcher creates a new config watcher @@ -144,6 +145,14 @@ func (cw *ConfigWatcher) SetMockReloadCallback(callback func()) { cw.onMockReload = callback } +// SetAPITokenReloadCallback sets the callback function to trigger when API tokens are reloaded. +// This allows the Monitor to rebuild token-to-agent bindings after tokens change on disk. +func (cw *ConfigWatcher) SetAPITokenReloadCallback(callback func()) { + cw.mu.Lock() + defer cw.mu.Unlock() + cw.onAPITokenReload = callback +} + // Start begins watching the config file func (cw *ConfigWatcher) Start() error { // Watch the directory for .env @@ -478,6 +487,10 @@ func (cw *ConfigWatcher) reloadConfig() { } func (cw *ConfigWatcher) reloadAPITokens() { + cw.mu.Lock() + callback := cw.onAPITokenReload + cw.mu.Unlock() + cw.mu.Lock() defer cw.mu.Unlock() @@ -525,10 +538,10 @@ func (cw *ConfigWatcher) reloadAPITokens() { // Only update if we successfully loaded tokens Mu.Lock() - defer Mu.Unlock() cw.config.APITokens = tokens cw.config.SortAPITokens() cw.config.APITokenEnabled = len(tokens) > 0 + Mu.Unlock() if cw.apiTokensPath != "" { if stat, err := os.Stat(cw.apiTokensPath); err == nil { @@ -540,6 +553,22 @@ func (cw *ConfigWatcher) reloadAPITokens() { Int("count", len(tokens)). Int("previousCount", existingCount). Msg("Reloaded API tokens from disk") + + // Notify Monitor to rebuild token bindings from current state. + // This ensures agent bindings remain consistent after token list changes. + if callback != nil { + go func() { + defer func() { + if r := recover(); r != nil { + log.Error(). + Interface("panic", r). + Stack(). + Msg("Recovered from panic in API token reload callback") + } + }() + callback() + }() + } } // reloadMockConfig handles mock.env file changes diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 81e5ef2..91a074c 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -1290,6 +1290,81 @@ func (m *Monitor) GetDockerHosts() []models.DockerHost { return m.state.GetDockerHosts() } +// RebuildTokenBindings reconstructs agent-to-token binding maps from the current +// state of Docker hosts and host agents. This should be called after API tokens +// are reloaded from disk to ensure bindings remain consistent with the new token set. +// It preserves bindings for tokens that still exist and removes orphaned entries. +func (m *Monitor) RebuildTokenBindings() { + if m == nil || m.state == nil || m.config == nil { + return + } + + m.mu.Lock() + defer m.mu.Unlock() + + // Build a set of valid token IDs from the current config + validTokens := make(map[string]struct{}) + for _, token := range m.config.APITokens { + if token.ID != "" { + validTokens[token.ID] = struct{}{} + } + } + + // Rebuild Docker token bindings + newDockerBindings := make(map[string]string) + dockerHosts := m.state.GetDockerHosts() + for _, host := range dockerHosts { + tokenID := strings.TrimSpace(host.TokenID) + if tokenID == "" { + continue + } + // Only keep bindings for tokens that still exist in config + if _, valid := validTokens[tokenID]; !valid { + continue + } + // Use AgentID if available, otherwise fall back to host ID + agentID := strings.TrimSpace(host.AgentID) + if agentID == "" { + agentID = host.ID + } + if agentID != "" { + newDockerBindings[tokenID] = agentID + } + } + + // Rebuild Host agent token bindings + newHostBindings := make(map[string]string) + hosts := m.state.GetHosts() + for _, host := range hosts { + tokenID := strings.TrimSpace(host.TokenID) + if tokenID == "" { + continue + } + // Only keep bindings for tokens that still exist in config + if _, valid := validTokens[tokenID]; !valid { + continue + } + // Use host ID as the binding identifier + if host.ID != "" { + newHostBindings[tokenID] = host.ID + } + } + + // Log what changed + oldDockerCount := len(m.dockerTokenBindings) + oldHostCount := len(m.hostTokenBindings) + m.dockerTokenBindings = newDockerBindings + m.hostTokenBindings = newHostBindings + + log.Info(). + Int("dockerBindings", len(newDockerBindings)). + Int("hostBindings", len(newHostBindings)). + Int("previousDockerBindings", oldDockerCount). + Int("previousHostBindings", oldHostCount). + Int("validTokens", len(validTokens)). + Msg("Rebuilt agent token bindings after API token reload") +} + // QueueDockerHostStop queues a stop command for the specified docker host. func (m *Monitor) QueueDockerHostStop(hostID string) (models.DockerHostCommandStatus, error) { return m.queueDockerStopCommand(hostID)