diff --git a/cmd/pulse/main.go b/cmd/pulse/main.go index 2797da5..afb0c32 100644 --- a/cmd/pulse/main.go +++ b/cmd/pulse/main.go @@ -201,6 +201,14 @@ func runServer() { } }) + // Set callback to rebuild token bindings when API tokens are reloaded from disk. + // This fixes issue #773 where agent token bindings become orphaned after config reload. + configWatcher.SetAPITokenReloadCallback(func() { + if monitor := reloadableMonitor.GetMonitor(); monitor != nil { + monitor.RebuildTokenBindings() + } + }) + if err := configWatcher.Start(); err != nil { log.Warn().Err(err).Msg("Failed to start config watcher") } diff --git a/internal/config/watcher.go b/internal/config/watcher.go index 6439288..0e7f1fd 100644 --- a/internal/config/watcher.go +++ b/internal/config/watcher.go @@ -32,6 +32,7 @@ type ConfigWatcher struct { apiTokensLastModTime time.Time mu sync.RWMutex onMockReload func() // Callback to trigger backend restart + onAPITokenReload func() // Callback when API tokens are reloaded from disk } // NewConfigWatcher creates a new config watcher @@ -144,6 +145,14 @@ func (cw *ConfigWatcher) SetMockReloadCallback(callback func()) { cw.onMockReload = callback } +// SetAPITokenReloadCallback sets the callback function to trigger when API tokens are reloaded. +// This allows the Monitor to rebuild token-to-agent bindings after tokens change on disk. +func (cw *ConfigWatcher) SetAPITokenReloadCallback(callback func()) { + cw.mu.Lock() + defer cw.mu.Unlock() + cw.onAPITokenReload = callback +} + // Start begins watching the config file func (cw *ConfigWatcher) Start() error { // Watch the directory for .env @@ -478,6 +487,10 @@ func (cw *ConfigWatcher) reloadConfig() { } func (cw *ConfigWatcher) reloadAPITokens() { + cw.mu.Lock() + callback := cw.onAPITokenReload + cw.mu.Unlock() + cw.mu.Lock() defer cw.mu.Unlock() @@ -525,10 +538,10 @@ func (cw *ConfigWatcher) reloadAPITokens() { // Only update if we successfully loaded tokens Mu.Lock() - defer Mu.Unlock() cw.config.APITokens = tokens cw.config.SortAPITokens() cw.config.APITokenEnabled = len(tokens) > 0 + Mu.Unlock() if cw.apiTokensPath != "" { if stat, err := os.Stat(cw.apiTokensPath); err == nil { @@ -540,6 +553,22 @@ func (cw *ConfigWatcher) reloadAPITokens() { Int("count", len(tokens)). Int("previousCount", existingCount). Msg("Reloaded API tokens from disk") + + // Notify Monitor to rebuild token bindings from current state. + // This ensures agent bindings remain consistent after token list changes. + if callback != nil { + go func() { + defer func() { + if r := recover(); r != nil { + log.Error(). + Interface("panic", r). + Stack(). + Msg("Recovered from panic in API token reload callback") + } + }() + callback() + }() + } } // reloadMockConfig handles mock.env file changes diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 81e5ef2..91a074c 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -1290,6 +1290,81 @@ func (m *Monitor) GetDockerHosts() []models.DockerHost { return m.state.GetDockerHosts() } +// RebuildTokenBindings reconstructs agent-to-token binding maps from the current +// state of Docker hosts and host agents. This should be called after API tokens +// are reloaded from disk to ensure bindings remain consistent with the new token set. +// It preserves bindings for tokens that still exist and removes orphaned entries. +func (m *Monitor) RebuildTokenBindings() { + if m == nil || m.state == nil || m.config == nil { + return + } + + m.mu.Lock() + defer m.mu.Unlock() + + // Build a set of valid token IDs from the current config + validTokens := make(map[string]struct{}) + for _, token := range m.config.APITokens { + if token.ID != "" { + validTokens[token.ID] = struct{}{} + } + } + + // Rebuild Docker token bindings + newDockerBindings := make(map[string]string) + dockerHosts := m.state.GetDockerHosts() + for _, host := range dockerHosts { + tokenID := strings.TrimSpace(host.TokenID) + if tokenID == "" { + continue + } + // Only keep bindings for tokens that still exist in config + if _, valid := validTokens[tokenID]; !valid { + continue + } + // Use AgentID if available, otherwise fall back to host ID + agentID := strings.TrimSpace(host.AgentID) + if agentID == "" { + agentID = host.ID + } + if agentID != "" { + newDockerBindings[tokenID] = agentID + } + } + + // Rebuild Host agent token bindings + newHostBindings := make(map[string]string) + hosts := m.state.GetHosts() + for _, host := range hosts { + tokenID := strings.TrimSpace(host.TokenID) + if tokenID == "" { + continue + } + // Only keep bindings for tokens that still exist in config + if _, valid := validTokens[tokenID]; !valid { + continue + } + // Use host ID as the binding identifier + if host.ID != "" { + newHostBindings[tokenID] = host.ID + } + } + + // Log what changed + oldDockerCount := len(m.dockerTokenBindings) + oldHostCount := len(m.hostTokenBindings) + m.dockerTokenBindings = newDockerBindings + m.hostTokenBindings = newHostBindings + + log.Info(). + Int("dockerBindings", len(newDockerBindings)). + Int("hostBindings", len(newHostBindings)). + Int("previousDockerBindings", oldDockerCount). + Int("previousHostBindings", oldHostCount). + Int("validTokens", len(validTokens)). + Msg("Rebuilt agent token bindings after API token reload") +} + // QueueDockerHostStop queues a stop command for the specified docker host. func (m *Monitor) QueueDockerHostStop(hostID string) (models.DockerHostCommandStatus, error) { return m.queueDockerStopCommand(hostID)