Clarify Docker agent cycling troubleshooting for cloned VMs/LXCs (related to #648)

Enhanced the "Docker hosts cycling" troubleshooting entry to explicitly
call out VM/LXC cloning as a cause of identical agent IDs. Added specific
remediation steps for regenerating machine IDs on cloned systems.

This addresses the resolution path discovered in discussion #648 where a
user cloned a Proxmox LXC and encountered cycling behavior even with
separate API tokens because the agent IDs were duplicated.

docs/DOCKER_MONITORING.md

@@ -257,9 +257,11 @@ Need the alerts but at a different tone? The same Containers tab exposes global
 
 **Symptom:** The Docker tab switches between hosts instead of showing all of them simultaneously.
 
-**Cause:** Multiple agents using the same API token. Pulse matches incoming reports by agent ID first and falls back to the API token when IDs are missing or identical. Shared tokens make agents indistinguishable.
+**Cause:** Multiple agents using the same API token or identical agent IDs. Pulse matches incoming reports by agent ID first and falls back to the API token when IDs are missing or identical. Cloning a VM or LXC that already runs `pulse-docker-agent` copies the agent ID, so the two hosts replace each other even with different tokens.
 
-**Fix:** Create a dedicated API token for each Docker host in **Settings → API Tokens** and update the agents with their unique tokens. See the [Quick install section](#quick-install-from-your-pulse-server-recommended) for token setup details.
+**Fix:**
+- If agents share the same API token: Create a dedicated API token for each Docker host in **Settings → API Tokens** and update the agents with their unique tokens. See the [Quick install section](#quick-install-from-your-pulse-server-recommended) for token setup details.
+- If a cloned VM/LXC has a duplicate agent ID: Regenerate the machine ID on the clone by running `sudo rm /etc/machine-id /var/lib/dbus/machine-id && sudo systemd-machine-id-setup`, then restart `pulse-docker-agent`. Alternatively, set a unique `--agent-id` or `PULSE_AGENT_ID` environment variable when starting the agent.
 
 ### Agent rejected after host removal
 

internal/config/docker_metadata.go

@@ -18,18 +18,31 @@ type DockerMetadata struct {
 	Tags        []string `json:"tags"`        // Optional tags for categorization
 }
 
+// DockerHostMetadata holds additional metadata for a Docker host
+type DockerHostMetadata struct {
+	CustomDisplayName string `json:"customDisplayName,omitempty"` // User-defined custom display name
+}
+
+// dockerMetadataFile represents the on-disk format for Docker metadata
+type dockerMetadataFile struct {
+	Containers map[string]*DockerMetadata     `json:"containers,omitempty"` // Container/service metadata (legacy: may be top-level)
+	Hosts      map[string]*DockerHostMetadata `json:"hosts,omitempty"`      // Host-level metadata
+}
+
 // DockerMetadataStore manages Docker resource metadata
 type DockerMetadataStore struct {
-	mu       sync.RWMutex
-	metadata map[string]*DockerMetadata // keyed by resource ID
-	dataPath string
+	mu           sync.RWMutex
+	metadata     map[string]*DockerMetadata     // keyed by resource ID (containers/services)
+	hostMetadata map[string]*DockerHostMetadata // keyed by host ID
+	dataPath     string
 }
 
 // NewDockerMetadataStore creates a new metadata store
 func NewDockerMetadataStore(dataPath string) *DockerMetadataStore {
 	store := &DockerMetadataStore{
-		metadata: make(map[string]*DockerMetadata),
-		dataPath: dataPath,
+		metadata:     make(map[string]*DockerMetadata),
+		hostMetadata: make(map[string]*DockerHostMetadata),
+		dataPath:     dataPath,
 	}
 
 	// Load existing metadata
@@ -64,6 +77,46 @@ func (s *DockerMetadataStore) GetAll() map[string]*DockerMetadata {
 	return result
 }
 
+// GetHostMetadata retrieves metadata for a Docker host
+func (s *DockerMetadataStore) GetHostMetadata(hostID string) *DockerHostMetadata {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if meta, exists := s.hostMetadata[hostID]; exists {
+		return meta
+	}
+	return nil
+}
+
+// GetAllHostMetadata retrieves all Docker host metadata
+func (s *DockerMetadataStore) GetAllHostMetadata() map[string]*DockerHostMetadata {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	// Return a copy to prevent external modifications
+	result := make(map[string]*DockerHostMetadata)
+	for k, v := range s.hostMetadata {
+		result[k] = v
+	}
+	return result
+}
+
+// SetHostMetadata updates or creates metadata for a Docker host
+func (s *DockerMetadataStore) SetHostMetadata(hostID string, meta *DockerHostMetadata) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if meta == nil || meta.CustomDisplayName == "" {
+		// If metadata is nil or custom display name is empty, delete the entry
+		delete(s.hostMetadata, hostID)
+	} else {
+		s.hostMetadata[hostID] = meta
+	}
+
+	// Save to disk
+	return s.save()
+}
+
 // Set updates or creates metadata for a Docker resource
 func (s *DockerMetadataStore) Set(resourceID string, meta *DockerMetadata) error {
 	s.mu.Lock()
@@ -136,11 +189,40 @@ func (s *DockerMetadataStore) Load() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if err := json.Unmarshal(data, &s.metadata); err != nil {
+	// Try to load as versioned format first
+	var fileData dockerMetadataFile
+	if err := json.Unmarshal(data, &fileData); err != nil {
 		return fmt.Errorf("failed to unmarshal metadata: %w", err)
 	}
 
-	log.Info().Int("count", len(s.metadata)).Msg("Loaded Docker metadata")
+	// Check if this is the new format (has "hosts" or "containers" keys)
+	if fileData.Hosts != nil || fileData.Containers != nil {
+		// New versioned format
+		if fileData.Containers != nil {
+			s.metadata = fileData.Containers
+		} else {
+			s.metadata = make(map[string]*DockerMetadata)
+		}
+		if fileData.Hosts != nil {
+			s.hostMetadata = fileData.Hosts
+		} else {
+			s.hostMetadata = make(map[string]*DockerHostMetadata)
+		}
+		log.Info().
+			Int("containerCount", len(s.metadata)).
+			Int("hostCount", len(s.hostMetadata)).
+			Msg("Loaded Docker metadata (versioned format)")
+	} else {
+		// Legacy format: top-level map is container metadata
+		if err := json.Unmarshal(data, &s.metadata); err != nil {
+			return fmt.Errorf("failed to unmarshal legacy metadata: %w", err)
+		}
+		s.hostMetadata = make(map[string]*DockerHostMetadata)
+		log.Info().
+			Int("containerCount", len(s.metadata)).
+			Msg("Loaded Docker metadata (legacy format)")
+	}
+
 	return nil
 }
 
@@ -150,7 +232,13 @@ func (s *DockerMetadataStore) save() error {
 
 	log.Debug().Str("path", filePath).Msg("Saving Docker metadata to disk")
 
-	data, err := json.Marshal(s.metadata)
+	// Use versioned format
+	fileData := dockerMetadataFile{
+		Containers: s.metadata,
+		Hosts:      s.hostMetadata,
+	}
+
+	data, err := json.Marshal(fileData)
 	if err != nil {
 		return fmt.Errorf("failed to marshal metadata: %w", err)
 	}
@@ -171,7 +259,7 @@ func (s *DockerMetadataStore) save() error {
 		return fmt.Errorf("failed to rename metadata file: %w", err)
 	}
 
-	log.Debug().Str("path", filePath).Int("entries", len(s.metadata)).Msg("Docker metadata saved successfully")
+	log.Debug().Str("path", filePath).Int("containers", len(s.metadata)).Int("hosts", len(s.hostMetadata)).Msg("Docker metadata saved successfully")
 
 	return nil
 }

internal/monitoring/monitor.go

@@ -529,6 +529,7 @@ type Monitor struct {
 	maxRetryAttempts           int
 	tempCollector              *TemperatureCollector // SSH-based temperature collector
 	guestMetadataStore         *config.GuestMetadataStore
+	dockerMetadataStore        *config.DockerMetadataStore
 	mu                         sync.RWMutex
 	startTime                  time.Time
 	rateTracker                *RateTracker
@@ -1170,7 +1171,22 @@ func (m *Monitor) SetDockerHostCustomDisplayName(hostID string, customName strin
 		return models.DockerHost{}, fmt.Errorf("docker host id is required")
 	}
 
-	host, ok := m.state.SetDockerHostCustomDisplayName(hostID, strings.TrimSpace(customName))
+	customName = strings.TrimSpace(customName)
+
+	// Persist to Docker metadata store first
+	var hostMeta *config.DockerHostMetadata
+	if customName != "" {
+		hostMeta = &config.DockerHostMetadata{
+			CustomDisplayName: customName,
+		}
+	}
+	if err := m.dockerMetadataStore.SetHostMetadata(hostID, hostMeta); err != nil {
+		log.Error().Err(err).Str("hostID", hostID).Msg("Failed to persist Docker host metadata")
+		return models.DockerHost{}, fmt.Errorf("failed to persist custom display name: %w", err)
+	}
+
+	// Update in-memory state
+	host, ok := m.state.SetDockerHostCustomDisplayName(hostID, customName)
 	if !ok {
 		return models.DockerHost{}, fmt.Errorf("docker host %q not found", hostID)
 	}
@@ -1838,6 +1854,13 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
 		host.TokenLastUsedAt = previous.TokenLastUsedAt
 	}
 
+	// Load custom display name from metadata store if not already set
+	if host.CustomDisplayName == "" {
+		if hostMeta := m.dockerMetadataStore.GetHostMetadata(identifier); hostMeta != nil {
+			host.CustomDisplayName = hostMeta.CustomDisplayName
+		}
+	}
+
 	m.state.UpsertDockerHost(host)
 	m.state.SetConnectionHealth(dockerConnectionPrefix+host.ID, true)
 
@@ -3432,6 +3455,7 @@ func New(cfg *config.Config) (*Monitor, error) {
 		maxRetryAttempts:           5,
 		tempCollector:              tempCollector,
 		guestMetadataStore:         config.NewGuestMetadataStore(cfg.DataPath),
+		dockerMetadataStore:        config.NewDockerMetadataStore(cfg.DataPath),
 		startTime:                  time.Now(),
 		rateTracker:                NewRateTracker(),
 		metricsHistory:             NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours