Add collision-safe Docker host identifiers (#590)
This commit is contained in:
parent
971d55f334
commit
f83caf8933
2 changed files with 372 additions and 12 deletions
|
|
@ -2,6 +2,8 @@ package monitoring
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha1"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
stderrors "errors"
|
stderrors "errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
@ -17,6 +19,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
|
|
@ -788,16 +791,273 @@ func tokenHintFromRecord(record *config.APITokenRecord) string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveDockerHostIdentifier(report agentsdocker.Report, tokenRecord *config.APITokenRecord, hosts []models.DockerHost) (string, []string, models.DockerHost, bool) {
|
||||||
|
base := strings.TrimSpace(report.AgentKey())
|
||||||
|
fallbacks := uniqueNonEmptyStrings(
|
||||||
|
base,
|
||||||
|
strings.TrimSpace(report.Agent.ID),
|
||||||
|
strings.TrimSpace(report.Host.MachineID),
|
||||||
|
strings.TrimSpace(report.Host.Hostname),
|
||||||
|
)
|
||||||
|
|
||||||
|
if existing, ok := findMatchingDockerHost(hosts, report, tokenRecord); ok {
|
||||||
|
return existing.ID, fallbacks, existing, true
|
||||||
|
}
|
||||||
|
|
||||||
|
identifier := base
|
||||||
|
if identifier == "" {
|
||||||
|
identifier = strings.TrimSpace(report.Host.MachineID)
|
||||||
|
}
|
||||||
|
if identifier == "" {
|
||||||
|
identifier = strings.TrimSpace(report.Host.Hostname)
|
||||||
|
}
|
||||||
|
if identifier == "" {
|
||||||
|
identifier = strings.TrimSpace(report.Agent.ID)
|
||||||
|
}
|
||||||
|
if identifier == "" {
|
||||||
|
identifier = fallbackDockerHostID(report, tokenRecord)
|
||||||
|
}
|
||||||
|
if identifier == "" {
|
||||||
|
identifier = "docker-host"
|
||||||
|
}
|
||||||
|
|
||||||
|
if dockerHostIDExists(identifier, hosts) {
|
||||||
|
identifier = generateDockerHostIdentifier(identifier, report, tokenRecord, hosts)
|
||||||
|
}
|
||||||
|
|
||||||
|
return identifier, fallbacks, models.DockerHost{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func findMatchingDockerHost(hosts []models.DockerHost, report agentsdocker.Report, tokenRecord *config.APITokenRecord) (models.DockerHost, bool) {
|
||||||
|
agentID := strings.TrimSpace(report.Agent.ID)
|
||||||
|
tokenID := ""
|
||||||
|
if tokenRecord != nil {
|
||||||
|
tokenID = strings.TrimSpace(tokenRecord.ID)
|
||||||
|
}
|
||||||
|
machineID := strings.TrimSpace(report.Host.MachineID)
|
||||||
|
hostname := strings.TrimSpace(report.Host.Hostname)
|
||||||
|
|
||||||
|
if agentID != "" {
|
||||||
|
for _, host := range hosts {
|
||||||
|
if strings.TrimSpace(host.AgentID) == agentID {
|
||||||
|
return host, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if tokenID != "" {
|
||||||
|
for _, host := range hosts {
|
||||||
|
if strings.TrimSpace(host.TokenID) == tokenID {
|
||||||
|
return host, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if machineID != "" && hostname != "" {
|
||||||
|
for _, host := range hosts {
|
||||||
|
if strings.TrimSpace(host.MachineID) == machineID && strings.TrimSpace(host.Hostname) == hostname {
|
||||||
|
if tokenID == "" || strings.TrimSpace(host.TokenID) == tokenID {
|
||||||
|
return host, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if machineID != "" && tokenID == "" {
|
||||||
|
for _, host := range hosts {
|
||||||
|
if strings.TrimSpace(host.MachineID) == machineID && strings.TrimSpace(host.TokenID) == "" {
|
||||||
|
return host, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if hostname != "" && tokenID == "" {
|
||||||
|
for _, host := range hosts {
|
||||||
|
if strings.TrimSpace(host.Hostname) == hostname && strings.TrimSpace(host.TokenID) == "" {
|
||||||
|
return host, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return models.DockerHost{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerHostIDExists(id string, hosts []models.DockerHost) bool {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, host := range hosts {
|
||||||
|
if host.ID == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateDockerHostIdentifier(base string, report agentsdocker.Report, tokenRecord *config.APITokenRecord, hosts []models.DockerHost) string {
|
||||||
|
if strings.TrimSpace(base) == "" {
|
||||||
|
base = fallbackDockerHostID(report, tokenRecord)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(base) == "" {
|
||||||
|
base = "docker-host"
|
||||||
|
}
|
||||||
|
|
||||||
|
used := make(map[string]struct{}, len(hosts))
|
||||||
|
for _, host := range hosts {
|
||||||
|
used[host.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
suffixes := dockerHostSuffixCandidates(report, tokenRecord)
|
||||||
|
for _, suffix := range suffixes {
|
||||||
|
candidate := fmt.Sprintf("%s::%s", base, suffix)
|
||||||
|
if _, exists := used[candidate]; !exists {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
seed := strings.Join(suffixes, "|")
|
||||||
|
if strings.TrimSpace(seed) == "" {
|
||||||
|
seed = base
|
||||||
|
}
|
||||||
|
sum := sha1.Sum([]byte(seed))
|
||||||
|
hashSuffix := fmt.Sprintf("hash-%s", hex.EncodeToString(sum[:6]))
|
||||||
|
candidate := fmt.Sprintf("%s::%s", base, hashSuffix)
|
||||||
|
if _, exists := used[candidate]; !exists {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
|
||||||
|
for idx := 2; ; idx++ {
|
||||||
|
candidate = fmt.Sprintf("%s::%d", base, idx)
|
||||||
|
if _, exists := used[candidate]; !exists {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func dockerHostSuffixCandidates(report agentsdocker.Report, tokenRecord *config.APITokenRecord) []string {
|
||||||
|
candidates := make([]string, 0, 5)
|
||||||
|
|
||||||
|
if tokenRecord != nil {
|
||||||
|
if sanitized := sanitizeDockerHostSuffix(tokenRecord.ID); sanitized != "" {
|
||||||
|
candidates = append(candidates, "token-"+sanitized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if agentID := sanitizeDockerHostSuffix(report.Agent.ID); agentID != "" {
|
||||||
|
candidates = append(candidates, "agent-"+agentID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if machineID := sanitizeDockerHostSuffix(report.Host.MachineID); machineID != "" {
|
||||||
|
candidates = append(candidates, "machine-"+machineID)
|
||||||
|
}
|
||||||
|
|
||||||
|
hostNameSanitized := sanitizeDockerHostSuffix(report.Host.Hostname)
|
||||||
|
if hostNameSanitized != "" {
|
||||||
|
candidates = append(candidates, "host-"+hostNameSanitized)
|
||||||
|
}
|
||||||
|
|
||||||
|
hostDisplay := sanitizeDockerHostSuffix(report.Host.Name)
|
||||||
|
if hostDisplay != "" && hostDisplay != hostNameSanitized {
|
||||||
|
candidates = append(candidates, "name-"+hostDisplay)
|
||||||
|
}
|
||||||
|
|
||||||
|
return uniqueNonEmptyStrings(candidates...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeDockerHostSuffix(value string) string {
|
||||||
|
value = strings.ToLower(strings.TrimSpace(value))
|
||||||
|
if value == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var builder strings.Builder
|
||||||
|
builder.Grow(len(value))
|
||||||
|
lastHyphen := false
|
||||||
|
runeCount := 0
|
||||||
|
|
||||||
|
for _, r := range value {
|
||||||
|
if runeCount >= 48 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||||||
|
builder.WriteRune(r)
|
||||||
|
lastHyphen = false
|
||||||
|
runeCount++
|
||||||
|
default:
|
||||||
|
if !lastHyphen {
|
||||||
|
builder.WriteRune('-')
|
||||||
|
lastHyphen = true
|
||||||
|
runeCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result := strings.Trim(builder.String(), "-")
|
||||||
|
if result == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func fallbackDockerHostID(report agentsdocker.Report, tokenRecord *config.APITokenRecord) string {
|
||||||
|
seedParts := dockerHostSuffixCandidates(report, tokenRecord)
|
||||||
|
if len(seedParts) == 0 {
|
||||||
|
seedParts = uniqueNonEmptyStrings(
|
||||||
|
report.Host.Hostname,
|
||||||
|
report.Host.MachineID,
|
||||||
|
report.Agent.ID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if len(seedParts) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
seed := strings.Join(seedParts, "|")
|
||||||
|
sum := sha1.Sum([]byte(seed))
|
||||||
|
return fmt.Sprintf("docker-host-%s", hex.EncodeToString(sum[:6]))
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueNonEmptyStrings(values ...string) []string {
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
result := make([]string, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[value]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
result = append(result, value)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// ApplyDockerReport ingests a docker agent report into the shared state.
|
// ApplyDockerReport ingests a docker agent report into the shared state.
|
||||||
func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *config.APITokenRecord) (models.DockerHost, error) {
|
func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *config.APITokenRecord) (models.DockerHost, error) {
|
||||||
identifier := strings.TrimSpace(report.AgentKey())
|
hostsSnapshot := m.state.GetDockerHosts()
|
||||||
if identifier == "" {
|
identifier, legacyIDs, previous, hasPrevious := resolveDockerHostIdentifier(report, tokenRecord, hostsSnapshot)
|
||||||
|
if strings.TrimSpace(identifier) == "" {
|
||||||
return models.DockerHost{}, fmt.Errorf("docker report missing agent identifier")
|
return models.DockerHost{}, fmt.Errorf("docker report missing agent identifier")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if this host was deliberately removed - reject report to prevent resurrection
|
// Check if this host was deliberately removed - reject report to prevent resurrection
|
||||||
m.mu.RLock()
|
m.mu.RLock()
|
||||||
removedAt, wasRemoved := m.removedDockerHosts[identifier]
|
removedAt, wasRemoved := m.removedDockerHosts[identifier]
|
||||||
|
if !wasRemoved {
|
||||||
|
for _, legacyID := range legacyIDs {
|
||||||
|
if legacyID == "" || legacyID == identifier {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ts, ok := m.removedDockerHosts[legacyID]; ok {
|
||||||
|
removedAt = ts
|
||||||
|
wasRemoved = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
if wasRemoved {
|
if wasRemoved {
|
||||||
|
|
@ -828,16 +1088,6 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
|
||||||
displayName = hostname
|
displayName = hostname
|
||||||
}
|
}
|
||||||
|
|
||||||
var previous models.DockerHost
|
|
||||||
var hasPrevious bool
|
|
||||||
for _, existing := range m.state.GetDockerHosts() {
|
|
||||||
if existing.ID == identifier {
|
|
||||||
previous = existing
|
|
||||||
hasPrevious = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
containers := make([]models.DockerContainer, 0, len(report.Containers))
|
containers := make([]models.DockerContainer, 0, len(report.Containers))
|
||||||
for _, payload := range report.Containers {
|
for _, payload := range report.Containers {
|
||||||
container := models.DockerContainer{
|
container := models.DockerContainer{
|
||||||
|
|
|
||||||
110
internal/monitoring/monitor_docker_test.go
Normal file
110
internal/monitoring/monitor_docker_test.go
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
package monitoring
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||||
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||||
|
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newTestMonitor(t *testing.T) *Monitor {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return &Monitor{
|
||||||
|
state: models.NewState(),
|
||||||
|
alertManager: alerts.NewManager(),
|
||||||
|
removedDockerHosts: make(map[string]time.Time),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyDockerReportGeneratesUniqueIDsForCollidingHosts(t *testing.T) {
|
||||||
|
monitor := newTestMonitor(t)
|
||||||
|
|
||||||
|
baseTimestamp := time.Now().UTC()
|
||||||
|
baseReport := agentsdocker.Report{
|
||||||
|
Agent: agentsdocker.AgentInfo{
|
||||||
|
Version: "1.0.0",
|
||||||
|
IntervalSeconds: 30,
|
||||||
|
},
|
||||||
|
Host: agentsdocker.HostInfo{
|
||||||
|
Hostname: "docker-host",
|
||||||
|
Name: "Docker Host",
|
||||||
|
MachineID: "machine-duplicate",
|
||||||
|
DockerVersion: "26.0.0",
|
||||||
|
TotalCPU: 4,
|
||||||
|
TotalMemoryBytes: 8 << 30,
|
||||||
|
UptimeSeconds: 120,
|
||||||
|
},
|
||||||
|
Containers: []agentsdocker.Container{
|
||||||
|
{ID: "container-1", Name: "nginx"},
|
||||||
|
},
|
||||||
|
Timestamp: baseTimestamp,
|
||||||
|
}
|
||||||
|
|
||||||
|
token1 := &config.APITokenRecord{ID: "token-host-1", Name: "Host 1"}
|
||||||
|
host1, err := monitor.ApplyDockerReport(baseReport, token1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ApplyDockerReport host1: %v", err)
|
||||||
|
}
|
||||||
|
if host1.ID == "" {
|
||||||
|
t.Fatalf("expected host1 to have an identifier")
|
||||||
|
}
|
||||||
|
|
||||||
|
hosts := monitor.state.GetDockerHosts()
|
||||||
|
if len(hosts) != 1 {
|
||||||
|
t.Fatalf("expected 1 host after first report, got %d", len(hosts))
|
||||||
|
}
|
||||||
|
|
||||||
|
secondReport := baseReport
|
||||||
|
secondReport.Host.Name = "Docker Host Clone"
|
||||||
|
secondReport.Timestamp = baseTimestamp.Add(45 * time.Second)
|
||||||
|
|
||||||
|
token2 := &config.APITokenRecord{ID: "token-host-2", Name: "Host 2"}
|
||||||
|
host2, err := monitor.ApplyDockerReport(secondReport, token2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ApplyDockerReport host2: %v", err)
|
||||||
|
}
|
||||||
|
if host2.ID == "" {
|
||||||
|
t.Fatalf("expected host2 to have an identifier")
|
||||||
|
}
|
||||||
|
if host2.ID == host1.ID {
|
||||||
|
t.Fatalf("expected unique identifiers, but both hosts share %q", host2.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
hosts = monitor.state.GetDockerHosts()
|
||||||
|
if len(hosts) != 2 {
|
||||||
|
t.Fatalf("expected 2 hosts after second report, got %d", len(hosts))
|
||||||
|
}
|
||||||
|
|
||||||
|
secondReport.Timestamp = secondReport.Timestamp.Add(45 * time.Second)
|
||||||
|
secondReport.Containers = append(secondReport.Containers, agentsdocker.Container{
|
||||||
|
ID: "container-2",
|
||||||
|
Name: "redis",
|
||||||
|
})
|
||||||
|
|
||||||
|
updatedHost2, err := monitor.ApplyDockerReport(secondReport, token2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ApplyDockerReport host2 update: %v", err)
|
||||||
|
}
|
||||||
|
if updatedHost2.ID != host2.ID {
|
||||||
|
t.Fatalf("expected host2 to retain identifier %q, got %q", host2.ID, updatedHost2.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
hosts = monitor.state.GetDockerHosts()
|
||||||
|
var found models.DockerHost
|
||||||
|
for _, h := range hosts {
|
||||||
|
if h.ID == host2.ID {
|
||||||
|
found = h
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found.ID == "" {
|
||||||
|
t.Fatalf("failed to locate host2 in state after update")
|
||||||
|
}
|
||||||
|
if len(found.Containers) != 2 {
|
||||||
|
t.Fatalf("expected host2 to have 2 containers after update, got %d", len(found.Containers))
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue