test: Add Docker container health and OOM kill alert tests
Coverage improvements: - checkDockerContainerHealth: 21.1% -> 94.7% - checkDockerContainerOOMKill: 19.2% -> 96.2% Tests cover: - Health states (healthy, empty, none, starting, unhealthy, degraded) - Health alert recovery when container becomes healthy - OOM kill detection (exit code 137) - OOM alert deduplication (repeated 137 doesn't re-alert) - OOM alert clearing when container recovers or exits with different code
This commit is contained in:
rcourtman
parent
1ed8b18c12
commit
b6624e32cf
1 changed files with 501 additions and 0 deletions
|
|
@ -9448,3 +9448,504 @@ func TestCheckPMGNodeQueues(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDockerContainerHealthAlert(t *testing.T) {
|
||||
t.Run("healthy container - no alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-health-1",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-1",
|
||||
Name: "healthy-app",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes",
|
||||
Health: "healthy",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected no health alert for healthy container")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("container with empty health - no alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-health-2",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-2",
|
||||
Name: "no-health-check",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes",
|
||||
Health: "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected no health alert for container with empty health")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("container with none health - no alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-health-3",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-3",
|
||||
Name: "no-health-check",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes",
|
||||
Health: "none",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected no health alert for container with none health")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("container starting - no alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-health-4",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-4",
|
||||
Name: "starting-app",
|
||||
State: "running",
|
||||
Status: "Up 5 seconds",
|
||||
Health: "starting",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected no health alert for starting container")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unhealthy container - critical alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-health-5",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-5",
|
||||
Name: "unhealthy-app",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes (unhealthy)",
|
||||
Health: "unhealthy",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
alert, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatal("expected health alert for unhealthy container")
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("expected critical alert for unhealthy container, got %s", alert.Level)
|
||||
}
|
||||
if alert.Type != "docker-container-health" {
|
||||
t.Fatalf("expected alert type docker-container-health, got %s", alert.Type)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("container with other health status - warning alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-health-6",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-6",
|
||||
Name: "degraded-app",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes",
|
||||
Health: "degraded",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
alert, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatal("expected health alert for degraded container")
|
||||
}
|
||||
if alert.Level != AlertLevelWarning {
|
||||
t.Fatalf("expected warning alert for non-unhealthy bad status, got %s", alert.Level)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("alert cleared when container becomes healthy", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
hostID := "host-health-7"
|
||||
containerID := "container-7"
|
||||
|
||||
// First check with unhealthy container
|
||||
hostUnhealthy := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "recovering-app",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes (unhealthy)",
|
||||
Health: "unhealthy",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(hostUnhealthy)
|
||||
|
||||
resourceID := dockerResourceID(hostID, containerID)
|
||||
alertID := fmt.Sprintf("docker-container-health-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; !exists {
|
||||
t.Fatal("expected health alert to be raised")
|
||||
}
|
||||
|
||||
// Now container becomes healthy
|
||||
hostHealthy := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "recovering-app",
|
||||
State: "running",
|
||||
Status: "Up 15 minutes",
|
||||
Health: "healthy",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(hostHealthy)
|
||||
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected health alert to be cleared when container became healthy")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDockerContainerOOMKillAlert(t *testing.T) {
|
||||
t.Run("running container - no alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-oom-1",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-1",
|
||||
Name: "running-app",
|
||||
State: "running",
|
||||
Status: "Up 10 minutes",
|
||||
ExitCode: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected no OOM alert for running container")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("exited container with non-137 exit code - no alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-oom-2",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-2",
|
||||
Name: "normal-exit-app",
|
||||
State: "exited",
|
||||
Status: "Exited (1) 5 minutes ago",
|
||||
ExitCode: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected no OOM alert for container with exit code 1")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("exited container with exit code 137 - critical OOM alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-oom-3",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-3",
|
||||
Name: "oom-killed-app",
|
||||
State: "exited",
|
||||
Status: "Exited (137) 1 minute ago",
|
||||
ExitCode: 137,
|
||||
MemoryUsage: 512 * 1024 * 1024,
|
||||
MemoryLimit: 512 * 1024 * 1024,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
alert, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatal("expected OOM alert for container with exit code 137")
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("expected critical OOM alert, got %s", alert.Level)
|
||||
}
|
||||
if alert.Type != "docker-container-oom-kill" {
|
||||
t.Fatalf("expected alert type docker-container-oom-kill, got %s", alert.Type)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dead container with exit code 137 - critical OOM alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-oom-dead",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: "container-dead",
|
||||
Name: "dead-oom-app",
|
||||
State: "dead",
|
||||
Status: "Dead",
|
||||
ExitCode: 137,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(host.ID, host.Containers[0].ID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
alert, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatal("expected OOM alert for dead container with exit code 137")
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("expected critical OOM alert, got %s", alert.Level)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("repeated 137 exit code - no new alert", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
hostID := "host-oom-4"
|
||||
containerID := "container-4"
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "oom-killed-app",
|
||||
State: "exited",
|
||||
Status: "Exited (137) 1 minute ago",
|
||||
ExitCode: 137,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// First check - should create alert
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerResourceID(hostID, containerID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
alert1, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatal("expected OOM alert on first check")
|
||||
}
|
||||
startTime := alert1.StartTime
|
||||
|
||||
// Second check with same exit code - should not create new alert
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
alert2, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatal("expected OOM alert to still exist on second check")
|
||||
}
|
||||
if alert2.StartTime != startTime {
|
||||
t.Fatal("expected alert start time to be preserved (not a new alert)")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("container recovers - alert cleared", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
hostID := "host-oom-5"
|
||||
containerID := "container-5"
|
||||
|
||||
// First check with OOM killed container
|
||||
hostOOM := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "recovering-app",
|
||||
State: "exited",
|
||||
Status: "Exited (137) 1 minute ago",
|
||||
ExitCode: 137,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(hostOOM)
|
||||
|
||||
resourceID := dockerResourceID(hostID, containerID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; !exists {
|
||||
t.Fatal("expected OOM alert to be raised")
|
||||
}
|
||||
|
||||
// Container is restarted and running again
|
||||
hostRunning := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "recovering-app",
|
||||
State: "running",
|
||||
Status: "Up 30 seconds",
|
||||
ExitCode: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(hostRunning)
|
||||
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected OOM alert to be cleared when container started running")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("container exits with different code - alert cleared", func(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
hostID := "host-oom-6"
|
||||
containerID := "container-6"
|
||||
|
||||
// First check with OOM killed container
|
||||
hostOOM := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "multi-exit-app",
|
||||
State: "exited",
|
||||
Status: "Exited (137) 1 minute ago",
|
||||
ExitCode: 137,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(hostOOM)
|
||||
|
||||
resourceID := dockerResourceID(hostID, containerID)
|
||||
alertID := fmt.Sprintf("docker-container-oom-%s", resourceID)
|
||||
if _, exists := m.activeAlerts[alertID]; !exists {
|
||||
t.Fatal("expected OOM alert to be raised")
|
||||
}
|
||||
|
||||
// Container exits with different exit code (normal error)
|
||||
hostNormalExit := models.DockerHost{
|
||||
ID: hostID,
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
Containers: []models.DockerContainer{
|
||||
{
|
||||
ID: containerID,
|
||||
Name: "multi-exit-app",
|
||||
State: "exited",
|
||||
Status: "Exited (1) 30 seconds ago",
|
||||
ExitCode: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(hostNormalExit)
|
||||
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatal("expected OOM alert to be cleared when container exited with different code")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue