Handle ignored docker containers during alert reevaluation (related to #561)
This commit is contained in:
parent
6de8df9281
commit
6ff54beb38
2 changed files with 65 additions and 0 deletions
|
|
@ -1422,6 +1422,20 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
||||||
alertsToResolve = append(alertsToResolve, alertID)
|
alertsToResolve = append(alertsToResolve, alertID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
containerName := strings.ToLower(strings.TrimSpace(alert.ResourceName))
|
||||||
|
containerID := ""
|
||||||
|
if alert.Metadata != nil {
|
||||||
|
if val, ok := alert.Metadata["containerId"].(string); ok {
|
||||||
|
containerID = strings.ToLower(strings.TrimSpace(val))
|
||||||
|
}
|
||||||
|
if val, ok := alert.Metadata["containerName"].(string); ok && containerName == "" {
|
||||||
|
containerName = strings.ToLower(strings.TrimSpace(val))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if matchesDockerIgnoredPrefix(containerName, containerID, m.config.DockerIgnoredContainerPrefixes) {
|
||||||
|
alertsToResolve = append(alertsToResolve, alertID)
|
||||||
|
continue
|
||||||
|
}
|
||||||
thresholds := ThresholdConfig{
|
thresholds := ThresholdConfig{
|
||||||
CPU: cloneThreshold(&m.config.DockerDefaults.CPU),
|
CPU: cloneThreshold(&m.config.DockerDefaults.CPU),
|
||||||
Memory: cloneThreshold(&m.config.DockerDefaults.Memory),
|
Memory: cloneThreshold(&m.config.DockerDefaults.Memory),
|
||||||
|
|
|
||||||
|
|
@ -1845,3 +1845,54 @@ func TestReevaluateClearsDockerContainerAlertWhenOverrideDisabled(t *testing.T)
|
||||||
t.Fatalf("expected docker container alert to be cleared when override is disabled")
|
t.Fatalf("expected docker container alert to be cleared when override is disabled")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReevaluateClearsDockerContainerAlertWhenIgnoredPrefixAdded(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
|
||||||
|
resourceID := "docker:host-2/container-abc123"
|
||||||
|
alertID := resourceID + "-cpu"
|
||||||
|
|
||||||
|
resolved := make(chan string, 1)
|
||||||
|
m.SetResolvedCallback(func(id string) {
|
||||||
|
resolved <- id
|
||||||
|
})
|
||||||
|
|
||||||
|
m.mu.Lock()
|
||||||
|
m.activeAlerts[alertID] = &Alert{
|
||||||
|
ID: alertID,
|
||||||
|
Type: "cpu",
|
||||||
|
ResourceID: resourceID,
|
||||||
|
ResourceName: "qbittorrentvpn",
|
||||||
|
Instance: "Docker",
|
||||||
|
Metadata: map[string]interface{}{
|
||||||
|
"resourceType": "Docker Container",
|
||||||
|
"containerId": "abc123",
|
||||||
|
"containerName": "qbittorrentvpn",
|
||||||
|
},
|
||||||
|
Threshold: 80,
|
||||||
|
Value: 95,
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
config := m.GetConfig()
|
||||||
|
config.DockerIgnoredContainerPrefixes = []string{"qbit"}
|
||||||
|
config.ActivationState = ActivationActive
|
||||||
|
|
||||||
|
m.UpdateConfig(config)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-resolved:
|
||||||
|
if got != alertID {
|
||||||
|
t.Fatalf("resolved callback fired for unexpected alert %s", got)
|
||||||
|
}
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
t.Fatalf("expected alert to be resolved after adding ignored prefix")
|
||||||
|
}
|
||||||
|
|
||||||
|
m.mu.RLock()
|
||||||
|
_, exists := m.activeAlerts[alertID]
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if exists {
|
||||||
|
t.Fatalf("expected docker container alert to be cleared when ignored prefix is configured")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue