b70dc3d00d
Add documentation to explain how transport-level and queue-level retries interact: - Email: MaxRetries (transport) * MaxAttempts (queue) = total SMTP attempts - Webhooks: RetryCount (transport) * MaxAttempts (queue) = total HTTP attempts - Example: 3 * 3 = 9 total delivery attempts for a single notification This clarifies the multiplicative retry behavior and helps operators understand the actual retry counts when using the persistent queue.
619 lines
18 KiB
Go
619 lines
18 KiB
Go
package notifications
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// EnhancedWebhookConfig extends WebhookConfig with template support
|
|
type EnhancedWebhookConfig struct {
|
|
WebhookConfig
|
|
Service string `json:"service"` // discord, slack, teams, pagerduty, generic
|
|
PayloadTemplate string `json:"payloadTemplate"` // Go template for payload
|
|
RetryEnabled bool `json:"retryEnabled"`
|
|
RetryCount int `json:"retryCount"`
|
|
FilterRules WebhookFilterRules `json:"filterRules"`
|
|
CustomFields map[string]interface{} `json:"customFields"` // For template variables
|
|
ResponseLogging bool `json:"responseLogging"` // Log response for debugging
|
|
}
|
|
|
|
// WebhookFilterRules defines filtering for this webhook
|
|
type WebhookFilterRules struct {
|
|
Levels []string `json:"levels"` // Only send these levels
|
|
Types []string `json:"types"` // Only send these alert types
|
|
Nodes []string `json:"nodes"` // Only send from these nodes
|
|
ResourceTypes []string `json:"resourceTypes"` // vm, container, storage, etc
|
|
}
|
|
|
|
// WebhookPayloadData contains all data available for templates
|
|
type WebhookPayloadData struct {
|
|
// Alert fields
|
|
ID string
|
|
Level string
|
|
Type string
|
|
ResourceName string
|
|
ResourceID string
|
|
Node string
|
|
Instance string
|
|
Message string
|
|
Value float64
|
|
Threshold float64
|
|
ValueFormatted string
|
|
ThresholdFormatted string
|
|
StartTime string
|
|
Duration string
|
|
Timestamp string
|
|
ResourceType string
|
|
Acknowledged bool
|
|
AckTime string
|
|
AckUser string
|
|
|
|
// Additional context
|
|
Metadata map[string]interface{}
|
|
CustomFields map[string]interface{}
|
|
AlertCount int
|
|
Alerts []*alerts.Alert // For grouped alerts
|
|
ChatID string // For Telegram webhooks
|
|
}
|
|
|
|
// SendEnhancedWebhook sends a webhook with template support
|
|
func (n *NotificationManager) SendEnhancedWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) error {
|
|
// Check filters
|
|
if !n.shouldSendWebhook(webhook, alert) {
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Str("alertID", alert.ID).
|
|
Msg("Alert filtered out by webhook rules")
|
|
return nil
|
|
}
|
|
|
|
// Prepare template data
|
|
data := n.prepareWebhookData(alert, webhook.CustomFields)
|
|
|
|
// Render URL template when placeholders are present
|
|
renderedURL, renderErr := renderWebhookURL(webhook.URL, data)
|
|
if renderErr != nil {
|
|
return fmt.Errorf("failed to render webhook URL template: %w", renderErr)
|
|
}
|
|
webhook.URL = renderedURL
|
|
|
|
// Service-specific enrichment
|
|
switch webhook.Service {
|
|
case "telegram":
|
|
chatID, chatErr := extractTelegramChatID(webhook.URL)
|
|
if chatErr != nil {
|
|
return fmt.Errorf("failed to extract Telegram chat_id: %w", chatErr)
|
|
}
|
|
if chatID != "" {
|
|
data.ChatID = chatID
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Str("chatID", chatID).
|
|
Msg("Extracted Telegram chat_id from rendered URL for enhanced webhook")
|
|
}
|
|
case "pagerduty":
|
|
if data.CustomFields == nil {
|
|
data.CustomFields = make(map[string]interface{})
|
|
}
|
|
if routingKey, ok := webhook.Headers["routing_key"]; ok {
|
|
data.CustomFields["routing_key"] = routingKey
|
|
}
|
|
}
|
|
|
|
// Generate payload from template with service-specific handling
|
|
payload, err := n.generatePayloadFromTemplateWithService(webhook.PayloadTemplate, data, webhook.Service)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate payload: %w", err)
|
|
}
|
|
|
|
// Send with retry logic
|
|
if webhook.RetryEnabled {
|
|
return n.sendWebhookWithRetry(webhook, payload)
|
|
}
|
|
|
|
return n.sendWebhookOnce(webhook, payload)
|
|
}
|
|
|
|
// NOTE: prepareWebhookData is now defined in notifications.go to avoid duplication
|
|
// NOTE: generatePayloadFromTemplate is now defined in notifications.go to avoid duplication
|
|
|
|
// shouldSendWebhook checks if alert matches webhook filter rules
|
|
func (n *NotificationManager) shouldSendWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) bool {
|
|
rules := webhook.FilterRules
|
|
|
|
// Check level filter
|
|
if len(rules.Levels) > 0 {
|
|
found := false
|
|
for _, level := range rules.Levels {
|
|
if string(alert.Level) == level {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Check type filter
|
|
if len(rules.Types) > 0 {
|
|
found := false
|
|
for _, alertType := range rules.Types {
|
|
if alert.Type == alertType {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Check node filter
|
|
if len(rules.Nodes) > 0 {
|
|
found := false
|
|
for _, node := range rules.Nodes {
|
|
if alert.Node == node {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Check resource type filter
|
|
if len(rules.ResourceTypes) > 0 {
|
|
resourceType, ok := alert.Metadata["resourceType"].(string)
|
|
if !ok {
|
|
return false
|
|
}
|
|
found := false
|
|
for _, rt := range rules.ResourceTypes {
|
|
if resourceType == rt {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// sendWebhookWithRetry implements exponential backoff retry with enhanced error tracking
|
|
// Note: When used with the persistent queue, retry behavior is layered:
|
|
// - Transport retries (this function): up to RetryCount attempts with exponential backoff
|
|
// - Queue retries: up to MaxAttempts (default 3) with exponential backoff
|
|
// Total attempts = RetryCount * MaxAttempts (e.g., 3 * 3 = 9 HTTP calls for a single notification)
|
|
// This ensures delivery even during transient failures at either layer.
|
|
func (n *NotificationManager) sendWebhookWithRetry(webhook EnhancedWebhookConfig, payload []byte) error {
|
|
maxRetries := webhook.RetryCount
|
|
if maxRetries <= 0 {
|
|
maxRetries = WebhookDefaultRetries
|
|
}
|
|
|
|
var lastErr error
|
|
var lastResp *http.Response
|
|
backoff := WebhookInitialBackoff
|
|
retryableErrors := 0
|
|
|
|
for attempt := 0; attempt <= maxRetries; attempt++ {
|
|
if attempt > 0 {
|
|
// Check for Retry-After header from previous response (overrides backoff)
|
|
if lastResp != nil && lastResp.StatusCode == 429 {
|
|
if retryAfter := lastResp.Header.Get("Retry-After"); retryAfter != "" {
|
|
if seconds, err := strconv.Atoi(retryAfter); err == nil {
|
|
customBackoff := time.Duration(seconds) * time.Second
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Dur("retryAfter", customBackoff).
|
|
Msg("Using Retry-After header for backoff")
|
|
time.Sleep(customBackoff)
|
|
backoff = customBackoff // Use this for next iteration
|
|
}
|
|
} else {
|
|
// No Retry-After, use exponential backoff
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Int("attempt", attempt).
|
|
Int("maxRetries", maxRetries).
|
|
Dur("backoff", backoff).
|
|
Msg("Retrying webhook after backoff")
|
|
time.Sleep(backoff)
|
|
}
|
|
} else {
|
|
// Not a 429, use exponential backoff
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Int("attempt", attempt).
|
|
Int("maxRetries", maxRetries).
|
|
Dur("backoff", backoff).
|
|
Msg("Retrying webhook after backoff")
|
|
time.Sleep(backoff)
|
|
}
|
|
|
|
// Exponential backoff for next iteration
|
|
backoff *= 2
|
|
if backoff > WebhookMaxBackoff {
|
|
backoff = WebhookMaxBackoff
|
|
}
|
|
}
|
|
|
|
resp, err := n.sendWebhookOnceWithResponse(webhook, payload)
|
|
lastResp = resp
|
|
if err == nil {
|
|
if attempt > 0 {
|
|
log.Info().
|
|
Str("webhook", webhook.Name).
|
|
Int("attempt", attempt).
|
|
Int("totalAttempts", attempt+1).
|
|
Msg("Webhook succeeded after retry")
|
|
}
|
|
// Log successful delivery
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Str("service", webhook.Service).
|
|
Int("payloadSize", len(payload)).
|
|
Msg("Webhook delivered successfully")
|
|
|
|
// Track successful delivery
|
|
delivery := WebhookDelivery{
|
|
WebhookName: webhook.Name,
|
|
WebhookURL: webhook.URL,
|
|
Service: webhook.Service,
|
|
AlertID: "enhanced", // This is for enhanced webhooks, alertID might not be available
|
|
Timestamp: time.Now(),
|
|
StatusCode: 200, // Assume success
|
|
Success: true,
|
|
RetryAttempts: attempt,
|
|
PayloadSize: len(payload),
|
|
}
|
|
n.addWebhookDelivery(delivery)
|
|
|
|
return nil
|
|
}
|
|
|
|
lastErr = err
|
|
|
|
// Determine if error is retryable
|
|
isRetryable := isRetryableWebhookError(err)
|
|
if isRetryable {
|
|
retryableErrors++
|
|
}
|
|
|
|
log.Warn().
|
|
Err(err).
|
|
Str("webhook", webhook.Name).
|
|
Str("service", webhook.Service).
|
|
Int("attempt", attempt+1).
|
|
Int("maxRetries", maxRetries+1).
|
|
Bool("retryable", isRetryable).
|
|
Msg("Webhook attempt failed")
|
|
|
|
// If error is not retryable, break early
|
|
if !isRetryable && attempt == 0 {
|
|
log.Error().
|
|
Err(err).
|
|
Str("webhook", webhook.Name).
|
|
Msg("Non-retryable webhook error - not attempting retry")
|
|
break
|
|
}
|
|
}
|
|
|
|
// Final error logging with summary
|
|
log.Error().
|
|
Err(lastErr).
|
|
Str("webhook", webhook.Name).
|
|
Str("service", webhook.Service).
|
|
Int("totalAttempts", maxRetries+1).
|
|
Int("retryableErrors", retryableErrors).
|
|
Msg("Webhook delivery failed after all retry attempts")
|
|
|
|
// Track failed delivery
|
|
delivery := WebhookDelivery{
|
|
WebhookName: webhook.Name,
|
|
WebhookURL: webhook.URL,
|
|
Service: webhook.Service,
|
|
AlertID: "enhanced", // This is for enhanced webhooks, alertID might not be available
|
|
Timestamp: time.Now(),
|
|
StatusCode: 0, // Unknown status
|
|
Success: false,
|
|
ErrorMessage: lastErr.Error(),
|
|
RetryAttempts: maxRetries,
|
|
PayloadSize: len(payload),
|
|
}
|
|
n.addWebhookDelivery(delivery)
|
|
|
|
return fmt.Errorf("webhook failed after %d attempts: %w", maxRetries+1, lastErr)
|
|
}
|
|
|
|
// isRetryableWebhookError determines if a webhook error should trigger a retry
|
|
func isRetryableWebhookError(err error) bool {
|
|
errStr := strings.ToLower(err.Error())
|
|
|
|
// Network-related errors that should be retried
|
|
if strings.Contains(errStr, "timeout") ||
|
|
strings.Contains(errStr, "connection refused") ||
|
|
strings.Contains(errStr, "connection reset") ||
|
|
strings.Contains(errStr, "no such host") ||
|
|
strings.Contains(errStr, "network unreachable") {
|
|
return true
|
|
}
|
|
|
|
// HTTP status codes that should be retried
|
|
if strings.Contains(errStr, "status 429") || // Rate limited
|
|
strings.Contains(errStr, "status 502") || // Bad Gateway
|
|
strings.Contains(errStr, "status 503") || // Service Unavailable
|
|
strings.Contains(errStr, "status 504") { // Gateway Timeout
|
|
return true
|
|
}
|
|
|
|
// 5xx server errors are generally retryable
|
|
for i := 500; i <= 599; i++ {
|
|
if strings.Contains(errStr, fmt.Sprintf("status %d", i)) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
// 4xx client errors are generally not retryable
|
|
for i := 400; i <= 499; i++ {
|
|
if strings.Contains(errStr, fmt.Sprintf("status %d", i)) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Default to retryable for unknown errors
|
|
return true
|
|
}
|
|
|
|
// sendWebhookOnceWithResponse sends a single webhook request and returns the response
|
|
func (n *NotificationManager) sendWebhookOnceWithResponse(webhook EnhancedWebhookConfig, payload []byte) (*http.Response, error) {
|
|
method := webhook.Method
|
|
if method == "" {
|
|
method = "POST"
|
|
}
|
|
|
|
req, err := http.NewRequest(method, webhook.URL, bytes.NewBuffer(payload))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
// Set headers
|
|
for key, value := range webhook.Headers {
|
|
req.Header.Set(key, value)
|
|
}
|
|
if req.Header.Get("Content-Type") == "" {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
req.Header.Set("User-Agent", "Pulse-Monitoring/2.0")
|
|
|
|
// Send request with secure client
|
|
client := createSecureWebhookClient(WebhookTimeout)
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Read response body with size limit
|
|
limitedReader := io.LimitReader(resp.Body, WebhookMaxResponseSize)
|
|
var respBody bytes.Buffer
|
|
bytesRead, err := respBody.ReadFrom(limitedReader)
|
|
if err != nil {
|
|
return resp, fmt.Errorf("failed to read webhook response body: %w", err)
|
|
}
|
|
|
|
if bytesRead >= WebhookMaxResponseSize {
|
|
log.Warn().
|
|
Str("webhook", webhook.Name).
|
|
Int64("bytesRead", bytesRead).
|
|
Msg("Webhook response exceeded size limit")
|
|
}
|
|
|
|
responseBody := respBody.String()
|
|
|
|
// Log response if enabled or if there's an error
|
|
if webhook.ResponseLogging || resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
log.Debug().
|
|
Str("webhook", webhook.Name).
|
|
Int("status", resp.StatusCode).
|
|
Str("response", responseBody).
|
|
Msg("Webhook response")
|
|
}
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return resp, fmt.Errorf("webhook returned status %d: %s", resp.StatusCode, responseBody)
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
// sendWebhookOnce sends a single webhook request (compatibility wrapper)
|
|
func (n *NotificationManager) sendWebhookOnce(webhook EnhancedWebhookConfig, payload []byte) error {
|
|
_, err := n.sendWebhookOnceWithResponse(webhook, payload)
|
|
return err
|
|
}
|
|
|
|
// NOTE: formatWebhookDuration is now defined in notifications.go to avoid duplication
|
|
|
|
// TestEnhancedWebhook tests a webhook with a specific payload
|
|
func (n *NotificationManager) TestEnhancedWebhook(webhook EnhancedWebhookConfig) (int, string, error) {
|
|
// Use the configured publicURL if available, otherwise use a placeholder
|
|
instanceURL := n.publicURL
|
|
if instanceURL == "" {
|
|
instanceURL = "https://192.168.1.100:8006"
|
|
}
|
|
|
|
// Create test alert
|
|
testAlert := &alerts.Alert{
|
|
ID: "test-" + time.Now().Format("20060102-150405"),
|
|
Type: "cpu",
|
|
Level: "warning",
|
|
ResourceID: "100",
|
|
ResourceName: "Test VM",
|
|
Node: "pve-node-01",
|
|
Instance: instanceURL,
|
|
Message: "Test webhook notification from Pulse Monitoring",
|
|
Value: 85.5,
|
|
Threshold: 80.0,
|
|
StartTime: time.Now().Add(-2 * time.Minute),
|
|
LastSeen: time.Now(),
|
|
Metadata: map[string]interface{}{
|
|
"resourceType": "vm",
|
|
},
|
|
}
|
|
|
|
// Prepare data
|
|
data := n.prepareWebhookData(testAlert, webhook.CustomFields)
|
|
|
|
// Render webhook URL using template data
|
|
renderedURL, renderErr := renderWebhookURL(webhook.URL, data)
|
|
if renderErr != nil {
|
|
return 0, "", fmt.Errorf("failed to render webhook URL template: %w", renderErr)
|
|
}
|
|
webhook.URL = renderedURL
|
|
|
|
// For Telegram, extract chat_id from URL if present
|
|
if webhook.Service == "telegram" {
|
|
if chatID, err := extractTelegramChatID(webhook.URL); err == nil && chatID != "" {
|
|
data.ChatID = chatID
|
|
} else if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("webhook", webhook.Name).
|
|
Msg("Failed to extract Telegram chat_id during enhanced webhook test")
|
|
}
|
|
// Note: For test webhooks, we don't fail if chat_id is missing
|
|
// as this may be intentional during testing
|
|
} else if webhook.Service == "pagerduty" {
|
|
if data.CustomFields == nil {
|
|
data.CustomFields = make(map[string]interface{})
|
|
}
|
|
if routingKey, ok := webhook.Headers["routing_key"]; ok {
|
|
data.CustomFields["routing_key"] = routingKey
|
|
}
|
|
}
|
|
|
|
// Generate payload with service-specific handling
|
|
payload, err := n.generatePayloadFromTemplateWithService(webhook.PayloadTemplate, data, webhook.Service)
|
|
if err != nil {
|
|
return 0, "", fmt.Errorf("failed to generate payload: %w", err)
|
|
}
|
|
|
|
// Send request
|
|
method := webhook.Method
|
|
if method == "" {
|
|
method = "POST"
|
|
}
|
|
|
|
// For Telegram webhooks, strip chat_id from URL if present
|
|
webhookURL := webhook.URL
|
|
if webhook.Service == "telegram" && strings.Contains(webhookURL, "chat_id=") {
|
|
if u, err := url.Parse(webhookURL); err == nil {
|
|
q := u.Query()
|
|
q.Del("chat_id") // Remove chat_id from query params
|
|
u.RawQuery = q.Encode()
|
|
webhookURL = u.String()
|
|
}
|
|
}
|
|
|
|
req, err := http.NewRequest(method, webhookURL, bytes.NewBuffer(payload))
|
|
if err != nil {
|
|
return 0, "", fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
// Set headers
|
|
// Special handling for ntfy service - add dynamic headers based on test alert level
|
|
if webhook.Service == "ntfy" {
|
|
// Set Content-Type for ntfy (plain text)
|
|
req.Header.Set("Content-Type", "text/plain")
|
|
|
|
// Set dynamic headers based on alert level
|
|
title := fmt.Sprintf("%s: %s",
|
|
func() string {
|
|
switch testAlert.Level {
|
|
case alerts.AlertLevelCritical:
|
|
return "CRITICAL"
|
|
case alerts.AlertLevelWarning:
|
|
return "WARNING"
|
|
default:
|
|
return "INFO"
|
|
}
|
|
}(),
|
|
testAlert.ResourceName,
|
|
)
|
|
req.Header.Set("Title", title)
|
|
|
|
priority := func() string {
|
|
switch testAlert.Level {
|
|
case alerts.AlertLevelCritical:
|
|
return "urgent"
|
|
case alerts.AlertLevelWarning:
|
|
return "high"
|
|
default:
|
|
return "default"
|
|
}
|
|
}()
|
|
req.Header.Set("Priority", priority)
|
|
|
|
tags := fmt.Sprintf("%s,pulse,%s",
|
|
func() string {
|
|
switch testAlert.Level {
|
|
case alerts.AlertLevelCritical:
|
|
return "rotating_light"
|
|
case alerts.AlertLevelWarning:
|
|
return "warning"
|
|
default:
|
|
return "white_check_mark"
|
|
}
|
|
}(),
|
|
testAlert.Type,
|
|
)
|
|
req.Header.Set("Tags", tags)
|
|
}
|
|
|
|
// Apply any custom headers from webhook config (these override defaults)
|
|
for key, value := range webhook.Headers {
|
|
// Skip template-like headers (those with {{)
|
|
if !strings.Contains(value, "{{") {
|
|
req.Header.Set(key, value)
|
|
}
|
|
}
|
|
|
|
if webhook.Service != "ntfy" && req.Header.Get("Content-Type") == "" {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
req.Header.Set("User-Agent", "Pulse-Monitoring/2.0 (Test)")
|
|
|
|
// Send with shorter timeout for testing
|
|
client := createSecureWebhookClient(WebhookTestTimeout)
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return 0, "", fmt.Errorf("request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Read response with size limit
|
|
limitedReader := io.LimitReader(resp.Body, WebhookMaxResponseSize)
|
|
var respBody bytes.Buffer
|
|
if _, err := respBody.ReadFrom(limitedReader); err != nil {
|
|
return 0, "", fmt.Errorf("failed to read webhook response body: %w", err)
|
|
}
|
|
|
|
return resp.StatusCode, respBody.String(), nil
|
|
}
|