This commit addresses 9 critical issues identified during the monitoring system audit: **Race Conditions Fixed:** - PBS backup pollers: Moved lock earlier to eliminate check-then-act race (lines 7316-7378) - PVE backup poll timing: Fixed double write to lastPVEBackupPoll with proper synchronization (lines 5927-5977) - Docker hosts cleanup: Refactored to avoid holding both m.mu and s.mu locks simultaneously (lines 1911-1937) **Context Propagation Fixed:** - Replaced all context.Background() calls with parent context for proper cancellation chain: - PBS backup poller (line 7367) - PVE backup poller (line 5955) - PBS fallback check (line 7154) **Memory Leak Prevention:** - Added cleanup for guest metadata cache (10 minute TTL, lines 1942-1957) - Added cleanup for diagnostic snapshots (1 hour TTL, lines 1959-1987) - Added cleanup for RRD cache (1 minute TTL, lines 1989-2007) - All cleanup methods called on 10-second ticker (lines 3791-3793) **Panic Recovery:** - Added recoverFromPanic helper to log panics with stack traces (lines 1910-1920) - Protected all critical goroutines: - poll (line 4020) - taskWorker (line 4200) - retryFailedConnections (line 3851) - checkMockAlerts (line 8896) - pollPVEInstance (line 4886) - pollPBSInstance (line 7164) - pollPMGInstance (line 7498) **Import Fixes:** - Added missing sync import to email_enhanced.go - Added missing os import to queue.go All fixes maintain proper lock ordering and release locks before calling methods that acquire other locks to prevent deadlocks.
424 lines
11 KiB
Go
424 lines
11 KiB
Go
package notifications
|
|
|
|
import (
|
|
"crypto/tls"
|
|
"fmt"
|
|
"net"
|
|
"net/smtp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// EnhancedEmailManager extends email functionality with provider support
|
|
type EnhancedEmailManager struct {
|
|
config EmailProviderConfig
|
|
rateLimit *RateLimiter
|
|
}
|
|
|
|
// RateLimiter implements a simple rate limiter
|
|
type RateLimiter struct {
|
|
mu sync.Mutex
|
|
rate int
|
|
lastSent time.Time
|
|
sentCount int
|
|
}
|
|
|
|
// NewEnhancedEmailManager creates an enhanced email manager
|
|
func NewEnhancedEmailManager(config EmailProviderConfig) *EnhancedEmailManager {
|
|
return &EnhancedEmailManager{
|
|
config: config,
|
|
rateLimit: &RateLimiter{
|
|
rate: config.RateLimit,
|
|
},
|
|
}
|
|
}
|
|
|
|
// SendEmailWithRetry sends email with retry logic
|
|
// Note: When used with the persistent queue, retry behavior is layered:
|
|
// - Transport retries (this function): up to MaxRetries attempts with RetryDelay between
|
|
// - Queue retries: up to MaxAttempts (default 3) with exponential backoff
|
|
// Total attempts = MaxRetries * MaxAttempts (e.g., 3 * 3 = 9 SMTP calls for a single notification)
|
|
// This ensures delivery even during transient failures at either layer.
|
|
func (e *EnhancedEmailManager) SendEmailWithRetry(subject, htmlBody, textBody string) error {
|
|
var lastErr error
|
|
|
|
for attempt := 0; attempt <= e.config.MaxRetries; attempt++ {
|
|
if attempt > 0 {
|
|
delay := time.Duration(e.config.RetryDelay) * time.Second
|
|
log.Debug().
|
|
Int("attempt", attempt).
|
|
Dur("delay", delay).
|
|
Msg("Retrying email send after delay")
|
|
time.Sleep(delay)
|
|
}
|
|
|
|
// Check rate limit
|
|
if err := e.checkRateLimit(); err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
|
|
// Try to send
|
|
err := e.sendEmailOnce(subject, htmlBody, textBody)
|
|
if err == nil {
|
|
if attempt > 0 {
|
|
log.Info().
|
|
Int("attempt", attempt).
|
|
Msg("Email sent successfully after retry")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
lastErr = err
|
|
log.Warn().
|
|
Err(err).
|
|
Int("attempt", attempt).
|
|
Str("provider", e.config.Provider).
|
|
Msg("Email send attempt failed")
|
|
}
|
|
|
|
return fmt.Errorf("email failed after %d attempts: %w", e.config.MaxRetries+1, lastErr)
|
|
}
|
|
|
|
// checkRateLimit enforces rate limiting
|
|
func (e *EnhancedEmailManager) checkRateLimit() error {
|
|
if e.config.RateLimit <= 0 {
|
|
return nil // No rate limit
|
|
}
|
|
|
|
e.rateLimit.mu.Lock()
|
|
defer e.rateLimit.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
if now.Sub(e.rateLimit.lastSent) >= time.Minute {
|
|
// Reset counter after a minute
|
|
e.rateLimit.sentCount = 0
|
|
e.rateLimit.lastSent = now
|
|
}
|
|
|
|
if e.rateLimit.sentCount >= e.config.RateLimit {
|
|
return fmt.Errorf("rate limit exceeded: %d emails per minute", e.config.RateLimit)
|
|
}
|
|
|
|
e.rateLimit.sentCount++
|
|
return nil
|
|
}
|
|
|
|
// sendEmailOnce sends a single email
|
|
func (e *EnhancedEmailManager) sendEmailOnce(subject, htmlBody, textBody string) error {
|
|
// Build message with enhanced headers
|
|
boundary := fmt.Sprintf("===============%d==", time.Now().UnixNano())
|
|
|
|
msg := fmt.Sprintf("From: %s\r\n", e.config.From)
|
|
msg += fmt.Sprintf("To: %s\r\n", strings.Join(e.config.To, ", "))
|
|
if e.config.ReplyTo != "" {
|
|
msg += fmt.Sprintf("Reply-To: %s\r\n", e.config.ReplyTo)
|
|
}
|
|
msg += fmt.Sprintf("Subject: %s\r\n", subject)
|
|
msg += fmt.Sprintf("Date: %s\r\n", time.Now().Format(time.RFC1123Z))
|
|
msg += fmt.Sprintf("Message-ID: <%d@pulse-monitoring>\r\n", time.Now().UnixNano())
|
|
msg += "MIME-Version: 1.0\r\n"
|
|
msg += fmt.Sprintf("Content-Type: multipart/alternative; boundary=\"%s\"\r\n", boundary)
|
|
msg += "X-Mailer: Pulse Monitoring System\r\n"
|
|
msg += "\r\n"
|
|
|
|
// Text part
|
|
msg += fmt.Sprintf("--%s\r\n", boundary)
|
|
msg += "Content-Type: text/plain; charset=\"UTF-8\"\r\n"
|
|
msg += "Content-Transfer-Encoding: 7bit\r\n"
|
|
msg += "\r\n"
|
|
msg += textBody + "\r\n"
|
|
|
|
// HTML part
|
|
msg += fmt.Sprintf("--%s\r\n", boundary)
|
|
msg += "Content-Type: text/html; charset=\"UTF-8\"\r\n"
|
|
msg += "Content-Transfer-Encoding: 7bit\r\n"
|
|
msg += "\r\n"
|
|
msg += htmlBody + "\r\n"
|
|
|
|
// End boundary
|
|
msg += fmt.Sprintf("--%s--\r\n", boundary)
|
|
|
|
// Send based on provider configuration
|
|
return e.sendViaProvider([]byte(msg))
|
|
}
|
|
|
|
// sendViaProvider sends email using provider-specific settings
|
|
func (e *EnhancedEmailManager) sendViaProvider(msg []byte) error {
|
|
addr := net.JoinHostPort(e.config.SMTPHost, strconv.Itoa(e.config.SMTPPort))
|
|
|
|
// Special handling for specific providers
|
|
switch e.config.Provider {
|
|
case "SendGrid":
|
|
// SendGrid uses "apikey" as username
|
|
if e.config.Username == "" {
|
|
e.config.Username = "apikey"
|
|
}
|
|
case "Postmark":
|
|
// Postmark uses API token for both username and password
|
|
if e.config.Password != "" && e.config.Username == "" {
|
|
e.config.Username = e.config.Password
|
|
}
|
|
case "SparkPost":
|
|
// SparkPost uses specific username
|
|
if e.config.Username == "" {
|
|
e.config.Username = "SMTP_Injection"
|
|
}
|
|
case "Resend":
|
|
// Resend uses "resend" as username
|
|
if e.config.Username == "" {
|
|
e.config.Username = "resend"
|
|
}
|
|
}
|
|
|
|
// Configure authentication
|
|
var auth smtp.Auth
|
|
if e.config.AuthRequired && e.config.Username != "" && e.config.Password != "" {
|
|
auth = smtp.PlainAuth("", e.config.Username, e.config.Password, e.config.SMTPHost)
|
|
}
|
|
|
|
// Send with TLS configuration
|
|
if e.config.TLS || e.config.SMTPPort == 465 {
|
|
return e.sendTLS(addr, auth, msg)
|
|
} else if e.config.StartTLS {
|
|
return e.sendStartTLS(addr, auth, msg)
|
|
} else {
|
|
// Use sendPlain for non-TLS connections with timeout
|
|
return e.sendPlain(addr, auth, msg)
|
|
}
|
|
}
|
|
|
|
// sendTLS sends email over TLS connection
|
|
func (e *EnhancedEmailManager) sendTLS(addr string, auth smtp.Auth, msg []byte) error {
|
|
tlsConfig := &tls.Config{
|
|
ServerName: e.config.SMTPHost,
|
|
InsecureSkipVerify: e.config.SkipTLSVerify,
|
|
}
|
|
|
|
// Use DialWithDialer with timeout
|
|
dialer := &net.Dialer{
|
|
Timeout: 10 * time.Second,
|
|
}
|
|
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("TLS dial failed: %w", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
// Set overall connection timeout
|
|
if err := conn.SetDeadline(time.Now().Add(30 * time.Second)); err != nil {
|
|
return fmt.Errorf("failed to set connection deadline: %w", err)
|
|
}
|
|
|
|
client, err := smtp.NewClient(conn, e.config.SMTPHost)
|
|
if err != nil {
|
|
return fmt.Errorf("SMTP client creation failed: %w", err)
|
|
}
|
|
defer client.Close()
|
|
|
|
if auth != nil {
|
|
if err = client.Auth(auth); err != nil {
|
|
return fmt.Errorf("SMTP auth failed: %w", err)
|
|
}
|
|
}
|
|
|
|
if err = client.Mail(e.config.From); err != nil {
|
|
return fmt.Errorf("MAIL FROM failed: %w", err)
|
|
}
|
|
|
|
for _, to := range e.config.To {
|
|
if err = client.Rcpt(to); err != nil {
|
|
return fmt.Errorf("RCPT TO failed for %s: %w", to, err)
|
|
}
|
|
}
|
|
|
|
w, err := client.Data()
|
|
if err != nil {
|
|
return fmt.Errorf("DATA command failed: %w", err)
|
|
}
|
|
|
|
_, err = w.Write(msg)
|
|
if err != nil {
|
|
return fmt.Errorf("message write failed: %w", err)
|
|
}
|
|
|
|
err = w.Close()
|
|
if err != nil {
|
|
return fmt.Errorf("message close failed: %w", err)
|
|
}
|
|
|
|
return client.Quit()
|
|
}
|
|
|
|
// sendStartTLS sends email using STARTTLS
|
|
func (e *EnhancedEmailManager) sendStartTLS(addr string, auth smtp.Auth, msg []byte) error {
|
|
// Use DialTimeout to prevent hanging on unreachable servers
|
|
conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
|
|
if err != nil {
|
|
return fmt.Errorf("TCP dial failed: %w", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
// Set overall connection timeout
|
|
if err := conn.SetDeadline(time.Now().Add(30 * time.Second)); err != nil {
|
|
return fmt.Errorf("failed to set connection deadline: %w", err)
|
|
}
|
|
|
|
client, err := smtp.NewClient(conn, e.config.SMTPHost)
|
|
if err != nil {
|
|
return fmt.Errorf("SMTP client creation failed: %w", err)
|
|
}
|
|
defer client.Close()
|
|
|
|
// STARTTLS
|
|
tlsConfig := &tls.Config{
|
|
ServerName: e.config.SMTPHost,
|
|
InsecureSkipVerify: e.config.SkipTLSVerify,
|
|
}
|
|
|
|
if err = client.StartTLS(tlsConfig); err != nil {
|
|
return fmt.Errorf("STARTTLS failed: %w", err)
|
|
}
|
|
|
|
if auth != nil {
|
|
if err = client.Auth(auth); err != nil {
|
|
return fmt.Errorf("SMTP auth failed: %w", err)
|
|
}
|
|
}
|
|
|
|
if err = client.Mail(e.config.From); err != nil {
|
|
return fmt.Errorf("MAIL FROM failed: %w", err)
|
|
}
|
|
|
|
for _, to := range e.config.To {
|
|
if err = client.Rcpt(to); err != nil {
|
|
return fmt.Errorf("RCPT TO failed for %s: %w", to, err)
|
|
}
|
|
}
|
|
|
|
w, err := client.Data()
|
|
if err != nil {
|
|
return fmt.Errorf("DATA command failed: %w", err)
|
|
}
|
|
|
|
_, err = w.Write(msg)
|
|
if err != nil {
|
|
return fmt.Errorf("message write failed: %w", err)
|
|
}
|
|
|
|
err = w.Close()
|
|
if err != nil {
|
|
return fmt.Errorf("message close failed: %w", err)
|
|
}
|
|
|
|
return client.Quit()
|
|
}
|
|
|
|
// TestConnection tests the email server connection
|
|
func (e *EnhancedEmailManager) TestConnection() error {
|
|
addr := net.JoinHostPort(e.config.SMTPHost, strconv.Itoa(e.config.SMTPPort))
|
|
|
|
// Try to connect
|
|
var conn net.Conn
|
|
var err error
|
|
|
|
if e.config.TLS || e.config.SMTPPort == 465 {
|
|
tlsConfig := &tls.Config{
|
|
ServerName: e.config.SMTPHost,
|
|
InsecureSkipVerify: e.config.SkipTLSVerify,
|
|
}
|
|
conn, err = tls.Dial("tcp", addr, tlsConfig)
|
|
} else {
|
|
conn, err = net.DialTimeout("tcp", addr, 10*time.Second)
|
|
}
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("connection failed: %w", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
client, err := smtp.NewClient(conn, e.config.SMTPHost)
|
|
if err != nil {
|
|
return fmt.Errorf("SMTP handshake failed: %w", err)
|
|
}
|
|
defer client.Close()
|
|
|
|
// Test STARTTLS if configured
|
|
if e.config.StartTLS && !e.config.TLS {
|
|
tlsConfig := &tls.Config{
|
|
ServerName: e.config.SMTPHost,
|
|
InsecureSkipVerify: e.config.SkipTLSVerify,
|
|
}
|
|
if err = client.StartTLS(tlsConfig); err != nil {
|
|
return fmt.Errorf("STARTTLS failed: %w", err)
|
|
}
|
|
}
|
|
|
|
// Test authentication if configured
|
|
if e.config.AuthRequired && e.config.Username != "" && e.config.Password != "" {
|
|
auth := smtp.PlainAuth("", e.config.Username, e.config.Password, e.config.SMTPHost)
|
|
if err = client.Auth(auth); err != nil {
|
|
return fmt.Errorf("authentication failed: %w", err)
|
|
}
|
|
}
|
|
|
|
return client.Quit()
|
|
}
|
|
|
|
// sendPlain sends email over plain SMTP connection with timeout
|
|
func (e *EnhancedEmailManager) sendPlain(addr string, auth smtp.Auth, msg []byte) error {
|
|
// Use DialTimeout to prevent hanging on unreachable servers
|
|
conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
|
|
if err != nil {
|
|
return fmt.Errorf("TCP dial failed: %w", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
// Set overall connection timeout
|
|
if err := conn.SetDeadline(time.Now().Add(30 * time.Second)); err != nil {
|
|
return fmt.Errorf("failed to set connection deadline: %w", err)
|
|
}
|
|
|
|
client, err := smtp.NewClient(conn, e.config.SMTPHost)
|
|
if err != nil {
|
|
return fmt.Errorf("SMTP client creation failed: %w", err)
|
|
}
|
|
defer client.Close()
|
|
|
|
if auth != nil {
|
|
if err = client.Auth(auth); err != nil {
|
|
return fmt.Errorf("SMTP auth failed: %w", err)
|
|
}
|
|
}
|
|
|
|
if err = client.Mail(e.config.From); err != nil {
|
|
return fmt.Errorf("MAIL FROM failed: %w", err)
|
|
}
|
|
|
|
for _, to := range e.config.To {
|
|
if err = client.Rcpt(to); err != nil {
|
|
return fmt.Errorf("RCPT TO failed for %s: %w", to, err)
|
|
}
|
|
}
|
|
|
|
w, err := client.Data()
|
|
if err != nil {
|
|
return fmt.Errorf("DATA command failed: %w", err)
|
|
}
|
|
|
|
_, err = w.Write(msg)
|
|
if err != nil {
|
|
return fmt.Errorf("message write failed: %w", err)
|
|
}
|
|
|
|
err = w.Close()
|
|
if err != nil {
|
|
return fmt.Errorf("message close failed: %w", err)
|
|
}
|
|
|
|
return client.Quit()
|
|
}
|