From fe1533ea13a9f2970ab711a93b4f5f22d483b921 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 22 Oct 2025 18:15:27 +0000 Subject: [PATCH] Improve PMG metric ingestion refs #551 --- .../src/components/Settings/NodeModal.tsx | 49 +++--- internal/monitoring/monitor.go | 87 +++++------ pkg/pmg/client.go | 90 +++++------ pkg/pmg/client_test.go | 144 +++++++++++++++++- pkg/pmg/json_types.go | 104 +++++++++++++ 5 files changed, 361 insertions(+), 113 deletions(-) create mode 100644 pkg/pmg/json_types.go diff --git a/frontend-modern/src/components/Settings/NodeModal.tsx b/frontend-modern/src/components/Settings/NodeModal.tsx index 17270eb..05ce315 100644 --- a/frontend-modern/src/components/Settings/NodeModal.tsx +++ b/frontend-modern/src/components/Settings/NodeModal.tsx @@ -35,10 +35,10 @@ export const NodeModal: Component = (props) => { const [isTesting, setIsTesting] = createSignal(false); // Function to get clean form data - const getCleanFormData = () => ({ + const getCleanFormData = (nodeType: 'pve' | 'pbs' | 'pmg' = props.nodeType) => ({ name: '', host: '', - authType: 'token' as 'password' | 'token', + authType: nodeType === 'pmg' ? 'password' : ('token' as 'password' | 'token'), setupMode: 'auto' as 'auto' | 'manual', user: '', password: '', @@ -70,7 +70,7 @@ export const NodeModal: Component = (props) => { // Force reset if resetKey changed if (key !== undefined && key !== previousResetKey) { previousResetKey = key; - setFormData(() => getCleanFormData()); + setFormData(() => getCleanFormData(props.nodeType)); setQuickSetupCommand(''); setTestResult(null); return; @@ -79,7 +79,7 @@ export const NodeModal: Component = (props) => { // Force reset if node type changed if (nodeType !== previousNodeType && previousNodeType !== undefined) { previousNodeType = nodeType; - setFormData(() => getCleanFormData()); + setFormData(() => getCleanFormData(props.nodeType)); setQuickSetupCommand(''); setTestResult(null); return; @@ -88,7 +88,7 @@ export const NodeModal: Component = (props) => { // Reset when opening for new node if (isOpen && !editingNode) { - setFormData(() => getCleanFormData()); + setFormData(() => getCleanFormData(props.nodeType)); setQuickSetupCommand(''); setTestResult(null); } @@ -437,23 +437,32 @@ export const NodeModal: Component = (props) => { Username & Password - + + + +

+ Proxmox Mail Gateway does not support API tokens. Use a service account with + password authentication (for example root@pam or a dedicated{' '} + api@pmg user). +

+ {/* Password Auth Fields */} diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 6f597e9..137b5ea 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -5891,14 +5891,14 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie Msg("Failed to fetch PMG queue status") } } else if queueData != nil { - total := queueData.Active + queueData.Deferred + queueData.Hold + queueData.Incoming + total := queueData.Active.Int64() + queueData.Deferred.Int64() + queueData.Hold.Int64() + queueData.Incoming.Int64() node.QueueStatus = &models.PMGQueueStatus{ - Active: queueData.Active, - Deferred: queueData.Deferred, - Hold: queueData.Hold, - Incoming: queueData.Incoming, - Total: total, - OldestAge: queueData.OldestAge, + Active: queueData.Active.Int(), + Deferred: queueData.Deferred.Int(), + Hold: queueData.Hold.Int(), + Incoming: queueData.Incoming.Int(), + Total: int(total), + OldestAge: queueData.OldestAge.Int64(), UpdatedAt: time.Now(), } } @@ -5935,8 +5935,9 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie } for _, b := range backups { - backupTime := time.Unix(b.Timestamp, 0) - id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, b.Timestamp) + timestamp := b.Timestamp.Int64() + backupTime := time.Unix(timestamp, 0) + id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, timestamp) if _, exists := seenBackupIDs[id]; exists { continue } @@ -5947,7 +5948,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie Node: nodeName, Filename: b.Filename, BackupTime: backupTime, - Size: b.Size, + Size: b.Size.Int64(), }) } } @@ -5964,22 +5965,22 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie } else if stats != nil { pmgInst.MailStats = &models.PMGMailStats{ Timeframe: "day", - CountTotal: stats.Count, - CountIn: stats.CountIn, - CountOut: stats.CountOut, - SpamIn: stats.SpamIn, - SpamOut: stats.SpamOut, - VirusIn: stats.VirusIn, - VirusOut: stats.VirusOut, - BouncesIn: stats.BouncesIn, - BouncesOut: stats.BouncesOut, - BytesIn: stats.BytesIn, - BytesOut: stats.BytesOut, - GreylistCount: stats.GreylistCount, - JunkIn: stats.JunkIn, - AverageProcessTimeMs: stats.AvgProcessSec * 1000, - RBLRejects: stats.RBLRejects, - PregreetRejects: stats.Pregreet, + CountTotal: stats.Count.Float64(), + CountIn: stats.CountIn.Float64(), + CountOut: stats.CountOut.Float64(), + SpamIn: stats.SpamIn.Float64(), + SpamOut: stats.SpamOut.Float64(), + VirusIn: stats.VirusIn.Float64(), + VirusOut: stats.VirusOut.Float64(), + BouncesIn: stats.BouncesIn.Float64(), + BouncesOut: stats.BouncesOut.Float64(), + BytesIn: stats.BytesIn.Float64(), + BytesOut: stats.BytesOut.Float64(), + GreylistCount: stats.GreylistCount.Float64(), + JunkIn: stats.JunkIn.Float64(), + AverageProcessTimeMs: stats.AvgProcessSec.Float64() * 1000, + RBLRejects: stats.RBLRejects.Float64(), + PregreetRejects: stats.Pregreet.Float64(), UpdatedAt: time.Now(), } } @@ -5991,22 +5992,22 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie } else if len(counts) > 0 { points := make([]models.PMGMailCountPoint, 0, len(counts)) for _, entry := range counts { - ts := time.Unix(entry.Time, 0) + ts := time.Unix(entry.Time.Int64(), 0) points = append(points, models.PMGMailCountPoint{ Timestamp: ts, - Count: entry.Count, - CountIn: entry.CountIn, - CountOut: entry.CountOut, - SpamIn: entry.SpamIn, - SpamOut: entry.SpamOut, - VirusIn: entry.VirusIn, - VirusOut: entry.VirusOut, - RBLRejects: entry.RBLRejects, - Pregreet: entry.PregreetReject, - BouncesIn: entry.BouncesIn, - BouncesOut: entry.BouncesOut, - Greylist: entry.GreylistCount, - Index: entry.Index, + Count: entry.Count.Float64(), + CountIn: entry.CountIn.Float64(), + CountOut: entry.CountOut.Float64(), + SpamIn: entry.SpamIn.Float64(), + SpamOut: entry.SpamOut.Float64(), + VirusIn: entry.VirusIn.Float64(), + VirusOut: entry.VirusOut.Float64(), + RBLRejects: entry.RBLRejects.Float64(), + Pregreet: entry.PregreetReject.Float64(), + BouncesIn: entry.BouncesIn.Float64(), + BouncesOut: entry.BouncesOut.Float64(), + Greylist: entry.GreylistCount.Float64(), + Index: entry.Index.Int(), Timeframe: "hour", WindowStart: ts, }) @@ -6023,7 +6024,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie for _, bucket := range scores { buckets = append(buckets, models.PMGSpamBucket{ Score: bucket.Level, - Count: float64(bucket.Count), + Count: float64(bucket.Count.Int()), }) } pmgInst.SpamDistribution = buckets @@ -6031,10 +6032,10 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie quarantine := models.PMGQuarantineTotals{} if spamStatus, err := client.GetQuarantineStatus(ctx, "spam"); err == nil && spamStatus != nil { - quarantine.Spam = spamStatus.Count + quarantine.Spam = int(spamStatus.Count.Int64()) } if virusStatus, err := client.GetQuarantineStatus(ctx, "virus"); err == nil && virusStatus != nil { - quarantine.Virus = virusStatus.Count + quarantine.Virus = int(virusStatus.Count.Int64()) } pmgInst.Quarantine = &quarantine diff --git a/pkg/pmg/client.go b/pkg/pmg/client.go index f1ce729..568dc52 100644 --- a/pkg/pmg/client.go +++ b/pkg/pmg/client.go @@ -55,45 +55,45 @@ type VersionInfo struct { } type MailStatistics struct { - Count float64 `json:"count"` - CountIn float64 `json:"count_in"` - CountOut float64 `json:"count_out"` - SpamIn float64 `json:"spamcount_in"` - SpamOut float64 `json:"spamcount_out"` - VirusIn float64 `json:"viruscount_in"` - VirusOut float64 `json:"viruscount_out"` - BouncesIn float64 `json:"bounces_in"` - BouncesOut float64 `json:"bounces_out"` - BytesIn float64 `json:"bytes_in"` - BytesOut float64 `json:"bytes_out"` - GreylistCount float64 `json:"glcount"` - JunkIn float64 `json:"junk_in"` - RBLRejects float64 `json:"rbl_rejects"` - Pregreet float64 `json:"pregreet_rejects"` - AvgProcessSec float64 `json:"avptime"` + Count flexibleFloat `json:"count"` + CountIn flexibleFloat `json:"count_in"` + CountOut flexibleFloat `json:"count_out"` + SpamIn flexibleFloat `json:"spamcount_in"` + SpamOut flexibleFloat `json:"spamcount_out"` + VirusIn flexibleFloat `json:"viruscount_in"` + VirusOut flexibleFloat `json:"viruscount_out"` + BouncesIn flexibleFloat `json:"bounces_in"` + BouncesOut flexibleFloat `json:"bounces_out"` + BytesIn flexibleFloat `json:"bytes_in"` + BytesOut flexibleFloat `json:"bytes_out"` + GreylistCount flexibleFloat `json:"glcount"` + JunkIn flexibleFloat `json:"junk_in"` + RBLRejects flexibleFloat `json:"rbl_rejects"` + Pregreet flexibleFloat `json:"pregreet_rejects"` + AvgProcessSec flexibleFloat `json:"avptime"` } type MailCountEntry struct { - Index int `json:"index"` - Time int64 `json:"time"` - Count float64 `json:"count"` - CountIn float64 `json:"count_in"` - CountOut float64 `json:"count_out"` - SpamIn float64 `json:"spamcount_in"` - SpamOut float64 `json:"spamcount_out"` - VirusIn float64 `json:"viruscount_in"` - VirusOut float64 `json:"viruscount_out"` - BouncesIn float64 `json:"bounces_in"` - BouncesOut float64 `json:"bounces_out"` - RBLRejects float64 `json:"rbl_rejects"` - PregreetReject float64 `json:"pregreet_rejects"` - GreylistCount float64 `json:"glcount"` + Index flexibleInt `json:"index"` + Time flexibleInt `json:"time"` + Count flexibleFloat `json:"count"` + CountIn flexibleFloat `json:"count_in"` + CountOut flexibleFloat `json:"count_out"` + SpamIn flexibleFloat `json:"spamcount_in"` + SpamOut flexibleFloat `json:"spamcount_out"` + VirusIn flexibleFloat `json:"viruscount_in"` + VirusOut flexibleFloat `json:"viruscount_out"` + BouncesIn flexibleFloat `json:"bounces_in"` + BouncesOut flexibleFloat `json:"bounces_out"` + RBLRejects flexibleFloat `json:"rbl_rejects"` + PregreetReject flexibleFloat `json:"pregreet_rejects"` + GreylistCount flexibleFloat `json:"glcount"` } type SpamScore struct { - Level string `json:"level"` - Count int `json:"count"` - Ratio float64 `json:"ratio"` + Level string `json:"level"` + Count flexibleInt `json:"count"` + Ratio flexibleFloat `json:"ratio"` } type ClusterStatusEntry struct { @@ -105,25 +105,25 @@ type ClusterStatusEntry struct { } type QuarantineStatus struct { - Count int `json:"count"` - AvgBytes float64 `json:"avgbytes"` - AvgSpam float64 `json:"avgspam,omitempty"` - Megabytes float64 `json:"mbytes"` + Count flexibleInt `json:"count"` + AvgBytes flexibleFloat `json:"avgbytes"` + AvgSpam flexibleFloat `json:"avgspam,omitempty"` + Megabytes flexibleFloat `json:"mbytes"` } type QueueStatusEntry struct { - Active int `json:"active"` - Deferred int `json:"deferred"` - Hold int `json:"hold"` - Incoming int `json:"incoming"` - OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds + Active flexibleInt `json:"active"` + Deferred flexibleInt `json:"deferred"` + Hold flexibleInt `json:"hold"` + Incoming flexibleInt `json:"incoming"` + OldestAge flexibleInt `json:"oldest_age,omitempty"` // Age of oldest message in seconds } // BackupEntry represents a PMG configuration backup stored on a node. type BackupEntry struct { - Filename string `json:"filename"` - Size int64 `json:"size"` - Timestamp int64 `json:"timestamp"` + Filename string `json:"filename"` + Size flexibleInt `json:"size"` + Timestamp flexibleInt `json:"timestamp"` } func NewClient(cfg ClientConfig) (*Client, error) { diff --git a/pkg/pmg/client_test.go b/pkg/pmg/client_test.go index b282e05..c97b641 100644 --- a/pkg/pmg/client_test.go +++ b/pkg/pmg/client_test.go @@ -222,7 +222,7 @@ func TestClientUsesTokenAuthorizationHeader(t *testing.T) { t.Fatalf("get mail statistics failed: %v", err) } - if stats == nil || stats.Count != 42 { + if stats == nil || stats.Count.Float64() != 42 { t.Fatalf("expected statistics count 42, got %+v", stats) } @@ -276,10 +276,144 @@ func TestListBackups(t *testing.T) { if backup.Filename != "pmg-backup_2024-01-01.tgz" { t.Fatalf("unexpected filename: %s", backup.Filename) } - if backup.Size != 123456 { - t.Fatalf("unexpected size: %d", backup.Size) + if backup.Size.Int64() != 123456 { + t.Fatalf("unexpected size: %d", backup.Size.Int64()) } - if backup.Timestamp != 1704096000 { - t.Fatalf("unexpected timestamp: %d", backup.Timestamp) + if backup.Timestamp.Int64() != 1704096000 { + t.Fatalf("unexpected timestamp: %d", backup.Timestamp.Int64()) + } +} + +func TestMailEndpointsHandleNullAndStringValues(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api2/json/access/ticket": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{"ticket":"ticket","CSRFPreventionToken":"csrf"}}`) + case "/api2/json/version": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{"version":"9.0","release":"1"}}`) + case "/api2/json/statistics/mail": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{ + "count":null, + "count_in":"25", + "count_out":"5", + "spamcount_in":"7", + "spamcount_out":null, + "viruscount_in":"0", + "viruscount_out":"0", + "bounces_in":null, + "bounces_out":"", + "bytes_in":"1024", + "bytes_out":"2048", + "glcount":"2", + "junk_in":"", + "rbl_rejects":"1", + "pregreet_rejects":null, + "avptime":"0.75" + }}`) + case "/api2/json/statistics/mailcount": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{ + "index":"0", + "time":"1713724800", + "count":"12", + "count_in":"8", + "count_out":"4", + "spamcount_in":null, + "spamcount_out":"1", + "viruscount_in":"", + "viruscount_out":"0", + "bounces_in":"1", + "bounces_out":"0", + "rbl_rejects":"2", + "pregreet_rejects":"", + "glcount":"3" + }]}`) + case "/api2/json/quarantine/spamstatus": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{"count":null,"avgbytes":"512","mbytes":"0.5"}}`) + case "/api2/json/quarantine/virusstatus": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{"count":"4","avgbytes":"256","mbytes":"0.25"}}`) + case "/api2/json/nodes/mail/postfix/queue": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{"active":"3","deferred":null,"hold":"1","incoming":"2","oldest_age":"600"}}`) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewClient(ClientConfig{ + Host: server.URL, + User: "api@pmg", + Password: "secret", + VerifySSL: false, + }) + if err != nil { + t.Fatalf("unexpected error creating client: %v", err) + } + + ctx := context.Background() + + stats, err := client.GetMailStatistics(ctx, "") + if err != nil { + t.Fatalf("GetMailStatistics failed: %v", err) + } + if stats == nil { + t.Fatal("expected statistics data") + } + if stats.Count.Float64() != 0 { + t.Fatalf("expected count to default to 0, got %v", stats.Count.Float64()) + } + if stats.CountIn.Float64() != 25 { + t.Fatalf("expected CountIn 25, got %v", stats.CountIn.Float64()) + } + if stats.BytesIn.Float64() != 1024 { + t.Fatalf("expected BytesIn 1024, got %v", stats.BytesIn.Float64()) + } + if stats.AvgProcessSec.Float64() != 0.75 { + t.Fatalf("expected AvgProcessSec 0.75, got %v", stats.AvgProcessSec.Float64()) + } + + counts, err := client.GetMailCount(ctx, 12) + if err != nil { + t.Fatalf("GetMailCount failed: %v", err) + } + if len(counts) != 1 { + t.Fatalf("expected 1 mail count entry, got %d", len(counts)) + } + entry := counts[0] + if entry.Time.Int64() != 1713724800 { + t.Fatalf("expected unix time 1713724800, got %d", entry.Time.Int64()) + } + if entry.Count.Float64() != 12 { + t.Fatalf("expected count 12, got %v", entry.Count.Float64()) + } + if entry.GreylistCount.Float64() != 3 { + t.Fatalf("expected greylist 3, got %v", entry.GreylistCount.Float64()) + } + + quarantine, err := client.GetQuarantineStatus(ctx, "spam") + if err != nil { + t.Fatalf("GetQuarantineStatus failed: %v", err) + } + if quarantine.Count.Int64() != 0 { + t.Fatalf("expected spam quarantine count default to 0, got %d", quarantine.Count.Int64()) + } + + queue, err := client.GetQueueStatus(ctx, "mail") + if err != nil { + t.Fatalf("GetQueueStatus failed: %v", err) + } + if queue.Active.Int() != 3 || queue.Deferred.Int() != 0 || queue.Hold.Int() != 1 || queue.Incoming.Int() != 2 { + t.Fatalf("unexpected queue values: %+v", queue) + } + if queue.OldestAge.Int64() != 600 { + t.Fatalf("expected oldest age 600, got %d", queue.OldestAge.Int64()) } } diff --git a/pkg/pmg/json_types.go b/pkg/pmg/json_types.go new file mode 100644 index 0000000..cee0cd4 --- /dev/null +++ b/pkg/pmg/json_types.go @@ -0,0 +1,104 @@ +package pmg + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// flexibleFloat handles numeric values that may arrive as numbers, strings, or nulls. +type flexibleFloat float64 + +func (f *flexibleFloat) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = 0 + return nil + } + + // Try decoding as a JSON number first + var num float64 + if err := json.Unmarshal(data, &num); err == nil { + *f = flexibleFloat(num) + return nil + } + + // Fall back to string decoding and parsing + var str string + if err := json.Unmarshal(data, &str); err != nil { + return fmt.Errorf("invalid numeric value %q: %w", string(data), err) + } + + str = strings.TrimSpace(str) + if str == "" || strings.EqualFold(str, "null") { + *f = 0 + return nil + } + + parsed, err := strconv.ParseFloat(str, 64) + if err != nil { + return fmt.Errorf("failed to parse numeric string %q: %w", str, err) + } + + *f = flexibleFloat(parsed) + return nil +} + +func (f flexibleFloat) Float64() float64 { + return float64(f) +} + +// flexibleInt handles integer values that may arrive as ints, floats, strings, or nulls. +type flexibleInt int64 + +func (i *flexibleInt) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *i = 0 + return nil + } + + // Try integer decoding + var num int64 + if err := json.Unmarshal(data, &num); err == nil { + *i = flexibleInt(num) + return nil + } + + // Try float decoding (round towards zero) + var floatNum float64 + if err := json.Unmarshal(data, &floatNum); err == nil { + *i = flexibleInt(int64(floatNum)) + return nil + } + + // Fall back to string parsing + var str string + if err := json.Unmarshal(data, &str); err != nil { + return fmt.Errorf("invalid integer value %q: %w", string(data), err) + } + + str = strings.TrimSpace(str) + if str == "" || strings.EqualFold(str, "null") { + *i = 0 + return nil + } + + parsed, err := strconv.ParseFloat(str, 64) + if err != nil { + return fmt.Errorf("failed to parse integer string %q: %w", str, err) + } + + *i = flexibleInt(int64(parsed)) + return nil +} + +func (i flexibleInt) Int64() int64 { + return int64(i) +} + +func (i flexibleInt) Int() int { + return int(i) +}