Improve PMG metric ingestion refs #551

This commit is contained in:
rcourtman 2025-10-22 18:15:27 +00:00
parent d813f2396f
commit fe1533ea13
5 changed files with 361 additions and 113 deletions

View file

@ -35,10 +35,10 @@ export const NodeModal: Component<NodeModalProps> = (props) => {
const [isTesting, setIsTesting] = createSignal(false);
// Function to get clean form data
const getCleanFormData = () => ({
const getCleanFormData = (nodeType: 'pve' | 'pbs' | 'pmg' = props.nodeType) => ({
name: '',
host: '',
authType: 'token' as 'password' | 'token',
authType: nodeType === 'pmg' ? 'password' : ('token' as 'password' | 'token'),
setupMode: 'auto' as 'auto' | 'manual',
user: '',
password: '',
@ -70,7 +70,7 @@ export const NodeModal: Component<NodeModalProps> = (props) => {
// Force reset if resetKey changed
if (key !== undefined && key !== previousResetKey) {
previousResetKey = key;
setFormData(() => getCleanFormData());
setFormData(() => getCleanFormData(props.nodeType));
setQuickSetupCommand('');
setTestResult(null);
return;
@ -79,7 +79,7 @@ export const NodeModal: Component<NodeModalProps> = (props) => {
// Force reset if node type changed
if (nodeType !== previousNodeType && previousNodeType !== undefined) {
previousNodeType = nodeType;
setFormData(() => getCleanFormData());
setFormData(() => getCleanFormData(props.nodeType));
setQuickSetupCommand('');
setTestResult(null);
return;
@ -88,7 +88,7 @@ export const NodeModal: Component<NodeModalProps> = (props) => {
// Reset when opening for new node
if (isOpen && !editingNode) {
setFormData(() => getCleanFormData());
setFormData(() => getCleanFormData(props.nodeType));
setQuickSetupCommand('');
setTestResult(null);
}
@ -437,23 +437,32 @@ export const NodeModal: Component<NodeModalProps> = (props) => {
Username & Password
</span>
</label>
<label class="flex items-center">
<input
type="radio"
name="authType"
value="token"
checked={formData().authType === 'token'}
onChange={() => updateField('authType', 'token')}
class="mr-2"
/>
<span class="text-sm text-gray-700 dark:text-gray-300">
API Token{' '}
<span class="text-green-600 dark:text-green-400 text-xs ml-1">
(Recommended)
<Show when={props.nodeType !== 'pmg'}>
<label class="flex items-center">
<input
type="radio"
name="authType"
value="token"
checked={formData().authType === 'token'}
onChange={() => updateField('authType', 'token')}
class="mr-2"
/>
<span class="text-sm text-gray-700 dark:text-gray-300">
API Token{' '}
<span class="text-green-600 dark:text-green-400 text-xs ml-1">
(Recommended)
</span>
</span>
</span>
</label>
</label>
</Show>
</div>
<Show when={props.nodeType === 'pmg'}>
<p class="text-xs text-gray-500 dark:text-gray-400 mt-2">
Proxmox Mail Gateway does not support API tokens. Use a service account with
password authentication (for example <code>root@pam</code> or a dedicated{' '}
<code>api@pmg</code> user).
</p>
</Show>
</div>
{/* Password Auth Fields */}

View file

@ -5891,14 +5891,14 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
Msg("Failed to fetch PMG queue status")
}
} else if queueData != nil {
total := queueData.Active + queueData.Deferred + queueData.Hold + queueData.Incoming
total := queueData.Active.Int64() + queueData.Deferred.Int64() + queueData.Hold.Int64() + queueData.Incoming.Int64()
node.QueueStatus = &models.PMGQueueStatus{
Active: queueData.Active,
Deferred: queueData.Deferred,
Hold: queueData.Hold,
Incoming: queueData.Incoming,
Total: total,
OldestAge: queueData.OldestAge,
Active: queueData.Active.Int(),
Deferred: queueData.Deferred.Int(),
Hold: queueData.Hold.Int(),
Incoming: queueData.Incoming.Int(),
Total: int(total),
OldestAge: queueData.OldestAge.Int64(),
UpdatedAt: time.Now(),
}
}
@ -5935,8 +5935,9 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
}
for _, b := range backups {
backupTime := time.Unix(b.Timestamp, 0)
id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, b.Timestamp)
timestamp := b.Timestamp.Int64()
backupTime := time.Unix(timestamp, 0)
id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, timestamp)
if _, exists := seenBackupIDs[id]; exists {
continue
}
@ -5947,7 +5948,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
Node: nodeName,
Filename: b.Filename,
BackupTime: backupTime,
Size: b.Size,
Size: b.Size.Int64(),
})
}
}
@ -5964,22 +5965,22 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
} else if stats != nil {
pmgInst.MailStats = &models.PMGMailStats{
Timeframe: "day",
CountTotal: stats.Count,
CountIn: stats.CountIn,
CountOut: stats.CountOut,
SpamIn: stats.SpamIn,
SpamOut: stats.SpamOut,
VirusIn: stats.VirusIn,
VirusOut: stats.VirusOut,
BouncesIn: stats.BouncesIn,
BouncesOut: stats.BouncesOut,
BytesIn: stats.BytesIn,
BytesOut: stats.BytesOut,
GreylistCount: stats.GreylistCount,
JunkIn: stats.JunkIn,
AverageProcessTimeMs: stats.AvgProcessSec * 1000,
RBLRejects: stats.RBLRejects,
PregreetRejects: stats.Pregreet,
CountTotal: stats.Count.Float64(),
CountIn: stats.CountIn.Float64(),
CountOut: stats.CountOut.Float64(),
SpamIn: stats.SpamIn.Float64(),
SpamOut: stats.SpamOut.Float64(),
VirusIn: stats.VirusIn.Float64(),
VirusOut: stats.VirusOut.Float64(),
BouncesIn: stats.BouncesIn.Float64(),
BouncesOut: stats.BouncesOut.Float64(),
BytesIn: stats.BytesIn.Float64(),
BytesOut: stats.BytesOut.Float64(),
GreylistCount: stats.GreylistCount.Float64(),
JunkIn: stats.JunkIn.Float64(),
AverageProcessTimeMs: stats.AvgProcessSec.Float64() * 1000,
RBLRejects: stats.RBLRejects.Float64(),
PregreetRejects: stats.Pregreet.Float64(),
UpdatedAt: time.Now(),
}
}
@ -5991,22 +5992,22 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
} else if len(counts) > 0 {
points := make([]models.PMGMailCountPoint, 0, len(counts))
for _, entry := range counts {
ts := time.Unix(entry.Time, 0)
ts := time.Unix(entry.Time.Int64(), 0)
points = append(points, models.PMGMailCountPoint{
Timestamp: ts,
Count: entry.Count,
CountIn: entry.CountIn,
CountOut: entry.CountOut,
SpamIn: entry.SpamIn,
SpamOut: entry.SpamOut,
VirusIn: entry.VirusIn,
VirusOut: entry.VirusOut,
RBLRejects: entry.RBLRejects,
Pregreet: entry.PregreetReject,
BouncesIn: entry.BouncesIn,
BouncesOut: entry.BouncesOut,
Greylist: entry.GreylistCount,
Index: entry.Index,
Count: entry.Count.Float64(),
CountIn: entry.CountIn.Float64(),
CountOut: entry.CountOut.Float64(),
SpamIn: entry.SpamIn.Float64(),
SpamOut: entry.SpamOut.Float64(),
VirusIn: entry.VirusIn.Float64(),
VirusOut: entry.VirusOut.Float64(),
RBLRejects: entry.RBLRejects.Float64(),
Pregreet: entry.PregreetReject.Float64(),
BouncesIn: entry.BouncesIn.Float64(),
BouncesOut: entry.BouncesOut.Float64(),
Greylist: entry.GreylistCount.Float64(),
Index: entry.Index.Int(),
Timeframe: "hour",
WindowStart: ts,
})
@ -6023,7 +6024,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
for _, bucket := range scores {
buckets = append(buckets, models.PMGSpamBucket{
Score: bucket.Level,
Count: float64(bucket.Count),
Count: float64(bucket.Count.Int()),
})
}
pmgInst.SpamDistribution = buckets
@ -6031,10 +6032,10 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
quarantine := models.PMGQuarantineTotals{}
if spamStatus, err := client.GetQuarantineStatus(ctx, "spam"); err == nil && spamStatus != nil {
quarantine.Spam = spamStatus.Count
quarantine.Spam = int(spamStatus.Count.Int64())
}
if virusStatus, err := client.GetQuarantineStatus(ctx, "virus"); err == nil && virusStatus != nil {
quarantine.Virus = virusStatus.Count
quarantine.Virus = int(virusStatus.Count.Int64())
}
pmgInst.Quarantine = &quarantine

View file

@ -55,45 +55,45 @@ type VersionInfo struct {
}
type MailStatistics struct {
Count float64 `json:"count"`
CountIn float64 `json:"count_in"`
CountOut float64 `json:"count_out"`
SpamIn float64 `json:"spamcount_in"`
SpamOut float64 `json:"spamcount_out"`
VirusIn float64 `json:"viruscount_in"`
VirusOut float64 `json:"viruscount_out"`
BouncesIn float64 `json:"bounces_in"`
BouncesOut float64 `json:"bounces_out"`
BytesIn float64 `json:"bytes_in"`
BytesOut float64 `json:"bytes_out"`
GreylistCount float64 `json:"glcount"`
JunkIn float64 `json:"junk_in"`
RBLRejects float64 `json:"rbl_rejects"`
Pregreet float64 `json:"pregreet_rejects"`
AvgProcessSec float64 `json:"avptime"`
Count flexibleFloat `json:"count"`
CountIn flexibleFloat `json:"count_in"`
CountOut flexibleFloat `json:"count_out"`
SpamIn flexibleFloat `json:"spamcount_in"`
SpamOut flexibleFloat `json:"spamcount_out"`
VirusIn flexibleFloat `json:"viruscount_in"`
VirusOut flexibleFloat `json:"viruscount_out"`
BouncesIn flexibleFloat `json:"bounces_in"`
BouncesOut flexibleFloat `json:"bounces_out"`
BytesIn flexibleFloat `json:"bytes_in"`
BytesOut flexibleFloat `json:"bytes_out"`
GreylistCount flexibleFloat `json:"glcount"`
JunkIn flexibleFloat `json:"junk_in"`
RBLRejects flexibleFloat `json:"rbl_rejects"`
Pregreet flexibleFloat `json:"pregreet_rejects"`
AvgProcessSec flexibleFloat `json:"avptime"`
}
type MailCountEntry struct {
Index int `json:"index"`
Time int64 `json:"time"`
Count float64 `json:"count"`
CountIn float64 `json:"count_in"`
CountOut float64 `json:"count_out"`
SpamIn float64 `json:"spamcount_in"`
SpamOut float64 `json:"spamcount_out"`
VirusIn float64 `json:"viruscount_in"`
VirusOut float64 `json:"viruscount_out"`
BouncesIn float64 `json:"bounces_in"`
BouncesOut float64 `json:"bounces_out"`
RBLRejects float64 `json:"rbl_rejects"`
PregreetReject float64 `json:"pregreet_rejects"`
GreylistCount float64 `json:"glcount"`
Index flexibleInt `json:"index"`
Time flexibleInt `json:"time"`
Count flexibleFloat `json:"count"`
CountIn flexibleFloat `json:"count_in"`
CountOut flexibleFloat `json:"count_out"`
SpamIn flexibleFloat `json:"spamcount_in"`
SpamOut flexibleFloat `json:"spamcount_out"`
VirusIn flexibleFloat `json:"viruscount_in"`
VirusOut flexibleFloat `json:"viruscount_out"`
BouncesIn flexibleFloat `json:"bounces_in"`
BouncesOut flexibleFloat `json:"bounces_out"`
RBLRejects flexibleFloat `json:"rbl_rejects"`
PregreetReject flexibleFloat `json:"pregreet_rejects"`
GreylistCount flexibleFloat `json:"glcount"`
}
type SpamScore struct {
Level string `json:"level"`
Count int `json:"count"`
Ratio float64 `json:"ratio"`
Level string `json:"level"`
Count flexibleInt `json:"count"`
Ratio flexibleFloat `json:"ratio"`
}
type ClusterStatusEntry struct {
@ -105,25 +105,25 @@ type ClusterStatusEntry struct {
}
type QuarantineStatus struct {
Count int `json:"count"`
AvgBytes float64 `json:"avgbytes"`
AvgSpam float64 `json:"avgspam,omitempty"`
Megabytes float64 `json:"mbytes"`
Count flexibleInt `json:"count"`
AvgBytes flexibleFloat `json:"avgbytes"`
AvgSpam flexibleFloat `json:"avgspam,omitempty"`
Megabytes flexibleFloat `json:"mbytes"`
}
type QueueStatusEntry struct {
Active int `json:"active"`
Deferred int `json:"deferred"`
Hold int `json:"hold"`
Incoming int `json:"incoming"`
OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds
Active flexibleInt `json:"active"`
Deferred flexibleInt `json:"deferred"`
Hold flexibleInt `json:"hold"`
Incoming flexibleInt `json:"incoming"`
OldestAge flexibleInt `json:"oldest_age,omitempty"` // Age of oldest message in seconds
}
// BackupEntry represents a PMG configuration backup stored on a node.
type BackupEntry struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
Timestamp int64 `json:"timestamp"`
Filename string `json:"filename"`
Size flexibleInt `json:"size"`
Timestamp flexibleInt `json:"timestamp"`
}
func NewClient(cfg ClientConfig) (*Client, error) {

View file

@ -222,7 +222,7 @@ func TestClientUsesTokenAuthorizationHeader(t *testing.T) {
t.Fatalf("get mail statistics failed: %v", err)
}
if stats == nil || stats.Count != 42 {
if stats == nil || stats.Count.Float64() != 42 {
t.Fatalf("expected statistics count 42, got %+v", stats)
}
@ -276,10 +276,144 @@ func TestListBackups(t *testing.T) {
if backup.Filename != "pmg-backup_2024-01-01.tgz" {
t.Fatalf("unexpected filename: %s", backup.Filename)
}
if backup.Size != 123456 {
t.Fatalf("unexpected size: %d", backup.Size)
if backup.Size.Int64() != 123456 {
t.Fatalf("unexpected size: %d", backup.Size.Int64())
}
if backup.Timestamp != 1704096000 {
t.Fatalf("unexpected timestamp: %d", backup.Timestamp)
if backup.Timestamp.Int64() != 1704096000 {
t.Fatalf("unexpected timestamp: %d", backup.Timestamp.Int64())
}
}
func TestMailEndpointsHandleNullAndStringValues(t *testing.T) {
t.Parallel()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api2/json/access/ticket":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"ticket":"ticket","CSRFPreventionToken":"csrf"}}`)
case "/api2/json/version":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"version":"9.0","release":"1"}}`)
case "/api2/json/statistics/mail":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{
"count":null,
"count_in":"25",
"count_out":"5",
"spamcount_in":"7",
"spamcount_out":null,
"viruscount_in":"0",
"viruscount_out":"0",
"bounces_in":null,
"bounces_out":"",
"bytes_in":"1024",
"bytes_out":"2048",
"glcount":"2",
"junk_in":"",
"rbl_rejects":"1",
"pregreet_rejects":null,
"avptime":"0.75"
}}`)
case "/api2/json/statistics/mailcount":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":[{
"index":"0",
"time":"1713724800",
"count":"12",
"count_in":"8",
"count_out":"4",
"spamcount_in":null,
"spamcount_out":"1",
"viruscount_in":"",
"viruscount_out":"0",
"bounces_in":"1",
"bounces_out":"0",
"rbl_rejects":"2",
"pregreet_rejects":"",
"glcount":"3"
}]}`)
case "/api2/json/quarantine/spamstatus":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":null,"avgbytes":"512","mbytes":"0.5"}}`)
case "/api2/json/quarantine/virusstatus":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":"4","avgbytes":"256","mbytes":"0.25"}}`)
case "/api2/json/nodes/mail/postfix/queue":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"active":"3","deferred":null,"hold":"1","incoming":"2","oldest_age":"600"}}`)
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
defer server.Close()
client, err := NewClient(ClientConfig{
Host: server.URL,
User: "api@pmg",
Password: "secret",
VerifySSL: false,
})
if err != nil {
t.Fatalf("unexpected error creating client: %v", err)
}
ctx := context.Background()
stats, err := client.GetMailStatistics(ctx, "")
if err != nil {
t.Fatalf("GetMailStatistics failed: %v", err)
}
if stats == nil {
t.Fatal("expected statistics data")
}
if stats.Count.Float64() != 0 {
t.Fatalf("expected count to default to 0, got %v", stats.Count.Float64())
}
if stats.CountIn.Float64() != 25 {
t.Fatalf("expected CountIn 25, got %v", stats.CountIn.Float64())
}
if stats.BytesIn.Float64() != 1024 {
t.Fatalf("expected BytesIn 1024, got %v", stats.BytesIn.Float64())
}
if stats.AvgProcessSec.Float64() != 0.75 {
t.Fatalf("expected AvgProcessSec 0.75, got %v", stats.AvgProcessSec.Float64())
}
counts, err := client.GetMailCount(ctx, 12)
if err != nil {
t.Fatalf("GetMailCount failed: %v", err)
}
if len(counts) != 1 {
t.Fatalf("expected 1 mail count entry, got %d", len(counts))
}
entry := counts[0]
if entry.Time.Int64() != 1713724800 {
t.Fatalf("expected unix time 1713724800, got %d", entry.Time.Int64())
}
if entry.Count.Float64() != 12 {
t.Fatalf("expected count 12, got %v", entry.Count.Float64())
}
if entry.GreylistCount.Float64() != 3 {
t.Fatalf("expected greylist 3, got %v", entry.GreylistCount.Float64())
}
quarantine, err := client.GetQuarantineStatus(ctx, "spam")
if err != nil {
t.Fatalf("GetQuarantineStatus failed: %v", err)
}
if quarantine.Count.Int64() != 0 {
t.Fatalf("expected spam quarantine count default to 0, got %d", quarantine.Count.Int64())
}
queue, err := client.GetQueueStatus(ctx, "mail")
if err != nil {
t.Fatalf("GetQueueStatus failed: %v", err)
}
if queue.Active.Int() != 3 || queue.Deferred.Int() != 0 || queue.Hold.Int() != 1 || queue.Incoming.Int() != 2 {
t.Fatalf("unexpected queue values: %+v", queue)
}
if queue.OldestAge.Int64() != 600 {
t.Fatalf("expected oldest age 600, got %d", queue.OldestAge.Int64())
}
}

104
pkg/pmg/json_types.go Normal file
View file

@ -0,0 +1,104 @@
package pmg
import (
"bytes"
"encoding/json"
"fmt"
"strconv"
"strings"
)
// flexibleFloat handles numeric values that may arrive as numbers, strings, or nulls.
type flexibleFloat float64
func (f *flexibleFloat) UnmarshalJSON(data []byte) error {
data = bytes.TrimSpace(data)
if len(data) == 0 || bytes.Equal(data, []byte("null")) {
*f = 0
return nil
}
// Try decoding as a JSON number first
var num float64
if err := json.Unmarshal(data, &num); err == nil {
*f = flexibleFloat(num)
return nil
}
// Fall back to string decoding and parsing
var str string
if err := json.Unmarshal(data, &str); err != nil {
return fmt.Errorf("invalid numeric value %q: %w", string(data), err)
}
str = strings.TrimSpace(str)
if str == "" || strings.EqualFold(str, "null") {
*f = 0
return nil
}
parsed, err := strconv.ParseFloat(str, 64)
if err != nil {
return fmt.Errorf("failed to parse numeric string %q: %w", str, err)
}
*f = flexibleFloat(parsed)
return nil
}
func (f flexibleFloat) Float64() float64 {
return float64(f)
}
// flexibleInt handles integer values that may arrive as ints, floats, strings, or nulls.
type flexibleInt int64
func (i *flexibleInt) UnmarshalJSON(data []byte) error {
data = bytes.TrimSpace(data)
if len(data) == 0 || bytes.Equal(data, []byte("null")) {
*i = 0
return nil
}
// Try integer decoding
var num int64
if err := json.Unmarshal(data, &num); err == nil {
*i = flexibleInt(num)
return nil
}
// Try float decoding (round towards zero)
var floatNum float64
if err := json.Unmarshal(data, &floatNum); err == nil {
*i = flexibleInt(int64(floatNum))
return nil
}
// Fall back to string parsing
var str string
if err := json.Unmarshal(data, &str); err != nil {
return fmt.Errorf("invalid integer value %q: %w", string(data), err)
}
str = strings.TrimSpace(str)
if str == "" || strings.EqualFold(str, "null") {
*i = 0
return nil
}
parsed, err := strconv.ParseFloat(str, 64)
if err != nil {
return fmt.Errorf("failed to parse integer string %q: %w", str, err)
}
*i = flexibleInt(int64(parsed))
return nil
}
func (i flexibleInt) Int64() int64 {
return int64(i)
}
func (i flexibleInt) Int() int {
return int(i)
}