danvics/Pulse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add snapshot size alert thresholds (#585)

Browse source
This commit is contained in:
rcourtman 2025-10-22 13:30:40 +00:00
parent 30879c3b7b
commit be26f957c0
20 changed files with 2306 additions and 874 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
README.md
View file

@ -320,11 +320,14 @@ Quick start - most settings are in the web UI:
- **Settings → Security**: Authentication and API tokens
- **Alerts**: Thresholds and notifications
### Apprise CLI Notifications
### Apprise Notifications
Pulse can broadcast grouped alerts through the [Apprise](https://github.com/caronc/apprise) CLI. Install Apprise on the Pulse host (for example with `pip install apprise`) and configure the targets under **Alerts → Notifications**. Each target URL should be a valid Apprise destination (Discord, Slack, email, SMS, etc.).
Pulse can broadcast grouped alerts through [Apprise](https://github.com/caronc/apprise) using either the local CLI or a remote Apprise API gateway. Configure everything under **Alerts → Notifications → Apprise**.
You can also override the CLI path and execution timeout if Apprise is installed in a non-standard location. Pulse automatically skips Apprise delivery when no targets are configured.
- **Local CLI** – Install Apprise on the Pulse host (for example `pip install apprise`) and enter one Apprise URL per line in the delivery targets field. You can override the CLI path and timeout if the executable lives outside of `$PATH`. Pulse skips CLI delivery automatically when no targets are configured.
- **Remote API** – Point Pulse at an Apprise API server by providing the base URL (such as `https://apprise-api.local:8000`). Optionally include a configuration key (for `/notify/{key}` routes), an API key header/value pair, and allow self-signed certificates for lab deployments. Targets remain optional in API mode—leave the list empty to let the Apprise server use its stored defaults.
For both modes, delivery targets accept any Apprise URL (Discord, Slack, email, SMS, etc.). The timeout applies to the CLI process or HTTP request respectively.
### Configuration Files

8
frontend-modern/src/api/notifications.ts
View file

@ -59,9 +59,15 @@ export interface Webhook {
export interface AppriseConfig {
enabled: boolean;
targets: string[];
mode?: 'cli' | 'http';
targets?: string[];
cliPath?: string;
timeoutSeconds?: number;
serverUrl?: string;
configKey?: string;
apiKey?: string;
apiKeyHeader?: string;
skipTlsVerify?: boolean;
}
export interface NotificationTestRequest {

242
frontend-modern/src/components/Alerts/ResourceTable.tsx
View file

@ -25,7 +25,8 @@ const COLUMN_TOOLTIP_LOOKUP: Record<string, string> = {
// PMG (Proxmox Mail Gateway) thresholds
'queue warn': 'Early warning when total mail queue exceeds this message count.',
'queue crit': 'Critical alert requiring urgent action when queue reaches this size.',
'deferred warn': 'Early warning for messages stuck in deferred queue (waiting to retry delivery).',
'deferred warn':
'Early warning for messages stuck in deferred queue (waiting to retry delivery).',
'deferred crit': 'Critical threshold for deferred messages indicating serious delivery problems.',
'hold warn': 'Early warning when administratively held messages exceed this count.',
'hold crit': 'Critical alert for held messages requiring immediate moderation attention.',
@ -38,7 +39,10 @@ const COLUMN_TOOLTIP_LOOKUP: Record<string, string> = {
'growth warn %': 'Early warning when quarantine growth rate exceeds this percentage.',
'growth warn min': 'Minimum new messages required before growth percentage triggers warning.',
'growth crit %': 'Critical quarantine growth rate requiring immediate investigation.',
'growth crit min': 'Minimum new messages required before growth percentage triggers critical alert.',
'growth crit min':
'Minimum new messages required before growth percentage triggers critical alert.',
'warning size (gib)': 'Total snapshot size in GiB that raises a warning.',
'critical size (gib)': 'Total snapshot size in GiB that raises a critical alert.',
};
const OFFLINE_ALERTS_TOOLTIP =
@ -117,7 +121,11 @@ interface ResourceTableProps {
formatMetricValue: (metric: string, value: number | undefined) => string;
hasActiveAlert: (resourceId: string, metric: string) => boolean;
globalDefaults?: Record<string, number | undefined>;
setGlobalDefaults?: (value: Record<string, number | undefined> | ((prev: Record<string, number | undefined>) => Record<string, number | undefined>)) => void;
setGlobalDefaults?: (
value:
| Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void;
setHasUnsavedChanges?: (value: boolean) => void;
globalDisableFlag?: () => boolean;
onToggleGlobalDisable?: () => void;
@ -150,7 +158,10 @@ export function ResourceTable(props: ResourceTableProps) {
return Boolean(props.globalDefaults);
};
const [activeMetricInput, setActiveMetricInput] = createSignal<{ resourceId: string; metric: string } | null>(null);
const [activeMetricInput, setActiveMetricInput] = createSignal<{
resourceId: string;
metric: string;
} | null>(null);
const [showDelayRow, setShowDelayRow] = createSignal(false);
// Track changes to global defaults and factory defaults for debugging
@ -174,12 +185,14 @@ export function ResourceTable(props: ResourceTableProps) {
console.log('[ResourceTable] Missing props, returning false');
return false;
}
const result = Object.keys(props.factoryDefaults).some(key => {
const result = Object.keys(props.factoryDefaults).some((key) => {
const current = props.globalDefaults?.[key];
const factory = props.factoryDefaults?.[key];
const differs = current !== undefined && current !== factory;
if (differs) {
console.log(`[ResourceTable] Difference found: ${key} current=${current} factory=${factory}`);
console.log(
`[ResourceTable] Difference found: ${key} current=${current} factory=${factory}`,
);
}
return differs;
});
@ -189,8 +202,7 @@ export function ResourceTable(props: ResourceTableProps) {
const normalizeMetricKey = (column: string): string => {
const key = column.trim().toLowerCase();
const mapped = (
new Map<string, string>([
const mapped = new Map<string, string>([
['cpu %', 'cpu'],
['memory %', 'memory'],
['disk %', 'disk'],
@ -207,8 +219,9 @@ export function ResourceTable(props: ResourceTableProps) {
['restart window (s)', 'restartWindow'],
['memory warn %', 'memoryWarnPct'],
['memory critical %', 'memoryCriticalPct'],
])
).get(key);
['warning size (gib)', 'warningSizeGiB'],
['critical size (gib)', 'criticalSizeGiB'],
]).get(key);
if (mapped) {
return mapped;
}
@ -233,6 +246,9 @@ export function ResourceTable(props: ResourceTableProps) {
if (['cpu', 'memory', 'disk', 'usage', 'memoryWarnPct', 'memoryCriticalPct'].includes(metric)) {
return { min: -1, max: 100 };
}
if (['warningSizeGiB', 'criticalSizeGiB'].includes(metric)) {
return { min: -1, max: 100000 };
}
if (metric === 'restartCount') {
return { min: -1, max: 50 };
}
@ -246,6 +262,9 @@ export function ResourceTable(props: ResourceTableProps) {
if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
return 'any';
}
if (['warningSizeGiB', 'criticalSizeGiB'].includes(metric)) {
return 'any';
}
return 1;
};
@ -280,10 +299,7 @@ export function ResourceTable(props: ResourceTableProps) {
return value;
};
const totalColumnCount = () =>
props.columns.length +
3 +
(props.showOfflineAlertsColumn ? 1 : 0);
const totalColumnCount = () => props.columns.length + 3 + (props.showOfflineAlertsColumn ? 1 : 0);
const getColumnHeaderTooltip = (column: string): string | undefined => {
const normalized = column.trim().toLowerCase();
@ -324,9 +340,14 @@ export function ResourceTable(props: ResourceTableProps) {
return (
<div class="flex flex-wrap items-center gap-3">
<Show when={meta.host} fallback={
<span class="text-sm font-medium text-gray-900 dark:text-gray-100">{meta.displayName || groupKey}</span>
}>
<Show
when={meta.host}
fallback={
<span class="text-sm font-medium text-gray-900 dark:text-gray-100">
{meta.displayName || groupKey}
</span>
}
>
{(host) => (
<a
href={host() as string}
@ -397,7 +418,10 @@ export function ResourceTable(props: ResourceTableProps) {
const offlineStateOrder: OfflineState[] = ['off', 'warning', 'critical'];
const offlineStateConfig: Record<OfflineState, { label: string; className: string; title: string }> = {
const offlineStateConfig: Record<
OfflineState,
{ label: string; className: string; title: string }
> = {
off: {
label: 'Off',
className:
@ -423,7 +447,11 @@ export function ResourceTable(props: ResourceTableProps) {
return offlineStateOrder[(idx + 1) % offlineStateOrder.length];
};
const renderOfflineStateButton = (state: OfflineState, disabled: boolean, onToggle: () => void) => {
const renderOfflineStateButton = (
state: OfflineState,
disabled: boolean,
onToggle: () => void,
) => {
const config = offlineStateConfig[state];
return (
<button
@ -485,10 +513,17 @@ export function ResourceTable(props: ResourceTableProps) {
</thead>
<tbody class="divide-y divide-gray-200 dark:divide-gray-700">
{/* Global Defaults Row */}
<Show when={props.globalDefaults && props.setGlobalDefaults && props.setHasUnsavedChanges}>
<tr class={`bg-gray-50 dark:bg-gray-800/50 border-b border-gray-300 dark:border-gray-600 ${props.globalDisableFlag?.() ? 'opacity-40' : ''}`}>
<Show
when={props.globalDefaults && props.setGlobalDefaults && props.setHasUnsavedChanges}
>
<tr
class={`bg-gray-50 dark:bg-gray-800/50 border-b border-gray-300 dark:border-gray-600 ${props.globalDisableFlag?.() ? 'opacity-40' : ''}`}
>
<td class="p-1 px-2 text-center align-middle">
<Show when={props.onToggleGlobalDisable} fallback={<span class="text-sm text-gray-400">-</span>}>
<Show
when={props.onToggleGlobalDisable}
fallback={<span class="text-sm text-gray-400">-</span>}
>
<div class="flex items-center justify-center">
<TogglePrimitive
size="sm"
@ -551,7 +586,11 @@ export function ResourceTable(props: ResourceTableProps) {
? 'border-gray-300 dark:border-gray-600 bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-500 italic placeholder:text-gray-400 dark:placeholder:text-gray-500 placeholder:opacity-60 pointer-events-none'
: 'border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 focus:border-blue-500 focus:ring-1 focus:ring-blue-500'
}`}
title={isOff() ? 'Click to enable this metric' : 'Set to -1 to disable alerts for this metric'}
title={
isOff()
? 'Click to enable this metric'
: 'Set to -1 to disable alerts for this metric'
}
/>
<Show when={isOff()}>
<button
@ -577,8 +616,13 @@ export function ResourceTable(props: ResourceTableProps) {
</For>
<Show when={props.showOfflineAlertsColumn}>
<td class="p-1 px-2 text-center align-middle">
<Show when={props.onSetGlobalOfflineState} fallback={
<Show when={props.onToggleGlobalDisableOffline} fallback={<span class="text-sm text-gray-400">-</span>}>
<Show
when={props.onSetGlobalOfflineState}
fallback={
<Show
when={props.onToggleGlobalDisableOffline}
fallback={<span class="text-sm text-gray-400">-</span>}
>
{(() => {
const defaultDisabled = props.globalDisableOfflineFlag?.() ?? false;
return renderToggleBadge({
@ -590,12 +634,15 @@ export function ResourceTable(props: ResourceTableProps) {
},
labelEnabled: 'On',
labelDisabled: 'Off',
titleEnabled: 'Offline alerts currently enabled by default. Click to disable.',
titleDisabled: 'Offline alerts currently disabled by default. Click to enable.',
titleEnabled:
'Offline alerts currently enabled by default. Click to disable.',
titleDisabled:
'Offline alerts currently disabled by default. Click to enable.',
});
})()}
</Show>
}>
}
>
{(() => {
const disabledGlobally = props.globalDisableFlag?.() ?? false;
const defaultDisabled = props.globalDisableOfflineFlag?.() ?? false;
@ -617,12 +664,18 @@ export function ResourceTable(props: ResourceTableProps) {
</Show>
<td class="p-1 px-2 text-center align-middle">
<div class="flex items-center justify-center gap-1">
<Show when={props.showDelayColumn && typeof props.onMetricDelayChange === 'function'}>
<Show
when={
props.showDelayColumn && typeof props.onMetricDelayChange === 'function'
}
>
<button
type="button"
onClick={() => setShowDelayRow(!showDelayRow())}
class="p-1 text-gray-600 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-300 transition-colors"
title={showDelayRow() ? 'Hide alert delay settings' : 'Show alert delay settings'}
title={
showDelayRow() ? 'Hide alert delay settings' : 'Show alert delay settings'
}
>
<svg
class={`w-4 h-4 transition-transform ${showDelayRow() ? 'rotate-180' : ''}`}
@ -646,12 +699,7 @@ export function ResourceTable(props: ResourceTableProps) {
class="p-1 text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300 transition-colors"
title="Reset to factory defaults"
>
<svg
class="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path
stroke-linecap="round"
stroke-linejoin="round"
@ -668,8 +716,16 @@ export function ResourceTable(props: ResourceTableProps) {
</td>
</tr>
</Show>
<Show when={showDelayRow() && props.showDelayColumn && typeof props.onMetricDelayChange === 'function'}>
<tr class={`bg-gray-50 dark:bg-gray-800/50 border-b border-gray-300 dark:border-gray-600 ${props.globalDisableFlag?.() ? 'opacity-40' : ''}`}>
<Show
when={
showDelayRow() &&
props.showDelayColumn &&
typeof props.onMetricDelayChange === 'function'
}
>
<tr
class={`bg-gray-50 dark:bg-gray-800/50 border-b border-gray-300 dark:border-gray-600 ${props.globalDisableFlag?.() ? 'opacity-40' : ''}`}
>
<td class="p-1 px-2 text-center align-middle">
<span class="text-sm text-gray-400">-</span>
</td>
@ -772,7 +828,9 @@ export function ResourceTable(props: ResourceTableProps) {
const extract = (source: Record<string, unknown> | undefined) =>
parseNumeric(source?.[metric]);
const defaults = resource.defaults as Record<string, unknown> | undefined;
const defaults = resource.defaults as
| Record<string, unknown>
| undefined;
if (isEditing()) {
const edited = extract(thresholds() as Record<string, unknown>);
@ -783,7 +841,9 @@ export function ResourceTable(props: ResourceTableProps) {
return fallback !== undefined ? fallback : 0;
}
const liveValue = extract(resource.thresholds as Record<string, unknown> | undefined);
const liveValue = extract(
resource.thresholds as Record<string, unknown> | undefined,
);
if (liveValue !== undefined) {
return liveValue;
}
@ -814,7 +874,10 @@ export function ResourceTable(props: ResourceTableProps) {
size="sm"
checked={isChecked}
disabled={globallyDisabled}
onToggle={() => !globallyDisabled && props.onToggleDisabled?.(resource.id)}
onToggle={() =>
!globallyDisabled &&
props.onToggleDisabled?.(resource.id)
}
class="my-[1px]"
title={
globallyDisabled
@ -823,7 +886,11 @@ export function ResourceTable(props: ResourceTableProps) {
? 'Click to enable alerts'
: 'Click to disable alerts'
}
ariaLabel={isChecked ? 'Alerts enabled for this resource' : 'Alerts disabled for this resource'}
ariaLabel={
isChecked
? 'Alerts enabled for this resource'
: 'Alerts disabled for this resource'
}
/>
</div>
);
@ -842,7 +909,10 @@ export function ResourceTable(props: ResourceTableProps) {
</span>
}
>
<div class="flex flex-wrap items-center gap-3" title={resource.status || undefined}>
<div
class="flex flex-wrap items-center gap-3"
title={resource.status || undefined}
>
<Show
when={resource.host}
fallback={
@ -897,7 +967,8 @@ export function ResourceTable(props: ResourceTableProps) {
<For each={props.columns}>
{(column) => {
const metric = normalizeMetricKey(column);
const showMetric = () => resourceSupportsMetric(resource.type, metric);
const showMetric = () =>
resourceSupportsMetric(resource.type, metric);
const bounds = metricBounds(metric);
const isDisabled = () => thresholds()?.[metric] === -1;
@ -1005,11 +1076,14 @@ export function ResourceTable(props: ResourceTableProps) {
const disabledGlobally = props.globalDisableFlag?.() ?? false;
const supportsTriState =
typeof props.onSetOfflineState === 'function' &&
(resource.type === 'guest' || resource.type === 'dockerContainer');
(resource.type === 'guest' ||
resource.type === 'dockerContainer');
if (supportsTriState) {
const defaultDisabled = props.globalDisableOfflineFlag?.() ?? false;
const defaultSeverity = props.globalOfflineSeverity ?? 'warning';
const defaultDisabled =
props.globalDisableOfflineFlag?.() ?? false;
const defaultSeverity =
props.globalOfflineSeverity ?? 'warning';
let state: OfflineState;
if (resource.disableConnectivity) {
@ -1019,30 +1093,39 @@ export function ResourceTable(props: ResourceTableProps) {
} else if (defaultDisabled) {
state = 'off';
} else {
state = defaultSeverity === 'critical' ? 'critical' : 'warning';
state =
defaultSeverity === 'critical' ? 'critical' : 'warning';
}
return renderOfflineStateButton(state, disabledGlobally, () => {
return renderOfflineStateButton(
state,
disabledGlobally,
() => {
if (disabledGlobally) return;
const next = nextOfflineState(state);
props.onSetOfflineState?.(resource.id, next);
});
},
);
}
if (!props.onToggleNodeConnectivity) {
return <span class="text-sm text-gray-400">-</span>;
}
const globalOfflineDisabled = props.globalDisableOfflineFlag?.() ?? false;
const globalOfflineDisabled =
props.globalDisableOfflineFlag?.() ?? false;
return renderToggleBadge({
isEnabled: !globalOfflineDisabled && !resource.disableConnectivity,
isEnabled:
!globalOfflineDisabled && !resource.disableConnectivity,
disabled: disabledGlobally,
onToggle: () => {
if (disabledGlobally) return;
props.onToggleNodeConnectivity?.(resource.id);
},
titleEnabled: 'Offline alerts enabled. Click to disable for this resource.',
titleDisabled: 'Offline alerts disabled. Click to enable for this resource.',
titleEnabled:
'Offline alerts enabled. Click to disable for this resource.',
titleDisabled:
'Offline alerts disabled. Click to enable for this resource.',
titleWhenDisabled: 'Offline alerts controlled globally',
});
})()}
@ -1111,7 +1194,8 @@ export function ResourceTable(props: ResourceTableProps) {
<Show
when={
resource.hasOverride ||
((resource.type === 'node' || resource.type === 'dockerHost') &&
((resource.type === 'node' ||
resource.type === 'dockerHost') &&
resource.disableConnectivity)
}
>
@ -1183,7 +1267,9 @@ export function ResourceTable(props: ResourceTableProps) {
return fallback !== undefined ? fallback : 0;
}
const liveValue = extract(resource.thresholds as Record<string, unknown> | undefined);
const liveValue = extract(
resource.thresholds as Record<string, unknown> | undefined,
);
if (liveValue !== undefined) {
return liveValue;
}
@ -1214,7 +1300,9 @@ export function ResourceTable(props: ResourceTableProps) {
size="sm"
checked={isChecked}
disabled={globallyDisabled}
onToggle={() => !globallyDisabled && props.onToggleDisabled?.(resource.id)}
onToggle={() =>
!globallyDisabled && props.onToggleDisabled?.(resource.id)
}
class="my-[1px]"
title={
globallyDisabled
@ -1223,7 +1311,11 @@ export function ResourceTable(props: ResourceTableProps) {
? 'Click to enable alerts'
: 'Click to disable alerts'
}
ariaLabel={isChecked ? 'Alerts enabled for this resource' : 'Alerts disabled for this resource'}
ariaLabel={
isChecked
? 'Alerts enabled for this resource'
: 'Alerts disabled for this resource'
}
/>
</div>
);
@ -1231,7 +1323,9 @@ export function ResourceTable(props: ResourceTableProps) {
</Show>
</td>
<td class="p-1 px-2">
<Show when={resource.type === 'node'} fallback={
<Show
when={resource.type === 'node'}
fallback={
<div class="flex items-center gap-2">
<span
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
@ -1244,9 +1338,15 @@ export function ResourceTable(props: ResourceTableProps) {
</span>
</Show>
</div>
}>
<div class="flex flex-wrap items-center gap-3" title={resource.status || undefined}>
<Show when={resource.host} fallback={
}
>
<div
class="flex flex-wrap items-center gap-3"
title={resource.status || undefined}
>
<Show
when={resource.host}
fallback={
<span
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
>
@ -1254,7 +1354,8 @@ export function ResourceTable(props: ResourceTableProps) {
? resource.name
: resource.displayName || resource.name}
</span>
}>
}
>
{(host) => (
<a
href={host() as string}
@ -1424,8 +1525,11 @@ export function ResourceTable(props: ResourceTableProps) {
<td class="p-1 px-2 text-center align-middle">
<Show when={props.onToggleNodeConnectivity}>
{(() => {
const defaultOfflineDisabled = props.globalDisableOfflineFlag?.() ?? false;
const isEnabled = !(resource.disableConnectivity || defaultOfflineDisabled);
const defaultOfflineDisabled =
props.globalDisableOfflineFlag?.() ?? false;
const isEnabled = !(
resource.disableConnectivity || defaultOfflineDisabled
);
const disabledGlobally = props.globalDisableFlag?.() ?? false;
return (
<StatusBadge
@ -1455,8 +1559,12 @@ export function ResourceTable(props: ResourceTableProps) {
/>
</Show>
<Show
when={resource.editable !== false && typeof props.onEdit === 'function'}
fallback={<span class="text-xs text-gray-400 dark:text-gray-600">—</span>}
when={
resource.editable !== false && typeof props.onEdit === 'function'
}
fallback={
<span class="text-xs text-gray-400 dark:text-gray-600">—</span>
}
>
<Show
when={!isEditing()}

373
frontend-modern/src/components/Alerts/ThresholdsTable.tsx
View file

@ -19,17 +19,15 @@ import type {
PMGBackup,
Backups,
} from '@/types/api';
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
import type {
RawOverrideConfig,
PMGThresholdDefaults,
SnapshotAlertConfig,
BackupAlertConfig,
} from '@/types/alerts';
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
import { useAlertsActivation } from '@/stores/alertsActivation';
type OverrideType =
| 'guest'
| 'node'
| 'storage'
| 'pbs'
| 'pmg'
| 'dockerHost'
| 'dockerContainer';
type OverrideType = 'guest' | 'node' | 'storage' | 'pbs' | 'pmg' | 'dockerHost' | 'dockerContainer';
type OfflineState = 'off' | 'warning' | 'critical';
@ -110,6 +108,8 @@ export const normalizeDockerIgnoredInput = (value: string): string[] =>
const DEFAULT_SNAPSHOT_WARNING = 30;
const DEFAULT_SNAPSHOT_CRITICAL = 45;
const DEFAULT_SNAPSHOT_WARNING_SIZE = 0;
const DEFAULT_SNAPSHOT_CRITICAL_SIZE = 0;
const DEFAULT_BACKUP_WARNING = 7;
const DEFAULT_BACKUP_CRITICAL = 14;
@ -143,13 +143,13 @@ interface ThresholdsTableProps {
pmgBackups?: PMGBackup[];
pmgThresholds: () => PMGThresholdDefaults;
setPMGThresholds: (
value:
| PMGThresholdDefaults
| ((prev: PMGThresholdDefaults) => PMGThresholdDefaults),
value: PMGThresholdDefaults | ((prev: PMGThresholdDefaults) => PMGThresholdDefaults),
) => void;
guestDefaults: SimpleThresholds;
setGuestDefaults: (
value: Record<string, number | undefined> | ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
value:
| Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void;
guestDisableConnectivity: () => boolean;
setGuestDisableConnectivity: (value: boolean) => void;
@ -157,11 +157,43 @@ interface ThresholdsTableProps {
setGuestPoweredOffSeverity: (value: 'warning' | 'critical') => void;
nodeDefaults: SimpleThresholds;
setNodeDefaults: (
value: Record<string, number | undefined> | ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
value:
| Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void;
dockerDefaults: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number };
dockerDefaults: {
cpu: number;
memory: number;
restartCount: number;
restartWindow: number;
memoryWarnPct: number;
memoryCriticalPct: number;
};
setDockerDefaults: (
value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }),
value:
| {
cpu: number;
memory: number;
restartCount: number;
restartWindow: number;
memoryWarnPct: number;
memoryCriticalPct: number;
}
| ((prev: {
cpu: number;
memory: number;
restartCount: number;
restartWindow: number;
memoryWarnPct: number;
memoryCriticalPct: number;
}) => {
cpu: number;
memory: number;
restartCount: number;
restartWindow: number;
memoryWarnPct: number;
memoryCriticalPct: number;
}),
) => void;
dockerIgnoredPrefixes: () => string[];
setDockerIgnoredPrefixes: (value: string[] | ((prev: string[]) => string[])) => void;
@ -185,17 +217,13 @@ interface ThresholdsTableProps {
) => void;
snapshotDefaults: () => SnapshotAlertConfig;
setSnapshotDefaults: (
value:
| SnapshotAlertConfig
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
value: SnapshotAlertConfig | ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
) => void;
snapshotFactoryDefaults?: SnapshotAlertConfig;
resetSnapshotDefaults?: () => void;
backupDefaults: () => BackupAlertConfig;
setBackupDefaults: (
value:
| BackupAlertConfig
| ((prev: BackupAlertConfig) => BackupAlertConfig),
value: BackupAlertConfig | ((prev: BackupAlertConfig) => BackupAlertConfig),
) => void;
backupFactoryDefaults?: BackupAlertConfig;
resetBackupDefaults?: () => void;
@ -306,10 +334,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
if (!el) return false;
const tag = el.tagName;
return (
tag === 'INPUT' ||
tag === 'TEXTAREA' ||
tag === 'SELECT' ||
el.contentEditable === 'true'
tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || el.contentEditable === 'true'
);
};
@ -381,6 +406,11 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
return String(value);
}
if (metric === 'warningSizeGiB' || metric === 'criticalSizeGiB') {
const rounded = Math.round(value * 10) / 10;
return `${rounded} GiB`;
}
// MB/s metrics
if (
metric === 'diskRead' ||
@ -403,14 +433,17 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
};
// Process nodes with their overrides
const getFriendlyNodeName = (value: string, clusterName?: string): string => {
const getFriendlyNodeName = (value: string, clusterName?: string): string => {
if (!value) return value;
const clusterLower = clusterName?.toLowerCase().trim();
const normalizeToken = (token?: string | null): string => {
if (!token) return '';
let result = token.replace(/\(.*?\)/g, ' ').replace(/\s+/g, ' ').trim();
let result = token
.replace(/\(.*?\)/g, ' ')
.replace(/\s+/g, ' ')
.trim();
if (clusterLower) {
result = result
.split(' ')
@ -420,7 +453,9 @@ const getFriendlyNodeName = (value: string, clusterName?: string): string => {
}
if (!result) return '';
const firstWord = result.split(/\s+/)[0] || result;
const withoutDomain = firstWord.includes('.') ? firstWord.split('.')[0] ?? firstWord : firstWord;
const withoutDomain = firstWord.includes('.')
? (firstWord.split('.')[0] ?? firstWord)
: firstWord;
return withoutDomain.trim();
};
@ -438,9 +473,9 @@ const getFriendlyNodeName = (value: string, clusterName?: string): string => {
}
return base;
};
};
const buildNodeHeaderMeta = (node: Node) => {
const buildNodeHeaderMeta = (node: Node) => {
const originalDisplayName = node.displayName?.trim() || node.name;
const friendlyName = getFriendlyNodeName(originalDisplayName, node.clusterName);
const hostValue = node.host?.trim();
@ -471,7 +506,7 @@ const buildNodeHeaderMeta = (node: Node) => {
});
return { headerMeta, keys };
};
};
const nodesWithOverrides = createMemo<Resource[]>((prev = []) => {
// If we're currently editing, return the previous value to avoid re-renders
@ -496,14 +531,14 @@ const buildNodeHeaderMeta = (node: Node) => {
);
});
const originalDisplayName = node.displayName?.trim() || node.name;
const friendlyName = getFriendlyNodeName(originalDisplayName, node.clusterName);
const rawName = node.name;
const sanitizedName = friendlyName || originalDisplayName || rawName.split('.')[0] || rawName;
// Build a best-effort management URL for the node
const hostValue = node.host?.trim() || rawName;
const normalizedHost = hostValue.startsWith('http://') || hostValue.startsWith('https://')
const normalizedHost =
hostValue.startsWith('http://') || hostValue.startsWith('https://')
? hostValue
: `https://${hostValue.includes(':') ? hostValue : `${hostValue}:8006`}`;
@ -602,7 +637,7 @@ const buildNodeHeaderMeta = (node: Node) => {
}, []);
// Process Docker containers grouped by host
const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((prev = {}) => {
const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((prev = {}) => {
if (editingId()) {
return prev;
}
@ -646,9 +681,12 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
);
});
const hasOverride =
hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false;
hasCustomThresholds ||
override?.disabled ||
override?.disableConnectivity ||
overrideSeverity !== undefined ||
false;
const containerName = normalizeContainerName(container);
const containerNameLower = containerName.toLowerCase();
@ -770,8 +808,9 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
});
const countOverrides = (resources: Resource[] | undefined) =>
resources?.filter((resource) => resource.hasOverride || resource.disabled || resource.disableConnectivity)
.length ?? 0;
resources?.filter(
(resource) => resource.hasOverride || resource.disabled || resource.disableConnectivity,
).length ?? 0;
const registerSection = (_key: string) => (_el: HTMLDivElement | null) => {
/* no-op placeholder for future scroll restoration */
@ -782,6 +821,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
enabled: false,
warningDays: DEFAULT_SNAPSHOT_WARNING,
criticalDays: DEFAULT_SNAPSHOT_CRITICAL,
warningSizeGiB: DEFAULT_SNAPSHOT_WARNING_SIZE,
criticalSizeGiB: DEFAULT_SNAPSHOT_CRITICAL_SIZE,
};
const sanitizeSnapshotConfig = (config: SnapshotAlertConfig): SnapshotAlertConfig => {
@ -795,17 +836,36 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
critical = warning;
}
const rawWarningSize = Number.isFinite(config.warningSizeGiB)
? Number(config.warningSizeGiB)
: DEFAULT_SNAPSHOT_WARNING_SIZE;
const rawCriticalSize = Number.isFinite(config.criticalSizeGiB)
? Number(config.criticalSizeGiB)
: DEFAULT_SNAPSHOT_CRITICAL_SIZE;
const roundSize = (value: number) => Math.round(Math.max(0, value) * 10) / 10;
let warningSize = roundSize(rawWarningSize);
let criticalSize = roundSize(rawCriticalSize);
if (criticalSize > 0 && warningSize > criticalSize) {
warningSize = criticalSize;
}
if (criticalSize === 0 && warningSize > 0) {
criticalSize = warningSize;
}
return {
enabled: !!config.enabled,
warningDays: warning,
criticalDays: critical,
warningSizeGiB: warningSize,
criticalSizeGiB: criticalSize,
};
};
const updateSnapshotDefaults = (
updater:
| SnapshotAlertConfig
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
updater: SnapshotAlertConfig | ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
) => {
props.setSnapshotDefaults((prev) => {
const next =
@ -822,6 +882,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
return {
'warning days': current.warningDays ?? 0,
'critical days': current.criticalDays ?? 0,
'warning size (gib)': current.warningSizeGiB ?? 0,
'critical size (gib)': current.criticalSizeGiB ?? 0,
};
});
@ -830,6 +892,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
return {
'warning days': factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING,
'critical days': factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL,
'warning size (gib)': factory.warningSizeGiB ?? DEFAULT_SNAPSHOT_WARNING_SIZE,
'critical size (gib)': factory.criticalSizeGiB ?? DEFAULT_SNAPSHOT_CRITICAL_SIZE,
};
});
@ -859,9 +923,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
};
const updateBackupDefaults = (
updater:
| BackupAlertConfig
| ((prev: BackupAlertConfig) => BackupAlertConfig),
updater: BackupAlertConfig | ((prev: BackupAlertConfig) => BackupAlertConfig),
) => {
props.setBackupDefaults((prev) => {
const next =
@ -889,18 +951,21 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
};
});
const snapshotOverridesCount = createMemo(() => {
const snapshotOverridesCount = createMemo(() => {
const current = props.snapshotDefaults();
const factory = snapshotFactoryConfig();
return current.enabled !== factory.enabled ||
const differs =
current.enabled !== factory.enabled ||
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !==
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) ||
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !==
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
? 1
: 0;
});
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) ||
(current.warningSizeGiB ?? DEFAULT_SNAPSHOT_WARNING_SIZE) !==
(factory.warningSizeGiB ?? DEFAULT_SNAPSHOT_WARNING_SIZE) ||
(current.criticalSizeGiB ?? DEFAULT_SNAPSHOT_CRITICAL_SIZE) !==
(factory.criticalSizeGiB ?? DEFAULT_SNAPSHOT_CRITICAL_SIZE);
return differs ? 1 : 0;
});
const backupOverridesCount = createMemo(() => {
const backupCurrent = props.backupDefaults();
@ -914,7 +979,6 @@ const snapshotOverridesCount = createMemo(() => {
: 0;
});
// Process guests with their overrides and group by node
const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => {
// If we're currently editing, return the previous value to avoid re-renders
@ -941,10 +1005,13 @@ const snapshotOverridesCount = createMemo(() => {
);
});
// A guest has an override if it has custom thresholds OR is disabled OR has connectivity disabled
const hasOverride =
hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false;
hasCustomThresholds ||
override?.disabled ||
override?.disableConnectivity ||
overrideSeverity !== undefined ||
false;
return {
id: guestId,
@ -1079,11 +1146,10 @@ const snapshotOverridesCount = createMemo(() => {
const record: Record<string, number> = {};
PMG_THRESHOLD_COLUMNS.forEach(({ key, normalized }) => {
const value = defaults[key];
record[normalized] =
typeof value === 'number' && Number.isFinite(value) ? value : 0;
record[normalized] = typeof value === 'number' && Number.isFinite(value) ? value : 0;
});
return record;
});
});
const setPMGGlobalDefaults = (
value:
@ -1344,9 +1410,13 @@ const snapshotOverridesCount = createMemo(() => {
if (resource.editScope === 'backup') {
const currentBackupDefaults = props.backupDefaults();
const nextWarning =
editedThresholds['warning days'] ?? currentBackupDefaults.warningDays ?? DEFAULT_BACKUP_WARNING;
editedThresholds['warning days'] ??
currentBackupDefaults.warningDays ??
DEFAULT_BACKUP_WARNING;
const nextCritical =
editedThresholds['critical days'] ?? currentBackupDefaults.criticalDays ?? DEFAULT_BACKUP_CRITICAL;
editedThresholds['critical days'] ??
currentBackupDefaults.criticalDays ??
DEFAULT_BACKUP_CRITICAL;
updateBackupDefaults({
enabled: currentBackupDefaults.enabled,
@ -1361,14 +1431,28 @@ const snapshotOverridesCount = createMemo(() => {
if (resource.editScope === 'snapshot') {
const currentSnapshotDefaults = props.snapshotDefaults();
const nextWarning =
editedThresholds['warning days'] ?? currentSnapshotDefaults.warningDays ?? DEFAULT_SNAPSHOT_WARNING;
editedThresholds['warning days'] ??
currentSnapshotDefaults.warningDays ??
DEFAULT_SNAPSHOT_WARNING;
const nextCritical =
editedThresholds['critical days'] ?? currentSnapshotDefaults.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL;
editedThresholds['critical days'] ??
currentSnapshotDefaults.criticalDays ??
DEFAULT_SNAPSHOT_CRITICAL;
const nextWarningSize =
editedThresholds['warning size (gib)'] ??
currentSnapshotDefaults.warningSizeGiB ??
DEFAULT_SNAPSHOT_WARNING_SIZE;
const nextCriticalSize =
editedThresholds['critical size (gib)'] ??
currentSnapshotDefaults.criticalSizeGiB ??
DEFAULT_SNAPSHOT_CRITICAL_SIZE;
updateSnapshotDefaults({
enabled: currentSnapshotDefaults.enabled,
warningDays: nextWarning,
criticalDays: nextCritical,
warningSizeGiB: nextWarningSize,
criticalSizeGiB: nextCriticalSize,
});
cancelEdit();
@ -1467,7 +1551,10 @@ const snapshotOverridesCount = createMemo(() => {
hysteresisThresholds.disableConnectivity = true;
delete hysteresisThresholds.poweredOffSeverity;
} else {
if ((resource.type === 'guest' || resource.type === 'dockerContainer') && props.guestDisableConnectivity()) {
if (
(resource.type === 'guest' || resource.type === 'dockerContainer') &&
props.guestDisableConnectivity()
) {
hysteresisThresholds.disableConnectivity = false;
} else {
delete hysteresisThresholds.disableConnectivity;
@ -1491,7 +1578,11 @@ const snapshotOverridesCount = createMemo(() => {
setEditingThresholds({});
};
const updateMetricDelay = (typeKey: 'guest' | 'node' | 'storage' | 'pbs', metricKey: string, value: number | null) => {
const updateMetricDelay = (
typeKey: 'guest' | 'node' | 'storage' | 'pbs',
metricKey: string,
value: number | null,
) => {
const normalizedMetric = metricKey.trim().toLowerCase();
if (!normalizedMetric) return;
@ -1576,10 +1667,7 @@ const snapshotOverridesCount = createMemo(() => {
delete (cleanThresholds as Record<string, unknown>).disabled;
// If enabling (disabled = false) and no custom thresholds exist, remove the override entirely
if (
!newDisabledState &&
(!existingOverride || Object.keys(cleanThresholds).length === 0)
) {
if (!newDisabledState && (!existingOverride || Object.keys(cleanThresholds).length === 0)) {
// Remove the override completely
props.setOverrides(props.overrides().filter((o) => o.id !== resourceId));
@ -1646,8 +1734,7 @@ const snapshotOverridesCount = createMemo(() => {
const offlineId = `pbs-offline-${resourceId}`;
props.removeAlerts(
(alert) =>
alert.resourceId === resourceId &&
(alert.id === offlineId || alert.type === 'offline'),
alert.resourceId === resourceId && (alert.id === offlineId || alert.type === 'offline'),
);
} else if (resource.type === 'dockerContainer') {
props.removeAlerts(
@ -1755,9 +1842,7 @@ const snapshotOverridesCount = createMemo(() => {
if (props.removeAlerts && resource.type === 'dockerHost') {
const offlineId = `docker-host-offline-${resourceId}`;
const resourceKey = `docker:${resourceId}`;
props.removeAlerts(
(alert) => alert.id === offlineId || alert.resourceId === resourceKey,
);
props.removeAlerts((alert) => alert.id === offlineId || alert.resourceId === resourceKey);
}
};
@ -1865,7 +1950,9 @@ const snapshotOverridesCount = createMemo(() => {
if (props.removeAlerts && newDisableConnectivity) {
if (resource.type === 'guest') {
props.removeAlerts((alert) => alert.resourceId === resourceId && alert.type === 'powered-off');
props.removeAlerts(
(alert) => alert.resourceId === resourceId && alert.type === 'powered-off',
);
} else if (resource.type === 'dockerContainer') {
props.removeAlerts(
(alert) =>
@ -1922,11 +2009,30 @@ const snapshotOverridesCount = createMemo(() => {
{/* Help Banner */}
<div class="rounded-lg border border-blue-200 bg-blue-50 dark:border-blue-800 dark:bg-blue-950/30 p-3">
<div class="flex items-start gap-2">
<svg class="w-5 h-5 text-blue-600 dark:text-blue-400 flex-shrink-0 mt-0.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
<svg
class="w-5 h-5 text-blue-600 dark:text-blue-400 flex-shrink-0 mt-0.5"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
/>
</svg>
<div class="text-sm text-blue-900 dark:text-blue-100">
<span class="font-medium">Quick tips:</span> Set any threshold to <code class="px-1 py-0.5 bg-blue-100 dark:bg-blue-900/50 rounded text-xs font-mono">0</code> to disable alerts for that metric. Click on disabled thresholds showing <span class="italic">Off</span> to re-enable them. Resources with custom settings show a <span class="inline-flex items-center px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded text-xs">Custom</span> badge.
<span class="font-medium">Quick tips:</span> Set any threshold to{' '}
<code class="px-1 py-0.5 bg-blue-100 dark:bg-blue-900/50 rounded text-xs font-mono">
0
</code>{' '}
to disable alerts for that metric. Click on disabled thresholds showing{' '}
<span class="italic">Off</span> to re-enable them. Resources with custom settings show a{' '}
<span class="inline-flex items-center px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded text-xs">
Custom
</span>{' '}
badge.
</div>
</div>
</div>
@ -1998,7 +2104,9 @@ const snapshotOverridesCount = createMemo(() => {
globalDisableFlag={props.disableAllNodes}
onToggleGlobalDisable={() => props.setDisableAllNodes(!props.disableAllNodes())}
globalDisableOfflineFlag={props.disableAllNodesOffline}
onToggleGlobalDisableOffline={() => props.setDisableAllNodesOffline(!props.disableAllNodesOffline())}
onToggleGlobalDisableOffline={() =>
props.setDisableAllNodesOffline(!props.disableAllNodesOffline())
}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().node}
metricDelaySeconds={props.metricTimeThresholds().node ?? {}}
@ -2032,22 +2140,42 @@ const snapshotOverridesCount = createMemo(() => {
globalDefaults={{ cpu: props.nodeDefaults.cpu, memory: props.nodeDefaults.memory }}
setGlobalDefaults={(value) => {
if (typeof value === 'function') {
const newValue = value({ cpu: props.nodeDefaults.cpu, memory: props.nodeDefaults.memory });
props.setNodeDefaults((prev) => ({ ...prev, cpu: newValue.cpu ?? prev.cpu, memory: newValue.memory ?? prev.memory }));
const newValue = value({
cpu: props.nodeDefaults.cpu,
memory: props.nodeDefaults.memory,
});
props.setNodeDefaults((prev) => ({
...prev,
cpu: newValue.cpu ?? prev.cpu,
memory: newValue.memory ?? prev.memory,
}));
} else {
props.setNodeDefaults((prev) => ({ ...prev, cpu: value.cpu ?? prev.cpu, memory: value.memory ?? prev.memory }));
props.setNodeDefaults((prev) => ({
...prev,
cpu: value.cpu ?? prev.cpu,
memory: value.memory ?? prev.memory,
}));
}
}}
setHasUnsavedChanges={props.setHasUnsavedChanges}
globalDisableFlag={props.disableAllPBS}
onToggleGlobalDisable={() => props.setDisableAllPBS(!props.disableAllPBS())}
globalDisableOfflineFlag={props.disableAllPBSOffline}
onToggleGlobalDisableOffline={() => props.setDisableAllPBSOffline(!props.disableAllPBSOffline())}
onToggleGlobalDisableOffline={() =>
props.setDisableAllPBSOffline(!props.disableAllPBSOffline())
}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().pbs}
metricDelaySeconds={props.metricTimeThresholds().pbs ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('pbs', metric, value)}
factoryDefaults={props.factoryNodeDefaults ? { cpu: props.factoryNodeDefaults.cpu, memory: props.factoryNodeDefaults.memory } : undefined}
factoryDefaults={
props.factoryNodeDefaults
? {
cpu: props.factoryNodeDefaults.cpu,
memory: props.factoryNodeDefaults.memory,
}
: undefined
}
onResetDefaults={props.resetNodeDefaults}
/>
</div>
@ -2059,7 +2187,15 @@ const snapshotOverridesCount = createMemo(() => {
title="VMs & Containers"
groupedResources={guestsGroupedByNode()}
groupHeaderMeta={guestGroupHeaderMeta()}
columns={['CPU %', 'Memory %', 'Disk %', 'Disk R MB/s', 'Disk W MB/s', 'Net In MB/s', 'Net Out MB/s']}
columns={[
'CPU %',
'Memory %',
'Disk %',
'Disk R MB/s',
'Disk W MB/s',
'Net In MB/s',
'Net Out MB/s',
]}
activeAlerts={props.activeAlerts}
emptyMessage="No VMs or containers match the current filters."
onEdit={startEditing}
@ -2108,8 +2244,22 @@ const snapshotOverridesCount = createMemo(() => {
<div ref={registerSection('backups')} class="scroll-mt-24">
<ResourceTable
title="Backups"
resources={[{ id: "backups-defaults", name: "Global Defaults", thresholds: backupDefaultsRecord(), defaults: backupDefaultsRecord(), editable: true, editScope: "backup" }]}
columns={['Warning Days', 'Critical Days']}
resources={[
{
id: 'backups-defaults',
name: 'Global Defaults',
thresholds: backupDefaultsRecord(),
defaults: backupDefaultsRecord(),
editable: true,
editScope: 'backup',
},
]}
columns={[
'Warning Days',
'Critical Days',
'Warning Size (GiB)',
'Critical Size (GiB)',
]}
activeAlerts={props.activeAlerts}
emptyMessage=""
onEdit={startEditing}
@ -2171,7 +2321,16 @@ const snapshotOverridesCount = createMemo(() => {
<div ref={registerSection('snapshots')} class="scroll-mt-24">
<ResourceTable
title="Snapshot Age"
resources={[{ id: "snapshots-defaults", name: "Global Defaults", thresholds: snapshotDefaultsRecord(), defaults: snapshotDefaultsRecord(), editable: true, editScope: "snapshot" }]}
resources={[
{
id: 'snapshots-defaults',
name: 'Global Defaults',
thresholds: snapshotDefaultsRecord(),
defaults: snapshotDefaultsRecord(),
editable: true,
editScope: 'snapshot',
},
]}
columns={['Warning Days', 'Critical Days']}
activeAlerts={props.activeAlerts}
emptyMessage=""
@ -2191,6 +2350,8 @@ const snapshotOverridesCount = createMemo(() => {
const currentRecord = {
'warning days': prev.warningDays ?? 0,
'critical days': prev.criticalDays ?? 0,
'warning size (gib)': prev.warningSizeGiB ?? 0,
'critical size (gib)': prev.criticalSizeGiB ?? 0,
};
const nextRecord =
typeof value === 'function'
@ -2206,6 +2367,14 @@ const snapshotOverridesCount = createMemo(() => {
typeof nextRecord['critical days'] === 'number'
? nextRecord['critical days']
: prev.criticalDays,
warningSizeGiB:
typeof nextRecord['warning size (gib)'] === 'number'
? nextRecord['warning size (gib)']
: prev.warningSizeGiB,
criticalSizeGiB:
typeof nextRecord['critical size (gib)'] === 'number'
? nextRecord['critical size (gib)']
: prev.criticalSizeGiB,
};
});
}}
@ -2266,12 +2435,15 @@ const snapshotOverridesCount = createMemo(() => {
globalDelaySeconds={props.timeThresholds().storage}
metricDelaySeconds={props.metricTimeThresholds().storage ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('storage', metric, value)}
factoryDefaults={props.factoryStorageDefault !== undefined ? { usage: props.factoryStorageDefault } : undefined}
factoryDefaults={
props.factoryStorageDefault !== undefined
? { usage: props.factoryStorageDefault }
: undefined
}
onResetDefaults={props.resetStorageDefault}
/>
</div>
</Show>
</Show>
<Show when={activeTab() === 'pmg'}>
@ -2279,7 +2451,8 @@ const snapshotOverridesCount = createMemo(() => {
when={pmgServersWithOverrides().length > 0}
fallback={
<div class="rounded-lg border border-gray-200 bg-white p-6 text-sm text-gray-600 dark:border-gray-700 dark:bg-gray-900 dark:text-gray-300">
No mail gateways configured yet. Add a PMG instance in Settings to manage thresholds.
No mail gateways configured yet. Add a PMG instance in Settings to manage
thresholds.
</div>
}
>
@ -2325,7 +2498,9 @@ const snapshotOverridesCount = createMemo(() => {
globalDisableFlag={props.disableAllPMG}
onToggleGlobalDisable={() => props.setDisableAllPMG(!props.disableAllPMG())}
globalDisableOfflineFlag={props.disableAllPMGOffline}
onToggleGlobalDisableOffline={() => props.setDisableAllPMGOffline(!props.disableAllPMGOffline())}
onToggleGlobalDisableOffline={() =>
props.setDisableAllPMGOffline(!props.disableAllPMGOffline())
}
/>
</div>
</Show>
@ -2383,9 +2558,13 @@ const snapshotOverridesCount = createMemo(() => {
formatMetricValue={formatMetricValue}
hasActiveAlert={hasActiveAlert}
globalDisableFlag={props.disableAllDockerHosts}
onToggleGlobalDisable={() => props.setDisableAllDockerHosts(!props.disableAllDockerHosts())}
onToggleGlobalDisable={() =>
props.setDisableAllDockerHosts(!props.disableAllDockerHosts())
}
globalDisableOfflineFlag={props.disableAllDockerHostsOffline}
onToggleGlobalDisableOffline={() => props.setDisableAllDockerHostsOffline(!props.disableAllDockerHostsOffline())}
onToggleGlobalDisableOffline={() =>
props.setDisableAllDockerHostsOffline(!props.disableAllDockerHostsOffline())
}
/>
</div>
</Show>
@ -2435,9 +2614,7 @@ const snapshotOverridesCount = createMemo(() => {
memoryCriticalPct: props.dockerDefaults.memoryCriticalPct,
};
const next =
typeof value === 'function'
? value(current)
: { ...current, ...value };
typeof value === 'function' ? value(current) : { ...current, ...value };
props.setDockerDefaults((prev) => ({
...prev,
@ -2451,7 +2628,9 @@ const snapshotOverridesCount = createMemo(() => {
}}
setHasUnsavedChanges={props.setHasUnsavedChanges}
globalDisableFlag={props.disableAllDockerContainers}
onToggleGlobalDisable={() => props.setDisableAllDockerContainers(!props.disableAllDockerContainers())}
onToggleGlobalDisable={() =>
props.setDisableAllDockerContainers(!props.disableAllDockerContainers())
}
globalDisableOfflineFlag={() => props.guestDisableConnectivity()}
onToggleGlobalDisableOffline={() =>
props.setGuestDisableConnectivity(!props.guestDisableConnectivity())

34
frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx
View file

@ -2,10 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest';
import { render, fireEvent, screen, cleanup } from '@solidjs/testing-library';
import { createSignal } from 'solid-js';
import {
ThresholdsTable,
normalizeDockerIgnoredInput,
} from '../ThresholdsTable';
import { ThresholdsTable, normalizeDockerIgnoredInput } from '../ThresholdsTable';
import type { PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
vi.mock('@solidjs/router', () => ({
@ -91,9 +88,21 @@ const baseProps = () => ({
setBackupDefaults: vi.fn(),
backupFactoryDefaults: { enabled: false, warningDays: 7, criticalDays: 14 } as BackupAlertConfig,
resetBackupDefaults: vi.fn(),
snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }),
snapshotDefaults: () => ({
enabled: false,
warningDays: 30,
criticalDays: 45,
warningSizeGiB: 0,
criticalSizeGiB: 0,
}),
setSnapshotDefaults: vi.fn(),
snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig,
snapshotFactoryDefaults: {
enabled: false,
warningDays: 30,
criticalDays: 45,
warningSizeGiB: 0,
criticalSizeGiB: 0,
} as SnapshotAlertConfig,
resetSnapshotDefaults: vi.fn(),
timeThresholds: () => ({ guest: 5, node: 5, storage: 5, pbs: 5 }),
metricTimeThresholds: () => ({}),
@ -126,7 +135,10 @@ const baseProps = () => ({
setDisableAllDockerHostsOffline: vi.fn(),
});
const renderThresholdsTable = (options?: { initialPrefixes?: string[]; includeReset?: boolean }) => {
const renderThresholdsTable = (options?: {
initialPrefixes?: string[];
includeReset?: boolean;
}) => {
let setDockerIgnoredPrefixesMock!: ReturnType<typeof vi.fn>;
let resetDockerIgnoredPrefixesMock: ReturnType<typeof vi.fn> | undefined;
let setHasUnsavedChangesMock!: ReturnType<typeof vi.fn>;
@ -177,9 +189,11 @@ const renderThresholdsTable = (options?: { initialPrefixes?: string[]; includeRe
describe('normalizeDockerIgnoredInput', () => {
it('trims whitespace and removes empty lines', () => {
expect(
normalizeDockerIgnoredInput(' runner- \n\n #system \n\t \njob-'),
).toEqual(['runner-', '#system', 'job-']);
expect(normalizeDockerIgnoredInput(' runner- \n\n #system \n\t \njob-')).toEqual([
'runner-',
'#system',
'job-',
]);
});
it('returns empty array for blank input', () => {

67
frontend-modern/src/components/Backups/UnifiedBackups.tsx
View file

@ -16,7 +16,16 @@ import { usePersistentSignal } from '@/hooks/usePersistentSignal';
type BackupSortKey = keyof Pick<
UnifiedBackup,
'backupTime' | 'name' | 'node' | 'vmid' | 'backupType' | 'size' | 'storage' | 'verified' | 'type' | 'owner'
| 'backupTime'
| 'name'
| 'node'
| 'vmid'
| 'backupType'
| 'size'
| 'storage'
| 'verified'
| 'type'
| 'owner'
>;
const BACKUP_SORT_KEY_VALUES: readonly BackupSortKey[] = [
'backupTime',
@ -70,16 +79,10 @@ const UnifiedBackups: Component = () => {
else if (value === 'pve') setBackupTypeFilter('local');
else if (value === 'pbs') setBackupTypeFilter('remote');
};
const [sortKey, setSortKey] = usePersistentSignal<BackupSortKey>(
'backupsSortKey',
'backupTime',
{
const [sortKey, setSortKey] = usePersistentSignal<BackupSortKey>('backupsSortKey', 'backupTime', {
deserialize: (raw) =>
BACKUP_SORT_KEY_VALUES.includes(raw as BackupSortKey)
? (raw as BackupSortKey)
: 'backupTime',
},
);
BACKUP_SORT_KEY_VALUES.includes(raw as BackupSortKey) ? (raw as BackupSortKey) : 'backupTime',
});
const [sortDirection, setSortDirection] = usePersistentSignal<'asc' | 'desc'>(
'backupsSortDirection',
'desc',
@ -188,7 +191,9 @@ const UnifiedBackups: Component = () => {
pveBackupsState()?.guestSnapshots?.forEach((snapshot) => {
// Try to find the guest name by matching VMID and instance (not hostname)
let guestName = '';
const vm = state.vms?.find((v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance);
const vm = state.vms?.find(
(v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance,
);
const ct = state.containers?.find(
(c) => c.vmid === snapshot.vmid && c.instance === snapshot.instance,
);
@ -209,7 +214,7 @@ const UnifiedBackups: Component = () => {
backupName: snapshot.name, // This is the snapshot name like "current", "pre-upgrade"
description: snapshot.description || '',
status: 'ok',
size: null,
size: typeof snapshot.sizeBytes === 'number' ? snapshot.sizeBytes : null,
storage: null,
datastore: null,
namespace: null,
@ -291,7 +296,7 @@ const UnifiedBackups: Component = () => {
unified.push({
backupType: 'remote',
vmid: displayType === 'Host' ? backup.vmid : (parseInt(backup.vmid) || 0),
vmid: displayType === 'Host' ? backup.vmid : parseInt(backup.vmid) || 0,
name: backup.comment || '',
type: displayType,
node: backup.instance || 'PBS',
@ -402,7 +407,7 @@ const UnifiedBackups: Component = () => {
// For regular backups: show Proxmox node in Node column, local storage in Location
unified.push({
backupType: backupType,
vmid: displayType === 'Host' ? backup.vmid : (backup.vmid || 0),
vmid: displayType === 'Host' ? backup.vmid : backup.vmid || 0,
name: backup.notes || backup.volid?.split('/').pop() || '',
type: displayType,
node: backup.node || '', // Proxmox node that has access to this backup
@ -532,9 +537,7 @@ const UnifiedBackups: Component = () => {
if (nodeFilter) {
const node = state.nodes?.find((n) => n.id === nodeFilter);
if (node) {
data = data.filter(
(item) => item.instance === node.instance && item.node === node.name,
);
data = data.filter((item) => item.instance === node.instance && item.node === node.name);
}
}
@ -1364,17 +1367,27 @@ const UnifiedBackups: Component = () => {
aria-label={availableBackupsTooltipText}
onMouseEnter={(e) => {
const rect = e.currentTarget.getBoundingClientRect();
showTooltip(availableBackupsTooltipText, rect.left + rect.width / 2, rect.top, {
showTooltip(
availableBackupsTooltipText,
rect.left + rect.width / 2,
rect.top,
{
align: 'center',
direction: 'up',
});
},
);
}}
onFocus={(e) => {
const rect = e.currentTarget.getBoundingClientRect();
showTooltip(availableBackupsTooltipText, rect.left + rect.width / 2, rect.top, {
showTooltip(
availableBackupsTooltipText,
rect.left + rect.width / 2,
rect.top,
{
align: 'center',
direction: 'up',
});
},
);
}}
onMouseLeave={() => hideTooltip()}
onBlur={() => hideTooltip()}
@ -1780,12 +1793,9 @@ const UnifiedBackups: Component = () => {
}`;
const breakdown: string[] = [];
if (d.snapshots > 0)
breakdown.push(`Snapshots: ${d.snapshots}`);
if (d.pve > 0)
breakdown.push(`PVE: ${d.pve}`);
if (d.pbs > 0)
breakdown.push(`PBS: ${d.pbs}`);
if (d.snapshots > 0) breakdown.push(`Snapshots: ${d.snapshots}`);
if (d.pve > 0) breakdown.push(`PVE: ${d.pve}`);
if (d.pbs > 0) breakdown.push(`PBS: ${d.pbs}`);
if (breakdown.length > 0) {
tooltipText += `\n${breakdown.join(' • ')}`;
@ -2121,7 +2131,8 @@ const UnifiedBackups: Component = () => {
onClick={() => handleSort('vmid')}
style="width: 60px;"
>
{hasHostBackups() ? 'VMID/Host' : 'VMID'} {sortKey() === 'vmid' && (sortDirection() === 'asc' ? '▲' : '▼')}
{hasHostBackups() ? 'VMID/Host' : 'VMID'}{' '}
{sortKey() === 'vmid' && (sortDirection() === 'asc' ? '▲' : '▼')}
</th>
<th
class="px-2 py-1.5 text-left text-[11px] sm:text-xs font-medium uppercase tracking-wider cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-600"

36
frontend-modern/src/pages/Alerts.tsx
View file

@ -157,9 +157,15 @@ interface UIEmailConfig {
interface UIAppriseConfig {
enabled: boolean;
mode: 'cli' | 'http';
targetsText: string;
cliPath: string;
timeoutSeconds: number;
serverUrl: string;
configKey: string;
apiKey: string;
apiKeyHeader: string;
skipTlsVerify: boolean;
}
interface QuietHoursConfig {
@ -239,9 +245,15 @@ export const createDefaultGrouping = (): GroupingConfig => ({
const createDefaultAppriseConfig = (): UIAppriseConfig => ({
enabled: false,
mode: 'cli',
targetsText: '',
cliPath: 'apprise',
timeoutSeconds: 15,
serverUrl: '',
configKey: '',
apiKey: '',
apiKeyHeader: 'X-API-KEY',
skipTlsVerify: false,
});
const parseAppriseTargets = (value: string): string[] =>
@ -486,9 +498,15 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
const config = appriseConfig();
return {
enabled: config.enabled,
mode: config.mode,
targets: parseAppriseTargets(config.targetsText),
cliPath: config.cliPath,
timeoutSeconds: config.timeoutSeconds,
serverUrl: config.serverUrl,
configKey: config.configKey,
apiKey: config.apiKey,
apiKeyHeader: config.apiKeyHeader,
skipTlsVerify: config.skipTlsVerify,
} as AppriseConfig;
};
@ -1012,12 +1030,18 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
const appriseData = await NotificationsAPI.getAppriseConfig();
setAppriseConfig({
enabled: appriseData.enabled ?? false,
mode: appriseData.mode === 'http' ? 'http' : 'cli',
targetsText: formatAppriseTargets(appriseData.targets),
cliPath: appriseData.cliPath || 'apprise',
timeoutSeconds:
typeof appriseData.timeoutSeconds === 'number' && appriseData.timeoutSeconds > 0
? appriseData.timeoutSeconds
: 15,
serverUrl: appriseData.serverUrl || '',
configKey: appriseData.configKey || '',
apiKey: appriseData.apiKey || '',
apiKeyHeader: appriseData.apiKeyHeader || 'X-API-KEY',
skipTlsVerify: Boolean(appriseData.skipTlsVerify),
});
} catch (appriseErr) {
console.error('Failed to load Apprise configuration:', appriseErr);
@ -1072,12 +1096,18 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
.then((appriseData) => {
setAppriseConfig({
enabled: appriseData.enabled ?? false,
mode: appriseData.mode === 'http' ? 'http' : 'cli',
targetsText: formatAppriseTargets(appriseData.targets),
cliPath: appriseData.cliPath || 'apprise',
timeoutSeconds:
typeof appriseData.timeoutSeconds === 'number' && appriseData.timeoutSeconds > 0
? appriseData.timeoutSeconds
: 15,
serverUrl: appriseData.serverUrl || '',
configKey: appriseData.configKey || '',
apiKey: appriseData.apiKey || '',
apiKeyHeader: appriseData.apiKeyHeader || 'X-API-KEY',
skipTlsVerify: Boolean(appriseData.skipTlsVerify),
});
})
.catch((err) => {
@ -1484,12 +1514,18 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
const updatedApprise = await NotificationsAPI.updateAppriseConfig(appriseData);
setAppriseConfig({
enabled: updatedApprise.enabled ?? false,
mode: updatedApprise.mode === 'http' ? 'http' : 'cli',
targetsText: formatAppriseTargets(updatedApprise.targets),
cliPath: updatedApprise.cliPath || 'apprise',
timeoutSeconds:
typeof updatedApprise.timeoutSeconds === 'number' && updatedApprise.timeoutSeconds > 0
? updatedApprise.timeoutSeconds
: 15,
serverUrl: updatedApprise.serverUrl || '',
configKey: updatedApprise.configKey || '',
apiKey: updatedApprise.apiKey || '',
apiKeyHeader: updatedApprise.apiKeyHeader || 'X-API-KEY',
skipTlsVerify: Boolean(updatedApprise.skipTlsVerify),
});
}

2
frontend-modern/src/types/alerts.ts
View file

@ -89,6 +89,8 @@ export interface SnapshotAlertConfig {
enabled: boolean;
warningDays: number;
criticalDays: number;
warningSizeGiB?: number;
criticalSizeGiB?: number;
}
export interface BackupAlertConfig {

1
frontend-modern/src/types/api.ts
View file

@ -607,6 +607,7 @@ export interface GuestSnapshot {
description: string;
parent: string;
vmstate: boolean;
sizeBytes?: number;
}
export interface Performance {

123
internal/alerts/alerts.go
View file

@ -297,6 +297,8 @@ type SnapshotAlertConfig struct {
Enabled bool `json:"enabled"`
WarningDays int `json:"warningDays"`
CriticalDays int `json:"criticalDays"`
WarningSizeGiB float64 `json:"warningSizeGiB,omitempty"`
CriticalSizeGiB float64 `json:"criticalSizeGiB,omitempty"`
}
// BackupAlertConfig represents backup age alert configuration
@ -516,6 +518,8 @@ func NewManager() *Manager {
Enabled: false,
WarningDays: 30,
CriticalDays: 45,
WarningSizeGiB: 0,
CriticalSizeGiB: 0,
},
BackupDefaults: BackupAlertConfig{
Enabled: false,
@ -836,6 +840,21 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
}
if config.SnapshotDefaults.CriticalDays == 0 && config.SnapshotDefaults.WarningDays > 0 {
config.SnapshotDefaults.CriticalDays = config.SnapshotDefaults.WarningDays
}
if config.SnapshotDefaults.WarningSizeGiB < 0 {
config.SnapshotDefaults.WarningSizeGiB = 0
}
if config.SnapshotDefaults.CriticalSizeGiB < 0 {
config.SnapshotDefaults.CriticalSizeGiB = 0
}
if config.SnapshotDefaults.CriticalSizeGiB > 0 && config.SnapshotDefaults.WarningSizeGiB > config.SnapshotDefaults.CriticalSizeGiB {
config.SnapshotDefaults.WarningSizeGiB = config.SnapshotDefaults.CriticalSizeGiB
}
if config.SnapshotDefaults.CriticalSizeGiB == 0 && config.SnapshotDefaults.WarningSizeGiB > 0 {
config.SnapshotDefaults.CriticalSizeGiB = config.SnapshotDefaults.WarningSizeGiB
}
if config.BackupDefaults.WarningDays < 0 {
config.BackupDefaults.WarningDays = 0
}
@ -2991,19 +3010,63 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
}
ageDays := ageHours / 24
var level AlertLevel
var threshold int
const gib = 1024.0 * 1024 * 1024
sizeGiB := 0.0
if snapshot.SizeBytes > 0 {
sizeGiB = float64(snapshot.SizeBytes) / gib
}
var (
ageLevel AlertLevel
ageThreshold int
sizeLevel AlertLevel
sizeThreshold float64
triggeredStats []string
)
if snapshotCfg.CriticalDays > 0 && ageDays >= float64(snapshotCfg.CriticalDays) {
level = AlertLevelCritical
threshold = snapshotCfg.CriticalDays
ageLevel = AlertLevelCritical
ageThreshold = snapshotCfg.CriticalDays
triggeredStats = append(triggeredStats, "age")
} else if snapshotCfg.WarningDays > 0 && ageDays >= float64(snapshotCfg.WarningDays) {
level = AlertLevelWarning
threshold = snapshotCfg.WarningDays
} else {
ageLevel = AlertLevelWarning
ageThreshold = snapshotCfg.WarningDays
triggeredStats = append(triggeredStats, "age")
}
if snapshot.SizeBytes > 0 {
if snapshotCfg.CriticalSizeGiB > 0 && sizeGiB >= snapshotCfg.CriticalSizeGiB {
sizeLevel = AlertLevelCritical
sizeThreshold = snapshotCfg.CriticalSizeGiB
triggeredStats = append(triggeredStats, "size")
} else if snapshotCfg.WarningSizeGiB > 0 && sizeGiB >= snapshotCfg.WarningSizeGiB {
sizeLevel = AlertLevelWarning
sizeThreshold = snapshotCfg.WarningSizeGiB
triggeredStats = append(triggeredStats, "size")
}
}
if ageLevel == "" && sizeLevel == "" {
continue
}
var level AlertLevel
switch {
case ageLevel == AlertLevelCritical || sizeLevel == AlertLevelCritical:
level = AlertLevelCritical
case ageLevel == AlertLevelWarning || sizeLevel == AlertLevelWarning:
level = AlertLevelWarning
default:
continue
}
useSizePrimary := false
if sizeLevel == AlertLevelCritical && ageLevel != AlertLevelCritical {
useSizePrimary = true
} else if sizeLevel != "" && ageLevel == "" {
useSizePrimary = true
}
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
validAlerts[alertID] = struct{}{}
@ -3030,32 +3093,60 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
}
ageDaysRounded := math.Round(ageDays*10) / 10
sizeGiBRounded := math.Round(sizeGiB*10) / 10
reasons := make([]string, 0, 2)
if ageLevel != "" {
reasons = append(reasons, fmt.Sprintf("%.1f days old (threshold %d days)", ageDaysRounded, ageThreshold))
}
if sizeLevel != "" {
reasons = append(reasons, fmt.Sprintf("%.1f GiB (threshold %.1f GiB)", sizeGiBRounded, sizeThreshold))
}
reasonText := strings.Join(reasons, " and ")
message := fmt.Sprintf(
"%s snapshot '%s' for %s is %.1f days old on %s (threshold: %d days)",
"%s snapshot '%s' for %s is %s on %s",
guestType,
snapshotName,
guestName,
ageDaysRounded,
reasonText,
snapshot.Node,
threshold,
)
thresholdTime := snapshot.Time.Add(time.Duration(threshold) * 24 * time.Hour)
alertValue := ageDays
alertThreshold := float64(ageThreshold)
thresholdTime := now
if useSizePrimary {
alertValue = sizeGiB
alertThreshold = sizeThreshold
} else if ageThreshold > 0 {
thresholdTime = snapshot.Time.Add(time.Duration(ageThreshold) * 24 * time.Hour)
if thresholdTime.After(now) {
thresholdTime = now
}
}
metadata := map[string]interface{}{
"snapshotName": snapshot.Name,
"snapshotCreatedAt": snapshot.Time,
"snapshotAgeDays": ageDays,
"snapshotAgeHours": ageHours,
"snapshotSizeBytes": snapshot.SizeBytes,
"snapshotSizeGiB": sizeGiB,
"guestName": guestName,
"guestType": guestType,
"guestInstance": snapshot.Instance,
"guestNode": snapshot.Node,
"guestVmid": snapshot.VMID,
"thresholdDays": threshold,
"triggeredMetrics": triggeredStats,
"primaryMetric": "age",
}
if useSizePrimary {
metadata["primaryMetric"] = "size"
}
if ageLevel != "" {
metadata["thresholdDays"] = ageThreshold
}
if sizeLevel != "" {
metadata["thresholdSizeGiB"] = sizeThreshold
}
resourceName := fmt.Sprintf("%s snapshot '%s'", guestName, snapshotName)
@ -3064,8 +3155,8 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
if existing, exists := m.activeAlerts[alertID]; exists {
existing.LastSeen = now
existing.Level = level
existing.Value = ageDays
existing.Threshold = float64(threshold)
existing.Value = alertValue
existing.Threshold = alertThreshold
existing.Message = message
existing.ResourceName = resourceName
if existing.Metadata == nil {
@ -3087,8 +3178,8 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
Node: snapshot.Node,
Instance: snapshot.Instance,
Message: message,
Value: ageDays,
Threshold: float64(threshold),
Value: alertValue,
Threshold: alertThreshold,
StartTime: thresholdTime,
LastSeen: now,
Metadata: metadata,

152
internal/alerts/alerts_test.go
View file

@ -224,6 +224,8 @@ func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
WarningSizeGiB: 0,
CriticalSizeGiB: 0,
},
Overrides: make(map[string]ThresholdConfig),
}
@ -243,6 +245,7 @@ func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
Type: "qemu",
VMID: 100,
Time: now.Add(-15 * 24 * time.Hour),
SizeBytes: 60 << 30,
},
}
guestNames := map[string]string{
@ -274,6 +277,155 @@ func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
}
}
func TestCheckSnapshotsForInstanceTriggersOnSnapshotSize(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
SnapshotDefaults: SnapshotAlertConfig{
Enabled: true,
WarningDays: 0,
CriticalDays: 0,
WarningSizeGiB: 50,
CriticalSizeGiB: 100,
},
Overrides: make(map[string]ThresholdConfig),
}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
now := time.Now()
snapshots := []models.GuestSnapshot{
{
ID: "inst-node-200-sizey",
Name: "pre-maintenance",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 200,
Time: now.Add(-2 * time.Hour),
SizeBytes: int64(120) << 30,
},
}
guestNames := map[string]string{
"inst-node-200": "db-server",
}
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
m.mu.RLock()
alert, exists := m.activeAlerts["snapshot-age-inst-node-200-sizey"]
m.mu.RUnlock()
if !exists {
t.Fatalf("expected snapshot size alert to be created")
}
if alert.Level != AlertLevelCritical {
t.Fatalf("expected critical level for large snapshot, got %s", alert.Level)
}
if alert.Value < 119.5 || alert.Value > 120.5 {
t.Fatalf("expected alert value near 120 GiB, got %.2f", alert.Value)
}
if alert.Threshold != 100 {
t.Fatalf("expected threshold 100 GiB, got %.2f", alert.Threshold)
}
if alert.Metadata == nil {
t.Fatalf("expected metadata for snapshot alert")
}
if metric, ok := alert.Metadata["primaryMetric"].(string); !ok || metric != "size" {
t.Fatalf("expected primary metric size, got %#v", alert.Metadata["primaryMetric"])
}
if sizeBytes, ok := alert.Metadata["snapshotSizeBytes"].(int64); !ok || sizeBytes == 0 {
t.Fatalf("expected snapshotSizeBytes in metadata")
}
metrics, ok := alert.Metadata["triggeredMetrics"].([]string)
if !ok {
t.Fatalf("expected triggeredMetrics slice, got %#v", alert.Metadata["triggeredMetrics"])
}
foundSize := false
for _, metric := range metrics {
if metric == "size" {
foundSize = true
break
}
}
if !foundSize {
t.Fatalf("expected size metric recorded in metadata")
}
}
func TestCheckSnapshotsForInstanceIncludesAgeAndSizeReasons(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
SnapshotDefaults: SnapshotAlertConfig{
Enabled: true,
WarningDays: 5,
CriticalDays: 10,
WarningSizeGiB: 40,
CriticalSizeGiB: 80,
},
Overrides: make(map[string]ThresholdConfig),
}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
now := time.Now()
snapshots := []models.GuestSnapshot{
{
ID: "inst-node-300-combined",
Name: "long-running",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 300,
Time: now.Add(-15 * 24 * time.Hour),
SizeBytes: int64(90) << 30,
},
}
guestNames := map[string]string{
"inst-node-300": "app-server",
}
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
m.mu.RLock()
alert, exists := m.activeAlerts["snapshot-age-inst-node-300-combined"]
m.mu.RUnlock()
if !exists {
t.Fatalf("expected combined snapshot alert to be created")
}
if alert.Level != AlertLevelCritical {
t.Fatalf("expected critical level, got %s", alert.Level)
}
if !strings.Contains(alert.Message, "days old") || !strings.Contains(strings.ToLower(alert.Message), "gib") {
t.Fatalf("expected alert message to reference age and size, got %q", alert.Message)
}
if alert.Metadata == nil {
t.Fatalf("expected metadata for combined alert")
}
metrics, ok := alert.Metadata["triggeredMetrics"].([]string)
if !ok {
t.Fatalf("expected triggeredMetrics slice, got %#v", alert.Metadata["triggeredMetrics"])
}
if len(metrics) < 2 {
t.Fatalf("expected both age and size metrics recorded, got %v", metrics)
}
if metric, ok := alert.Metadata["primaryMetric"].(string); !ok || metric != "age" {
t.Fatalf("expected primary metric age, got %#v", alert.Metadata["primaryMetric"])
}
}
func TestCheckBackupsCreatesAndClearsAlerts(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()

6
internal/api/notifications.go
View file

@ -113,8 +113,14 @@ func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *htt
log.Info().
Bool("enabled", config.Enabled).
Str("mode", string(config.Mode)).
Int("targetCount", len(config.Targets)).
Str("cliPath", config.CLIPath).
Str("serverUrl", config.ServerURL).
Str("configKey", config.ConfigKey).
Bool("hasApiKey", config.APIKey != "").
Str("apiKeyHeader", config.APIKeyHeader).
Bool("skipTlsVerify", config.SkipTLSVerify).
Int("timeoutSeconds", config.TimeoutSeconds).
Msg("Parsed Apprise configuration update")

31
internal/config/persistence.go
View file

@ -215,6 +215,21 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
}
if config.SnapshotDefaults.CriticalDays == 0 && config.SnapshotDefaults.WarningDays > 0 {
config.SnapshotDefaults.CriticalDays = config.SnapshotDefaults.WarningDays
}
if config.SnapshotDefaults.WarningSizeGiB < 0 {
config.SnapshotDefaults.WarningSizeGiB = 0
}
if config.SnapshotDefaults.CriticalSizeGiB < 0 {
config.SnapshotDefaults.CriticalSizeGiB = 0
}
if config.SnapshotDefaults.CriticalSizeGiB > 0 && config.SnapshotDefaults.WarningSizeGiB > config.SnapshotDefaults.CriticalSizeGiB {
config.SnapshotDefaults.WarningSizeGiB = config.SnapshotDefaults.CriticalSizeGiB
}
if config.SnapshotDefaults.CriticalSizeGiB == 0 && config.SnapshotDefaults.WarningSizeGiB > 0 {
config.SnapshotDefaults.CriticalSizeGiB = config.SnapshotDefaults.WarningSizeGiB
}
if config.BackupDefaults.WarningDays < 0 {
config.BackupDefaults.WarningDays = 0
}
@ -280,6 +295,8 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
Enabled: false,
WarningDays: 30,
CriticalDays: 45,
WarningSizeGiB: 0,
CriticalSizeGiB: 0,
},
BackupDefaults: alerts.BackupAlertConfig{
Enabled: false,
@ -345,6 +362,18 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
}
if config.SnapshotDefaults.WarningSizeGiB < 0 {
config.SnapshotDefaults.WarningSizeGiB = 0
}
if config.SnapshotDefaults.CriticalSizeGiB < 0 {
config.SnapshotDefaults.CriticalSizeGiB = 0
}
if config.SnapshotDefaults.CriticalSizeGiB > 0 && config.SnapshotDefaults.WarningSizeGiB > config.SnapshotDefaults.CriticalSizeGiB {
config.SnapshotDefaults.WarningSizeGiB = config.SnapshotDefaults.CriticalSizeGiB
}
if config.SnapshotDefaults.CriticalSizeGiB == 0 && config.SnapshotDefaults.WarningSizeGiB > 0 {
config.SnapshotDefaults.CriticalSizeGiB = config.SnapshotDefaults.WarningSizeGiB
}
if config.BackupDefaults.WarningDays < 0 {
config.BackupDefaults.WarningDays = 0
}
@ -500,9 +529,11 @@ func (c *ConfigPersistence) LoadAppriseConfig() (*notifications.AppriseConfig, e
if os.IsNotExist(err) {
defaultCfg := notifications.AppriseConfig{
Enabled: false,
Mode: notifications.AppriseModeCLI,
Targets: []string{},
CLIPath: "apprise",
TimeoutSeconds: 15,
APIKeyHeader: "X-API-KEY",
}
return &defaultCfg, nil
}

8
internal/config/persistence_test.go
View file

@ -148,6 +148,8 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
Enabled: true,
WarningDays: 20,
CriticalDays: 10,
WarningSizeGiB: 15,
CriticalSizeGiB: 8,
},
BackupDefaults: alerts.BackupAlertConfig{
Enabled: true,
@ -210,6 +212,12 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
if loaded.SnapshotDefaults.CriticalDays != 10 {
t.Fatalf("expected snapshot critical days preserved at 10, got %d", loaded.SnapshotDefaults.CriticalDays)
}
if loaded.SnapshotDefaults.WarningSizeGiB != 8 {
t.Fatalf("expected snapshot warning size normalized to 8, got %.1f", loaded.SnapshotDefaults.WarningSizeGiB)
}
if loaded.SnapshotDefaults.CriticalSizeGiB != 8 {
t.Fatalf("expected snapshot critical size preserved at 8, got %.1f", loaded.SnapshotDefaults.CriticalSizeGiB)
}
}
func TestAppriseConfigPersistence(t *testing.T) {

2
internal/mock/generator.go
View file

@ -2515,6 +2515,7 @@ func generateSnapshots(vms []models.VM, containers []models.Container) []models.
Time: snapshotTime,
Description: fmt.Sprintf("Snapshot of %s taken on %s", vm.Name, snapshotTime.Format("2006-01-02")),
VMState: rand.Float64() > 0.5, // 50% include VM state
SizeBytes: int64(10+rand.Intn(90)) << 30, // 10-99 GiB
}
// Add parent relationship for some snapshots
@ -2547,6 +2548,7 @@ func generateSnapshots(vms []models.VM, containers []models.Container) []models.
Time: snapshotTime,
Description: fmt.Sprintf("Container snapshot for %s", ct.Name),
VMState: false, // Containers don't have VM state
SizeBytes: int64(5+rand.Intn(40)) << 30, // 5-44 GiB
}
snapshots = append(snapshots, snapshot)

133
internal/models/models.go
View file

@ -14,6 +14,7 @@ type State struct {
VMs []VM `json:"vms"`
Containers []Container `json:"containers"`
DockerHosts []DockerHost `json:"dockerHosts"`
Hosts []Host `json:"hosts"`
Storage []Storage `json:"storage"`
CephClusters []CephCluster `json:"cephClusters"`
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
@ -137,6 +138,52 @@ type Container struct {
LastSeen time.Time `json:"lastSeen"`
}
// Host represents a generic infrastructure host reporting via external agents.
type Host struct {
ID string `json:"id"`
Hostname string `json:"hostname"`
DisplayName string `json:"displayName,omitempty"`
Platform string `json:"platform,omitempty"`
OSName string `json:"osName,omitempty"`
OSVersion string `json:"osVersion,omitempty"`
KernelVersion string `json:"kernelVersion,omitempty"`
Architecture string `json:"architecture,omitempty"`
CPUCount int `json:"cpuCount,omitempty"`
CPUUsage float64 `json:"cpuUsage,omitempty"`
Memory Memory `json:"memory"`
LoadAverage []float64 `json:"loadAverage,omitempty"`
Disks []Disk `json:"disks,omitempty"`
NetworkInterfaces []HostNetworkInterface `json:"networkInterfaces,omitempty"`
Sensors HostSensorSummary `json:"sensors,omitempty"`
Status string `json:"status"`
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
IntervalSeconds int `json:"intervalSeconds,omitempty"`
LastSeen time.Time `json:"lastSeen"`
AgentVersion string `json:"agentVersion,omitempty"`
TokenID string `json:"tokenId,omitempty"`
TokenName string `json:"tokenName,omitempty"`
TokenHint string `json:"tokenHint,omitempty"`
TokenLastUsedAt *time.Time `json:"tokenLastUsedAt,omitempty"`
Tags []string `json:"tags,omitempty"`
}
// HostNetworkInterface describes a host network adapter summary.
type HostNetworkInterface struct {
Name string `json:"name"`
MAC string `json:"mac,omitempty"`
Addresses []string `json:"addresses,omitempty"`
RXBytes uint64 `json:"rxBytes,omitempty"`
TXBytes uint64 `json:"txBytes,omitempty"`
SpeedMbps *int64 `json:"speedMbps,omitempty"`
}
// HostSensorSummary captures optional per-host sensor readings.
type HostSensorSummary struct {
TemperatureCelsius map[string]float64 `json:"temperatureCelsius,omitempty"`
FanRPM map[string]float64 `json:"fanRpm,omitempty"`
Additional map[string]float64 `json:"additional,omitempty"`
}
// DockerHost represents a Docker host reporting metrics via the external agent.
type DockerHost struct {
ID string `json:"id"`
@ -674,6 +721,7 @@ type GuestSnapshot struct {
Description string `json:"description,omitempty"`
Parent string `json:"parent,omitempty"`
VMState bool `json:"vmstate"`
SizeBytes int64 `json:"sizeBytes,omitempty"`
}
// Performance represents performance metrics
@ -1046,6 +1094,91 @@ func (s *State) GetDockerHosts() []DockerHost {
return hosts
}
// UpsertHost inserts or updates a generic host in state.
func (s *State) UpsertHost(host Host) {
s.mu.Lock()
defer s.mu.Unlock()
updated := false
for i, existing := range s.Hosts {
if existing.ID == host.ID {
s.Hosts[i] = host
updated = true
break
}
}
if !updated {
s.Hosts = append(s.Hosts, host)
}
sort.Slice(s.Hosts, func(i, j int) bool {
return s.Hosts[i].Hostname < s.Hosts[j].Hostname
})
s.LastUpdate = time.Now()
}
// GetHosts returns a copy of all generic hosts.
func (s *State) GetHosts() []Host {
s.mu.RLock()
defer s.mu.RUnlock()
hosts := make([]Host, len(s.Hosts))
copy(hosts, s.Hosts)
return hosts
}
// RemoveHost removes a host by ID and returns the removed entry.
func (s *State) RemoveHost(hostID string) (Host, bool) {
s.mu.Lock()
defer s.mu.Unlock()
for i, host := range s.Hosts {
if host.ID == hostID {
s.Hosts = append(s.Hosts[:i], s.Hosts[i+1:]...)
s.LastUpdate = time.Now()
return host, true
}
}
return Host{}, false
}
// SetHostStatus updates the status of a host if present.
func (s *State) SetHostStatus(hostID, status string) bool {
s.mu.Lock()
defer s.mu.Unlock()
for i, host := range s.Hosts {
if host.ID == hostID {
if host.Status != status {
host.Status = status
s.Hosts[i] = host
s.LastUpdate = time.Now()
}
return true
}
}
return false
}
// TouchHost updates the last seen timestamp for a host.
func (s *State) TouchHost(hostID string, ts time.Time) bool {
s.mu.Lock()
defer s.mu.Unlock()
for i, host := range s.Hosts {
if host.ID == hostID {
host.LastSeen = ts
s.Hosts[i] = host
s.LastUpdate = time.Now()
return true
}
}
return false
}
// UpdateStorage updates the storage in the state
func (s *State) UpdateStorage(storage []Storage) {
s.mu.Lock()

121
internal/monitoring/monitor.go
View file

@ -6431,6 +6431,17 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
return
}
if len(allSnapshots) > 0 {
sizeMap := m.collectSnapshotSizes(snapshotCtx, instanceName, client, allSnapshots)
if len(sizeMap) > 0 {
for i := range allSnapshots {
if size, ok := sizeMap[allSnapshots[i].ID]; ok && size > 0 {
allSnapshots[i].SizeBytes = size
}
}
}
}
// Update state with guest snapshots for this instance
m.state.UpdateGuestSnapshotsForInstance(instanceName, allSnapshots)
@ -6444,6 +6455,116 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
Msg("Guest snapshots polled")
}
func (m *Monitor) collectSnapshotSizes(ctx context.Context, instanceName string, client PVEClientInterface, snapshots []models.GuestSnapshot) map[string]int64 {
sizes := make(map[string]int64, len(snapshots))
if len(snapshots) == 0 {
return sizes
}
validSnapshots := make(map[string]struct{}, len(snapshots))
nodes := make(map[string]struct{})
for _, snap := range snapshots {
validSnapshots[snap.ID] = struct{}{}
if snap.Node != "" {
nodes[snap.Node] = struct{}{}
}
}
if len(nodes) == 0 {
return sizes
}
seenVolids := make(map[string]struct{})
for nodeName := range nodes {
if ctx.Err() != nil {
break
}
storages, err := client.GetStorage(ctx, nodeName)
if err != nil {
log.Debug().
Err(err).
Str("node", nodeName).
Str("instance", instanceName).
Msg("Failed to get storage list for snapshot sizing")
continue
}
for _, storage := range storages {
if ctx.Err() != nil {
break
}
contentTypes := strings.ToLower(storage.Content)
if !strings.Contains(contentTypes, "images") && !strings.Contains(contentTypes, "rootdir") {
continue
}
contents, err := client.GetStorageContent(ctx, nodeName, storage.Storage)
if err != nil {
log.Debug().
Err(err).
Str("node", nodeName).
Str("storage", storage.Storage).
Str("instance", instanceName).
Msg("Failed to get storage content for snapshot sizing")
continue
}
for _, item := range contents {
if item.VMID <= 0 {
continue
}
if _, seen := seenVolids[item.Volid]; seen {
continue
}
snapName := extractSnapshotName(item.Volid)
if snapName == "" {
continue
}
key := fmt.Sprintf("%s-%s-%d-%s", instanceName, nodeName, item.VMID, snapName)
if _, ok := validSnapshots[key]; !ok {
continue
}
seenVolids[item.Volid] = struct{}{}
size := int64(item.Size)
if size < 0 {
size = 0
}
sizes[key] += size
}
}
}
return sizes
}
func extractSnapshotName(volid string) string {
if volid == "" {
return ""
}
parts := strings.SplitN(volid, ":", 2)
remainder := volid
if len(parts) == 2 {
remainder = parts[1]
}
if idx := strings.Index(remainder, "@"); idx >= 0 && idx+1 < len(remainder) {
return strings.TrimSpace(remainder[idx+1:])
}
return ""
}
// Stop gracefully stops the monitor
func (m *Monitor) Stop() {
log.Info().Msg("Stopping monitor")

122
internal/monitoring/monitor_snapshots_test.go Normal file
View file

@ -0,0 +1,122 @@
package monitoring
import (
"context"
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
type fakeSnapshotClient struct {
storages map[string][]proxmox.Storage
contents map[string]map[string][]proxmox.StorageContent
}
func (f fakeSnapshotClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) { return nil, nil }
func (f fakeSnapshotClient) GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
return f.storages[node], nil
}
func (f fakeSnapshotClient) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
if storageContents, ok := f.contents[node]; ok {
return storageContents[storage], nil
}
return nil, nil
}
func (f fakeSnapshotClient) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error) {
return nil, nil
}
func (f fakeSnapshotClient) IsClusterMember(ctx context.Context) (bool, error) { return false, nil }
func (f fakeSnapshotClient) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
return map[string]interface{}{}, nil
}
func (f fakeSnapshotClient) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error) {
return nil, nil
}
func (f fakeSnapshotClient) GetCephDF(ctx context.Context) (*proxmox.CephDF, error) { return nil, nil }
func TestCollectSnapshotSizes(t *testing.T) {
m := &Monitor{}
snapshots := []models.GuestSnapshot{
{
ID: "inst-node1-100-pre",
Name: "pre",
Node: "node1",
Instance: "inst",
Type: "qemu",
VMID: 100,
},
}
client := fakeSnapshotClient{
storages: map[string][]proxmox.Storage{
"node1": {
{Storage: "local-zfs", Content: "images"},
},
},
contents: map[string]map[string][]proxmox.StorageContent{
"node1": {
"local-zfs": {
{Volid: "local-zfs:vm-100-disk-0@pre", VMID: 100, Size: 20 << 30},
// Duplicate entry should be deduped via volid tracking
{Volid: "local-zfs:vm-100-disk-0@pre", VMID: 100, Size: 20 << 30},
},
},
},
}
sizes := m.collectSnapshotSizes(context.Background(), "inst", client, snapshots)
got, ok := sizes[snapshots[0].ID]
if !ok {
t.Fatalf("expected size entry for snapshot")
}
want := int64(20 << 30)
if got != want {
t.Fatalf("unexpected size: got %d want %d", got, want)
}
}

269
internal/notifications/notifications.go
View file

@ -3,6 +3,7 @@ package notifications
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
@ -171,6 +172,15 @@ func copyAppriseConfig(cfg AppriseConfig) AppriseConfig {
// NormalizeAppriseConfig cleans and normalizes Apprise configuration values.
func NormalizeAppriseConfig(cfg AppriseConfig) AppriseConfig {
normalized := cfg
mode := strings.ToLower(strings.TrimSpace(string(normalized.Mode)))
switch mode {
case string(AppriseModeHTTP):
normalized.Mode = AppriseModeHTTP
default:
normalized.Mode = AppriseModeCLI
}
normalized.CLIPath = strings.TrimSpace(normalized.CLIPath)
if normalized.CLIPath == "" {
normalized.CLIPath = "apprise"
@ -198,11 +208,29 @@ func NormalizeAppriseConfig(cfg AppriseConfig) AppriseConfig {
seen[lower] = struct{}{}
cleanTargets = append(cleanTargets, trimmed)
}
normalized.Targets = cleanTargets
if len(cleanTargets) == 0 {
normalized.ServerURL = strings.TrimSpace(normalized.ServerURL)
normalized.ServerURL = strings.TrimRight(normalized.ServerURL, "/")
normalized.ConfigKey = strings.TrimSpace(normalized.ConfigKey)
normalized.APIKey = strings.TrimSpace(normalized.APIKey)
normalized.APIKeyHeader = strings.TrimSpace(normalized.APIKeyHeader)
if normalized.APIKeyHeader == "" {
normalized.APIKeyHeader = "X-API-KEY"
}
switch normalized.Mode {
case AppriseModeCLI:
if len(normalized.Targets) == 0 {
normalized.Enabled = false
}
case AppriseModeHTTP:
if normalized.ServerURL == "" {
normalized.Enabled = false
}
}
return normalized
}
@ -258,12 +286,26 @@ type WebhookConfig struct {
CustomFields map[string]string `json:"customFields,omitempty"`
}
// AppriseConfig holds Apprise CLI notification settings.
// AppriseMode identifies how Pulse should deliver notifications through Apprise.
type AppriseMode string
const (
AppriseModeCLI AppriseMode = "cli"
AppriseModeHTTP AppriseMode = "http"
)
// AppriseConfig holds Apprise notification settings.
type AppriseConfig struct {
Enabled bool `json:"enabled"`
Mode AppriseMode `json:"mode,omitempty"`
Targets []string `json:"targets"`
CLIPath string `json:"cliPath,omitempty"`
TimeoutSeconds int `json:"timeoutSeconds,omitempty"`
ServerURL string `json:"serverUrl,omitempty"`
ConfigKey string `json:"configKey,omitempty"`
APIKey string `json:"apiKey,omitempty"`
APIKeyHeader string `json:"apiKeyHeader,omitempty"`
SkipTLSVerify bool `json:"skipTlsVerify,omitempty"`
}
// NewNotificationManager creates a new notification manager
@ -281,9 +323,11 @@ func NewNotificationManager(publicURL string) *NotificationManager {
webhooks: []WebhookConfig{},
appriseConfig: AppriseConfig{
Enabled: false,
Mode: AppriseModeCLI,
Targets: []string{},
CLIPath: "apprise",
TimeoutSeconds: 15,
APIKeyHeader: "X-API-KEY",
},
groupWindow: 30 * time.Second,
pendingAlerts: make([]*alerts.Alert, 0),
@ -571,7 +615,7 @@ func (n *NotificationManager) sendGroupedAlerts() {
}
}
if appriseConfig.Enabled && len(appriseConfig.Targets) > 0 {
if appriseConfig.Enabled {
go n.sendGroupedApprise(appriseConfig, alertsToSend)
}
@ -604,23 +648,64 @@ func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList
}
cfg := NormalizeAppriseConfig(config)
if !cfg.Enabled || len(cfg.Targets) == 0 {
if !cfg.Enabled {
return
}
primary := alertList[0]
alertCount := len(alertList)
title, body, notifyType := buildApprisePayload(alertList, n.publicURL)
if title == "" && body == "" {
log.Warn().Msg("Apprise notification skipped: failed to build payload")
return
}
switch cfg.Mode {
case AppriseModeHTTP:
if err := n.sendAppriseViaHTTP(cfg, title, body, notifyType); err != nil {
log.Warn().
Err(err).
Str("mode", string(cfg.Mode)).
Str("serverUrl", cfg.ServerURL).
Msg("Failed to send Apprise notification via API")
}
default:
if err := n.sendAppriseViaCLI(cfg, title, body); err != nil {
log.Warn().
Err(err).
Str("mode", string(cfg.Mode)).
Str("cliPath", cfg.CLIPath).
Strs("targets", cfg.Targets).
Msg("Failed to send Apprise notification")
}
}
}
func buildApprisePayload(alertList []*alerts.Alert, publicURL string) (string, string, string) {
validAlerts := make([]*alerts.Alert, 0, len(alertList))
var primary *alerts.Alert
for _, alert := range alertList {
if alert == nil {
continue
}
if primary == nil {
primary = alert
}
validAlerts = append(validAlerts, alert)
}
if len(validAlerts) == 0 || primary == nil {
return "", "", "info"
}
title := fmt.Sprintf("Pulse alert: %s", primary.ResourceName)
if alertCount > 1 {
title = fmt.Sprintf("Pulse alerts (%d)", alertCount)
if len(validAlerts) > 1 {
title = fmt.Sprintf("Pulse alerts (%d)", len(validAlerts))
}
var bodyBuilder strings.Builder
bodyBuilder.WriteString(primary.Message)
bodyBuilder.WriteString("\n\n")
for _, alert := range alertList {
for _, alert := range validAlerts {
bodyBuilder.WriteString(fmt.Sprintf("[%s] %s", strings.ToUpper(string(alert.Level)), alert.ResourceName))
bodyBuilder.WriteString(fmt.Sprintf(" — value %.2f (threshold %.2f)\n", alert.Value, alert.Threshold))
if alert.Node != "" {
@ -632,11 +717,33 @@ func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList
bodyBuilder.WriteString("\n")
}
if n.publicURL != "" {
bodyBuilder.WriteString("Dashboard: " + n.publicURL + "\n")
if publicURL != "" {
bodyBuilder.WriteString("Dashboard: " + publicURL + "\n")
}
body := bodyBuilder.String()
return title, bodyBuilder.String(), resolveAppriseNotificationType(validAlerts)
}
func resolveAppriseNotificationType(alertList []*alerts.Alert) string {
notifyType := "info"
for _, alert := range alertList {
if alert == nil {
continue
}
switch alert.Level {
case alerts.AlertLevelCritical:
return "failure"
case alerts.AlertLevelWarning:
notifyType = "warning"
}
}
return notifyType
}
func (n *NotificationManager) sendAppriseViaCLI(cfg AppriseConfig, title, body string) error {
if len(cfg.Targets) == 0 {
return fmt.Errorf("no Apprise targets configured for CLI delivery")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.TimeoutSeconds)*time.Second)
defer cancel()
@ -651,12 +758,14 @@ func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList
output, err := execFn(ctx, cfg.CLIPath, args)
if err != nil {
log.Warn().
Err(err).
if len(output) > 0 {
log.Debug().
Str("cliPath", cfg.CLIPath).
Strs("targets", cfg.Targets).
Msg("Failed to send Apprise notification")
return
Str("output", string(output)).
Msg("Apprise CLI output (error)")
}
return err
}
if len(output) > 0 {
@ -666,6 +775,97 @@ func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList
Str("output", string(output)).
Msg("Apprise CLI output")
}
return nil
}
func (n *NotificationManager) sendAppriseViaHTTP(cfg AppriseConfig, title, body, notifyType string) error {
if cfg.ServerURL == "" {
return fmt.Errorf("apprise server URL is not configured")
}
serverURL := cfg.ServerURL
lowerURL := strings.ToLower(serverURL)
if !strings.HasPrefix(lowerURL, "http://") && !strings.HasPrefix(lowerURL, "https://") {
return fmt.Errorf("apprise server URL must start with http or https: %s", serverURL)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.TimeoutSeconds)*time.Second)
defer cancel()
notifyEndpoint := "/notify"
if cfg.ConfigKey != "" {
notifyEndpoint = "/notify/" + url.PathEscape(cfg.ConfigKey)
}
requestURL := strings.TrimRight(serverURL, "/") + notifyEndpoint
payload := map[string]any{
"body": body,
"title": title,
}
if len(cfg.Targets) > 0 {
payload["urls"] = cfg.Targets
}
if notifyType != "" {
payload["type"] = notifyType
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal Apprise payload: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL, bytes.NewReader(payloadBytes))
if err != nil {
return fmt.Errorf("failed to create Apprise request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
if cfg.APIKey != "" {
if cfg.APIKeyHeader == "" {
req.Header.Set("X-API-KEY", cfg.APIKey)
} else {
req.Header.Set(cfg.APIKeyHeader, cfg.APIKey)
}
}
client := &http.Client{
Timeout: time.Duration(cfg.TimeoutSeconds) * time.Second,
}
if strings.HasPrefix(lowerURL, "https://") && cfg.SkipTLSVerify {
client.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to reach Apprise server: %w", err)
}
defer resp.Body.Close()
limited := io.LimitReader(resp.Body, WebhookMaxResponseSize)
respBody, _ := io.ReadAll(limited)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
if len(respBody) > 0 {
return fmt.Errorf("apprise server returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))
}
return fmt.Errorf("apprise server returned HTTP %d", resp.StatusCode)
}
if len(respBody) > 0 {
log.Debug().
Str("mode", string(cfg.Mode)).
Str("serverUrl", cfg.ServerURL).
Str("response", string(respBody)).
Msg("Apprise API response")
}
return nil
}
// sendEmail sends an email notification
@ -878,6 +1078,8 @@ func (n *NotificationManager) sendGroupedWebhook(webhook WebhookConfig, alertLis
if routingKey, ok := webhook.Headers["routing_key"]; ok {
dataPtr.CustomFields["routing_key"] = routingKey
}
case "pushover":
dataPtr.CustomFields = ensurePushoverCustomFieldAliases(dataPtr.CustomFields)
}
serviceDataApplied = true
}
@ -1339,6 +1541,39 @@ func convertWebhookCustomFields(fields map[string]string) map[string]interface{}
return converted
}
func ensurePushoverCustomFieldAliases(fields map[string]interface{}) map[string]interface{} {
if fields == nil {
return nil
}
if _, ok := fields["token"]; !ok || isEmptyInterface(fields["token"]) {
if legacy, ok := fields["app_token"]; ok && !isEmptyInterface(legacy) {
fields["token"] = legacy
}
}
if _, ok := fields["user"]; !ok || isEmptyInterface(fields["user"]) {
if legacy, ok := fields["user_token"]; ok && !isEmptyInterface(legacy) {
fields["user"] = legacy
}
}
return fields
}
func isEmptyInterface(value interface{}) bool {
switch v := value.(type) {
case string:
return strings.TrimSpace(v) == ""
case fmt.Stringer:
return strings.TrimSpace(v.String()) == ""
case nil:
return true
default:
return false
}
}
// prepareWebhookData prepares data for template rendering
func (n *NotificationManager) prepareWebhookData(alert *alerts.Alert, customFields map[string]interface{}) WebhookPayloadData {
duration := time.Since(alert.StartTime)

171
internal/notifications/notifications_test.go
View file

@ -2,6 +2,11 @@ package notifications
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@ -24,10 +29,15 @@ func TestNormalizeAppriseConfig(t *testing.T) {
Targets: []string{" discord://token ", "", "DISCORD://TOKEN"},
CLIPath: " ",
TimeoutSeconds: -5,
APIKeyHeader: "",
}
normalized := NormalizeAppriseConfig(original)
if normalized.Mode != AppriseModeCLI {
t.Fatalf("expected default mode cli, got %q", normalized.Mode)
}
if normalized.CLIPath != "apprise" {
t.Fatalf("expected default CLI path 'apprise', got %q", normalized.CLIPath)
}
@ -44,11 +54,51 @@ func TestNormalizeAppriseConfig(t *testing.T) {
t.Fatalf("unexpected targets normalization result: %#v", normalized.Targets)
}
if normalized.APIKeyHeader != "X-API-KEY" {
t.Fatalf("expected default API key header, got %q", normalized.APIKeyHeader)
}
// When all targets removed, enabled should reset to false
empty := NormalizeAppriseConfig(AppriseConfig{Enabled: true})
if empty.Enabled {
t.Fatalf("expected enabled to be false when no targets configured")
}
httpConfig := NormalizeAppriseConfig(AppriseConfig{
Enabled: true,
Mode: AppriseModeHTTP,
ServerURL: "https://apprise.example.com/api/",
APIKey: " secret ",
APIKeyHeader: " X-Token ",
TimeoutSeconds: 200,
})
if httpConfig.Mode != AppriseModeHTTP {
t.Fatalf("expected HTTP mode, got %q", httpConfig.Mode)
}
if httpConfig.ServerURL != "https://apprise.example.com/api" {
t.Fatalf("expected server URL to be trimmed, got %q", httpConfig.ServerURL)
}
if httpConfig.APIKey != "secret" {
t.Fatalf("expected API key to be trimmed, got %q", httpConfig.APIKey)
}
if httpConfig.APIKeyHeader != "X-Token" {
t.Fatalf("expected API key header to be trimmed, got %q", httpConfig.APIKeyHeader)
}
if httpConfig.TimeoutSeconds != 120 {
t.Fatalf("expected timeout to clamp to 120, got %d", httpConfig.TimeoutSeconds)
}
if !httpConfig.Enabled {
t.Fatalf("expected HTTP config with server URL to remain enabled")
}
disabledHTTP := NormalizeAppriseConfig(AppriseConfig{
Enabled: true,
Mode: AppriseModeHTTP,
})
if disabledHTTP.Enabled {
t.Fatalf("expected HTTP config without server URL to disable notifications")
}
}
func TestSendGroupedAppriseInvokesExecutor(t *testing.T) {
@ -108,6 +158,127 @@ func TestSendGroupedAppriseInvokesExecutor(t *testing.T) {
}
}
func TestSendGroupedAppriseHTTP(t *testing.T) {
nm := NewNotificationManager("https://pulse.local")
nm.SetGroupingWindow(0)
nm.SetEmailConfig(EmailConfig{Enabled: false})
type apprisePayload struct {
Body string `json:"body"`
Title string `json:"title"`
Type string `json:"type"`
URLs []string `json:"urls"`
}
type capturedRequest struct {
Method string
Path string
ContentType string
APIKey string
Payload apprisePayload
}
requests := make(chan capturedRequest, 1)
errs := make(chan error, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
body, err := io.ReadAll(r.Body)
if err != nil {
errs <- err
w.WriteHeader(http.StatusInternalServerError)
return
}
var payload apprisePayload
if err := json.Unmarshal(body, &payload); err != nil {
errs <- err
w.WriteHeader(http.StatusBadRequest)
return
}
requests <- capturedRequest{
Method: r.Method,
Path: r.URL.Path,
ContentType: r.Header.Get("Content-Type"),
APIKey: r.Header.Get("X-Test-Key"),
Payload: payload,
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"ok":true}`))
}))
defer server.Close()
nm.SetAppriseConfig(AppriseConfig{
Enabled: true,
Mode: AppriseModeHTTP,
ServerURL: server.URL,
ConfigKey: "primary",
APIKey: "secret",
APIKeyHeader: "X-Test-Key",
Targets: []string{"discord://token"},
TimeoutSeconds: 10,
})
alert := &alerts.Alert{
ID: "test",
Type: "cpu",
Level: alerts.AlertLevelCritical,
ResourceID: "vm-100",
ResourceName: "vm-100",
Message: "CPU usage high",
Value: 95,
Threshold: 90,
StartTime: time.Now().Add(-time.Minute),
LastSeen: time.Now(),
}
nm.mu.Lock()
nm.pendingAlerts = append(nm.pendingAlerts, alert)
nm.mu.Unlock()
nm.sendGroupedAlerts()
var req capturedRequest
select {
case req = <-requests:
case err := <-errs:
t.Fatalf("server error: %v", err)
case <-time.After(2 * time.Second):
t.Fatalf("timed out waiting for Apprise API request")
}
if req.Method != http.MethodPost {
t.Fatalf("expected POST request, got %s", req.Method)
}
if req.Path != "/notify/primary" {
t.Fatalf("expected notify path with config key, got %s", req.Path)
}
if req.ContentType != "application/json" {
t.Fatalf("expected JSON content type, got %s", req.ContentType)
}
if req.APIKey != "secret" {
t.Fatalf("expected API key header to be set, got %q", req.APIKey)
}
if req.Payload.Title != "Pulse alert: vm-100" {
t.Fatalf("unexpected title: %s", req.Payload.Title)
}
if req.Payload.Type != "failure" {
t.Fatalf("expected failure notification type, got %s", req.Payload.Type)
}
if len(req.Payload.URLs) != 1 || req.Payload.URLs[0] != "discord://token" {
t.Fatalf("unexpected URLs in payload: %#v", req.Payload.URLs)
}
if !strings.Contains(req.Payload.Body, "CPU usage high") {
t.Fatalf("expected alert message in payload body, got %s", req.Payload.Body)
}
if !strings.Contains(req.Payload.Body, "Dashboard: https://pulse.local") {
t.Fatalf("expected dashboard link in payload body, got %s", req.Payload.Body)
}
}
func TestNotificationCooldownAllowsNewAlertInstance(t *testing.T) {
nm := NewNotificationManager("")
nm.SetCooldown(1) // 1 minute cooldown