danvics/Pulse
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adjust backup and snapshot alert handling

Browse source
This commit is contained in:
Pulse Automation Bot 2025-10-18 20:11:01 +00:00
parent 80b9d0602a
commit cfdfe896be
27 changed files with 1565 additions and 207 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
frontend-modern/src/App.tsx
View file

@ -98,15 +98,25 @@ function App() {
cephClusters: [], cephClusters: [],
physicalDisks: [], physicalDisks: [],
pbs: [], pbs: [],
pmg: [], pmg: [],
metrics: [], metrics: [],
pveBackups: { pveBackups: {
backupTasks: [],
storageBackups: [],
guestSnapshots: [],
},
pbsBackups: [],
pmgBackups: [],
backups: {
pve: {
backupTasks: [], backupTasks: [],
storageBackups: [], storageBackups: [],
guestSnapshots: [], guestSnapshots: [],
}, },
pbsBackups: [], pbs: [],
performance: { pmg: [],
},
performance: {
apiCallDuration: {}, apiCallDuration: {},
lastPollDuration: 0, lastPollDuration: 0,
pollingStartTime: '', pollingStartTime: '',

225
frontend-modern/src/components/Alerts/ResourceTable.tsx
View file

@ -54,6 +54,7 @@ export interface Resource {
host?: string; host?: string;
type?: string; type?: string;
resourceType?: string; resourceType?: string;
subtitle?: string;
thresholds?: Record<string, number | undefined>; thresholds?: Record<string, number | undefined>;
defaults?: Record<string, number | undefined>; defaults?: Record<string, number | undefined>;
disabled?: boolean; disabled?: boolean;
@ -68,6 +69,12 @@ export interface Resource {
clusterName?: string; clusterName?: string;
isClusterMember?: boolean; isClusterMember?: boolean;
delaySeconds?: number; delaySeconds?: number;
editScope?: 'snapshot';
isEnabled?: boolean;
toggleEnabled?: () => void;
toggleTitleEnabled?: string;
toggleTitleDisabled?: string;
editable?: boolean;
[key: string]: unknown; [key: string]: unknown;
} }
@ -386,9 +393,7 @@ export function ResourceTable(props: ResourceTableProps) {
titleEnabled?: string; titleEnabled?: string;
titleDisabled?: string; titleDisabled?: string;
titleWhenDisabled?: string; titleWhenDisabled?: string;
}) => { }) => <StatusBadge {...config} />;
return <StatusBadge {...config} />;
};
const offlineStateOrder: OfflineState[] = ['off', 'warning', 'critical']; const offlineStateOrder: OfflineState[] = ['off', 'warning', 'critical'];
@ -826,61 +831,67 @@ export function ResourceTable(props: ResourceTableProps) {
</Show> </Show>
</td> </td>
<td class="p-1 px-2"> <td class="p-1 px-2">
<Show when={resource.type === 'node'} fallback={ <div class="flex items-center gap-2">
<div class="flex items-center gap-2"> <Show
<span when={resource.type === 'node'}
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`} fallback={
>
{resource.name}
</span>
<Show when={resource.hasOverride || resource.disableConnectivity}>
<span class="text-xs px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded">
Custom
</span>
</Show>
</div>
}>
<div class="flex flex-wrap items-center gap-3" title={resource.status || undefined}>
<Show when={resource.host} fallback={
<span <span
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`} class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
> >
{resource.type === 'node' {resource.name}
? resource.name
: resource.displayName || resource.name}
</span> </span>
}> }
{(host) => ( >
<a <div class="flex flex-wrap items-center gap-3" title={resource.status || undefined}>
href={host() as string} <Show
target="_blank" when={resource.host}
rel="noopener noreferrer" fallback={
onClick={(e) => e.stopPropagation()} <span
class={`text-sm font-medium transition-colors duration-150 ${ class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
resource.disabled >
? 'text-gray-500 dark:text-gray-500' {resource.type === 'node'
: 'text-gray-900 dark:text-gray-100 hover:text-sky-600 dark:hover:text-sky-400' ? resource.name
}`} : resource.displayName || resource.name}
title={`Open ${resource.displayName || resource.name} web interface`} </span>
> }
{resource.type === 'node' >
? resource.name {(host) => (
: resource.displayName || resource.name} <a
</a> href={host() as string}
)} target="_blank"
</Show> rel="noopener noreferrer"
<Show when={resource.clusterName}> onClick={(e) => e.stopPropagation()}
<span class="rounded px-2 py-0.5 text-[10px] font-medium whitespace-nowrap bg-blue-100 text-blue-700 dark:bg-blue-900/40 dark:text-blue-300"> class={`text-sm font-medium transition-colors duration-150 ${
{resource.clusterName} resource.disabled
</span> ? 'text-gray-500 dark:text-gray-500'
</Show> : 'text-gray-900 dark:text-gray-100 hover:text-sky-600 dark:hover:text-sky-400'
<Show when={resource.hasOverride || resource.disableConnectivity}> }`}
<span class="text-xs px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded"> title={`Open ${resource.displayName || resource.name} web interface`}
Custom >
</span> {resource.type === 'node'
</Show> ? resource.name
</div> : resource.displayName || resource.name}
</Show> </a>
)}
</Show>
<Show when={resource.clusterName}>
<span class="rounded px-2 py-0.5 text-[10px] font-medium whitespace-nowrap bg-blue-100 text-blue-700 dark:bg-blue-900/40 dark:text-blue-300">
{resource.clusterName}
</span>
</Show>
</div>
</Show>
<Show when={resource.subtitle}>
<span class="text-xs text-gray-500 dark:text-gray-400">
{resource.subtitle as string}
</span>
</Show>
<Show when={resource.hasOverride || resource.disableConnectivity}>
<span class="text-xs px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded">
Custom
</span>
</Show>
</div>
</td> </td>
{/* Metric columns - dynamically rendered based on resource type */} {/* Metric columns - dynamically rendered based on resource type */}
<For each={props.columns}> <For each={props.columns}>
@ -1279,14 +1290,7 @@ export function ResourceTable(props: ResourceTableProps) {
{/* Metric columns - dynamically rendered based on resource type */} {/* Metric columns - dynamically rendered based on resource type */}
<For each={props.columns}> <For each={props.columns}>
{(column) => { {(column) => {
const metric = column const metric = normalizeMetricKey(column);
.toLowerCase()
.replace(' %', '')
.replace(' mb/s', '')
.replace('disk r', 'diskRead')
.replace('disk w', 'diskWrite')
.replace('net in', 'networkIn')
.replace('net out', 'networkOut');
// Check if this metric applies to this resource type // Check if this metric applies to this resource type
const showMetric = () => { const showMetric = () => {
@ -1312,6 +1316,9 @@ export function ResourceTable(props: ResourceTableProps) {
const openMetricEditor = (e: MouseEvent) => { const openMetricEditor = (e: MouseEvent) => {
e.stopPropagation(); e.stopPropagation();
if (resource.editable === false) {
return;
}
setActiveMetricInput({ resourceId: resource.id, metric }); setActiveMetricInput({ resourceId: resource.id, metric });
props.onEdit( props.onEdit(
resource.id, resource.id,
@ -1438,7 +1445,20 @@ export function ResourceTable(props: ResourceTableProps) {
{/* Actions column */} {/* Actions column */}
<td class="p-1 px-2"> <td class="p-1 px-2">
<div class="flex items-center justify-center gap-1"> <div class="flex items-center justify-center gap-1">
<Show when={typeof resource.toggleEnabled === 'function'}>
<StatusBadge
size="sm"
isEnabled={resource.isEnabled ?? true}
onToggle={resource.toggleEnabled}
titleEnabled={resource.toggleTitleEnabled}
titleDisabled={resource.toggleTitleDisabled}
/>
</Show>
<Show <Show
when={resource.editable !== false && typeof props.onEdit === 'function'}
fallback={<span class="text-xs text-gray-400 dark:text-gray-600">—</span>}
>
<Show
when={!isEditing()} when={!isEditing()}
fallback={ fallback={
<button <button
@ -1466,43 +1486,18 @@ export function ResourceTable(props: ResourceTableProps) {
</button> </button>
} }
> >
<button <div class="flex items-center gap-1">
type="button"
onClick={() =>
props.onEdit(
resource.id,
resource.thresholds ? { ...resource.thresholds } : {},
resource.defaults ? { ...resource.defaults } : {},
)
}
class="p-1 text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300"
title="Edit thresholds"
>
<svg
class="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"
/>
</svg>
</button>
<Show
when={
resource.hasOverride ||
(resource.type === 'node' && resource.disableConnectivity)
}
>
<button <button
type="button" type="button"
onClick={() => props.onRemoveOverride(resource.id)} onClick={() =>
class="p-1 text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300" props.onEdit(
title="Remove override" resource.id,
resource.thresholds ? { ...resource.thresholds } : {},
resource.defaults ? { ...resource.defaults } : {},
)
}
class="p-1 text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300"
title="Edit thresholds"
> >
<svg <svg
class="w-4 h-4" class="w-4 h-4"
@ -1514,14 +1509,42 @@ export function ResourceTable(props: ResourceTableProps) {
stroke-linecap="round" stroke-linecap="round"
stroke-linejoin="round" stroke-linejoin="round"
stroke-width="2" stroke-width="2"
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"
/> />
</svg> </svg>
</button> </button>
</Show> <Show
when={
resource.hasOverride ||
(resource.type === 'node' && resource.disableConnectivity)
}
>
<button
type="button"
onClick={() => props.onRemoveOverride(resource.id)}
class="p-1 text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300"
title="Remove override"
>
<svg
class="w-4 h-4"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
/>
</svg>
</button>
</Show>
</div>
</Show> </Show>
</div> </Show>
</td> </div>
</td>
</tr> </tr>
); );
}} }}

264
frontend-modern/src/components/Alerts/ThresholdsTable.tsx
View file

@ -14,8 +14,12 @@ import type {
PMGInstance, PMGInstance,
DockerHost, DockerHost,
DockerContainer, DockerContainer,
PVEBackups,
PBSBackup,
PMGBackup,
Backups,
} from '@/types/api'; } from '@/types/api';
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts'; import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable'; import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
type OverrideType = type OverrideType =
| 'guest' | 'guest'
@ -103,6 +107,11 @@ export const normalizeDockerIgnoredInput = (value: string): string[] =>
.map((entry) => entry.trim()) .map((entry) => entry.trim())
.filter((entry) => entry.length > 0); .filter((entry) => entry.length > 0);
const DEFAULT_SNAPSHOT_WARNING = 30;
const DEFAULT_SNAPSHOT_CRITICAL = 45;
const DEFAULT_BACKUP_WARNING = 7;
const DEFAULT_BACKUP_CRITICAL = 14;
// Simple threshold object for the UI // Simple threshold object for the UI
interface SimpleThresholds { interface SimpleThresholds {
cpu?: number; cpu?: number;
@ -127,6 +136,10 @@ interface ThresholdsTableProps {
dockerHosts: DockerHost[]; dockerHosts: DockerHost[];
pbsInstances?: PBSInstance[]; // PBS instances from state pbsInstances?: PBSInstance[]; // PBS instances from state
pmgInstances?: PMGInstance[]; // PMG instances from state pmgInstances?: PMGInstance[]; // PMG instances from state
backups?: Backups;
pveBackups?: PVEBackups;
pbsBackups?: PBSBackup[];
pmgBackups?: PMGBackup[];
pmgThresholds: () => PMGThresholdDefaults; pmgThresholds: () => PMGThresholdDefaults;
setPMGThresholds: ( setPMGThresholds: (
value: value:
@ -177,6 +190,14 @@ interface ThresholdsTableProps {
) => void; ) => void;
snapshotFactoryDefaults?: SnapshotAlertConfig; snapshotFactoryDefaults?: SnapshotAlertConfig;
resetSnapshotDefaults?: () => void; resetSnapshotDefaults?: () => void;
backupDefaults: () => BackupAlertConfig;
setBackupDefaults: (
value:
| BackupAlertConfig
| ((prev: BackupAlertConfig) => BackupAlertConfig),
) => void;
backupFactoryDefaults?: BackupAlertConfig;
resetBackupDefaults?: () => void;
setHasUnsavedChanges: (value: boolean) => void; setHasUnsavedChanges: (value: boolean) => void;
activeAlerts?: Record<string, Alert>; activeAlerts?: Record<string, Alert>;
removeAlerts?: (predicate: (alert: Alert) => boolean) => void; removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
@ -208,9 +229,6 @@ interface ThresholdsTableProps {
setDisableAllDockerHostsOffline: (value: boolean) => void; setDisableAllDockerHostsOffline: (value: boolean) => void;
} }
const DEFAULT_SNAPSHOT_WARNING = 30;
const DEFAULT_SNAPSHOT_CRITICAL = 45;
export function ThresholdsTable(props: ThresholdsTableProps) { export function ThresholdsTable(props: ThresholdsTableProps) {
const navigate = useNavigate(); const navigate = useNavigate();
const location = useLocation(); const location = useLocation();
@ -803,26 +821,88 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
}; };
}); });
const snapshotFactoryDefaultsRecord = createMemo(() => { const backupFactoryConfig = () =>
const factory = snapshotFactoryConfig(); props.backupFactoryDefaults ?? {
enabled: false,
warningDays: DEFAULT_BACKUP_WARNING,
criticalDays: DEFAULT_BACKUP_CRITICAL,
};
const sanitizeBackupConfig = (config: BackupAlertConfig): BackupAlertConfig => {
let warning = Math.max(0, Math.round(config.warningDays ?? 0));
let critical = Math.max(0, Math.round(config.criticalDays ?? 0));
if (critical > 0 && warning > critical) {
warning = critical;
}
if (critical === 0 && warning > 0) {
critical = warning;
}
return { return {
'warning days': factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING, enabled: !!config.enabled,
'critical days': factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL, warningDays: warning,
criticalDays: critical,
};
};
const updateBackupDefaults = (
updater:
| BackupAlertConfig
| ((prev: BackupAlertConfig) => BackupAlertConfig),
) => {
props.setBackupDefaults((prev) => {
const next =
typeof updater === 'function'
? (updater as (prev: BackupAlertConfig) => BackupAlertConfig)(prev)
: { ...prev, ...updater };
return sanitizeBackupConfig(next);
});
props.setHasUnsavedChanges(true);
};
const backupDefaultsRecord = createMemo(() => {
const current = props.backupDefaults();
return {
'warning days': current.warningDays ?? 0,
'critical days': current.criticalDays ?? 0,
}; };
}); });
const snapshotOverridesCount = createMemo(() => { const backupFactoryDefaultsRecord = createMemo(() => {
const current = props.snapshotDefaults(); const factory = backupFactoryConfig();
const factory = snapshotFactoryConfig(); return {
return current.enabled !== factory.enabled || 'warning days': factory.warningDays ?? DEFAULT_BACKUP_WARNING,
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !== 'critical days': factory.criticalDays ?? DEFAULT_BACKUP_CRITICAL,
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) || };
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !== });
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
const snapshotOverridesCount = createMemo(() => {
const current = props.snapshotDefaults();
const factory = snapshotFactoryConfig();
return current.enabled !== factory.enabled ||
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !==
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) ||
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !==
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
? 1
: 0;
});
const backupOverridesCount = createMemo(() => {
const backupCurrent = props.backupDefaults();
const backupFactory = backupFactoryConfig();
return backupCurrent.enabled !== backupFactory.enabled ||
(backupCurrent.warningDays ?? DEFAULT_BACKUP_WARNING) !==
(backupFactory.warningDays ?? DEFAULT_BACKUP_WARNING) ||
(backupCurrent.criticalDays ?? DEFAULT_BACKUP_CRITICAL) !==
(backupFactory.criticalDays ?? DEFAULT_BACKUP_CRITICAL)
? 1 ? 1
: 0; : 0;
}); });
// Process guests with their overrides and group by node // Process guests with their overrides and group by node
const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => { const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => {
// If we're currently editing, return the previous value to avoid re-renders // If we're currently editing, return the previous value to avoid re-renders
@ -1161,18 +1241,25 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
}, },
{ {
key: 'storage' as const, key: 'storage' as const,
label: 'Storage', label: 'Storage',
total: props.storage?.length ?? 0, total: props.storage?.length ?? 0,
overrides: countOverrides(storageWithOverrides()), overrides: countOverrides(storageWithOverrides()),
tab: 'proxmox' as const, tab: 'proxmox' as const,
}, },
{ {
key: 'snapshots' as const, key: 'backups' as const,
label: 'Snapshots', label: 'Backups',
total: 1, total: 1,
overrides: snapshotOverridesCount(), overrides: backupOverridesCount(),
tab: 'proxmox' as const, tab: 'proxmox' as const,
}, },
{
key: 'snapshots' as const,
label: 'Snapshot Age',
total: 1,
overrides: snapshotOverridesCount(),
tab: 'proxmox' as const,
},
{ {
key: 'pbs' as const, key: 'pbs' as const,
label: 'PBS Servers', label: 'PBS Servers',
@ -1240,6 +1327,41 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
if (!resource) return; if (!resource) return;
const editedThresholds = editingThresholds(); const editedThresholds = editingThresholds();
if (resource.editScope === 'backup') {
const currentBackupDefaults = props.backupDefaults();
const nextWarning =
editedThresholds['warning days'] ?? currentBackupDefaults.warningDays ?? DEFAULT_BACKUP_WARNING;
const nextCritical =
editedThresholds['critical days'] ?? currentBackupDefaults.criticalDays ?? DEFAULT_BACKUP_CRITICAL;
updateBackupDefaults({
enabled: currentBackupDefaults.enabled,
warningDays: nextWarning,
criticalDays: nextCritical,
});
cancelEdit();
return;
}
if (resource.editScope === 'snapshot') {
const currentSnapshotDefaults = props.snapshotDefaults();
const nextWarning =
editedThresholds['warning days'] ?? currentSnapshotDefaults.warningDays ?? DEFAULT_SNAPSHOT_WARNING;
const nextCritical =
editedThresholds['critical days'] ?? currentSnapshotDefaults.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL;
updateSnapshotDefaults({
enabled: currentSnapshotDefaults.enabled,
warningDays: nextWarning,
criticalDays: nextCritical,
});
cancelEdit();
return;
}
const defaultThresholds = (resource.defaults ?? {}) as Record<string, number | undefined>; const defaultThresholds = (resource.defaults ?? {}) as Record<string, number | undefined>;
// Only include values that differ from defaults // Only include values that differ from defaults
@ -1969,11 +2091,74 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
</div> </div>
</Show> </Show>
<Show when={hasSection('backups')}>
<div ref={registerSection('backups')} class="scroll-mt-24">
<ResourceTable
title="Backups"
resources={[{ id: "backups-defaults", name: "Global Defaults", thresholds: backupDefaultsRecord(), defaults: backupDefaultsRecord(), editable: true, editScope: "backup" }]}
columns={['Warning Days', 'Critical Days']}
activeAlerts={props.activeAlerts}
emptyMessage=""
onEdit={startEditing}
onSaveEdit={saveEdit}
onCancelEdit={cancelEdit}
onRemoveOverride={removeOverride}
showOfflineAlertsColumn={false}
editingId={editingId}
editingThresholds={editingThresholds}
setEditingThresholds={setEditingThresholds}
formatMetricValue={formatMetricValue}
hasActiveAlert={hasActiveAlert}
globalDefaults={backupDefaultsRecord()}
setGlobalDefaults={(value) => {
updateBackupDefaults((prev) => {
const currentRecord = {
'warning days': prev.warningDays ?? 0,
'critical days': prev.criticalDays ?? 0,
};
const nextRecord =
typeof value === 'function'
? value(currentRecord)
: { ...currentRecord, ...value };
return {
...prev,
warningDays:
typeof nextRecord['warning days'] === 'number'
? nextRecord['warning days']
: prev.warningDays,
criticalDays:
typeof nextRecord['critical days'] === 'number'
? nextRecord['critical days']
: prev.criticalDays,
};
});
}}
setHasUnsavedChanges={props.setHasUnsavedChanges}
globalDisableFlag={() => !props.backupDefaults().enabled}
onToggleGlobalDisable={() =>
updateBackupDefaults((prev) => ({
...prev,
enabled: !prev.enabled,
}))
}
factoryDefaults={backupFactoryDefaultsRecord()}
onResetDefaults={() => {
if (props.resetBackupDefaults) {
props.resetBackupDefaults();
props.setHasUnsavedChanges(true);
} else {
updateBackupDefaults(backupFactoryConfig());
}
}}
/>
</div>
</Show>
<Show when={hasSection('snapshots')}> <Show when={hasSection('snapshots')}>
<div ref={registerSection('snapshots')} class="scroll-mt-24"> <div ref={registerSection('snapshots')} class="scroll-mt-24">
<ResourceTable <ResourceTable
title="Snapshot Age" title="Snapshot Age"
resources={[]} resources={[{ id: "snapshots-defaults", name: "Global Defaults", thresholds: snapshotDefaultsRecord(), defaults: snapshotDefaultsRecord(), editable: true, editScope: "snapshot" }]}
columns={['Warning Days', 'Critical Days']} columns={['Warning Days', 'Critical Days']}
activeAlerts={props.activeAlerts} activeAlerts={props.activeAlerts}
emptyMessage="" emptyMessage=""
@ -2019,18 +2204,15 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
enabled: !prev.enabled, enabled: !prev.enabled,
})) }))
} }
factoryDefaults={snapshotFactoryDefaultsRecord()} factoryDefaults={snapshotFactoryConfig()}
onResetDefaults={ onResetDefaults={() => {
props.resetSnapshotDefaults if (props.resetSnapshotDefaults) {
? () => { props.resetSnapshotDefaults();
props.resetSnapshotDefaults?.(); props.setHasUnsavedChanges(true);
props.setHasUnsavedChanges(true); } else {
} updateSnapshotDefaults(snapshotFactoryConfig());
: () => }
updateSnapshotDefaults({ }}
...snapshotFactoryConfig(),
})
}
/> />
</div> </div>
</Show> </Show>

6
frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx
View file

@ -6,7 +6,7 @@ import {
ThresholdsTable, ThresholdsTable,
normalizeDockerIgnoredInput, normalizeDockerIgnoredInput,
} from '../ThresholdsTable'; } from '../ThresholdsTable';
import type { PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts'; import type { PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
vi.mock('@solidjs/router', () => ({ vi.mock('@solidjs/router', () => ({
useNavigate: () => vi.fn(), useNavigate: () => vi.fn(),
@ -87,6 +87,10 @@ const baseProps = () => ({
factoryNodeDefaults: {}, factoryNodeDefaults: {},
factoryDockerDefaults: {}, factoryDockerDefaults: {},
factoryStorageDefault: 85, factoryStorageDefault: 85,
backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }),
setBackupDefaults: vi.fn(),
backupFactoryDefaults: { enabled: false, warningDays: 7, criticalDays: 14 } as BackupAlertConfig,
resetBackupDefaults: vi.fn(),
snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }), snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }),
setSnapshotDefaults: vi.fn(), setSnapshotDefaults: vi.fn(),
snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig, snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig,

12
frontend-modern/src/components/Backups/Backups.tsx
View file

@ -13,7 +13,15 @@ const Backups: Component = () => {
<ProxmoxSectionNav current="backups" /> <ProxmoxSectionNav current="backups" />
{/* Loading State */} {/* Loading State */}
<Show when={connected() && !state.pveBackups && !state.pbs}> <Show
when={
connected() &&
!(state.backups?.pve?.guestSnapshots?.length ||
state.backups?.pve?.storageBackups?.length ||
state.backups?.pbs?.length ||
state.backups?.pmg?.length)
}
>
<Card padding="lg"> <Card padding="lg">
<EmptyState <EmptyState
icon={ icon={
@ -72,7 +80,7 @@ const Backups: Component = () => {
</Show> </Show>
{/* Main Content - Unified Backups View */} {/* Main Content - Unified Backups View */}
<Show when={connected() && (state.pveBackups || state.pbs)}> <Show when={connected() && (state.backups?.pve || state.backups?.pbs || state.pbs)}>
<UnifiedBackups /> <UnifiedBackups />
</Show> </Show>
</div> </div>

53
frontend-modern/src/components/Backups/UnifiedBackups.tsx
View file

@ -25,6 +25,9 @@ interface DateGroup {
const UnifiedBackups: Component = () => { const UnifiedBackups: Component = () => {
const { state } = useWebSocket(); const { state } = useWebSocket();
const pveBackupsState = createMemo(() => state.backups?.pve ?? state.pveBackups);
const pbsBackupsState = createMemo(() => state.backups?.pbs ?? state.pbsBackups);
const pmgBackupsState = createMemo(() => state.backups?.pmg ?? state.pmgBackups);
const [searchTerm, setSearchTerm] = createSignal(''); const [searchTerm, setSearchTerm] = createSignal('');
const [selectedNode, setSelectedNode] = createSignal<string | null>(null); const [selectedNode, setSelectedNode] = createSignal<string | null>(null);
const [typeFilter, setTypeFilter] = createSignal<'all' | FilterableGuestType>('all'); const [typeFilter, setTypeFilter] = createSignal<'all' | FilterableGuestType>('all');
@ -159,10 +162,14 @@ const UnifiedBackups: Component = () => {
// Check if we have any backup data yet // Check if we have any backup data yet
const isLoading = createMemo(() => { const isLoading = createMemo(() => {
const pve = pveBackupsState();
const pbs = pbsBackupsState();
const pmg = pmgBackupsState();
return ( return (
!state.pveBackups?.guestSnapshots && !(pve?.guestSnapshots?.length ?? 0) &&
!state.pveBackups?.storageBackups && !(pve?.storageBackups?.length ?? 0) &&
!state.pbsBackups?.length && !(pbs?.length ?? 0) &&
!(pmg?.length ?? 0) &&
!state.pbs?.length !state.pbs?.length
); );
}); });
@ -176,7 +183,7 @@ const UnifiedBackups: Component = () => {
const debugMode = typeof window !== 'undefined' && localStorage.getItem('debug-pmg') === 'true'; const debugMode = typeof window !== 'undefined' && localStorage.getItem('debug-pmg') === 'true';
// Normalize snapshots // Normalize snapshots
state.pveBackups?.guestSnapshots?.forEach((snapshot) => { pveBackupsState()?.guestSnapshots?.forEach((snapshot) => {
// Try to find the guest name by matching VMID and instance (not hostname) // Try to find the guest name by matching VMID and instance (not hostname)
let guestName = ''; let guestName = '';
const vm = state.vms?.find((v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance); const vm = state.vms?.find((v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance);
@ -213,8 +220,8 @@ const UnifiedBackups: Component = () => {
// This ensures we have the complete PBS data with namespaces // This ensures we have the complete PBS data with namespaces
// Filter by selected PBS instance if one is selected // Filter by selected PBS instance if one is selected
const pbsBackupsToProcess = selectedPBSInstance() const pbsBackupsToProcess = selectedPBSInstance()
? state.pbsBackups?.filter((b) => b.instance === selectedPBSInstance()) ? pbsBackupsState()?.filter((b) => b.instance === selectedPBSInstance())
: state.pbsBackups; : pbsBackupsState();
pbsBackupsToProcess?.forEach((backup) => { pbsBackupsToProcess?.forEach((backup) => {
const backupDate = new Date(backup.backupTime); const backupDate = new Date(backup.backupTime);
@ -302,8 +309,38 @@ const UnifiedBackups: Component = () => {
}); });
}); });
pmgBackupsState()?.forEach((backup) => {
const backupTimeMs = backup.backupTime ? Date.parse(backup.backupTime) : Number.NaN;
const backupTimeSeconds = Number.isFinite(backupTimeMs) ? Math.floor(backupTimeMs / 1000) : 0;
const backupKey = backup.id || `${backup.instance}:${backup.node}:${backup.filename}`;
if (seenBackups.has(backupKey)) {
return;
}
seenBackups.add(backupKey);
const displayName = backup.node || backup.filename || 'PMG Host Backup';
unified.push({
backupType: 'local',
vmid: backup.node || backup.filename,
name: displayName,
type: 'Host',
node: backup.node || 'PMG',
instance: backup.instance || 'PMG',
backupTime: backupTimeSeconds,
backupName: backup.filename || displayName,
description: backup.filename || '',
status: 'ok',
size: backup.size || null,
storage: 'PMG',
datastore: null,
namespace: null,
verified: null,
protected: false,
});
});
// Normalize local backups (including PBS through PVE storage) // Normalize local backups (including PBS through PVE storage)
state.pveBackups?.storageBackups?.forEach((backup) => { pveBackupsState()?.storageBackups?.forEach((backup) => {
// Skip templates and ISOs - they're not backups // Skip templates and ISOs - they're not backups
if (backup.type === 'vztmpl' || backup.type === 'iso') { if (backup.type === 'vztmpl' || backup.type === 'iso') {
return; return;
@ -1199,7 +1236,7 @@ const UnifiedBackups: Component = () => {
// Count backups for this PBS instance // Count backups for this PBS instance
const pbsBackups = () => const pbsBackups = () =>
state.pbsBackups?.filter((b) => b.instance === pbs.name).length || 0; pbsBackupsState()?.filter((b) => b.instance === pbs.name).length || 0;
const isSelected = () => selectedPBSInstance() === pbs.name; const isSelected = () => selectedPBSInstance() === pbs.name;

22
frontend-modern/src/components/Settings/Settings.tsx
View file

@ -317,6 +317,10 @@ const Settings: Component<SettingsProps> = (props) => {
description: 'Manage Pulse configuration.', description: 'Manage Pulse configuration.',
}; };
const pveBackupsState = () => state.backups?.pve ?? state.pveBackups;
const pbsBackupsState = () => state.backups?.pbs ?? state.pbsBackups;
const pmgBackupsState = () => state.backups?.pmg ?? state.pmgBackups;
// Keep tab state in sync with URL and handle /settings redirect without flicker // Keep tab state in sync with URL and handle /settings redirect without flicker
createEffect( createEffect(
on( on(
@ -5413,11 +5417,11 @@ const Settings: Component<SettingsProps> = (props) => {
storageCount: state.storage?.length || 0, storageCount: state.storage?.length || 0,
physicalDisksCount: state.physicalDisks?.length || 0, physicalDisksCount: state.physicalDisks?.length || 0,
pbsCount: state.pbs?.length || 0, pbsCount: state.pbs?.length || 0,
pbsBackupsCount: state.pbsBackups?.length || 0, pbsBackupsCount: pbsBackupsState()?.length || 0,
pveBackups: { pveBackups: {
backupTasksCount: state.pveBackups?.backupTasks?.length || 0, backupTasksCount: pveBackupsState()?.backupTasks?.length || 0,
storageBackupsCount: state.pveBackups?.storageBackups?.length || 0, storageBackupsCount: pveBackupsState()?.storageBackups?.length || 0,
guestSnapshotsCount: state.pveBackups?.guestSnapshots?.length || 0, guestSnapshotsCount: pveBackupsState()?.guestSnapshots?.length || 0,
}, },
}, },
// Node status details // Node status details
@ -5454,9 +5458,9 @@ const Settings: Component<SettingsProps> = (props) => {
} }
: undefined, : undefined,
hasBackups: hasBackups:
(state.pveBackups?.storageBackups?.filter( (pveBackupsState()?.storageBackups ?? []).filter(
(b) => b.storage === s.name, (b) => b.storage === s.name,
).length || 0) > 0, ).length > 0,
})) || [], })) || [],
// Physical disks - critical for troubleshooting // Physical disks - critical for troubleshooting
physicalDisks: physicalDisks:
@ -5472,10 +5476,10 @@ const Settings: Component<SettingsProps> = (props) => {
smart: d.smart ?? null, smart: d.smart ?? null,
})) || [], })) || [],
backups: { backups: {
pveBackupTasks: state.pveBackups?.backupTasks?.slice(0, 10) || [], pveBackupTasks: pveBackupsState()?.backupTasks?.slice(0, 10) || [],
pveStorageBackups: pveStorageBackups:
state.pveBackups?.storageBackups?.slice(0, 10) || [], pveBackupsState()?.storageBackups?.slice(0, 10) || [],
pbsBackups: state.pbsBackups?.slice(0, 10) || [], pbsBackups: pbsBackupsState()?.slice(0, 10) || [],
}, },
connectionHealth: state.connectionHealth || {}, connectionHealth: state.connectionHealth || {},
performance: { performance: {

16
frontend-modern/src/components/shared/UnifiedNodeSelector.tsx
View file

@ -44,6 +44,8 @@ export const UnifiedNodeSelector: Component<UnifiedNodeSelectorProps> = (props)
// This allows users to select a node AND search within it // This allows users to select a node AND search within it
// Calculate backup counts for nodes and PBS instances // Calculate backup counts for nodes and PBS instances
const pveBackupsState = () => state.backups?.pve ?? state.pveBackups;
const pbsBackupsState = () => state.backups?.pbs ?? state.pbsBackups;
const backupCounts = createMemo(() => { const backupCounts = createMemo(() => {
const counts: Record<string, number> = {}; const counts: Record<string, number> = {};
@ -54,15 +56,16 @@ export const UnifiedNodeSelector: Component<UnifiedNodeSelectorProps> = (props)
let count = 0; let count = 0;
// Count storage backups (excluding PBS backups which are counted separately) // Count storage backups (excluding PBS backups which are counted separately)
if (state.pveBackups?.storageBackups) { const pveBackups = pveBackupsState();
count += state.pveBackups.storageBackups.filter( if (pveBackups?.storageBackups) {
count += pveBackups.storageBackups.filter(
(b) => b.instance === node.instance && b.node === node.name && !b.isPBS, (b) => b.instance === node.instance && b.node === node.name && !b.isPBS,
).length; ).length;
} }
// Count snapshots // Count snapshots
if (state.pveBackups?.guestSnapshots) { if (pveBackups?.guestSnapshots) {
count += state.pveBackups.guestSnapshots.filter((s) => s.instance === node.instance && s.node === node.name).length; count += pveBackups.guestSnapshots.filter((s) => s.instance === node.instance && s.node === node.name).length;
} }
counts[node.id] = count; counts[node.id] = count;
@ -70,9 +73,10 @@ export const UnifiedNodeSelector: Component<UnifiedNodeSelectorProps> = (props)
} }
// Count PBS backups by instance // Count PBS backups by instance
if (state.pbs && state.pbsBackups) { const pbsBackups = pbsBackupsState();
if (state.pbs && pbsBackups) {
state.pbs.forEach((pbs) => { state.pbs.forEach((pbs) => {
counts[pbs.name] = state.pbsBackups?.filter((b) => b.instance === pbs.name).length || 0; counts[pbs.name] = pbsBackups.filter((b) => b.instance === pbs.name).length || 0;
}); });
} }

82
frontend-modern/src/pages/Alerts.tsx
View file

@ -3,7 +3,7 @@ import type { JSX } from 'solid-js';
import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect'; import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect';
import { WebhookConfig } from '@/components/Alerts/WebhookConfig'; import { WebhookConfig } from '@/components/Alerts/WebhookConfig';
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable'; import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts'; import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
import { Card } from '@/components/shared/Card'; import { Card } from '@/components/shared/Card';
import { SectionHeader } from '@/components/shared/SectionHeader'; import { SectionHeader } from '@/components/shared/SectionHeader';
import { SettingsPanel } from '@/components/shared/SettingsPanel'; import { SettingsPanel } from '@/components/shared/SettingsPanel';
@ -769,6 +769,25 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setMetricTimeThresholds({}); setMetricTimeThresholds({});
} }
// Load backup thresholds
if (config.backupDefaults) {
const enabled = Boolean(config.backupDefaults.enabled);
const rawWarning = config.backupDefaults.warningDays ?? FACTORY_BACKUP_DEFAULTS.warningDays;
const rawCritical = config.backupDefaults.criticalDays ?? FACTORY_BACKUP_DEFAULTS.criticalDays;
const safeCritical = Math.max(0, rawCritical);
const normalizedWarning = Math.max(0, rawWarning);
const warningDays =
safeCritical > 0 && normalizedWarning > safeCritical ? safeCritical : normalizedWarning;
const criticalDays = Math.max(safeCritical, warningDays);
setBackupDefaults({
enabled,
warningDays,
criticalDays,
});
} else {
setBackupDefaults({ ...FACTORY_BACKUP_DEFAULTS });
}
// Load snapshot thresholds // Load snapshot thresholds
if (config.snapshotDefaults) { if (config.snapshotDefaults) {
const enabled = Boolean(config.snapshotDefaults.enabled); const enabled = Boolean(config.snapshotDefaults.enabled);
@ -1058,6 +1077,11 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
warningDays: 30, warningDays: 30,
criticalDays: 45, criticalDays: 45,
}; };
const FACTORY_BACKUP_DEFAULTS: BackupAlertConfig = {
enabled: false,
warningDays: 7,
criticalDays: 14,
};
// Threshold states - using trigger values for display // Threshold states - using trigger values for display
const [guestDefaults, setGuestDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_GUEST_DEFAULTS }); const [guestDefaults, setGuestDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_GUEST_DEFAULTS });
@ -1070,6 +1094,9 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal<string[]>([]); const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal<string[]>([]);
const [storageDefault, setStorageDefault] = createSignal(FACTORY_STORAGE_DEFAULT); const [storageDefault, setStorageDefault] = createSignal(FACTORY_STORAGE_DEFAULT);
const [backupDefaults, setBackupDefaults] = createSignal<BackupAlertConfig>({
...FACTORY_BACKUP_DEFAULTS,
});
// Reset functions // Reset functions
const resetGuestDefaults = () => { const resetGuestDefaults = () => {
@ -1096,6 +1123,10 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setStorageDefault(FACTORY_STORAGE_DEFAULT); setStorageDefault(FACTORY_STORAGE_DEFAULT);
setHasUnsavedChanges(true); setHasUnsavedChanges(true);
}; };
const resetBackupDefaults = () => {
setBackupDefaults({ ...FACTORY_BACKUP_DEFAULTS });
setHasUnsavedChanges(true);
};
const resetSnapshotDefaults = () => { const resetSnapshotDefaults = () => {
setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS }); setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS });
setHasUnsavedChanges(true); setHasUnsavedChanges(true);
@ -1224,12 +1255,20 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
}; };
const snapshotConfig = snapshotDefaults(); const snapshotConfig = snapshotDefaults();
const normalizedWarningDays = Math.max(0, snapshotConfig.warningDays ?? 0); const normalizedSnapshotWarning = Math.max(0, snapshotConfig.warningDays ?? 0);
const normalizedCriticalDays = Math.max(0, snapshotConfig.criticalDays ?? 0); const normalizedSnapshotCritical = Math.max(0, snapshotConfig.criticalDays ?? 0);
const finalCriticalDays = const finalSnapshotCritical =
normalizedCriticalDays > 0 normalizedSnapshotCritical > 0
? Math.max(normalizedCriticalDays, normalizedWarningDays) ? Math.max(normalizedSnapshotCritical, normalizedSnapshotWarning)
: normalizedWarningDays; : normalizedSnapshotWarning;
const backupConfig = backupDefaults();
const normalizedBackupWarning = Math.max(0, backupConfig.warningDays ?? 0);
const normalizedBackupCritical = Math.max(0, backupConfig.criticalDays ?? 0);
const finalBackupCritical =
normalizedBackupCritical > 0
? Math.max(normalizedBackupCritical, normalizedBackupWarning)
: normalizedBackupWarning;
const alertConfig = { const alertConfig = {
enabled: true, enabled: true,
@ -1284,8 +1323,13 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()), metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
snapshotDefaults: { snapshotDefaults: {
enabled: snapshotConfig.enabled, enabled: snapshotConfig.enabled,
warningDays: normalizedWarningDays, warningDays: normalizedSnapshotWarning,
criticalDays: finalCriticalDays, criticalDays: finalSnapshotCritical,
},
backupDefaults: {
enabled: backupConfig.enabled,
warningDays: normalizedBackupWarning,
criticalDays: finalBackupCritical,
}, },
pmgDefaults: pmgThresholds(), pmgDefaults: pmgThresholds(),
// Use rawOverridesConfig which is already properly formatted with disabled flags // Use rawOverridesConfig which is already properly formatted with disabled flags
@ -1491,14 +1535,18 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes} resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes}
resetStorageDefault={resetStorageDefault} resetStorageDefault={resetStorageDefault}
resetSnapshotDefaults={resetSnapshotDefaults} resetSnapshotDefaults={resetSnapshotDefaults}
resetBackupDefaults={resetBackupDefaults}
factoryGuestDefaults={FACTORY_GUEST_DEFAULTS} factoryGuestDefaults={FACTORY_GUEST_DEFAULTS}
factoryNodeDefaults={FACTORY_NODE_DEFAULTS} factoryNodeDefaults={FACTORY_NODE_DEFAULTS}
factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS} factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS}
factoryStorageDefault={FACTORY_STORAGE_DEFAULT} factoryStorageDefault={FACTORY_STORAGE_DEFAULT}
snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS} snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS}
backupFactoryDefaults={FACTORY_BACKUP_DEFAULTS}
timeThresholds={timeThresholds} timeThresholds={timeThresholds}
metricTimeThresholds={metricTimeThresholds} metricTimeThresholds={metricTimeThresholds}
setMetricTimeThresholds={setMetricTimeThresholds} setMetricTimeThresholds={setMetricTimeThresholds}
backupDefaults={backupDefaults}
setBackupDefaults={setBackupDefaults}
snapshotDefaults={snapshotDefaults} snapshotDefaults={snapshotDefaults}
setSnapshotDefaults={setSnapshotDefaults} setSnapshotDefaults={setSnapshotDefaults}
pmgThresholds={pmgThresholds} pmgThresholds={pmgThresholds}
@ -2028,6 +2076,14 @@ interface ThresholdsTabProps {
) => void; ) => void;
snapshotFactoryDefaults: SnapshotAlertConfig; snapshotFactoryDefaults: SnapshotAlertConfig;
resetSnapshotDefaults: () => void; resetSnapshotDefaults: () => void;
backupDefaults: () => BackupAlertConfig;
setBackupDefaults: (
value:
| BackupAlertConfig
| ((prev: BackupAlertConfig) => BackupAlertConfig),
) => void;
backupFactoryDefaults: BackupAlertConfig;
resetBackupDefaults: () => void;
setOverrides: (value: Override[]) => void; setOverrides: (value: Override[]) => void;
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void; setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
activeAlerts: Record<string, Alert>; activeAlerts: Record<string, Alert>;
@ -2085,6 +2141,10 @@ function ThresholdsTab(props: ThresholdsTabProps) {
dockerHosts={props.state.dockerHosts || []} dockerHosts={props.state.dockerHosts || []}
pbsInstances={props.state.pbs || []} pbsInstances={props.state.pbs || []}
pmgInstances={props.state.pmg || []} pmgInstances={props.state.pmg || []}
backups={props.state.backups}
pveBackups={props.state.pveBackups}
pbsBackups={props.state.pbsBackups}
pmgBackups={props.state.pmgBackups}
pmgThresholds={props.pmgThresholds} pmgThresholds={props.pmgThresholds}
setPMGThresholds={props.setPMGThresholds} setPMGThresholds={props.setPMGThresholds}
guestDefaults={props.guestDefaults()} guestDefaults={props.guestDefaults()}
@ -2104,6 +2164,10 @@ function ThresholdsTab(props: ThresholdsTabProps) {
timeThresholds={props.timeThresholds} timeThresholds={props.timeThresholds}
metricTimeThresholds={props.metricTimeThresholds} metricTimeThresholds={props.metricTimeThresholds}
setMetricTimeThresholds={props.setMetricTimeThresholds} setMetricTimeThresholds={props.setMetricTimeThresholds}
backupDefaults={props.backupDefaults}
setBackupDefaults={props.setBackupDefaults}
backupFactoryDefaults={props.backupFactoryDefaults}
resetBackupDefaults={props.resetBackupDefaults}
snapshotDefaults={props.snapshotDefaults} snapshotDefaults={props.snapshotDefaults}
setSnapshotDefaults={props.setSnapshotDefaults} setSnapshotDefaults={props.setSnapshotDefaults}
snapshotFactoryDefaults={props.snapshotFactoryDefaults} snapshotFactoryDefaults={props.snapshotFactoryDefaults}

21
frontend-modern/src/stores/websocket.ts
View file

@ -36,6 +36,16 @@ export function createWebSocketStore(url: string) {
guestSnapshots: [], guestSnapshots: [],
} as PVEBackups, } as PVEBackups,
pbsBackups: [], pbsBackups: [],
pmgBackups: [],
backups: {
pve: {
backupTasks: [],
storageBackups: [],
guestSnapshots: [],
},
pbs: [],
pmg: [],
},
performance: { performance: {
apiCallDuration: {}, apiCallDuration: {},
lastPollDuration: 0, lastPollDuration: 0,
@ -282,8 +292,19 @@ export function createWebSocketStore(url: string) {
setState('cephClusters', message.data.cephClusters); setState('cephClusters', message.data.cephClusters);
if (message.data.pbs !== undefined) setState('pbs', message.data.pbs); if (message.data.pbs !== undefined) setState('pbs', message.data.pbs);
if (message.data.pmg !== undefined) setState('pmg', message.data.pmg); if (message.data.pmg !== undefined) setState('pmg', message.data.pmg);
if (message.data.backups !== undefined) {
setState('backups', message.data.backups);
if (message.data.backups.pve !== undefined)
setState('pveBackups', message.data.backups.pve);
if (message.data.backups.pbs !== undefined)
setState('pbsBackups', message.data.backups.pbs);
if (message.data.backups.pmg !== undefined)
setState('pmgBackups', message.data.backups.pmg);
}
if (message.data.pbsBackups !== undefined) if (message.data.pbsBackups !== undefined)
setState('pbsBackups', message.data.pbsBackups); setState('pbsBackups', message.data.pbsBackups);
if (message.data.pmgBackups !== undefined)
setState('pmgBackups', message.data.pmgBackups);
if (message.data.metrics !== undefined) setState('metrics', message.data.metrics); if (message.data.metrics !== undefined) setState('metrics', message.data.metrics);
if (message.data.pveBackups !== undefined) if (message.data.pveBackups !== undefined)
setState('pveBackups', message.data.pveBackups); setState('pveBackups', message.data.pveBackups);

7
frontend-modern/src/types/alerts.ts
View file

@ -91,6 +91,12 @@ export interface SnapshotAlertConfig {
criticalDays: number; criticalDays: number;
} }
export interface BackupAlertConfig {
enabled: boolean;
warningDays: number;
criticalDays: number;
}
export interface AlertConfig { export interface AlertConfig {
enabled: boolean; enabled: boolean;
guestDefaults: AlertThresholds; guestDefaults: AlertThresholds;
@ -100,6 +106,7 @@ export interface AlertConfig {
dockerIgnoredContainerPrefixes?: string[]; dockerIgnoredContainerPrefixes?: string[];
pmgDefaults?: PMGThresholdDefaults; pmgDefaults?: PMGThresholdDefaults;
snapshotDefaults?: SnapshotAlertConfig; snapshotDefaults?: SnapshotAlertConfig;
backupDefaults?: BackupAlertConfig;
customRules?: CustomAlertRule[]; customRules?: CustomAlertRule[];
overrides: Record<string, RawOverrideConfig>; // key: resource ID overrides: Record<string, RawOverrideConfig>; // key: resource ID
minimumDelta?: number; minimumDelta?: number;

17
frontend-modern/src/types/api.ts
View file

@ -11,6 +11,8 @@ export interface State {
pbs: PBSInstance[]; pbs: PBSInstance[];
pmg: PMGInstance[]; pmg: PMGInstance[];
pbsBackups: PBSBackup[]; pbsBackups: PBSBackup[];
pmgBackups: PMGBackup[];
backups: Backups;
metrics: Metric[]; metrics: Metric[];
pveBackups: PVEBackups; pveBackups: PVEBackups;
performance: Performance; performance: Performance;
@ -414,6 +416,21 @@ export interface PBSBackup {
owner?: string; owner?: string;
} }
export interface PMGBackup {
id: string;
instance: string;
node: string;
filename: string;
backupTime: string;
size: number;
}
export interface Backups {
pve: PVEBackups;
pbs: PBSBackup[];
pmg: PMGBackup[];
}
export interface PBSBackupJob { export interface PBSBackupJob {
id: string; id: string;
store: string; store: string;

403
internal/alerts/alerts.go
View file

@ -290,6 +290,22 @@ type SnapshotAlertConfig struct {
CriticalDays int `json:"criticalDays"` CriticalDays int `json:"criticalDays"`
} }
// BackupAlertConfig represents backup age alert configuration
type BackupAlertConfig struct {
Enabled bool `json:"enabled"`
WarningDays int `json:"warningDays"`
CriticalDays int `json:"criticalDays"`
}
// GuestLookup describes a guest identity used for snapshot/backup evaluations.
type GuestLookup struct {
Name string
Instance string
Node string
Type string
VMID int
}
// AlertConfig represents the complete alert configuration // AlertConfig represents the complete alert configuration
type AlertConfig struct { type AlertConfig struct {
Enabled bool `json:"enabled"` Enabled bool `json:"enabled"`
@ -300,6 +316,7 @@ type AlertConfig struct {
DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"` DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"`
PMGDefaults PMGThresholdConfig `json:"pmgDefaults"` PMGDefaults PMGThresholdConfig `json:"pmgDefaults"`
SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"` SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"`
BackupDefaults BackupAlertConfig `json:"backupDefaults"`
Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID
CustomRules []CustomAlertRule `json:"customRules,omitempty"` CustomRules []CustomAlertRule `json:"customRules,omitempty"`
Schedule ScheduleConfig `json:"schedule"` Schedule ScheduleConfig `json:"schedule"`
@ -486,6 +503,11 @@ func NewManager() *Manager {
WarningDays: 30, WarningDays: 30,
CriticalDays: 45, CriticalDays: 45,
}, },
BackupDefaults: BackupAlertConfig{
Enabled: false,
WarningDays: 7,
CriticalDays: 14,
},
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80}, StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
MinimumDelta: 2.0, // 2% minimum change MinimumDelta: 2.0, // 2% minimum change
SuppressionWindow: 5, // 5 minutes SuppressionWindow: 5, // 5 minutes
@ -702,6 +724,15 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays { if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
} }
if config.BackupDefaults.WarningDays < 0 {
config.BackupDefaults.WarningDays = 0
}
if config.BackupDefaults.CriticalDays < 0 {
config.BackupDefaults.CriticalDays = 0
}
if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays {
config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays
}
// Ensure minimums for other important fields // Ensure minimums for other important fields
if config.MinimumDelta <= 0 { if config.MinimumDelta <= 0 {
@ -764,6 +795,9 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
if !m.config.SnapshotDefaults.Enabled { if !m.config.SnapshotDefaults.Enabled {
m.clearSnapshotAlertsForInstanceLocked("") m.clearSnapshotAlertsForInstanceLocked("")
} }
if !m.config.BackupDefaults.Enabled {
m.clearBackupAlertsLocked()
}
m.applyGlobalOfflineSettingsLocked() m.applyGlobalOfflineSettingsLocked()
@ -2756,7 +2790,7 @@ func (m *Manager) CheckStorage(storage models.Storage) {
} }
} }
func buildSnapshotGuestKey(instance, node string, vmid int) string { func BuildGuestKey(instance, node string, vmid int) string {
instance = strings.TrimSpace(instance) instance = strings.TrimSpace(instance)
node = strings.TrimSpace(node) node = strings.TrimSpace(node)
if instance == "" { if instance == "" {
@ -2817,7 +2851,7 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID) alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
validAlerts[alertID] = struct{}{} validAlerts[alertID] = struct{}{}
guestKey := buildSnapshotGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID) guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
guestName := strings.TrimSpace(guestNames[guestKey]) guestName := strings.TrimSpace(guestNames[guestKey])
guestType := "VM" guestType := "VM"
@ -2966,6 +3000,356 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
m.mu.Unlock() m.mu.Unlock()
} }
// CheckBackups evaluates storage, PBS, and PMG backups for age-based alerts.
func (m *Manager) CheckBackups(
storageBackups []models.StorageBackup,
pbsBackups []models.PBSBackup,
pmgBackups []models.PMGBackup,
guestsByKey map[string]GuestLookup,
guestsByVMID map[string]GuestLookup,
) {
m.mu.RLock()
enabled := m.config.Enabled
backupCfg := m.config.BackupDefaults
m.mu.RUnlock()
if !enabled || !backupCfg.Enabled {
m.clearBackupAlerts()
return
}
if backupCfg.WarningDays <= 0 && backupCfg.CriticalDays <= 0 {
m.clearBackupAlerts()
return
}
type backupRecord struct {
key string
lookup GuestLookup
fallbackName string
instance string
node string
source string
storage string
datastore string
backupType string
filename string
lastTime time.Time
}
records := make(map[string]*backupRecord)
updateRecord := func(key string, candidate backupRecord) {
if key == "" {
return
}
if existing, ok := records[key]; ok {
if candidate.lastTime.After(existing.lastTime) {
*existing = candidate
}
return
}
record := candidate
records[key] = &record
}
now := time.Now()
for _, backup := range storageBackups {
if backup.Time.IsZero() {
continue
}
key := BuildGuestKey(backup.Instance, backup.Node, backup.VMID)
info := guestsByKey[key]
displayName := info.Name
if displayName == "" {
displayName = fmt.Sprintf("%s-%d", sanitizeAlertKey(backup.Node), backup.VMID)
}
updateRecord(key, backupRecord{
key: key,
lookup: info,
fallbackName: displayName,
instance: backup.Instance,
node: backup.Node,
source: "PVE storage",
storage: backup.Storage,
backupType: backup.Type,
lastTime: backup.Time,
})
}
for _, backup := range pbsBackups {
if backup.BackupTime.IsZero() {
continue
}
if backup.VMID == "0" {
// Host configuration backups - skip from age alerts
continue
}
info, exists := guestsByVMID[backup.VMID]
var key string
var displayName string
var instance string
var node string
if exists && info.Instance != "" && info.Node != "" {
key = BuildGuestKey(info.Instance, info.Node, info.VMID)
displayName = info.Name
instance = info.Instance
node = info.Node
} else {
key = fmt.Sprintf("pbs:%s:%s:%s", backup.Instance, backup.BackupType, backup.VMID)
displayName = fmt.Sprintf("VMID %s", backup.VMID)
instance = fmt.Sprintf("PBS:%s", backup.Instance)
node = backup.Datastore
}
updateRecord(key, backupRecord{
key: key,
lookup: info,
fallbackName: displayName,
instance: instance,
node: node,
source: "PBS",
datastore: backup.Datastore,
backupType: backup.BackupType,
lastTime: backup.BackupTime,
})
}
for _, backup := range pmgBackups {
if backup.BackupTime.IsZero() {
continue
}
instanceLabel := strings.TrimSpace(backup.Instance)
if instanceLabel == "" {
instanceLabel = "PMG"
}
nodeName := strings.TrimSpace(backup.Node)
keyComponent := nodeName
if keyComponent == "" {
keyComponent = strings.TrimSpace(backup.Filename)
}
if keyComponent == "" {
keyComponent = "unknown"
}
displayName := nodeName
if displayName == "" {
displayName = instanceLabel
}
if displayName == "" {
displayName = "PMG gateway"
} else {
displayName = fmt.Sprintf("PMG %s", displayName)
}
instanceField := fmt.Sprintf("PMG:%s", instanceLabel)
key := fmt.Sprintf("pmg:%s:%s", instanceLabel, keyComponent)
updateRecord(key, backupRecord{
key: key,
fallbackName: displayName,
instance: instanceField,
node: nodeName,
source: "PMG",
backupType: "pmg",
filename: backup.Filename,
lastTime: backup.BackupTime,
})
}
if len(records) == 0 {
m.clearBackupAlerts()
return
}
validAlerts := make(map[string]struct{})
for key, record := range records {
age := now.Sub(record.lastTime)
if age < 0 {
continue
}
ageDays := age.Hours() / 24
if ageDays < 0 {
continue
}
ageDaysRounded := math.Round(ageDays*10) / 10
var level AlertLevel
var threshold int
switch {
case backupCfg.CriticalDays > 0 && ageDays >= float64(backupCfg.CriticalDays):
level = AlertLevelCritical
threshold = backupCfg.CriticalDays
case backupCfg.WarningDays > 0 && ageDays >= float64(backupCfg.WarningDays):
level = AlertLevelWarning
threshold = backupCfg.WarningDays
default:
continue
}
alertKey := sanitizeAlertKey(key)
alertID := fmt.Sprintf("backup-age-%s", alertKey)
validAlerts[alertID] = struct{}{}
displayName := record.lookup.Name
if displayName == "" {
displayName = record.fallbackName
}
if displayName == "" {
displayName = "Unknown guest"
}
node := record.node
if node == "" {
node = record.lookup.Node
}
instance := record.instance
if instance == "" {
instance = record.lookup.Instance
}
thresholdTime := record.lastTime.Add(time.Duration(threshold) * 24 * time.Hour)
if thresholdTime.After(now) {
thresholdTime = now
}
var sourceLabel string
switch record.source {
case "PBS":
sourceLabel = fmt.Sprintf("PBS datastore %s on %s", record.datastore, strings.TrimPrefix(instance, "PBS:"))
case "PMG":
if node != "" {
sourceLabel = fmt.Sprintf("PMG node %s", node)
} else {
sourceLabel = "PMG"
}
default:
sourceLabel = fmt.Sprintf("storage %s on %s", record.storage, node)
}
message := fmt.Sprintf(
"%s backup via %s is %.1f days old (threshold: %d days)",
displayName,
sourceLabel,
ageDaysRounded,
threshold,
)
metadata := map[string]interface{}{
"source": record.source,
"lastBackupTime": record.lastTime,
"ageDays": ageDays,
"thresholdDays": threshold,
}
if record.storage != "" {
metadata["storage"] = record.storage
}
if record.datastore != "" {
metadata["datastore"] = record.datastore
}
if record.backupType != "" {
metadata["backupType"] = record.backupType
}
if record.filename != "" {
metadata["filename"] = record.filename
}
m.mu.Lock()
if existing, exists := m.activeAlerts[alertID]; exists {
existing.LastSeen = now
existing.Level = level
existing.Value = ageDays
existing.Threshold = float64(threshold)
existing.Message = message
if existing.Metadata == nil {
existing.Metadata = make(map[string]interface{})
}
for k, v := range metadata {
existing.Metadata[k] = v
}
m.mu.Unlock()
continue
}
alert := &Alert{
ID: alertID,
Type: "backup-age",
Level: level,
ResourceID: alertKey,
ResourceName: fmt.Sprintf("%s backup", displayName),
Node: node,
Instance: instance,
Message: message,
Value: ageDays,
Threshold: float64(threshold),
StartTime: thresholdTime,
LastSeen: now,
Metadata: metadata,
}
m.preserveAlertState(alertID, alert)
m.activeAlerts[alertID] = alert
m.recentAlerts[alertID] = alert
m.historyManager.AddAlert(*alert)
go func() {
defer func() {
if r := recover(); r != nil {
log.Error().Interface("panic", r).Msg("Panic in SaveActiveAlerts goroutine (backup)")
}
}()
if err := m.SaveActiveAlerts(); err != nil {
log.Error().Err(err).Msg("Failed to save active alerts after backup alert creation")
}
}()
if !m.checkRateLimit(alertID) {
m.mu.Unlock()
log.Debug().
Str("alertID", alertID).
Str("resource", displayName).
Msg("Backup alert suppressed due to rate limit")
continue
}
if m.onAlert != nil {
notified := now
alert.LastNotified = &notified
if m.dispatchAlert(alert, true) {
log.Info().
Str("alertID", alertID).
Str("resource", displayName).
Msg("Backup age alert dispatched")
} else {
alert.LastNotified = nil
}
}
m.mu.Unlock()
}
m.mu.Lock()
for alertID, alert := range m.activeAlerts {
if alert == nil || alert.Type != "backup-age" {
continue
}
if _, ok := validAlerts[alertID]; ok {
continue
}
m.clearAlertNoLock(alertID)
}
m.mu.Unlock()
}
// checkZFSPoolHealth checks ZFS pool for errors and degraded state // checkZFSPoolHealth checks ZFS pool for errors and degraded state
func (m *Manager) checkZFSPoolHealth(storage models.Storage) { func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
pool := storage.ZFSPool pool := storage.ZFSPool
@ -6463,3 +6847,18 @@ func (m *Manager) clearSnapshotAlertsForInstanceLocked(instance string) {
m.clearAlertNoLock(alertID) m.clearAlertNoLock(alertID)
} }
} }
func (m *Manager) clearBackupAlerts() {
m.mu.Lock()
m.clearBackupAlertsLocked()
m.mu.Unlock()
}
func (m *Manager) clearBackupAlertsLocked() {
for alertID, alert := range m.activeAlerts {
if alert == nil || alert.Type != "backup-age" {
continue
}
m.clearAlertNoLock(alertID)
}
}

155
internal/alerts/alerts_test.go
View file

@ -3,6 +3,7 @@ package alerts
import ( import (
"fmt" "fmt"
"reflect" "reflect"
"strings"
"testing" "testing"
"time" "time"
@ -273,6 +274,160 @@ func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
} }
} }
func TestCheckBackupsCreatesAndClearsAlerts(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
m.mu.Lock()
m.config.Enabled = true
m.config.BackupDefaults = BackupAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
}
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
now := time.Now()
storageBackups := []models.StorageBackup{
{
ID: "inst-node-100-backup",
Storage: "local",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 100,
Time: now.Add(-15 * 24 * time.Hour),
},
}
key := BuildGuestKey("inst", "node", 100)
guestsByKey := map[string]GuestLookup{
key: {
Name: "app-server",
Instance: "inst",
Node: "node",
Type: "qemu",
VMID: 100,
},
}
guestsByVMID := map[string]GuestLookup{
"100": guestsByKey[key],
}
m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID)
m.mu.RLock()
alert, exists := m.activeAlerts["backup-age-"+sanitizeAlertKey(key)]
m.mu.RUnlock()
if !exists {
t.Fatalf("expected backup age alert to be created")
}
if alert.Level != AlertLevelCritical {
t.Fatalf("expected critical backup alert, got %s", alert.Level)
}
// Recent backup clears alert
storageBackups[0].Time = now
m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID)
m.mu.RLock()
_, exists = m.activeAlerts["backup-age-"+sanitizeAlertKey(key)]
m.mu.RUnlock()
if exists {
t.Fatalf("expected backup-age alert to clear after fresh backup")
}
}
func TestCheckBackupsHandlesPbsOnlyGuests(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
m.mu.Lock()
m.config.Enabled = true
m.config.BackupDefaults = BackupAlertConfig{
Enabled: true,
WarningDays: 3,
CriticalDays: 5,
}
m.mu.Unlock()
now := time.Now()
pbsBackups := []models.PBSBackup{
{
ID: "pbs-backup-999-0",
Instance: "pbs-main",
Datastore: "backup-store",
BackupType: "qemu",
VMID: "999",
BackupTime: now.Add(-6 * 24 * time.Hour),
},
}
m.CheckBackups(nil, pbsBackups, nil, map[string]GuestLookup{}, map[string]GuestLookup{})
m.mu.RLock()
found := false
for id, alert := range m.activeAlerts {
if strings.HasPrefix(id, "backup-age-") {
found = true
if alert.Level != AlertLevelCritical {
t.Fatalf("expected PBS backup alert to be critical")
}
break
}
}
m.mu.RUnlock()
if !found {
t.Fatalf("expected PBS backup alert to be created")
}
}
func TestCheckBackupsHandlesPmgBackups(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
m.mu.Lock()
m.config.Enabled = true
m.config.BackupDefaults = BackupAlertConfig{
Enabled: true,
WarningDays: 5,
CriticalDays: 7,
}
m.mu.Unlock()
now := time.Now()
pmgBackups := []models.PMGBackup{
{
ID: "pmg-backup-mail-01",
Instance: "mail",
Node: "mail-gateway",
Filename: "pmg-backup_2024-01-01.tgz",
BackupTime: now.Add(-8 * 24 * time.Hour),
Size: 123456,
},
}
m.CheckBackups(nil, nil, pmgBackups, map[string]GuestLookup{}, map[string]GuestLookup{})
m.mu.RLock()
found := false
for id, alert := range m.activeAlerts {
if strings.HasPrefix(id, "backup-age-") {
found = true
if alert.Level != AlertLevelCritical {
t.Fatalf("expected PMG backup alert to be critical")
}
break
}
}
m.mu.RUnlock()
if !found {
t.Fatalf("expected PMG backup alert to be created")
}
}
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) { func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
m := NewManager() m := NewManager()

26
internal/api/router.go
View file

@ -2452,16 +2452,24 @@ func (r *Router) handleBackups(w http.ResponseWriter, req *http.Request) {
// Get current state // Get current state
state := r.monitor.GetState() state := r.monitor.GetState()
// Return backup data structure
backups := map[string]interface{}{
"backupTasks": state.PVEBackups.BackupTasks,
"storageBackups": state.PVEBackups.StorageBackups,
"guestSnapshots": state.PVEBackups.GuestSnapshots,
"pbsBackups": state.PBSBackups,
}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(backups) json.NewEncoder(w).Encode(struct {
Backups models.Backups `json:"backups"`
PVEBackups models.PVEBackups `json:"pveBackups"`
PBSBackups []models.PBSBackup `json:"pbsBackups"`
PMGBackups []models.PMGBackup `json:"pmgBackups"`
BackupTasks []models.BackupTask `json:"backupTasks"`
Storage []models.StorageBackup `json:"storageBackups"`
GuestSnaps []models.GuestSnapshot `json:"guestSnapshots"`
}{
Backups: state.Backups,
PVEBackups: state.PVEBackups,
PBSBackups: state.PBSBackups,
PMGBackups: state.PMGBackups,
BackupTasks: state.PVEBackups.BackupTasks,
Storage: state.PVEBackups.StorageBackups,
GuestSnaps: state.PVEBackups.GuestSnapshots,
})
} }
// handleBackupsPVE handles PVE backup requests // handleBackupsPVE handles PVE backup requests

2
internal/api/types.go
View file

@ -58,6 +58,8 @@ type StateResponse struct {
PBSInstances []models.PBSInstance `json:"pbs"` PBSInstances []models.PBSInstance `json:"pbs"`
PMGInstances []models.PMGInstance `json:"pmg"` PMGInstances []models.PMGInstance `json:"pmg"`
PBSBackups []models.PBSBackup `json:"pbsBackups"` PBSBackups []models.PBSBackup `json:"pbsBackups"`
PMGBackups []models.PMGBackup `json:"pmgBackups"`
Backups models.Backups `json:"backups"`
Metrics []models.Metric `json:"metrics"` Metrics []models.Metric `json:"metrics"`
PVEBackups models.PVEBackups `json:"pveBackups"` PVEBackups models.PVEBackups `json:"pveBackups"`
Performance models.Performance `json:"performance"` Performance models.Performance `json:"performance"`

23
internal/config/persistence.go
View file

@ -194,6 +194,15 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays { if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
} }
if config.BackupDefaults.WarningDays < 0 {
config.BackupDefaults.WarningDays = 0
}
if config.BackupDefaults.CriticalDays < 0 {
config.BackupDefaults.CriticalDays = 0
}
if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays {
config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays
}
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes) config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
data, err := json.MarshalIndent(config, "", " ") data, err := json.MarshalIndent(config, "", " ")
@ -251,6 +260,11 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
WarningDays: 30, WarningDays: 30,
CriticalDays: 45, CriticalDays: 45,
}, },
BackupDefaults: alerts.BackupAlertConfig{
Enabled: false,
WarningDays: 7,
CriticalDays: 14,
},
Overrides: make(map[string]alerts.ThresholdConfig), Overrides: make(map[string]alerts.ThresholdConfig),
}, nil }, nil
} }
@ -310,6 +324,15 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays { if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
} }
if config.BackupDefaults.WarningDays < 0 {
config.BackupDefaults.WarningDays = 0
}
if config.BackupDefaults.CriticalDays < 0 {
config.BackupDefaults.CriticalDays = 0
}
if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays {
config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays
}
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds) config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes) config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)

14
internal/config/persistence_test.go
View file

@ -140,6 +140,11 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
WarningDays: 20, WarningDays: 20,
CriticalDays: 10, CriticalDays: 10,
}, },
BackupDefaults: alerts.BackupAlertConfig{
Enabled: true,
WarningDays: 12,
CriticalDays: 8,
},
NodeDefaults: alerts.ThresholdConfig{ NodeDefaults: alerts.ThresholdConfig{
Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0}, Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0},
}, },
@ -174,6 +179,15 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
if loaded.NodeDefaults.Temperature.Trigger != 80 || loaded.NodeDefaults.Temperature.Clear != 75 { if loaded.NodeDefaults.Temperature.Trigger != 80 || loaded.NodeDefaults.Temperature.Clear != 75 {
t.Fatalf("expected temperature defaults 80/75, got %+v", loaded.NodeDefaults.Temperature) t.Fatalf("expected temperature defaults 80/75, got %+v", loaded.NodeDefaults.Temperature)
} }
if !loaded.BackupDefaults.Enabled {
t.Fatalf("expected backup defaults to remain enabled")
}
if loaded.BackupDefaults.WarningDays != 8 {
t.Fatalf("expected backup warning normalized to 8, got %d", loaded.BackupDefaults.WarningDays)
}
if loaded.BackupDefaults.CriticalDays != 8 {
t.Fatalf("expected backup critical normalized to 8, got %d", loaded.BackupDefaults.CriticalDays)
}
expectedPrefixes := []string{"Runner"} expectedPrefixes := []string{"Runner"}
if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) { if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) {
t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes) t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes)

47
internal/mock/generator.go
View file

@ -329,6 +329,12 @@ func GenerateMockData(config MockConfig) models.StateSnapshot {
StorageBackups: generateBackups(data.VMs, data.Containers), StorageBackups: generateBackups(data.VMs, data.Containers),
GuestSnapshots: generateSnapshots(data.VMs, data.Containers), GuestSnapshots: generateSnapshots(data.VMs, data.Containers),
} }
data.PMGBackups = extractPMGBackups(data.PVEBackups.StorageBackups)
data.Backups = models.Backups{
PVE: data.PVEBackups,
PBS: append([]models.PBSBackup(nil), data.PBSBackups...),
PMG: append([]models.PMGBackup(nil), data.PMGBackups...),
}
// Calculate stats // Calculate stats
data.Stats.StartTime = time.Now() data.Stats.StartTime = time.Now()
@ -2100,6 +2106,47 @@ func generateBackups(vms []models.VM, containers []models.Container) []models.St
return backups return backups
} }
func extractPMGBackups(storageBackups []models.StorageBackup) []models.PMGBackup {
pmgBackups := make([]models.PMGBackup, 0)
for _, backup := range storageBackups {
if !strings.EqualFold(backup.Type, "host") {
continue
}
format := strings.ToLower(backup.Format)
notes := strings.ToLower(backup.Notes)
if !strings.Contains(format, "pmg") && !strings.Contains(notes, "pmg") {
continue
}
filename := backup.Volid
if filename == "" {
filename = backup.Notes
}
instance := backup.Instance
if instance == "" && backup.Node != "" {
instance = fmt.Sprintf("PMG:%s", backup.Node)
}
if instance == "" {
instance = "PMG:mock"
}
pmgBackups = append(pmgBackups, models.PMGBackup{
ID: backup.ID,
Instance: instance,
Node: backup.Node,
Filename: filename,
BackupTime: backup.Time,
Size: backup.Size,
})
}
sort.Slice(pmgBackups, func(i, j int) bool {
return pmgBackups[i].BackupTime.After(pmgBackups[j].BackupTime)
})
return pmgBackups
}
// generatePBSInstances generates mock PBS instances // generatePBSInstances generates mock PBS instances
func generatePBSInstances() []models.PBSInstance { func generatePBSInstances() []models.PBSInstance {
pbsInstances := []models.PBSInstance{ pbsInstances := []models.PBSInstance{

10
internal/mock/integration.go
View file

@ -310,6 +310,7 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot {
PhysicalDisks: append([]models.PhysicalDisk(nil), state.PhysicalDisks...), PhysicalDisks: append([]models.PhysicalDisk(nil), state.PhysicalDisks...),
PBSInstances: append([]models.PBSInstance(nil), state.PBSInstances...), PBSInstances: append([]models.PBSInstance(nil), state.PBSInstances...),
PBSBackups: append([]models.PBSBackup(nil), state.PBSBackups...), PBSBackups: append([]models.PBSBackup(nil), state.PBSBackups...),
PMGBackups: append([]models.PMGBackup(nil), state.PMGBackups...),
Metrics: append([]models.Metric(nil), state.Metrics...), Metrics: append([]models.Metric(nil), state.Metrics...),
Performance: state.Performance, Performance: state.Performance,
Stats: state.Stats, Stats: state.Stats,
@ -324,6 +325,15 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot {
StorageBackups: append([]models.StorageBackup(nil), state.PVEBackups.StorageBackups...), StorageBackups: append([]models.StorageBackup(nil), state.PVEBackups.StorageBackups...),
GuestSnapshots: append([]models.GuestSnapshot(nil), state.PVEBackups.GuestSnapshots...), GuestSnapshots: append([]models.GuestSnapshot(nil), state.PVEBackups.GuestSnapshots...),
} }
copyState.Backups = models.Backups{
PVE: models.PVEBackups{
BackupTasks: append([]models.BackupTask(nil), state.Backups.PVE.BackupTasks...),
StorageBackups: append([]models.StorageBackup(nil), state.Backups.PVE.StorageBackups...),
GuestSnapshots: append([]models.GuestSnapshot(nil), state.Backups.PVE.GuestSnapshots...),
},
PBS: append([]models.PBSBackup(nil), state.Backups.PBS...),
PMG: append([]models.PMGBackup(nil), state.Backups.PMG...),
}
for k, v := range state.ConnectionHealth { for k, v := range state.ConnectionHealth {
copyState.ConnectionHealth[k] = v copyState.ConnectionHealth[k] = v

81
internal/models/models.go
View file

@ -20,6 +20,8 @@ type State struct {
PBSInstances []PBSInstance `json:"pbs"` PBSInstances []PBSInstance `json:"pbs"`
PMGInstances []PMGInstance `json:"pmg"` PMGInstances []PMGInstance `json:"pmg"`
PBSBackups []PBSBackup `json:"pbsBackups"` PBSBackups []PBSBackup `json:"pbsBackups"`
PMGBackups []PMGBackup `json:"pmgBackups"`
Backups Backups `json:"backups"`
Metrics []Metric `json:"metrics"` Metrics []Metric `json:"metrics"`
PVEBackups PVEBackups `json:"pveBackups"` PVEBackups PVEBackups `json:"pveBackups"`
Performance Performance `json:"performance"` Performance Performance `json:"performance"`
@ -461,6 +463,23 @@ type PMGNodeStatus struct {
QueueStatus *PMGQueueStatus `json:"queueStatus,omitempty"` // Postfix queue status for this node QueueStatus *PMGQueueStatus `json:"queueStatus,omitempty"` // Postfix queue status for this node
} }
// PMGBackup represents a configuration backup generated by a PMG node.
type PMGBackup struct {
ID string `json:"id"`
Instance string `json:"instance"`
Node string `json:"node"`
Filename string `json:"filename"`
BackupTime time.Time `json:"backupTime"`
Size int64 `json:"size"`
}
// Backups aggregates backup collections by source type.
type Backups struct {
PVE PVEBackups `json:"pve"`
PBS []PBSBackup `json:"pbs"`
PMG []PMGBackup `json:"pmg"`
}
// PMGMailStats summarizes aggregated mail statistics for a timeframe // PMGMailStats summarizes aggregated mail statistics for a timeframe
type PMGMailStats struct { type PMGMailStats struct {
Timeframe string `json:"timeframe"` Timeframe string `json:"timeframe"`
@ -677,7 +696,13 @@ type Stats struct {
// NewState creates a new State instance // NewState creates a new State instance
func NewState() *State { func NewState() *State {
return &State{ pveBackups := PVEBackups{
BackupTasks: make([]BackupTask, 0),
StorageBackups: make([]StorageBackup, 0),
GuestSnapshots: make([]GuestSnapshot, 0),
}
state := &State{
Nodes: make([]Node, 0), Nodes: make([]Node, 0),
VMs: make([]VM, 0), VMs: make([]VM, 0),
Containers: make([]Container, 0), Containers: make([]Container, 0),
@ -687,17 +712,31 @@ func NewState() *State {
PBSInstances: make([]PBSInstance, 0), PBSInstances: make([]PBSInstance, 0),
PMGInstances: make([]PMGInstance, 0), PMGInstances: make([]PMGInstance, 0),
PBSBackups: make([]PBSBackup, 0), PBSBackups: make([]PBSBackup, 0),
Metrics: make([]Metric, 0), PMGBackups: make([]PMGBackup, 0),
PVEBackups: PVEBackups{ Backups: Backups{
BackupTasks: make([]BackupTask, 0), PVE: pveBackups,
StorageBackups: make([]StorageBackup, 0), PBS: make([]PBSBackup, 0),
GuestSnapshots: make([]GuestSnapshot, 0), PMG: make([]PMGBackup, 0),
}, },
Metrics: make([]Metric, 0),
PVEBackups: pveBackups,
ConnectionHealth: make(map[string]bool), ConnectionHealth: make(map[string]bool),
ActiveAlerts: make([]Alert, 0), ActiveAlerts: make([]Alert, 0),
RecentlyResolved: make([]ResolvedAlert, 0), RecentlyResolved: make([]ResolvedAlert, 0),
LastUpdate: time.Now(), LastUpdate: time.Now(),
} }
state.syncBackupsLocked()
return state
}
// syncBackupsLocked updates the aggregated backups structure.
func (s *State) syncBackupsLocked() {
s.Backups = Backups{
PVE: s.PVEBackups,
PBS: append([]PBSBackup(nil), s.PBSBackups...),
PMG: append([]PMGBackup(nil), s.PMGBackups...),
}
} }
// UpdateActiveAlerts updates the active alerts in the state // UpdateActiveAlerts updates the active alerts in the state
@ -1211,6 +1250,7 @@ func (s *State) UpdateBackupTasksForInstance(instanceName string, tasks []Backup
}) })
s.PVEBackups.BackupTasks = newTasks s.PVEBackups.BackupTasks = newTasks
s.syncBackupsLocked()
s.LastUpdate = time.Now() s.LastUpdate = time.Now()
} }
@ -1245,6 +1285,7 @@ func (s *State) UpdateStorageBackupsForInstance(instanceName string, backups []S
}) })
s.PVEBackups.StorageBackups = newBackups s.PVEBackups.StorageBackups = newBackups
s.syncBackupsLocked()
s.LastUpdate = time.Now() s.LastUpdate = time.Now()
} }
@ -1279,6 +1320,7 @@ func (s *State) UpdateGuestSnapshotsForInstance(instanceName string, snapshots [
}) })
s.PVEBackups.GuestSnapshots = newSnapshots s.PVEBackups.GuestSnapshots = newSnapshots
s.syncBackupsLocked()
s.LastUpdate = time.Now() s.LastUpdate = time.Now()
} }
@ -1326,5 +1368,32 @@ func (s *State) UpdatePBSBackups(instanceName string, backups []PBSBackup) {
}) })
s.PBSBackups = newBackups s.PBSBackups = newBackups
s.syncBackupsLocked()
s.LastUpdate = time.Now()
}
// UpdatePMGBackups updates PMG backups for a specific instance.
func (s *State) UpdatePMGBackups(instanceName string, backups []PMGBackup) {
s.mu.Lock()
defer s.mu.Unlock()
combined := make([]PMGBackup, 0, len(s.PMGBackups)+len(backups))
for _, backup := range s.PMGBackups {
if backup.Instance != instanceName {
combined = append(combined, backup)
}
}
if len(backups) > 0 {
combined = append(combined, backups...)
}
if len(combined) > 1 {
sort.Slice(combined, func(i, j int) bool {
return combined[i].BackupTime.After(combined[j].BackupTime)
})
}
s.PMGBackups = combined
s.syncBackupsLocked()
s.LastUpdate = time.Now() s.LastUpdate = time.Now()
} }

3
internal/models/models_frontend.go
View file

@ -230,6 +230,9 @@ type StateFrontend struct {
PhysicalDisks []PhysicalDisk `json:"physicalDisks"` PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
PBS []PBSInstance `json:"pbs"` // Keep as is PBS []PBSInstance `json:"pbs"` // Keep as is
PMG []PMGInstance `json:"pmg"` PMG []PMGInstance `json:"pmg"`
PBSBackups []PBSBackup `json:"pbsBackups"`
PMGBackups []PMGBackup `json:"pmgBackups"`
Backups Backups `json:"backups"`
ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts
Metrics map[string]any `json:"metrics"` // Empty object for now Metrics map[string]any `json:"metrics"` // Empty object for now
PVEBackups PVEBackups `json:"pveBackups"` // Keep as is PVEBackups PVEBackups `json:"pveBackups"` // Keep as is

27
internal/models/state_snapshot.go
View file

@ -14,6 +14,8 @@ type StateSnapshot struct {
PBSInstances []PBSInstance `json:"pbs"` PBSInstances []PBSInstance `json:"pbs"`
PMGInstances []PMGInstance `json:"pmg"` PMGInstances []PMGInstance `json:"pmg"`
PBSBackups []PBSBackup `json:"pbsBackups"` PBSBackups []PBSBackup `json:"pbsBackups"`
PMGBackups []PMGBackup `json:"pmgBackups"`
Backups Backups `json:"backups"`
Metrics []Metric `json:"metrics"` Metrics []Metric `json:"metrics"`
PVEBackups PVEBackups `json:"pveBackups"` PVEBackups PVEBackups `json:"pveBackups"`
Performance Performance `json:"performance"` Performance Performance `json:"performance"`
@ -29,6 +31,14 @@ func (s *State) GetSnapshot() StateSnapshot {
s.mu.RLock() s.mu.RLock()
defer s.mu.RUnlock() defer s.mu.RUnlock()
pbsBackups := append([]PBSBackup{}, s.PBSBackups...)
pmgBackups := append([]PMGBackup{}, s.PMGBackups...)
pveBackups := PVEBackups{
BackupTasks: append([]BackupTask{}, s.PVEBackups.BackupTasks...),
StorageBackups: append([]StorageBackup{}, s.PVEBackups.StorageBackups...),
GuestSnapshots: append([]GuestSnapshot{}, s.PVEBackups.GuestSnapshots...),
}
// Create a snapshot without mutex // Create a snapshot without mutex
snapshot := StateSnapshot{ snapshot := StateSnapshot{
Nodes: append([]Node{}, s.Nodes...), Nodes: append([]Node{}, s.Nodes...),
@ -40,13 +50,15 @@ func (s *State) GetSnapshot() StateSnapshot {
PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...), PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...),
PBSInstances: append([]PBSInstance{}, s.PBSInstances...), PBSInstances: append([]PBSInstance{}, s.PBSInstances...),
PMGInstances: append([]PMGInstance{}, s.PMGInstances...), PMGInstances: append([]PMGInstance{}, s.PMGInstances...),
PBSBackups: append([]PBSBackup{}, s.PBSBackups...), PBSBackups: pbsBackups,
Metrics: append([]Metric{}, s.Metrics...), PMGBackups: pmgBackups,
PVEBackups: PVEBackups{ Backups: Backups{
BackupTasks: append([]BackupTask{}, s.PVEBackups.BackupTasks...), PVE: pveBackups,
StorageBackups: append([]StorageBackup{}, s.PVEBackups.StorageBackups...), PBS: pbsBackups,
GuestSnapshots: append([]GuestSnapshot{}, s.PVEBackups.GuestSnapshots...), PMG: pmgBackups,
}, },
Metrics: append([]Metric{}, s.Metrics...),
PVEBackups: pveBackups,
Performance: s.Performance, Performance: s.Performance,
ConnectionHealth: make(map[string]bool), ConnectionHealth: make(map[string]bool),
Stats: s.Stats, Stats: s.Stats,
@ -110,6 +122,9 @@ func (s StateSnapshot) ToFrontend() StateFrontend {
PhysicalDisks: s.PhysicalDisks, PhysicalDisks: s.PhysicalDisks,
PBS: s.PBSInstances, PBS: s.PBSInstances,
PMG: s.PMGInstances, PMG: s.PMGInstances,
PBSBackups: s.PBSBackups,
PMGBackups: s.PMGBackups,
Backups: s.Backups,
ActiveAlerts: s.ActiveAlerts, ActiveAlerts: s.ActiveAlerts,
Metrics: make(map[string]any), Metrics: make(map[string]any),
PVEBackups: s.PVEBackups, PVEBackups: s.PVEBackups,

153
internal/monitoring/monitor.go
View file

@ -1707,7 +1707,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
Int("vms", len(state.VMs)). Int("vms", len(state.VMs)).
Int("containers", len(state.Containers)). Int("containers", len(state.Containers)).
Int("pbs", len(state.PBSInstances)). Int("pbs", len(state.PBSInstances)).
Int("pbsBackups", len(state.PBSBackups)). Int("pbsBackups", len(state.Backups.PBS)).
Int("physicalDisks", len(state.PhysicalDisks)). Int("physicalDisks", len(state.PhysicalDisks)).
Msg("Broadcasting state update (ticker)") Msg("Broadcasting state update (ticker)")
// Convert to frontend format before broadcasting (converts time.Time to int64, etc.) // Convert to frontend format before broadcasting (converts time.Time to int64, etc.)
@ -5167,7 +5167,11 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
cluster, err := client.GetClusterStatus(ctx, true) cluster, err := client.GetClusterStatus(ctx, true)
if err != nil { if err != nil {
log.Debug().Err(err).Str("instance", instanceName).Msg("Failed to retrieve PMG cluster status") log.Debug().Err(err).Str("instance", instanceName).Msg("Failed to retrieve PMG cluster status")
} else if len(cluster) > 0 { }
backupNodes := make(map[string]struct{})
if len(cluster) > 0 {
nodes := make([]models.PMGNodeStatus, 0, len(cluster)) nodes := make([]models.PMGNodeStatus, 0, len(cluster))
for _, entry := range cluster { for _, entry := range cluster {
status := strings.ToLower(strings.TrimSpace(entry.Type)) status := strings.ToLower(strings.TrimSpace(entry.Type))
@ -5180,6 +5184,8 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
Role: entry.Type, Role: entry.Type,
} }
backupNodes[entry.Name] = struct{}{}
// Fetch queue status for this node // Fetch queue status for this node
if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil { if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil {
log.Debug().Err(qErr). log.Debug().Err(qErr).
@ -5204,6 +5210,53 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
pmgInst.Nodes = nodes pmgInst.Nodes = nodes
} }
if len(backupNodes) == 0 {
trimmed := strings.TrimSpace(instanceName)
if trimmed != "" {
backupNodes[trimmed] = struct{}{}
}
}
pmgBackups := make([]models.PMGBackup, 0)
seenBackupIDs := make(map[string]struct{})
for nodeName := range backupNodes {
if ctx.Err() != nil {
break
}
backups, backupErr := client.ListBackups(ctx, nodeName)
if backupErr != nil {
log.Debug().Err(backupErr).
Str("instance", instanceName).
Str("node", nodeName).
Msg("Failed to list PMG configuration backups")
continue
}
for _, b := range backups {
backupTime := time.Unix(b.Timestamp, 0)
id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, b.Timestamp)
if _, exists := seenBackupIDs[id]; exists {
continue
}
seenBackupIDs[id] = struct{}{}
pmgBackups = append(pmgBackups, models.PMGBackup{
ID: id,
Instance: instanceName,
Node: nodeName,
Filename: b.Filename,
BackupTime: backupTime,
Size: b.Size,
})
}
}
log.Debug().
Str("instance", instanceName).
Int("backupCount", len(pmgBackups)).
Msg("PMG backups polled")
if stats, err := client.GetMailStatistics(ctx, "day"); err != nil { if stats, err := client.GetMailStatistics(ctx, "day"); err != nil {
log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to fetch PMG mail statistics") log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to fetch PMG mail statistics")
} else if stats != nil { } else if stats != nil {
@ -5279,6 +5332,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
} }
pmgInst.Quarantine = &quarantine pmgInst.Quarantine = &quarantine
m.state.UpdatePMGBackups(instanceName, pmgBackups)
m.state.UpdatePMGInstance(pmgInst) m.state.UpdatePMGInstance(pmgInst)
log.Info(). log.Info().
Str("instance", instanceName). Str("instance", instanceName).
@ -5620,6 +5674,24 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// Update state with storage backups for this instance // Update state with storage backups for this instance
m.state.UpdateStorageBackupsForInstance(instanceName, allBackups) m.state.UpdateStorageBackupsForInstance(instanceName, allBackups)
if m.alertManager != nil {
snapshot := m.state.GetSnapshot()
guestsByKey, guestsByVMID := buildGuestLookups(snapshot)
pveStorage := snapshot.Backups.PVE.StorageBackups
if len(pveStorage) == 0 && len(snapshot.PVEBackups.StorageBackups) > 0 {
pveStorage = snapshot.PVEBackups.StorageBackups
}
pbsBackups := snapshot.Backups.PBS
if len(pbsBackups) == 0 && len(snapshot.PBSBackups) > 0 {
pbsBackups = snapshot.PBSBackups
}
pmgBackups := snapshot.Backups.PMG
if len(pmgBackups) == 0 && len(snapshot.PMGBackups) > 0 {
pmgBackups = snapshot.PMGBackups
}
m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID)
}
log.Debug(). log.Debug().
Str("instance", instanceName). Str("instance", instanceName).
Int("count", len(allBackups)). Int("count", len(allBackups)).
@ -5636,6 +5708,49 @@ func shouldPreserveBackups(nodeCount int, hadSuccessfulNode bool, storagesWithBa
return false return false
} }
func buildGuestLookups(snapshot models.StateSnapshot) (map[string]alerts.GuestLookup, map[string]alerts.GuestLookup) {
byKey := make(map[string]alerts.GuestLookup)
byVMID := make(map[string]alerts.GuestLookup)
for _, vm := range snapshot.VMs {
info := alerts.GuestLookup{
Name: vm.Name,
Instance: vm.Instance,
Node: vm.Node,
Type: vm.Type,
VMID: vm.VMID,
}
key := alerts.BuildGuestKey(vm.Instance, vm.Node, vm.VMID)
byKey[key] = info
vmidKey := fmt.Sprintf("%d", vm.VMID)
if _, exists := byVMID[vmidKey]; !exists {
byVMID[vmidKey] = info
}
}
for _, ct := range snapshot.Containers {
info := alerts.GuestLookup{
Name: ct.Name,
Instance: ct.Instance,
Node: ct.Node,
Type: ct.Type,
VMID: int(ct.VMID),
}
key := alerts.BuildGuestKey(ct.Instance, ct.Node, int(ct.VMID))
if _, exists := byKey[key]; !exists {
byKey[key] = info
}
vmidKey := fmt.Sprintf("%d", ct.VMID)
if _, exists := byVMID[vmidKey]; !exists {
byVMID[vmidKey] = info
}
}
return byKey, byVMID
}
func (m *Monitor) calculateBackupOperationTimeout(instanceName string) time.Duration { func (m *Monitor) calculateBackupOperationTimeout(instanceName string) time.Duration {
const ( const (
minTimeout = 2 * time.Minute minTimeout = 2 * time.Minute
@ -6042,6 +6157,7 @@ func (m *Monitor) removeFailedPMGInstance(instanceName string) {
} }
m.state.UpdatePMGInstances(updated) m.state.UpdatePMGInstances(updated)
m.state.UpdatePMGBackups(instanceName, nil)
m.state.SetConnectionHealth("pmg-"+instanceName, false) m.state.SetConnectionHealth("pmg-"+instanceName, false)
} }
@ -6149,6 +6265,24 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
// Update state // Update state
m.state.UpdatePBSBackups(instanceName, allBackups) m.state.UpdatePBSBackups(instanceName, allBackups)
if m.alertManager != nil {
snapshot := m.state.GetSnapshot()
guestsByKey, guestsByVMID := buildGuestLookups(snapshot)
pveStorage := snapshot.Backups.PVE.StorageBackups
if len(pveStorage) == 0 && len(snapshot.PVEBackups.StorageBackups) > 0 {
pveStorage = snapshot.PVEBackups.StorageBackups
}
pbsBackups := snapshot.Backups.PBS
if len(pbsBackups) == 0 && len(snapshot.PBSBackups) > 0 {
pbsBackups = snapshot.PBSBackups
}
pmgBackups := snapshot.Backups.PMG
if len(pmgBackups) == 0 && len(snapshot.PMGBackups) > 0 {
pmgBackups = snapshot.PMGBackups
}
m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID)
}
} }
// checkMockAlerts checks alerts for mock data // checkMockAlerts checks alerts for mock data
@ -6188,6 +6322,21 @@ func (m *Monitor) checkMockAlerts() {
Msg("Collecting resources for alert cleanup in mock mode") Msg("Collecting resources for alert cleanup in mock mode")
m.alertManager.CleanupAlertsForNodes(existingNodes) m.alertManager.CleanupAlertsForNodes(existingNodes)
guestsByKey, guestsByVMID := buildGuestLookups(state)
pveStorage := state.Backups.PVE.StorageBackups
if len(pveStorage) == 0 && len(state.PVEBackups.StorageBackups) > 0 {
pveStorage = state.PVEBackups.StorageBackups
}
pbsBackups := state.Backups.PBS
if len(pbsBackups) == 0 && len(state.PBSBackups) > 0 {
pbsBackups = state.PBSBackups
}
pmgBackups := state.Backups.PMG
if len(pmgBackups) == 0 && len(state.PMGBackups) > 0 {
pmgBackups = state.PMGBackups
}
m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID)
// Limit how many guests we check per cycle to prevent blocking with large datasets // Limit how many guests we check per cycle to prevent blocking with large datasets
const maxGuestsPerCycle = 50 const maxGuestsPerCycle = 50
guestsChecked := 0 guestsChecked := 0

14
internal/monitoring/monitor_pmg_test.go
View file

@ -66,6 +66,11 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`) fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`)
case "/api2/json/nodes/mail-gateway/backup":
w.Header().Set("Content-Type", "application/json")
timestamp := time.Now().Add(-8 * 24 * time.Hour).Unix()
fmt.Fprintf(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":%d}]}`, timestamp)
default: default:
t.Fatalf("unexpected path: %s", r.URL.Path) t.Fatalf("unexpected path: %s", r.URL.Path)
} }
@ -144,6 +149,15 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {
if failures := mon.authFailures["pmg-primary"]; failures != 0 { if failures := mon.authFailures["pmg-primary"]; failures != 0 {
t.Fatalf("expected no auth failures tracked, got %d", failures) t.Fatalf("expected no auth failures tracked, got %d", failures)
} }
if len(snapshot.PMGBackups) != 1 {
t.Fatalf("expected 1 PMG backup in state, got %d", len(snapshot.PMGBackups))
}
pmgBackup := snapshot.PMGBackups[0]
if pmgBackup.Node != "mail-gateway" {
t.Fatalf("expected PMG backup node mail-gateway, got %s", pmgBackup.Node)
}
} }
func TestPollPMGInstanceRecordsAuthFailures(t *testing.T) { func TestPollPMGInstanceRecordsAuthFailures(t *testing.T) {

17
pkg/pmg/client.go
View file

@ -119,6 +119,13 @@ type QueueStatusEntry struct {
OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds
} }
// BackupEntry represents a PMG configuration backup stored on a node.
type BackupEntry struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
Timestamp int64 `json:"timestamp"`
}
func NewClient(cfg ClientConfig) (*Client, error) { func NewClient(cfg ClientConfig) (*Client, error) {
if cfg.Timeout <= 0 { if cfg.Timeout <= 0 {
cfg.Timeout = 60 * time.Second cfg.Timeout = 60 * time.Second
@ -459,3 +466,13 @@ func (c *Client) GetQueueStatus(ctx context.Context, node string) (*QueueStatusE
} }
return &resp.Data, nil return &resp.Data, nil
} }
// ListBackups returns configuration backup archives available on a PMG node.
func (c *Client) ListBackups(ctx context.Context, node string) ([]BackupEntry, error) {
path := fmt.Sprintf("/nodes/%s/backup", url.PathEscape(node))
var resp apiResponse[[]BackupEntry]
if err := c.getJSON(ctx, path, nil, &resp); err != nil {
return nil, err
}
return resp.Data, nil
}

52
pkg/pmg/client_test.go
View file

@ -231,3 +231,55 @@ func TestClientUsesTokenAuthorizationHeader(t *testing.T) {
t.Fatalf("expected authorization header %q, got %q", expected, authHeader) t.Fatalf("expected authorization header %q, got %q", expected, authHeader)
} }
} }
func TestListBackups(t *testing.T) {
t.Parallel()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api2/json/access/ticket":
if r.Method != http.MethodPost {
t.Fatalf("expected POST for auth, got %s", r.Method)
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"ticket":"ticket789","CSRFPreventionToken":"csrf789"}}`)
case "/api2/json/nodes/node1/backup":
if r.Method != http.MethodGet {
t.Fatalf("expected GET for backup listing, got %s", r.Method)
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":1704096000}]}`)
default:
t.Fatalf("unexpected request path: %s", r.URL.Path)
}
}))
defer server.Close()
client, err := NewClient(ClientConfig{
Host: server.URL,
User: "api@pmg",
Password: "secret",
VerifySSL: false,
})
if err != nil {
t.Fatalf("unexpected error creating client: %v", err)
}
backups, err := client.ListBackups(context.Background(), "node1")
if err != nil {
t.Fatalf("ListBackups failed: %v", err)
}
if len(backups) != 1 {
t.Fatalf("expected 1 backup, got %d", len(backups))
}
backup := backups[0]
if backup.Filename != "pmg-backup_2024-01-01.tgz" {
t.Fatalf("unexpected filename: %s", backup.Filename)
}
if backup.Size != 123456 {
t.Fatalf("unexpected size: %d", backup.Size)
}
if backup.Timestamp != 1704096000 {
t.Fatalf("unexpected timestamp: %d", backup.Timestamp)
}
}