diff --git a/frontend-modern/src/App.tsx b/frontend-modern/src/App.tsx index 59eb6f9..987156a 100644 --- a/frontend-modern/src/App.tsx +++ b/frontend-modern/src/App.tsx @@ -98,15 +98,25 @@ function App() { cephClusters: [], physicalDisks: [], pbs: [], - pmg: [], - metrics: [], - pveBackups: { + pmg: [], + metrics: [], + pveBackups: { + backupTasks: [], + storageBackups: [], + guestSnapshots: [], + }, + pbsBackups: [], + pmgBackups: [], + backups: { + pve: { backupTasks: [], storageBackups: [], guestSnapshots: [], }, - pbsBackups: [], - performance: { + pbs: [], + pmg: [], + }, + performance: { apiCallDuration: {}, lastPollDuration: 0, pollingStartTime: '', diff --git a/frontend-modern/src/components/Alerts/ResourceTable.tsx b/frontend-modern/src/components/Alerts/ResourceTable.tsx index 5f2198c..d726faf 100644 --- a/frontend-modern/src/components/Alerts/ResourceTable.tsx +++ b/frontend-modern/src/components/Alerts/ResourceTable.tsx @@ -54,6 +54,7 @@ export interface Resource { host?: string; type?: string; resourceType?: string; + subtitle?: string; thresholds?: Record; defaults?: Record; disabled?: boolean; @@ -68,6 +69,12 @@ export interface Resource { clusterName?: string; isClusterMember?: boolean; delaySeconds?: number; + editScope?: 'snapshot'; + isEnabled?: boolean; + toggleEnabled?: () => void; + toggleTitleEnabled?: string; + toggleTitleDisabled?: string; + editable?: boolean; [key: string]: unknown; } @@ -386,9 +393,7 @@ export function ResourceTable(props: ResourceTableProps) { titleEnabled?: string; titleDisabled?: string; titleWhenDisabled?: string; - }) => { - return ; - }; + }) => ; const offlineStateOrder: OfflineState[] = ['off', 'warning', 'critical']; @@ -826,61 +831,67 @@ export function ResourceTable(props: ResourceTableProps) { - - - {resource.name} - - - - Custom - - - - }> -
- + - {resource.type === 'node' - ? resource.name - : resource.displayName || resource.name} + {resource.name} - }> - {(host) => ( - e.stopPropagation()} - class={`text-sm font-medium transition-colors duration-150 ${ - resource.disabled - ? 'text-gray-500 dark:text-gray-500' - : 'text-gray-900 dark:text-gray-100 hover:text-sky-600 dark:hover:text-sky-400' - }`} - title={`Open ${resource.displayName || resource.name} web interface`} - > - {resource.type === 'node' - ? resource.name - : resource.displayName || resource.name} - - )} - - - - {resource.clusterName} - - - - - Custom - - -
- + } + > +
+ + {resource.type === 'node' + ? resource.name + : resource.displayName || resource.name} + + } + > + {(host) => ( + e.stopPropagation()} + class={`text-sm font-medium transition-colors duration-150 ${ + resource.disabled + ? 'text-gray-500 dark:text-gray-500' + : 'text-gray-900 dark:text-gray-100 hover:text-sky-600 dark:hover:text-sky-400' + }`} + title={`Open ${resource.displayName || resource.name} web interface`} + > + {resource.type === 'node' + ? resource.name + : resource.displayName || resource.name} + + )} + + + + {resource.clusterName} + + +
+ + + + {resource.subtitle as string} + + + + + Custom + + + {/* Metric columns - dynamically rendered based on resource type */} @@ -1279,14 +1290,7 @@ export function ResourceTable(props: ResourceTableProps) { {/* Metric columns - dynamically rendered based on resource type */} {(column) => { - const metric = column - .toLowerCase() - .replace(' %', '') - .replace(' mb/s', '') - .replace('disk r', 'diskRead') - .replace('disk w', 'diskWrite') - .replace('net in', 'networkIn') - .replace('net out', 'networkOut'); + const metric = normalizeMetricKey(column); // Check if this metric applies to this resource type const showMetric = () => { @@ -1312,6 +1316,9 @@ export function ResourceTable(props: ResourceTableProps) { const openMetricEditor = (e: MouseEvent) => { e.stopPropagation(); + if (resource.editable === false) { + return; + } setActiveMetricInput({ resourceId: resource.id, metric }); props.onEdit( resource.id, @@ -1438,7 +1445,20 @@ export function ResourceTable(props: ResourceTableProps) { {/* Actions column */}
+ + + —} + > + } > - - +
- + + + +
-
- + + + ); }} diff --git a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx index 64de7e2..67d8b90 100644 --- a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx +++ b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx @@ -14,8 +14,12 @@ import type { PMGInstance, DockerHost, DockerContainer, + PVEBackups, + PBSBackup, + PMGBackup, + Backups, } from '@/types/api'; -import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts'; +import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts'; import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable'; type OverrideType = | 'guest' @@ -103,6 +107,11 @@ export const normalizeDockerIgnoredInput = (value: string): string[] => .map((entry) => entry.trim()) .filter((entry) => entry.length > 0); +const DEFAULT_SNAPSHOT_WARNING = 30; +const DEFAULT_SNAPSHOT_CRITICAL = 45; +const DEFAULT_BACKUP_WARNING = 7; +const DEFAULT_BACKUP_CRITICAL = 14; + // Simple threshold object for the UI interface SimpleThresholds { cpu?: number; @@ -127,6 +136,10 @@ interface ThresholdsTableProps { dockerHosts: DockerHost[]; pbsInstances?: PBSInstance[]; // PBS instances from state pmgInstances?: PMGInstance[]; // PMG instances from state + backups?: Backups; + pveBackups?: PVEBackups; + pbsBackups?: PBSBackup[]; + pmgBackups?: PMGBackup[]; pmgThresholds: () => PMGThresholdDefaults; setPMGThresholds: ( value: @@ -177,6 +190,14 @@ interface ThresholdsTableProps { ) => void; snapshotFactoryDefaults?: SnapshotAlertConfig; resetSnapshotDefaults?: () => void; + backupDefaults: () => BackupAlertConfig; + setBackupDefaults: ( + value: + | BackupAlertConfig + | ((prev: BackupAlertConfig) => BackupAlertConfig), + ) => void; + backupFactoryDefaults?: BackupAlertConfig; + resetBackupDefaults?: () => void; setHasUnsavedChanges: (value: boolean) => void; activeAlerts?: Record; removeAlerts?: (predicate: (alert: Alert) => boolean) => void; @@ -208,9 +229,6 @@ interface ThresholdsTableProps { setDisableAllDockerHostsOffline: (value: boolean) => void; } -const DEFAULT_SNAPSHOT_WARNING = 30; -const DEFAULT_SNAPSHOT_CRITICAL = 45; - export function ThresholdsTable(props: ThresholdsTableProps) { const navigate = useNavigate(); const location = useLocation(); @@ -803,26 +821,88 @@ const dockerContainersGroupedByHost = createMemo>((pr }; }); - const snapshotFactoryDefaultsRecord = createMemo(() => { - const factory = snapshotFactoryConfig(); + const backupFactoryConfig = () => + props.backupFactoryDefaults ?? { + enabled: false, + warningDays: DEFAULT_BACKUP_WARNING, + criticalDays: DEFAULT_BACKUP_CRITICAL, + }; + + const sanitizeBackupConfig = (config: BackupAlertConfig): BackupAlertConfig => { + let warning = Math.max(0, Math.round(config.warningDays ?? 0)); + let critical = Math.max(0, Math.round(config.criticalDays ?? 0)); + + if (critical > 0 && warning > critical) { + warning = critical; + } + if (critical === 0 && warning > 0) { + critical = warning; + } + return { - 'warning days': factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING, - 'critical days': factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL, + enabled: !!config.enabled, + warningDays: warning, + criticalDays: critical, + }; + }; + + const updateBackupDefaults = ( + updater: + | BackupAlertConfig + | ((prev: BackupAlertConfig) => BackupAlertConfig), + ) => { + props.setBackupDefaults((prev) => { + const next = + typeof updater === 'function' + ? (updater as (prev: BackupAlertConfig) => BackupAlertConfig)(prev) + : { ...prev, ...updater }; + return sanitizeBackupConfig(next); + }); + props.setHasUnsavedChanges(true); + }; + + const backupDefaultsRecord = createMemo(() => { + const current = props.backupDefaults(); + return { + 'warning days': current.warningDays ?? 0, + 'critical days': current.criticalDays ?? 0, }; }); - const snapshotOverridesCount = createMemo(() => { - const current = props.snapshotDefaults(); - const factory = snapshotFactoryConfig(); - return current.enabled !== factory.enabled || - (current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !== - (factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) || - (current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !== - (factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) +const backupFactoryDefaultsRecord = createMemo(() => { + const factory = backupFactoryConfig(); + return { + 'warning days': factory.warningDays ?? DEFAULT_BACKUP_WARNING, + 'critical days': factory.criticalDays ?? DEFAULT_BACKUP_CRITICAL, + }; +}); + +const snapshotOverridesCount = createMemo(() => { + const current = props.snapshotDefaults(); + const factory = snapshotFactoryConfig(); + return current.enabled !== factory.enabled || + (current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !== + (factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) || + (current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !== + (factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) + ? 1 + : 0; +}); + + + const backupOverridesCount = createMemo(() => { + const backupCurrent = props.backupDefaults(); + const backupFactory = backupFactoryConfig(); + return backupCurrent.enabled !== backupFactory.enabled || + (backupCurrent.warningDays ?? DEFAULT_BACKUP_WARNING) !== + (backupFactory.warningDays ?? DEFAULT_BACKUP_WARNING) || + (backupCurrent.criticalDays ?? DEFAULT_BACKUP_CRITICAL) !== + (backupFactory.criticalDays ?? DEFAULT_BACKUP_CRITICAL) ? 1 : 0; }); + // Process guests with their overrides and group by node const guestsGroupedByNode = createMemo>((prev = {}) => { // If we're currently editing, return the previous value to avoid re-renders @@ -1161,18 +1241,25 @@ const dockerContainersGroupedByHost = createMemo>((pr }, { key: 'storage' as const, - label: 'Storage', - total: props.storage?.length ?? 0, - overrides: countOverrides(storageWithOverrides()), - tab: 'proxmox' as const, - }, - { - key: 'snapshots' as const, - label: 'Snapshots', - total: 1, - overrides: snapshotOverridesCount(), - tab: 'proxmox' as const, - }, + label: 'Storage', + total: props.storage?.length ?? 0, + overrides: countOverrides(storageWithOverrides()), + tab: 'proxmox' as const, + }, + { + key: 'backups' as const, + label: 'Backups', + total: 1, + overrides: backupOverridesCount(), + tab: 'proxmox' as const, + }, + { + key: 'snapshots' as const, + label: 'Snapshot Age', + total: 1, + overrides: snapshotOverridesCount(), + tab: 'proxmox' as const, + }, { key: 'pbs' as const, label: 'PBS Servers', @@ -1240,6 +1327,41 @@ const dockerContainersGroupedByHost = createMemo>((pr if (!resource) return; const editedThresholds = editingThresholds(); + + if (resource.editScope === 'backup') { + const currentBackupDefaults = props.backupDefaults(); + const nextWarning = + editedThresholds['warning days'] ?? currentBackupDefaults.warningDays ?? DEFAULT_BACKUP_WARNING; + const nextCritical = + editedThresholds['critical days'] ?? currentBackupDefaults.criticalDays ?? DEFAULT_BACKUP_CRITICAL; + + updateBackupDefaults({ + enabled: currentBackupDefaults.enabled, + warningDays: nextWarning, + criticalDays: nextCritical, + }); + + cancelEdit(); + return; + } + + if (resource.editScope === 'snapshot') { + const currentSnapshotDefaults = props.snapshotDefaults(); + const nextWarning = + editedThresholds['warning days'] ?? currentSnapshotDefaults.warningDays ?? DEFAULT_SNAPSHOT_WARNING; + const nextCritical = + editedThresholds['critical days'] ?? currentSnapshotDefaults.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL; + + updateSnapshotDefaults({ + enabled: currentSnapshotDefaults.enabled, + warningDays: nextWarning, + criticalDays: nextCritical, + }); + + cancelEdit(); + return; + } + const defaultThresholds = (resource.defaults ?? {}) as Record; // Only include values that differ from defaults @@ -1969,11 +2091,74 @@ const dockerContainersGroupedByHost = createMemo>((pr + +
+ { + updateBackupDefaults((prev) => { + const currentRecord = { + 'warning days': prev.warningDays ?? 0, + 'critical days': prev.criticalDays ?? 0, + }; + const nextRecord = + typeof value === 'function' + ? value(currentRecord) + : { ...currentRecord, ...value }; + return { + ...prev, + warningDays: + typeof nextRecord['warning days'] === 'number' + ? nextRecord['warning days'] + : prev.warningDays, + criticalDays: + typeof nextRecord['critical days'] === 'number' + ? nextRecord['critical days'] + : prev.criticalDays, + }; + }); + }} + setHasUnsavedChanges={props.setHasUnsavedChanges} + globalDisableFlag={() => !props.backupDefaults().enabled} + onToggleGlobalDisable={() => + updateBackupDefaults((prev) => ({ + ...prev, + enabled: !prev.enabled, + })) + } + factoryDefaults={backupFactoryDefaultsRecord()} + onResetDefaults={() => { + if (props.resetBackupDefaults) { + props.resetBackupDefaults(); + props.setHasUnsavedChanges(true); + } else { + updateBackupDefaults(backupFactoryConfig()); + } + }} + /> +
+ +
>((pr enabled: !prev.enabled, })) } - factoryDefaults={snapshotFactoryDefaultsRecord()} - onResetDefaults={ - props.resetSnapshotDefaults - ? () => { - props.resetSnapshotDefaults?.(); - props.setHasUnsavedChanges(true); - } - : () => - updateSnapshotDefaults({ - ...snapshotFactoryConfig(), - }) - } + factoryDefaults={snapshotFactoryConfig()} + onResetDefaults={() => { + if (props.resetSnapshotDefaults) { + props.resetSnapshotDefaults(); + props.setHasUnsavedChanges(true); + } else { + updateSnapshotDefaults(snapshotFactoryConfig()); + } + }} />
diff --git a/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx b/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx index dd6df71..ef32ee1 100644 --- a/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx +++ b/frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx @@ -6,7 +6,7 @@ import { ThresholdsTable, normalizeDockerIgnoredInput, } from '../ThresholdsTable'; -import type { PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts'; +import type { PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts'; vi.mock('@solidjs/router', () => ({ useNavigate: () => vi.fn(), @@ -87,6 +87,10 @@ const baseProps = () => ({ factoryNodeDefaults: {}, factoryDockerDefaults: {}, factoryStorageDefault: 85, + backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }), + setBackupDefaults: vi.fn(), + backupFactoryDefaults: { enabled: false, warningDays: 7, criticalDays: 14 } as BackupAlertConfig, + resetBackupDefaults: vi.fn(), snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }), setSnapshotDefaults: vi.fn(), snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig, diff --git a/frontend-modern/src/components/Backups/Backups.tsx b/frontend-modern/src/components/Backups/Backups.tsx index 811e44d..636f4bd 100644 --- a/frontend-modern/src/components/Backups/Backups.tsx +++ b/frontend-modern/src/components/Backups/Backups.tsx @@ -13,7 +13,15 @@ const Backups: Component = () => { {/* Loading State */} - + { {/* Main Content - Unified Backups View */} - + diff --git a/frontend-modern/src/components/Backups/UnifiedBackups.tsx b/frontend-modern/src/components/Backups/UnifiedBackups.tsx index 5127e81..76b7790 100644 --- a/frontend-modern/src/components/Backups/UnifiedBackups.tsx +++ b/frontend-modern/src/components/Backups/UnifiedBackups.tsx @@ -25,6 +25,9 @@ interface DateGroup { const UnifiedBackups: Component = () => { const { state } = useWebSocket(); + const pveBackupsState = createMemo(() => state.backups?.pve ?? state.pveBackups); + const pbsBackupsState = createMemo(() => state.backups?.pbs ?? state.pbsBackups); + const pmgBackupsState = createMemo(() => state.backups?.pmg ?? state.pmgBackups); const [searchTerm, setSearchTerm] = createSignal(''); const [selectedNode, setSelectedNode] = createSignal(null); const [typeFilter, setTypeFilter] = createSignal<'all' | FilterableGuestType>('all'); @@ -159,10 +162,14 @@ const UnifiedBackups: Component = () => { // Check if we have any backup data yet const isLoading = createMemo(() => { + const pve = pveBackupsState(); + const pbs = pbsBackupsState(); + const pmg = pmgBackupsState(); return ( - !state.pveBackups?.guestSnapshots && - !state.pveBackups?.storageBackups && - !state.pbsBackups?.length && + !(pve?.guestSnapshots?.length ?? 0) && + !(pve?.storageBackups?.length ?? 0) && + !(pbs?.length ?? 0) && + !(pmg?.length ?? 0) && !state.pbs?.length ); }); @@ -176,7 +183,7 @@ const UnifiedBackups: Component = () => { const debugMode = typeof window !== 'undefined' && localStorage.getItem('debug-pmg') === 'true'; // Normalize snapshots - state.pveBackups?.guestSnapshots?.forEach((snapshot) => { + pveBackupsState()?.guestSnapshots?.forEach((snapshot) => { // Try to find the guest name by matching VMID and instance (not hostname) let guestName = ''; const vm = state.vms?.find((v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance); @@ -213,8 +220,8 @@ const UnifiedBackups: Component = () => { // This ensures we have the complete PBS data with namespaces // Filter by selected PBS instance if one is selected const pbsBackupsToProcess = selectedPBSInstance() - ? state.pbsBackups?.filter((b) => b.instance === selectedPBSInstance()) - : state.pbsBackups; + ? pbsBackupsState()?.filter((b) => b.instance === selectedPBSInstance()) + : pbsBackupsState(); pbsBackupsToProcess?.forEach((backup) => { const backupDate = new Date(backup.backupTime); @@ -302,8 +309,38 @@ const UnifiedBackups: Component = () => { }); }); + pmgBackupsState()?.forEach((backup) => { + const backupTimeMs = backup.backupTime ? Date.parse(backup.backupTime) : Number.NaN; + const backupTimeSeconds = Number.isFinite(backupTimeMs) ? Math.floor(backupTimeMs / 1000) : 0; + const backupKey = backup.id || `${backup.instance}:${backup.node}:${backup.filename}`; + if (seenBackups.has(backupKey)) { + return; + } + seenBackups.add(backupKey); + + const displayName = backup.node || backup.filename || 'PMG Host Backup'; + unified.push({ + backupType: 'local', + vmid: backup.node || backup.filename, + name: displayName, + type: 'Host', + node: backup.node || 'PMG', + instance: backup.instance || 'PMG', + backupTime: backupTimeSeconds, + backupName: backup.filename || displayName, + description: backup.filename || '', + status: 'ok', + size: backup.size || null, + storage: 'PMG', + datastore: null, + namespace: null, + verified: null, + protected: false, + }); + }); + // Normalize local backups (including PBS through PVE storage) - state.pveBackups?.storageBackups?.forEach((backup) => { + pveBackupsState()?.storageBackups?.forEach((backup) => { // Skip templates and ISOs - they're not backups if (backup.type === 'vztmpl' || backup.type === 'iso') { return; @@ -1199,7 +1236,7 @@ const UnifiedBackups: Component = () => { // Count backups for this PBS instance const pbsBackups = () => - state.pbsBackups?.filter((b) => b.instance === pbs.name).length || 0; + pbsBackupsState()?.filter((b) => b.instance === pbs.name).length || 0; const isSelected = () => selectedPBSInstance() === pbs.name; diff --git a/frontend-modern/src/components/Settings/Settings.tsx b/frontend-modern/src/components/Settings/Settings.tsx index c55b91d..7ea7b13 100644 --- a/frontend-modern/src/components/Settings/Settings.tsx +++ b/frontend-modern/src/components/Settings/Settings.tsx @@ -317,6 +317,10 @@ const Settings: Component = (props) => { description: 'Manage Pulse configuration.', }; + const pveBackupsState = () => state.backups?.pve ?? state.pveBackups; + const pbsBackupsState = () => state.backups?.pbs ?? state.pbsBackups; + const pmgBackupsState = () => state.backups?.pmg ?? state.pmgBackups; + // Keep tab state in sync with URL and handle /settings redirect without flicker createEffect( on( @@ -5413,11 +5417,11 @@ const Settings: Component = (props) => { storageCount: state.storage?.length || 0, physicalDisksCount: state.physicalDisks?.length || 0, pbsCount: state.pbs?.length || 0, - pbsBackupsCount: state.pbsBackups?.length || 0, + pbsBackupsCount: pbsBackupsState()?.length || 0, pveBackups: { - backupTasksCount: state.pveBackups?.backupTasks?.length || 0, - storageBackupsCount: state.pveBackups?.storageBackups?.length || 0, - guestSnapshotsCount: state.pveBackups?.guestSnapshots?.length || 0, + backupTasksCount: pveBackupsState()?.backupTasks?.length || 0, + storageBackupsCount: pveBackupsState()?.storageBackups?.length || 0, + guestSnapshotsCount: pveBackupsState()?.guestSnapshots?.length || 0, }, }, // Node status details @@ -5454,9 +5458,9 @@ const Settings: Component = (props) => { } : undefined, hasBackups: - (state.pveBackups?.storageBackups?.filter( + (pveBackupsState()?.storageBackups ?? []).filter( (b) => b.storage === s.name, - ).length || 0) > 0, + ).length > 0, })) || [], // Physical disks - critical for troubleshooting physicalDisks: @@ -5472,10 +5476,10 @@ const Settings: Component = (props) => { smart: d.smart ?? null, })) || [], backups: { - pveBackupTasks: state.pveBackups?.backupTasks?.slice(0, 10) || [], + pveBackupTasks: pveBackupsState()?.backupTasks?.slice(0, 10) || [], pveStorageBackups: - state.pveBackups?.storageBackups?.slice(0, 10) || [], - pbsBackups: state.pbsBackups?.slice(0, 10) || [], + pveBackupsState()?.storageBackups?.slice(0, 10) || [], + pbsBackups: pbsBackupsState()?.slice(0, 10) || [], }, connectionHealth: state.connectionHealth || {}, performance: { diff --git a/frontend-modern/src/components/shared/UnifiedNodeSelector.tsx b/frontend-modern/src/components/shared/UnifiedNodeSelector.tsx index 85c2e41..ccd8a0f 100644 --- a/frontend-modern/src/components/shared/UnifiedNodeSelector.tsx +++ b/frontend-modern/src/components/shared/UnifiedNodeSelector.tsx @@ -44,6 +44,8 @@ export const UnifiedNodeSelector: Component = (props) // This allows users to select a node AND search within it // Calculate backup counts for nodes and PBS instances + const pveBackupsState = () => state.backups?.pve ?? state.pveBackups; + const pbsBackupsState = () => state.backups?.pbs ?? state.pbsBackups; const backupCounts = createMemo(() => { const counts: Record = {}; @@ -54,15 +56,16 @@ export const UnifiedNodeSelector: Component = (props) let count = 0; // Count storage backups (excluding PBS backups which are counted separately) - if (state.pveBackups?.storageBackups) { - count += state.pveBackups.storageBackups.filter( + const pveBackups = pveBackupsState(); + if (pveBackups?.storageBackups) { + count += pveBackups.storageBackups.filter( (b) => b.instance === node.instance && b.node === node.name && !b.isPBS, ).length; } // Count snapshots - if (state.pveBackups?.guestSnapshots) { - count += state.pveBackups.guestSnapshots.filter((s) => s.instance === node.instance && s.node === node.name).length; + if (pveBackups?.guestSnapshots) { + count += pveBackups.guestSnapshots.filter((s) => s.instance === node.instance && s.node === node.name).length; } counts[node.id] = count; @@ -70,9 +73,10 @@ export const UnifiedNodeSelector: Component = (props) } // Count PBS backups by instance - if (state.pbs && state.pbsBackups) { + const pbsBackups = pbsBackupsState(); + if (state.pbs && pbsBackups) { state.pbs.forEach((pbs) => { - counts[pbs.name] = state.pbsBackups?.filter((b) => b.instance === pbs.name).length || 0; + counts[pbs.name] = pbsBackups.filter((b) => b.instance === pbs.name).length || 0; }); } diff --git a/frontend-modern/src/pages/Alerts.tsx b/frontend-modern/src/pages/Alerts.tsx index 2280cc2..c3cc531 100644 --- a/frontend-modern/src/pages/Alerts.tsx +++ b/frontend-modern/src/pages/Alerts.tsx @@ -3,7 +3,7 @@ import type { JSX } from 'solid-js'; import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect'; import { WebhookConfig } from '@/components/Alerts/WebhookConfig'; import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable'; -import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts'; +import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts'; import { Card } from '@/components/shared/Card'; import { SectionHeader } from '@/components/shared/SectionHeader'; import { SettingsPanel } from '@/components/shared/SettingsPanel'; @@ -769,6 +769,25 @@ const [appriseConfig, setAppriseConfig] = createSignal( setMetricTimeThresholds({}); } + // Load backup thresholds + if (config.backupDefaults) { + const enabled = Boolean(config.backupDefaults.enabled); + const rawWarning = config.backupDefaults.warningDays ?? FACTORY_BACKUP_DEFAULTS.warningDays; + const rawCritical = config.backupDefaults.criticalDays ?? FACTORY_BACKUP_DEFAULTS.criticalDays; + const safeCritical = Math.max(0, rawCritical); + const normalizedWarning = Math.max(0, rawWarning); + const warningDays = + safeCritical > 0 && normalizedWarning > safeCritical ? safeCritical : normalizedWarning; + const criticalDays = Math.max(safeCritical, warningDays); + setBackupDefaults({ + enabled, + warningDays, + criticalDays, + }); + } else { + setBackupDefaults({ ...FACTORY_BACKUP_DEFAULTS }); + } + // Load snapshot thresholds if (config.snapshotDefaults) { const enabled = Boolean(config.snapshotDefaults.enabled); @@ -1058,6 +1077,11 @@ const [appriseConfig, setAppriseConfig] = createSignal( warningDays: 30, criticalDays: 45, }; + const FACTORY_BACKUP_DEFAULTS: BackupAlertConfig = { + enabled: false, + warningDays: 7, + criticalDays: 14, + }; // Threshold states - using trigger values for display const [guestDefaults, setGuestDefaults] = createSignal>({ ...FACTORY_GUEST_DEFAULTS }); @@ -1070,6 +1094,9 @@ const [appriseConfig, setAppriseConfig] = createSignal( const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal([]); const [storageDefault, setStorageDefault] = createSignal(FACTORY_STORAGE_DEFAULT); + const [backupDefaults, setBackupDefaults] = createSignal({ + ...FACTORY_BACKUP_DEFAULTS, + }); // Reset functions const resetGuestDefaults = () => { @@ -1096,6 +1123,10 @@ const [appriseConfig, setAppriseConfig] = createSignal( setStorageDefault(FACTORY_STORAGE_DEFAULT); setHasUnsavedChanges(true); }; + const resetBackupDefaults = () => { + setBackupDefaults({ ...FACTORY_BACKUP_DEFAULTS }); + setHasUnsavedChanges(true); + }; const resetSnapshotDefaults = () => { setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS }); setHasUnsavedChanges(true); @@ -1224,12 +1255,20 @@ const [appriseConfig, setAppriseConfig] = createSignal( }; const snapshotConfig = snapshotDefaults(); - const normalizedWarningDays = Math.max(0, snapshotConfig.warningDays ?? 0); - const normalizedCriticalDays = Math.max(0, snapshotConfig.criticalDays ?? 0); - const finalCriticalDays = - normalizedCriticalDays > 0 - ? Math.max(normalizedCriticalDays, normalizedWarningDays) - : normalizedWarningDays; + const normalizedSnapshotWarning = Math.max(0, snapshotConfig.warningDays ?? 0); + const normalizedSnapshotCritical = Math.max(0, snapshotConfig.criticalDays ?? 0); + const finalSnapshotCritical = + normalizedSnapshotCritical > 0 + ? Math.max(normalizedSnapshotCritical, normalizedSnapshotWarning) + : normalizedSnapshotWarning; + + const backupConfig = backupDefaults(); + const normalizedBackupWarning = Math.max(0, backupConfig.warningDays ?? 0); + const normalizedBackupCritical = Math.max(0, backupConfig.criticalDays ?? 0); + const finalBackupCritical = + normalizedBackupCritical > 0 + ? Math.max(normalizedBackupCritical, normalizedBackupWarning) + : normalizedBackupWarning; const alertConfig = { enabled: true, @@ -1284,8 +1323,13 @@ const [appriseConfig, setAppriseConfig] = createSignal( metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()), snapshotDefaults: { enabled: snapshotConfig.enabled, - warningDays: normalizedWarningDays, - criticalDays: finalCriticalDays, + warningDays: normalizedSnapshotWarning, + criticalDays: finalSnapshotCritical, + }, + backupDefaults: { + enabled: backupConfig.enabled, + warningDays: normalizedBackupWarning, + criticalDays: finalBackupCritical, }, pmgDefaults: pmgThresholds(), // Use rawOverridesConfig which is already properly formatted with disabled flags @@ -1491,14 +1535,18 @@ const [appriseConfig, setAppriseConfig] = createSignal( resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes} resetStorageDefault={resetStorageDefault} resetSnapshotDefaults={resetSnapshotDefaults} + resetBackupDefaults={resetBackupDefaults} factoryGuestDefaults={FACTORY_GUEST_DEFAULTS} factoryNodeDefaults={FACTORY_NODE_DEFAULTS} factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS} factoryStorageDefault={FACTORY_STORAGE_DEFAULT} snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS} + backupFactoryDefaults={FACTORY_BACKUP_DEFAULTS} timeThresholds={timeThresholds} metricTimeThresholds={metricTimeThresholds} setMetricTimeThresholds={setMetricTimeThresholds} + backupDefaults={backupDefaults} + setBackupDefaults={setBackupDefaults} snapshotDefaults={snapshotDefaults} setSnapshotDefaults={setSnapshotDefaults} pmgThresholds={pmgThresholds} @@ -2028,6 +2076,14 @@ interface ThresholdsTabProps { ) => void; snapshotFactoryDefaults: SnapshotAlertConfig; resetSnapshotDefaults: () => void; + backupDefaults: () => BackupAlertConfig; + setBackupDefaults: ( + value: + | BackupAlertConfig + | ((prev: BackupAlertConfig) => BackupAlertConfig), + ) => void; + backupFactoryDefaults: BackupAlertConfig; + resetBackupDefaults: () => void; setOverrides: (value: Override[]) => void; setRawOverridesConfig: (value: Record) => void; activeAlerts: Record; @@ -2085,6 +2141,10 @@ function ThresholdsTab(props: ThresholdsTabProps) { dockerHosts={props.state.dockerHosts || []} pbsInstances={props.state.pbs || []} pmgInstances={props.state.pmg || []} + backups={props.state.backups} + pveBackups={props.state.pveBackups} + pbsBackups={props.state.pbsBackups} + pmgBackups={props.state.pmgBackups} pmgThresholds={props.pmgThresholds} setPMGThresholds={props.setPMGThresholds} guestDefaults={props.guestDefaults()} @@ -2104,6 +2164,10 @@ function ThresholdsTab(props: ThresholdsTabProps) { timeThresholds={props.timeThresholds} metricTimeThresholds={props.metricTimeThresholds} setMetricTimeThresholds={props.setMetricTimeThresholds} + backupDefaults={props.backupDefaults} + setBackupDefaults={props.setBackupDefaults} + backupFactoryDefaults={props.backupFactoryDefaults} + resetBackupDefaults={props.resetBackupDefaults} snapshotDefaults={props.snapshotDefaults} setSnapshotDefaults={props.setSnapshotDefaults} snapshotFactoryDefaults={props.snapshotFactoryDefaults} diff --git a/frontend-modern/src/stores/websocket.ts b/frontend-modern/src/stores/websocket.ts index e3810ec..57aed5b 100644 --- a/frontend-modern/src/stores/websocket.ts +++ b/frontend-modern/src/stores/websocket.ts @@ -36,6 +36,16 @@ export function createWebSocketStore(url: string) { guestSnapshots: [], } as PVEBackups, pbsBackups: [], + pmgBackups: [], + backups: { + pve: { + backupTasks: [], + storageBackups: [], + guestSnapshots: [], + }, + pbs: [], + pmg: [], + }, performance: { apiCallDuration: {}, lastPollDuration: 0, @@ -282,8 +292,19 @@ export function createWebSocketStore(url: string) { setState('cephClusters', message.data.cephClusters); if (message.data.pbs !== undefined) setState('pbs', message.data.pbs); if (message.data.pmg !== undefined) setState('pmg', message.data.pmg); + if (message.data.backups !== undefined) { + setState('backups', message.data.backups); + if (message.data.backups.pve !== undefined) + setState('pveBackups', message.data.backups.pve); + if (message.data.backups.pbs !== undefined) + setState('pbsBackups', message.data.backups.pbs); + if (message.data.backups.pmg !== undefined) + setState('pmgBackups', message.data.backups.pmg); + } if (message.data.pbsBackups !== undefined) setState('pbsBackups', message.data.pbsBackups); + if (message.data.pmgBackups !== undefined) + setState('pmgBackups', message.data.pmgBackups); if (message.data.metrics !== undefined) setState('metrics', message.data.metrics); if (message.data.pveBackups !== undefined) setState('pveBackups', message.data.pveBackups); diff --git a/frontend-modern/src/types/alerts.ts b/frontend-modern/src/types/alerts.ts index 0f0db1d..a2ac141 100644 --- a/frontend-modern/src/types/alerts.ts +++ b/frontend-modern/src/types/alerts.ts @@ -91,6 +91,12 @@ export interface SnapshotAlertConfig { criticalDays: number; } +export interface BackupAlertConfig { + enabled: boolean; + warningDays: number; + criticalDays: number; +} + export interface AlertConfig { enabled: boolean; guestDefaults: AlertThresholds; @@ -100,6 +106,7 @@ export interface AlertConfig { dockerIgnoredContainerPrefixes?: string[]; pmgDefaults?: PMGThresholdDefaults; snapshotDefaults?: SnapshotAlertConfig; + backupDefaults?: BackupAlertConfig; customRules?: CustomAlertRule[]; overrides: Record; // key: resource ID minimumDelta?: number; diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index a50fb3b..ab27d3f 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -11,6 +11,8 @@ export interface State { pbs: PBSInstance[]; pmg: PMGInstance[]; pbsBackups: PBSBackup[]; + pmgBackups: PMGBackup[]; + backups: Backups; metrics: Metric[]; pveBackups: PVEBackups; performance: Performance; @@ -414,6 +416,21 @@ export interface PBSBackup { owner?: string; } +export interface PMGBackup { + id: string; + instance: string; + node: string; + filename: string; + backupTime: string; + size: number; +} + +export interface Backups { + pve: PVEBackups; + pbs: PBSBackup[]; + pmg: PMGBackup[]; +} + export interface PBSBackupJob { id: string; store: string; diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 75904aa..89c8f64 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -290,6 +290,22 @@ type SnapshotAlertConfig struct { CriticalDays int `json:"criticalDays"` } +// BackupAlertConfig represents backup age alert configuration +type BackupAlertConfig struct { + Enabled bool `json:"enabled"` + WarningDays int `json:"warningDays"` + CriticalDays int `json:"criticalDays"` +} + +// GuestLookup describes a guest identity used for snapshot/backup evaluations. +type GuestLookup struct { + Name string + Instance string + Node string + Type string + VMID int +} + // AlertConfig represents the complete alert configuration type AlertConfig struct { Enabled bool `json:"enabled"` @@ -300,6 +316,7 @@ type AlertConfig struct { DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"` PMGDefaults PMGThresholdConfig `json:"pmgDefaults"` SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"` + BackupDefaults BackupAlertConfig `json:"backupDefaults"` Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID CustomRules []CustomAlertRule `json:"customRules,omitempty"` Schedule ScheduleConfig `json:"schedule"` @@ -486,6 +503,11 @@ func NewManager() *Manager { WarningDays: 30, CriticalDays: 45, }, + BackupDefaults: BackupAlertConfig{ + Enabled: false, + WarningDays: 7, + CriticalDays: 14, + }, StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80}, MinimumDelta: 2.0, // 2% minimum change SuppressionWindow: 5, // 5 minutes @@ -702,6 +724,15 @@ func (m *Manager) UpdateConfig(config AlertConfig) { if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays { config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays } + if config.BackupDefaults.WarningDays < 0 { + config.BackupDefaults.WarningDays = 0 + } + if config.BackupDefaults.CriticalDays < 0 { + config.BackupDefaults.CriticalDays = 0 + } + if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays { + config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays + } // Ensure minimums for other important fields if config.MinimumDelta <= 0 { @@ -764,6 +795,9 @@ func (m *Manager) UpdateConfig(config AlertConfig) { if !m.config.SnapshotDefaults.Enabled { m.clearSnapshotAlertsForInstanceLocked("") } + if !m.config.BackupDefaults.Enabled { + m.clearBackupAlertsLocked() + } m.applyGlobalOfflineSettingsLocked() @@ -2756,7 +2790,7 @@ func (m *Manager) CheckStorage(storage models.Storage) { } } -func buildSnapshotGuestKey(instance, node string, vmid int) string { +func BuildGuestKey(instance, node string, vmid int) string { instance = strings.TrimSpace(instance) node = strings.TrimSpace(node) if instance == "" { @@ -2817,7 +2851,7 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID) validAlerts[alertID] = struct{}{} - guestKey := buildSnapshotGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID) + guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID) guestName := strings.TrimSpace(guestNames[guestKey]) guestType := "VM" @@ -2966,6 +3000,356 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod m.mu.Unlock() } +// CheckBackups evaluates storage, PBS, and PMG backups for age-based alerts. +func (m *Manager) CheckBackups( + storageBackups []models.StorageBackup, + pbsBackups []models.PBSBackup, + pmgBackups []models.PMGBackup, + guestsByKey map[string]GuestLookup, + guestsByVMID map[string]GuestLookup, +) { + m.mu.RLock() + enabled := m.config.Enabled + backupCfg := m.config.BackupDefaults + m.mu.RUnlock() + + if !enabled || !backupCfg.Enabled { + m.clearBackupAlerts() + return + } + + if backupCfg.WarningDays <= 0 && backupCfg.CriticalDays <= 0 { + m.clearBackupAlerts() + return + } + + type backupRecord struct { + key string + lookup GuestLookup + fallbackName string + instance string + node string + source string + storage string + datastore string + backupType string + filename string + lastTime time.Time + } + + records := make(map[string]*backupRecord) + + updateRecord := func(key string, candidate backupRecord) { + if key == "" { + return + } + if existing, ok := records[key]; ok { + if candidate.lastTime.After(existing.lastTime) { + *existing = candidate + } + return + } + record := candidate + records[key] = &record + } + + now := time.Now() + + for _, backup := range storageBackups { + if backup.Time.IsZero() { + continue + } + + key := BuildGuestKey(backup.Instance, backup.Node, backup.VMID) + info := guestsByKey[key] + displayName := info.Name + if displayName == "" { + displayName = fmt.Sprintf("%s-%d", sanitizeAlertKey(backup.Node), backup.VMID) + } + + updateRecord(key, backupRecord{ + key: key, + lookup: info, + fallbackName: displayName, + instance: backup.Instance, + node: backup.Node, + source: "PVE storage", + storage: backup.Storage, + backupType: backup.Type, + lastTime: backup.Time, + }) + } + + for _, backup := range pbsBackups { + if backup.BackupTime.IsZero() { + continue + } + if backup.VMID == "0" { + // Host configuration backups - skip from age alerts + continue + } + + info, exists := guestsByVMID[backup.VMID] + var key string + var displayName string + var instance string + var node string + + if exists && info.Instance != "" && info.Node != "" { + key = BuildGuestKey(info.Instance, info.Node, info.VMID) + displayName = info.Name + instance = info.Instance + node = info.Node + } else { + key = fmt.Sprintf("pbs:%s:%s:%s", backup.Instance, backup.BackupType, backup.VMID) + displayName = fmt.Sprintf("VMID %s", backup.VMID) + instance = fmt.Sprintf("PBS:%s", backup.Instance) + node = backup.Datastore + } + + updateRecord(key, backupRecord{ + key: key, + lookup: info, + fallbackName: displayName, + instance: instance, + node: node, + source: "PBS", + datastore: backup.Datastore, + backupType: backup.BackupType, + lastTime: backup.BackupTime, + }) + } + + for _, backup := range pmgBackups { + if backup.BackupTime.IsZero() { + continue + } + + instanceLabel := strings.TrimSpace(backup.Instance) + if instanceLabel == "" { + instanceLabel = "PMG" + } + + nodeName := strings.TrimSpace(backup.Node) + keyComponent := nodeName + if keyComponent == "" { + keyComponent = strings.TrimSpace(backup.Filename) + } + if keyComponent == "" { + keyComponent = "unknown" + } + + displayName := nodeName + if displayName == "" { + displayName = instanceLabel + } + if displayName == "" { + displayName = "PMG gateway" + } else { + displayName = fmt.Sprintf("PMG %s", displayName) + } + + instanceField := fmt.Sprintf("PMG:%s", instanceLabel) + key := fmt.Sprintf("pmg:%s:%s", instanceLabel, keyComponent) + + updateRecord(key, backupRecord{ + key: key, + fallbackName: displayName, + instance: instanceField, + node: nodeName, + source: "PMG", + backupType: "pmg", + filename: backup.Filename, + lastTime: backup.BackupTime, + }) + } + + if len(records) == 0 { + m.clearBackupAlerts() + return + } + + validAlerts := make(map[string]struct{}) + + for key, record := range records { + age := now.Sub(record.lastTime) + if age < 0 { + continue + } + + ageDays := age.Hours() / 24 + if ageDays < 0 { + continue + } + ageDaysRounded := math.Round(ageDays*10) / 10 + + var level AlertLevel + var threshold int + switch { + case backupCfg.CriticalDays > 0 && ageDays >= float64(backupCfg.CriticalDays): + level = AlertLevelCritical + threshold = backupCfg.CriticalDays + case backupCfg.WarningDays > 0 && ageDays >= float64(backupCfg.WarningDays): + level = AlertLevelWarning + threshold = backupCfg.WarningDays + default: + continue + } + + alertKey := sanitizeAlertKey(key) + alertID := fmt.Sprintf("backup-age-%s", alertKey) + validAlerts[alertID] = struct{}{} + + displayName := record.lookup.Name + if displayName == "" { + displayName = record.fallbackName + } + if displayName == "" { + displayName = "Unknown guest" + } + + node := record.node + if node == "" { + node = record.lookup.Node + } + instance := record.instance + if instance == "" { + instance = record.lookup.Instance + } + + thresholdTime := record.lastTime.Add(time.Duration(threshold) * 24 * time.Hour) + if thresholdTime.After(now) { + thresholdTime = now + } + + var sourceLabel string + switch record.source { + case "PBS": + sourceLabel = fmt.Sprintf("PBS datastore %s on %s", record.datastore, strings.TrimPrefix(instance, "PBS:")) + case "PMG": + if node != "" { + sourceLabel = fmt.Sprintf("PMG node %s", node) + } else { + sourceLabel = "PMG" + } + default: + sourceLabel = fmt.Sprintf("storage %s on %s", record.storage, node) + } + + message := fmt.Sprintf( + "%s backup via %s is %.1f days old (threshold: %d days)", + displayName, + sourceLabel, + ageDaysRounded, + threshold, + ) + + metadata := map[string]interface{}{ + "source": record.source, + "lastBackupTime": record.lastTime, + "ageDays": ageDays, + "thresholdDays": threshold, + } + if record.storage != "" { + metadata["storage"] = record.storage + } + if record.datastore != "" { + metadata["datastore"] = record.datastore + } + if record.backupType != "" { + metadata["backupType"] = record.backupType + } + if record.filename != "" { + metadata["filename"] = record.filename + } + + m.mu.Lock() + if existing, exists := m.activeAlerts[alertID]; exists { + existing.LastSeen = now + existing.Level = level + existing.Value = ageDays + existing.Threshold = float64(threshold) + existing.Message = message + if existing.Metadata == nil { + existing.Metadata = make(map[string]interface{}) + } + for k, v := range metadata { + existing.Metadata[k] = v + } + m.mu.Unlock() + continue + } + + alert := &Alert{ + ID: alertID, + Type: "backup-age", + Level: level, + ResourceID: alertKey, + ResourceName: fmt.Sprintf("%s backup", displayName), + Node: node, + Instance: instance, + Message: message, + Value: ageDays, + Threshold: float64(threshold), + StartTime: thresholdTime, + LastSeen: now, + Metadata: metadata, + } + + m.preserveAlertState(alertID, alert) + + m.activeAlerts[alertID] = alert + m.recentAlerts[alertID] = alert + m.historyManager.AddAlert(*alert) + + go func() { + defer func() { + if r := recover(); r != nil { + log.Error().Interface("panic", r).Msg("Panic in SaveActiveAlerts goroutine (backup)") + } + }() + if err := m.SaveActiveAlerts(); err != nil { + log.Error().Err(err).Msg("Failed to save active alerts after backup alert creation") + } + }() + + if !m.checkRateLimit(alertID) { + m.mu.Unlock() + log.Debug(). + Str("alertID", alertID). + Str("resource", displayName). + Msg("Backup alert suppressed due to rate limit") + continue + } + + if m.onAlert != nil { + notified := now + alert.LastNotified = ¬ified + if m.dispatchAlert(alert, true) { + log.Info(). + Str("alertID", alertID). + Str("resource", displayName). + Msg("Backup age alert dispatched") + } else { + alert.LastNotified = nil + } + } + m.mu.Unlock() + } + + m.mu.Lock() + for alertID, alert := range m.activeAlerts { + if alert == nil || alert.Type != "backup-age" { + continue + } + if _, ok := validAlerts[alertID]; ok { + continue + } + m.clearAlertNoLock(alertID) + } + m.mu.Unlock() +} + // checkZFSPoolHealth checks ZFS pool for errors and degraded state func (m *Manager) checkZFSPoolHealth(storage models.Storage) { pool := storage.ZFSPool @@ -6463,3 +6847,18 @@ func (m *Manager) clearSnapshotAlertsForInstanceLocked(instance string) { m.clearAlertNoLock(alertID) } } + +func (m *Manager) clearBackupAlerts() { + m.mu.Lock() + m.clearBackupAlertsLocked() + m.mu.Unlock() +} + +func (m *Manager) clearBackupAlertsLocked() { + for alertID, alert := range m.activeAlerts { + if alert == nil || alert.Type != "backup-age" { + continue + } + m.clearAlertNoLock(alertID) + } +} diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index a2eefc0..3b6655d 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -3,6 +3,7 @@ package alerts import ( "fmt" "reflect" + "strings" "testing" "time" @@ -273,6 +274,160 @@ func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) { } } +func TestCheckBackupsCreatesAndClearsAlerts(t *testing.T) { + m := NewManager() + m.ClearActiveAlerts() + + m.mu.Lock() + m.config.Enabled = true + m.config.BackupDefaults = BackupAlertConfig{ + Enabled: true, + WarningDays: 7, + CriticalDays: 14, + } + m.config.TimeThreshold = 0 + m.config.TimeThresholds = map[string]int{} + m.mu.Unlock() + + now := time.Now() + storageBackups := []models.StorageBackup{ + { + ID: "inst-node-100-backup", + Storage: "local", + Node: "node", + Instance: "inst", + Type: "qemu", + VMID: 100, + Time: now.Add(-15 * 24 * time.Hour), + }, + } + + key := BuildGuestKey("inst", "node", 100) + guestsByKey := map[string]GuestLookup{ + key: { + Name: "app-server", + Instance: "inst", + Node: "node", + Type: "qemu", + VMID: 100, + }, + } + guestsByVMID := map[string]GuestLookup{ + "100": guestsByKey[key], + } + + m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID) + + m.mu.RLock() + alert, exists := m.activeAlerts["backup-age-"+sanitizeAlertKey(key)] + m.mu.RUnlock() + if !exists { + t.Fatalf("expected backup age alert to be created") + } + if alert.Level != AlertLevelCritical { + t.Fatalf("expected critical backup alert, got %s", alert.Level) + } + + // Recent backup clears alert + storageBackups[0].Time = now + m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID) + + m.mu.RLock() + _, exists = m.activeAlerts["backup-age-"+sanitizeAlertKey(key)] + m.mu.RUnlock() + if exists { + t.Fatalf("expected backup-age alert to clear after fresh backup") + } +} + +func TestCheckBackupsHandlesPbsOnlyGuests(t *testing.T) { + m := NewManager() + m.ClearActiveAlerts() + + m.mu.Lock() + m.config.Enabled = true + m.config.BackupDefaults = BackupAlertConfig{ + Enabled: true, + WarningDays: 3, + CriticalDays: 5, + } + m.mu.Unlock() + + now := time.Now() + pbsBackups := []models.PBSBackup{ + { + ID: "pbs-backup-999-0", + Instance: "pbs-main", + Datastore: "backup-store", + BackupType: "qemu", + VMID: "999", + BackupTime: now.Add(-6 * 24 * time.Hour), + }, + } + + m.CheckBackups(nil, pbsBackups, nil, map[string]GuestLookup{}, map[string]GuestLookup{}) + + m.mu.RLock() + found := false + for id, alert := range m.activeAlerts { + if strings.HasPrefix(id, "backup-age-") { + found = true + if alert.Level != AlertLevelCritical { + t.Fatalf("expected PBS backup alert to be critical") + } + break + } + } + m.mu.RUnlock() + if !found { + t.Fatalf("expected PBS backup alert to be created") + } +} + +func TestCheckBackupsHandlesPmgBackups(t *testing.T) { + m := NewManager() + m.ClearActiveAlerts() + + m.mu.Lock() + m.config.Enabled = true + m.config.BackupDefaults = BackupAlertConfig{ + Enabled: true, + WarningDays: 5, + CriticalDays: 7, + } + m.mu.Unlock() + + now := time.Now() + pmgBackups := []models.PMGBackup{ + { + ID: "pmg-backup-mail-01", + Instance: "mail", + Node: "mail-gateway", + Filename: "pmg-backup_2024-01-01.tgz", + BackupTime: now.Add(-8 * 24 * time.Hour), + Size: 123456, + }, + } + + m.CheckBackups(nil, nil, pmgBackups, map[string]GuestLookup{}, map[string]GuestLookup{}) + + m.mu.RLock() + found := false + for id, alert := range m.activeAlerts { + if strings.HasPrefix(id, "backup-age-") { + found = true + if alert.Level != AlertLevelCritical { + t.Fatalf("expected PMG backup alert to be critical") + } + break + } + } + m.mu.RUnlock() + if !found { + t.Fatalf("expected PMG backup alert to be created") + } +} + func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) { m := NewManager() diff --git a/internal/api/router.go b/internal/api/router.go index a75f380..6e6d964 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2452,16 +2452,24 @@ func (r *Router) handleBackups(w http.ResponseWriter, req *http.Request) { // Get current state state := r.monitor.GetState() - // Return backup data structure - backups := map[string]interface{}{ - "backupTasks": state.PVEBackups.BackupTasks, - "storageBackups": state.PVEBackups.StorageBackups, - "guestSnapshots": state.PVEBackups.GuestSnapshots, - "pbsBackups": state.PBSBackups, - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(backups) + json.NewEncoder(w).Encode(struct { + Backups models.Backups `json:"backups"` + PVEBackups models.PVEBackups `json:"pveBackups"` + PBSBackups []models.PBSBackup `json:"pbsBackups"` + PMGBackups []models.PMGBackup `json:"pmgBackups"` + BackupTasks []models.BackupTask `json:"backupTasks"` + Storage []models.StorageBackup `json:"storageBackups"` + GuestSnaps []models.GuestSnapshot `json:"guestSnapshots"` + }{ + Backups: state.Backups, + PVEBackups: state.PVEBackups, + PBSBackups: state.PBSBackups, + PMGBackups: state.PMGBackups, + BackupTasks: state.PVEBackups.BackupTasks, + Storage: state.PVEBackups.StorageBackups, + GuestSnaps: state.PVEBackups.GuestSnapshots, + }) } // handleBackupsPVE handles PVE backup requests diff --git a/internal/api/types.go b/internal/api/types.go index c1b1563..d99c4af 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -58,6 +58,8 @@ type StateResponse struct { PBSInstances []models.PBSInstance `json:"pbs"` PMGInstances []models.PMGInstance `json:"pmg"` PBSBackups []models.PBSBackup `json:"pbsBackups"` + PMGBackups []models.PMGBackup `json:"pmgBackups"` + Backups models.Backups `json:"backups"` Metrics []models.Metric `json:"metrics"` PVEBackups models.PVEBackups `json:"pveBackups"` Performance models.Performance `json:"performance"` diff --git a/internal/config/persistence.go b/internal/config/persistence.go index 4059995..f2a87ad 100644 --- a/internal/config/persistence.go +++ b/internal/config/persistence.go @@ -194,6 +194,15 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error { if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays { config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays } + if config.BackupDefaults.WarningDays < 0 { + config.BackupDefaults.WarningDays = 0 + } + if config.BackupDefaults.CriticalDays < 0 { + config.BackupDefaults.CriticalDays = 0 + } + if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays { + config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays + } config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes) data, err := json.MarshalIndent(config, "", " ") @@ -251,6 +260,11 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) { WarningDays: 30, CriticalDays: 45, }, + BackupDefaults: alerts.BackupAlertConfig{ + Enabled: false, + WarningDays: 7, + CriticalDays: 14, + }, Overrides: make(map[string]alerts.ThresholdConfig), }, nil } @@ -310,6 +324,15 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) { if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays { config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays } + if config.BackupDefaults.WarningDays < 0 { + config.BackupDefaults.WarningDays = 0 + } + if config.BackupDefaults.CriticalDays < 0 { + config.BackupDefaults.CriticalDays = 0 + } + if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays { + config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays + } config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds) config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes) diff --git a/internal/config/persistence_test.go b/internal/config/persistence_test.go index 0671d6d..b0aca3f 100644 --- a/internal/config/persistence_test.go +++ b/internal/config/persistence_test.go @@ -140,6 +140,11 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) { WarningDays: 20, CriticalDays: 10, }, + BackupDefaults: alerts.BackupAlertConfig{ + Enabled: true, + WarningDays: 12, + CriticalDays: 8, + }, NodeDefaults: alerts.ThresholdConfig{ Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0}, }, @@ -174,6 +179,15 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) { if loaded.NodeDefaults.Temperature.Trigger != 80 || loaded.NodeDefaults.Temperature.Clear != 75 { t.Fatalf("expected temperature defaults 80/75, got %+v", loaded.NodeDefaults.Temperature) } + if !loaded.BackupDefaults.Enabled { + t.Fatalf("expected backup defaults to remain enabled") + } + if loaded.BackupDefaults.WarningDays != 8 { + t.Fatalf("expected backup warning normalized to 8, got %d", loaded.BackupDefaults.WarningDays) + } + if loaded.BackupDefaults.CriticalDays != 8 { + t.Fatalf("expected backup critical normalized to 8, got %d", loaded.BackupDefaults.CriticalDays) + } expectedPrefixes := []string{"Runner"} if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) { t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes) diff --git a/internal/mock/generator.go b/internal/mock/generator.go index 724dc29..e19e0f6 100644 --- a/internal/mock/generator.go +++ b/internal/mock/generator.go @@ -329,6 +329,12 @@ func GenerateMockData(config MockConfig) models.StateSnapshot { StorageBackups: generateBackups(data.VMs, data.Containers), GuestSnapshots: generateSnapshots(data.VMs, data.Containers), } + data.PMGBackups = extractPMGBackups(data.PVEBackups.StorageBackups) + data.Backups = models.Backups{ + PVE: data.PVEBackups, + PBS: append([]models.PBSBackup(nil), data.PBSBackups...), + PMG: append([]models.PMGBackup(nil), data.PMGBackups...), + } // Calculate stats data.Stats.StartTime = time.Now() @@ -2100,6 +2106,47 @@ func generateBackups(vms []models.VM, containers []models.Container) []models.St return backups } +func extractPMGBackups(storageBackups []models.StorageBackup) []models.PMGBackup { + pmgBackups := make([]models.PMGBackup, 0) + for _, backup := range storageBackups { + if !strings.EqualFold(backup.Type, "host") { + continue + } + format := strings.ToLower(backup.Format) + notes := strings.ToLower(backup.Notes) + if !strings.Contains(format, "pmg") && !strings.Contains(notes, "pmg") { + continue + } + + filename := backup.Volid + if filename == "" { + filename = backup.Notes + } + instance := backup.Instance + if instance == "" && backup.Node != "" { + instance = fmt.Sprintf("PMG:%s", backup.Node) + } + if instance == "" { + instance = "PMG:mock" + } + + pmgBackups = append(pmgBackups, models.PMGBackup{ + ID: backup.ID, + Instance: instance, + Node: backup.Node, + Filename: filename, + BackupTime: backup.Time, + Size: backup.Size, + }) + } + + sort.Slice(pmgBackups, func(i, j int) bool { + return pmgBackups[i].BackupTime.After(pmgBackups[j].BackupTime) + }) + + return pmgBackups +} + // generatePBSInstances generates mock PBS instances func generatePBSInstances() []models.PBSInstance { pbsInstances := []models.PBSInstance{ diff --git a/internal/mock/integration.go b/internal/mock/integration.go index 5d3a42f..6c07a83 100644 --- a/internal/mock/integration.go +++ b/internal/mock/integration.go @@ -310,6 +310,7 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot { PhysicalDisks: append([]models.PhysicalDisk(nil), state.PhysicalDisks...), PBSInstances: append([]models.PBSInstance(nil), state.PBSInstances...), PBSBackups: append([]models.PBSBackup(nil), state.PBSBackups...), + PMGBackups: append([]models.PMGBackup(nil), state.PMGBackups...), Metrics: append([]models.Metric(nil), state.Metrics...), Performance: state.Performance, Stats: state.Stats, @@ -324,6 +325,15 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot { StorageBackups: append([]models.StorageBackup(nil), state.PVEBackups.StorageBackups...), GuestSnapshots: append([]models.GuestSnapshot(nil), state.PVEBackups.GuestSnapshots...), } + copyState.Backups = models.Backups{ + PVE: models.PVEBackups{ + BackupTasks: append([]models.BackupTask(nil), state.Backups.PVE.BackupTasks...), + StorageBackups: append([]models.StorageBackup(nil), state.Backups.PVE.StorageBackups...), + GuestSnapshots: append([]models.GuestSnapshot(nil), state.Backups.PVE.GuestSnapshots...), + }, + PBS: append([]models.PBSBackup(nil), state.Backups.PBS...), + PMG: append([]models.PMGBackup(nil), state.Backups.PMG...), + } for k, v := range state.ConnectionHealth { copyState.ConnectionHealth[k] = v diff --git a/internal/models/models.go b/internal/models/models.go index c68e361..918fc3a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -20,6 +20,8 @@ type State struct { PBSInstances []PBSInstance `json:"pbs"` PMGInstances []PMGInstance `json:"pmg"` PBSBackups []PBSBackup `json:"pbsBackups"` + PMGBackups []PMGBackup `json:"pmgBackups"` + Backups Backups `json:"backups"` Metrics []Metric `json:"metrics"` PVEBackups PVEBackups `json:"pveBackups"` Performance Performance `json:"performance"` @@ -461,6 +463,23 @@ type PMGNodeStatus struct { QueueStatus *PMGQueueStatus `json:"queueStatus,omitempty"` // Postfix queue status for this node } +// PMGBackup represents a configuration backup generated by a PMG node. +type PMGBackup struct { + ID string `json:"id"` + Instance string `json:"instance"` + Node string `json:"node"` + Filename string `json:"filename"` + BackupTime time.Time `json:"backupTime"` + Size int64 `json:"size"` +} + +// Backups aggregates backup collections by source type. +type Backups struct { + PVE PVEBackups `json:"pve"` + PBS []PBSBackup `json:"pbs"` + PMG []PMGBackup `json:"pmg"` +} + // PMGMailStats summarizes aggregated mail statistics for a timeframe type PMGMailStats struct { Timeframe string `json:"timeframe"` @@ -677,7 +696,13 @@ type Stats struct { // NewState creates a new State instance func NewState() *State { - return &State{ + pveBackups := PVEBackups{ + BackupTasks: make([]BackupTask, 0), + StorageBackups: make([]StorageBackup, 0), + GuestSnapshots: make([]GuestSnapshot, 0), + } + + state := &State{ Nodes: make([]Node, 0), VMs: make([]VM, 0), Containers: make([]Container, 0), @@ -687,17 +712,31 @@ func NewState() *State { PBSInstances: make([]PBSInstance, 0), PMGInstances: make([]PMGInstance, 0), PBSBackups: make([]PBSBackup, 0), - Metrics: make([]Metric, 0), - PVEBackups: PVEBackups{ - BackupTasks: make([]BackupTask, 0), - StorageBackups: make([]StorageBackup, 0), - GuestSnapshots: make([]GuestSnapshot, 0), + PMGBackups: make([]PMGBackup, 0), + Backups: Backups{ + PVE: pveBackups, + PBS: make([]PBSBackup, 0), + PMG: make([]PMGBackup, 0), }, + Metrics: make([]Metric, 0), + PVEBackups: pveBackups, ConnectionHealth: make(map[string]bool), ActiveAlerts: make([]Alert, 0), RecentlyResolved: make([]ResolvedAlert, 0), LastUpdate: time.Now(), } + + state.syncBackupsLocked() + return state +} + +// syncBackupsLocked updates the aggregated backups structure. +func (s *State) syncBackupsLocked() { + s.Backups = Backups{ + PVE: s.PVEBackups, + PBS: append([]PBSBackup(nil), s.PBSBackups...), + PMG: append([]PMGBackup(nil), s.PMGBackups...), + } } // UpdateActiveAlerts updates the active alerts in the state @@ -1211,6 +1250,7 @@ func (s *State) UpdateBackupTasksForInstance(instanceName string, tasks []Backup }) s.PVEBackups.BackupTasks = newTasks + s.syncBackupsLocked() s.LastUpdate = time.Now() } @@ -1245,6 +1285,7 @@ func (s *State) UpdateStorageBackupsForInstance(instanceName string, backups []S }) s.PVEBackups.StorageBackups = newBackups + s.syncBackupsLocked() s.LastUpdate = time.Now() } @@ -1279,6 +1320,7 @@ func (s *State) UpdateGuestSnapshotsForInstance(instanceName string, snapshots [ }) s.PVEBackups.GuestSnapshots = newSnapshots + s.syncBackupsLocked() s.LastUpdate = time.Now() } @@ -1326,5 +1368,32 @@ func (s *State) UpdatePBSBackups(instanceName string, backups []PBSBackup) { }) s.PBSBackups = newBackups + s.syncBackupsLocked() + s.LastUpdate = time.Now() +} + +// UpdatePMGBackups updates PMG backups for a specific instance. +func (s *State) UpdatePMGBackups(instanceName string, backups []PMGBackup) { + s.mu.Lock() + defer s.mu.Unlock() + + combined := make([]PMGBackup, 0, len(s.PMGBackups)+len(backups)) + for _, backup := range s.PMGBackups { + if backup.Instance != instanceName { + combined = append(combined, backup) + } + } + if len(backups) > 0 { + combined = append(combined, backups...) + } + + if len(combined) > 1 { + sort.Slice(combined, func(i, j int) bool { + return combined[i].BackupTime.After(combined[j].BackupTime) + }) + } + + s.PMGBackups = combined + s.syncBackupsLocked() s.LastUpdate = time.Now() } diff --git a/internal/models/models_frontend.go b/internal/models/models_frontend.go index 281aed6..7bc2166 100644 --- a/internal/models/models_frontend.go +++ b/internal/models/models_frontend.go @@ -230,6 +230,9 @@ type StateFrontend struct { PhysicalDisks []PhysicalDisk `json:"physicalDisks"` PBS []PBSInstance `json:"pbs"` // Keep as is PMG []PMGInstance `json:"pmg"` + PBSBackups []PBSBackup `json:"pbsBackups"` + PMGBackups []PMGBackup `json:"pmgBackups"` + Backups Backups `json:"backups"` ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts Metrics map[string]any `json:"metrics"` // Empty object for now PVEBackups PVEBackups `json:"pveBackups"` // Keep as is diff --git a/internal/models/state_snapshot.go b/internal/models/state_snapshot.go index e1f7ea9..af82895 100644 --- a/internal/models/state_snapshot.go +++ b/internal/models/state_snapshot.go @@ -14,6 +14,8 @@ type StateSnapshot struct { PBSInstances []PBSInstance `json:"pbs"` PMGInstances []PMGInstance `json:"pmg"` PBSBackups []PBSBackup `json:"pbsBackups"` + PMGBackups []PMGBackup `json:"pmgBackups"` + Backups Backups `json:"backups"` Metrics []Metric `json:"metrics"` PVEBackups PVEBackups `json:"pveBackups"` Performance Performance `json:"performance"` @@ -29,6 +31,14 @@ func (s *State) GetSnapshot() StateSnapshot { s.mu.RLock() defer s.mu.RUnlock() + pbsBackups := append([]PBSBackup{}, s.PBSBackups...) + pmgBackups := append([]PMGBackup{}, s.PMGBackups...) + pveBackups := PVEBackups{ + BackupTasks: append([]BackupTask{}, s.PVEBackups.BackupTasks...), + StorageBackups: append([]StorageBackup{}, s.PVEBackups.StorageBackups...), + GuestSnapshots: append([]GuestSnapshot{}, s.PVEBackups.GuestSnapshots...), + } + // Create a snapshot without mutex snapshot := StateSnapshot{ Nodes: append([]Node{}, s.Nodes...), @@ -40,13 +50,15 @@ func (s *State) GetSnapshot() StateSnapshot { PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...), PBSInstances: append([]PBSInstance{}, s.PBSInstances...), PMGInstances: append([]PMGInstance{}, s.PMGInstances...), - PBSBackups: append([]PBSBackup{}, s.PBSBackups...), - Metrics: append([]Metric{}, s.Metrics...), - PVEBackups: PVEBackups{ - BackupTasks: append([]BackupTask{}, s.PVEBackups.BackupTasks...), - StorageBackups: append([]StorageBackup{}, s.PVEBackups.StorageBackups...), - GuestSnapshots: append([]GuestSnapshot{}, s.PVEBackups.GuestSnapshots...), + PBSBackups: pbsBackups, + PMGBackups: pmgBackups, + Backups: Backups{ + PVE: pveBackups, + PBS: pbsBackups, + PMG: pmgBackups, }, + Metrics: append([]Metric{}, s.Metrics...), + PVEBackups: pveBackups, Performance: s.Performance, ConnectionHealth: make(map[string]bool), Stats: s.Stats, @@ -110,6 +122,9 @@ func (s StateSnapshot) ToFrontend() StateFrontend { PhysicalDisks: s.PhysicalDisks, PBS: s.PBSInstances, PMG: s.PMGInstances, + PBSBackups: s.PBSBackups, + PMGBackups: s.PMGBackups, + Backups: s.Backups, ActiveAlerts: s.ActiveAlerts, Metrics: make(map[string]any), PVEBackups: s.PVEBackups, diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index 10caaee..b06319f 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -1707,7 +1707,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) { Int("vms", len(state.VMs)). Int("containers", len(state.Containers)). Int("pbs", len(state.PBSInstances)). - Int("pbsBackups", len(state.PBSBackups)). + Int("pbsBackups", len(state.Backups.PBS)). Int("physicalDisks", len(state.PhysicalDisks)). Msg("Broadcasting state update (ticker)") // Convert to frontend format before broadcasting (converts time.Time to int64, etc.) @@ -5167,7 +5167,11 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie cluster, err := client.GetClusterStatus(ctx, true) if err != nil { log.Debug().Err(err).Str("instance", instanceName).Msg("Failed to retrieve PMG cluster status") - } else if len(cluster) > 0 { + } + + backupNodes := make(map[string]struct{}) + + if len(cluster) > 0 { nodes := make([]models.PMGNodeStatus, 0, len(cluster)) for _, entry := range cluster { status := strings.ToLower(strings.TrimSpace(entry.Type)) @@ -5180,6 +5184,8 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie Role: entry.Type, } + backupNodes[entry.Name] = struct{}{} + // Fetch queue status for this node if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil { log.Debug().Err(qErr). @@ -5204,6 +5210,53 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie pmgInst.Nodes = nodes } + if len(backupNodes) == 0 { + trimmed := strings.TrimSpace(instanceName) + if trimmed != "" { + backupNodes[trimmed] = struct{}{} + } + } + + pmgBackups := make([]models.PMGBackup, 0) + seenBackupIDs := make(map[string]struct{}) + + for nodeName := range backupNodes { + if ctx.Err() != nil { + break + } + + backups, backupErr := client.ListBackups(ctx, nodeName) + if backupErr != nil { + log.Debug().Err(backupErr). + Str("instance", instanceName). + Str("node", nodeName). + Msg("Failed to list PMG configuration backups") + continue + } + + for _, b := range backups { + backupTime := time.Unix(b.Timestamp, 0) + id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, b.Timestamp) + if _, exists := seenBackupIDs[id]; exists { + continue + } + seenBackupIDs[id] = struct{}{} + pmgBackups = append(pmgBackups, models.PMGBackup{ + ID: id, + Instance: instanceName, + Node: nodeName, + Filename: b.Filename, + BackupTime: backupTime, + Size: b.Size, + }) + } + } + + log.Debug(). + Str("instance", instanceName). + Int("backupCount", len(pmgBackups)). + Msg("PMG backups polled") + if stats, err := client.GetMailStatistics(ctx, "day"); err != nil { log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to fetch PMG mail statistics") } else if stats != nil { @@ -5279,6 +5332,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie } pmgInst.Quarantine = &quarantine + m.state.UpdatePMGBackups(instanceName, pmgBackups) m.state.UpdatePMGInstance(pmgInst) log.Info(). Str("instance", instanceName). @@ -5620,6 +5674,24 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName // Update state with storage backups for this instance m.state.UpdateStorageBackupsForInstance(instanceName, allBackups) + if m.alertManager != nil { + snapshot := m.state.GetSnapshot() + guestsByKey, guestsByVMID := buildGuestLookups(snapshot) + pveStorage := snapshot.Backups.PVE.StorageBackups + if len(pveStorage) == 0 && len(snapshot.PVEBackups.StorageBackups) > 0 { + pveStorage = snapshot.PVEBackups.StorageBackups + } + pbsBackups := snapshot.Backups.PBS + if len(pbsBackups) == 0 && len(snapshot.PBSBackups) > 0 { + pbsBackups = snapshot.PBSBackups + } + pmgBackups := snapshot.Backups.PMG + if len(pmgBackups) == 0 && len(snapshot.PMGBackups) > 0 { + pmgBackups = snapshot.PMGBackups + } + m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID) + } + log.Debug(). Str("instance", instanceName). Int("count", len(allBackups)). @@ -5636,6 +5708,49 @@ func shouldPreserveBackups(nodeCount int, hadSuccessfulNode bool, storagesWithBa return false } +func buildGuestLookups(snapshot models.StateSnapshot) (map[string]alerts.GuestLookup, map[string]alerts.GuestLookup) { + byKey := make(map[string]alerts.GuestLookup) + byVMID := make(map[string]alerts.GuestLookup) + + for _, vm := range snapshot.VMs { + info := alerts.GuestLookup{ + Name: vm.Name, + Instance: vm.Instance, + Node: vm.Node, + Type: vm.Type, + VMID: vm.VMID, + } + key := alerts.BuildGuestKey(vm.Instance, vm.Node, vm.VMID) + byKey[key] = info + + vmidKey := fmt.Sprintf("%d", vm.VMID) + if _, exists := byVMID[vmidKey]; !exists { + byVMID[vmidKey] = info + } + } + + for _, ct := range snapshot.Containers { + info := alerts.GuestLookup{ + Name: ct.Name, + Instance: ct.Instance, + Node: ct.Node, + Type: ct.Type, + VMID: int(ct.VMID), + } + key := alerts.BuildGuestKey(ct.Instance, ct.Node, int(ct.VMID)) + if _, exists := byKey[key]; !exists { + byKey[key] = info + } + + vmidKey := fmt.Sprintf("%d", ct.VMID) + if _, exists := byVMID[vmidKey]; !exists { + byVMID[vmidKey] = info + } + } + + return byKey, byVMID +} + func (m *Monitor) calculateBackupOperationTimeout(instanceName string) time.Duration { const ( minTimeout = 2 * time.Minute @@ -6042,6 +6157,7 @@ func (m *Monitor) removeFailedPMGInstance(instanceName string) { } m.state.UpdatePMGInstances(updated) + m.state.UpdatePMGBackups(instanceName, nil) m.state.SetConnectionHealth("pmg-"+instanceName, false) } @@ -6149,6 +6265,24 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien // Update state m.state.UpdatePBSBackups(instanceName, allBackups) + + if m.alertManager != nil { + snapshot := m.state.GetSnapshot() + guestsByKey, guestsByVMID := buildGuestLookups(snapshot) + pveStorage := snapshot.Backups.PVE.StorageBackups + if len(pveStorage) == 0 && len(snapshot.PVEBackups.StorageBackups) > 0 { + pveStorage = snapshot.PVEBackups.StorageBackups + } + pbsBackups := snapshot.Backups.PBS + if len(pbsBackups) == 0 && len(snapshot.PBSBackups) > 0 { + pbsBackups = snapshot.PBSBackups + } + pmgBackups := snapshot.Backups.PMG + if len(pmgBackups) == 0 && len(snapshot.PMGBackups) > 0 { + pmgBackups = snapshot.PMGBackups + } + m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID) + } } // checkMockAlerts checks alerts for mock data @@ -6188,6 +6322,21 @@ func (m *Monitor) checkMockAlerts() { Msg("Collecting resources for alert cleanup in mock mode") m.alertManager.CleanupAlertsForNodes(existingNodes) + guestsByKey, guestsByVMID := buildGuestLookups(state) + pveStorage := state.Backups.PVE.StorageBackups + if len(pveStorage) == 0 && len(state.PVEBackups.StorageBackups) > 0 { + pveStorage = state.PVEBackups.StorageBackups + } + pbsBackups := state.Backups.PBS + if len(pbsBackups) == 0 && len(state.PBSBackups) > 0 { + pbsBackups = state.PBSBackups + } + pmgBackups := state.Backups.PMG + if len(pmgBackups) == 0 && len(state.PMGBackups) > 0 { + pmgBackups = state.PMGBackups + } + m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID) + // Limit how many guests we check per cycle to prevent blocking with large datasets const maxGuestsPerCycle = 50 guestsChecked := 0 diff --git a/internal/monitoring/monitor_pmg_test.go b/internal/monitoring/monitor_pmg_test.go index c7fe529..9d499d4 100644 --- a/internal/monitoring/monitor_pmg_test.go +++ b/internal/monitoring/monitor_pmg_test.go @@ -66,6 +66,11 @@ func TestPollPMGInstancePopulatesState(t *testing.T) { w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`) + case "/api2/json/nodes/mail-gateway/backup": + w.Header().Set("Content-Type", "application/json") + timestamp := time.Now().Add(-8 * 24 * time.Hour).Unix() + fmt.Fprintf(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":%d}]}`, timestamp) + default: t.Fatalf("unexpected path: %s", r.URL.Path) } @@ -144,6 +149,15 @@ func TestPollPMGInstancePopulatesState(t *testing.T) { if failures := mon.authFailures["pmg-primary"]; failures != 0 { t.Fatalf("expected no auth failures tracked, got %d", failures) } + + if len(snapshot.PMGBackups) != 1 { + t.Fatalf("expected 1 PMG backup in state, got %d", len(snapshot.PMGBackups)) + } + + pmgBackup := snapshot.PMGBackups[0] + if pmgBackup.Node != "mail-gateway" { + t.Fatalf("expected PMG backup node mail-gateway, got %s", pmgBackup.Node) + } } func TestPollPMGInstanceRecordsAuthFailures(t *testing.T) { diff --git a/pkg/pmg/client.go b/pkg/pmg/client.go index 9ae5e61..f1ce729 100644 --- a/pkg/pmg/client.go +++ b/pkg/pmg/client.go @@ -119,6 +119,13 @@ type QueueStatusEntry struct { OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds } +// BackupEntry represents a PMG configuration backup stored on a node. +type BackupEntry struct { + Filename string `json:"filename"` + Size int64 `json:"size"` + Timestamp int64 `json:"timestamp"` +} + func NewClient(cfg ClientConfig) (*Client, error) { if cfg.Timeout <= 0 { cfg.Timeout = 60 * time.Second @@ -459,3 +466,13 @@ func (c *Client) GetQueueStatus(ctx context.Context, node string) (*QueueStatusE } return &resp.Data, nil } + +// ListBackups returns configuration backup archives available on a PMG node. +func (c *Client) ListBackups(ctx context.Context, node string) ([]BackupEntry, error) { + path := fmt.Sprintf("/nodes/%s/backup", url.PathEscape(node)) + var resp apiResponse[[]BackupEntry] + if err := c.getJSON(ctx, path, nil, &resp); err != nil { + return nil, err + } + return resp.Data, nil +} diff --git a/pkg/pmg/client_test.go b/pkg/pmg/client_test.go index 38f01a9..b282e05 100644 --- a/pkg/pmg/client_test.go +++ b/pkg/pmg/client_test.go @@ -231,3 +231,55 @@ func TestClientUsesTokenAuthorizationHeader(t *testing.T) { t.Fatalf("expected authorization header %q, got %q", expected, authHeader) } } + +func TestListBackups(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api2/json/access/ticket": + if r.Method != http.MethodPost { + t.Fatalf("expected POST for auth, got %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":{"ticket":"ticket789","CSRFPreventionToken":"csrf789"}}`) + case "/api2/json/nodes/node1/backup": + if r.Method != http.MethodGet { + t.Fatalf("expected GET for backup listing, got %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":1704096000}]}`) + default: + t.Fatalf("unexpected request path: %s", r.URL.Path) + } + })) + defer server.Close() + + client, err := NewClient(ClientConfig{ + Host: server.URL, + User: "api@pmg", + Password: "secret", + VerifySSL: false, + }) + if err != nil { + t.Fatalf("unexpected error creating client: %v", err) + } + + backups, err := client.ListBackups(context.Background(), "node1") + if err != nil { + t.Fatalf("ListBackups failed: %v", err) + } + if len(backups) != 1 { + t.Fatalf("expected 1 backup, got %d", len(backups)) + } + backup := backups[0] + if backup.Filename != "pmg-backup_2024-01-01.tgz" { + t.Fatalf("unexpected filename: %s", backup.Filename) + } + if backup.Size != 123456 { + t.Fatalf("unexpected size: %d", backup.Size) + } + if backup.Timestamp != 1704096000 { + t.Fatalf("unexpected timestamp: %d", backup.Timestamp) + } +}