Adjust backup and snapshot alert handling
This commit is contained in:
parent
80b9d0602a
commit
cfdfe896be
27 changed files with 1565 additions and 207 deletions
|
|
@ -98,15 +98,25 @@ function App() {
|
||||||
cephClusters: [],
|
cephClusters: [],
|
||||||
physicalDisks: [],
|
physicalDisks: [],
|
||||||
pbs: [],
|
pbs: [],
|
||||||
pmg: [],
|
pmg: [],
|
||||||
metrics: [],
|
metrics: [],
|
||||||
pveBackups: {
|
pveBackups: {
|
||||||
|
backupTasks: [],
|
||||||
|
storageBackups: [],
|
||||||
|
guestSnapshots: [],
|
||||||
|
},
|
||||||
|
pbsBackups: [],
|
||||||
|
pmgBackups: [],
|
||||||
|
backups: {
|
||||||
|
pve: {
|
||||||
backupTasks: [],
|
backupTasks: [],
|
||||||
storageBackups: [],
|
storageBackups: [],
|
||||||
guestSnapshots: [],
|
guestSnapshots: [],
|
||||||
},
|
},
|
||||||
pbsBackups: [],
|
pbs: [],
|
||||||
performance: {
|
pmg: [],
|
||||||
|
},
|
||||||
|
performance: {
|
||||||
apiCallDuration: {},
|
apiCallDuration: {},
|
||||||
lastPollDuration: 0,
|
lastPollDuration: 0,
|
||||||
pollingStartTime: '',
|
pollingStartTime: '',
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,7 @@ export interface Resource {
|
||||||
host?: string;
|
host?: string;
|
||||||
type?: string;
|
type?: string;
|
||||||
resourceType?: string;
|
resourceType?: string;
|
||||||
|
subtitle?: string;
|
||||||
thresholds?: Record<string, number | undefined>;
|
thresholds?: Record<string, number | undefined>;
|
||||||
defaults?: Record<string, number | undefined>;
|
defaults?: Record<string, number | undefined>;
|
||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
|
|
@ -68,6 +69,12 @@ export interface Resource {
|
||||||
clusterName?: string;
|
clusterName?: string;
|
||||||
isClusterMember?: boolean;
|
isClusterMember?: boolean;
|
||||||
delaySeconds?: number;
|
delaySeconds?: number;
|
||||||
|
editScope?: 'snapshot';
|
||||||
|
isEnabled?: boolean;
|
||||||
|
toggleEnabled?: () => void;
|
||||||
|
toggleTitleEnabled?: string;
|
||||||
|
toggleTitleDisabled?: string;
|
||||||
|
editable?: boolean;
|
||||||
[key: string]: unknown;
|
[key: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -386,9 +393,7 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
titleEnabled?: string;
|
titleEnabled?: string;
|
||||||
titleDisabled?: string;
|
titleDisabled?: string;
|
||||||
titleWhenDisabled?: string;
|
titleWhenDisabled?: string;
|
||||||
}) => {
|
}) => <StatusBadge {...config} />;
|
||||||
return <StatusBadge {...config} />;
|
|
||||||
};
|
|
||||||
|
|
||||||
const offlineStateOrder: OfflineState[] = ['off', 'warning', 'critical'];
|
const offlineStateOrder: OfflineState[] = ['off', 'warning', 'critical'];
|
||||||
|
|
||||||
|
|
@ -826,61 +831,67 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
</Show>
|
</Show>
|
||||||
</td>
|
</td>
|
||||||
<td class="p-1 px-2">
|
<td class="p-1 px-2">
|
||||||
<Show when={resource.type === 'node'} fallback={
|
<div class="flex items-center gap-2">
|
||||||
<div class="flex items-center gap-2">
|
<Show
|
||||||
<span
|
when={resource.type === 'node'}
|
||||||
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
|
fallback={
|
||||||
>
|
|
||||||
{resource.name}
|
|
||||||
</span>
|
|
||||||
<Show when={resource.hasOverride || resource.disableConnectivity}>
|
|
||||||
<span class="text-xs px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded">
|
|
||||||
Custom
|
|
||||||
</span>
|
|
||||||
</Show>
|
|
||||||
</div>
|
|
||||||
}>
|
|
||||||
<div class="flex flex-wrap items-center gap-3" title={resource.status || undefined}>
|
|
||||||
<Show when={resource.host} fallback={
|
|
||||||
<span
|
<span
|
||||||
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
|
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
|
||||||
>
|
>
|
||||||
{resource.type === 'node'
|
{resource.name}
|
||||||
? resource.name
|
|
||||||
: resource.displayName || resource.name}
|
|
||||||
</span>
|
</span>
|
||||||
}>
|
}
|
||||||
{(host) => (
|
>
|
||||||
<a
|
<div class="flex flex-wrap items-center gap-3" title={resource.status || undefined}>
|
||||||
href={host() as string}
|
<Show
|
||||||
target="_blank"
|
when={resource.host}
|
||||||
rel="noopener noreferrer"
|
fallback={
|
||||||
onClick={(e) => e.stopPropagation()}
|
<span
|
||||||
class={`text-sm font-medium transition-colors duration-150 ${
|
class={`text-sm font-medium ${resource.disabled ? 'text-gray-500 dark:text-gray-500' : 'text-gray-900 dark:text-gray-100'}`}
|
||||||
resource.disabled
|
>
|
||||||
? 'text-gray-500 dark:text-gray-500'
|
{resource.type === 'node'
|
||||||
: 'text-gray-900 dark:text-gray-100 hover:text-sky-600 dark:hover:text-sky-400'
|
? resource.name
|
||||||
}`}
|
: resource.displayName || resource.name}
|
||||||
title={`Open ${resource.displayName || resource.name} web interface`}
|
</span>
|
||||||
>
|
}
|
||||||
{resource.type === 'node'
|
>
|
||||||
? resource.name
|
{(host) => (
|
||||||
: resource.displayName || resource.name}
|
<a
|
||||||
</a>
|
href={host() as string}
|
||||||
)}
|
target="_blank"
|
||||||
</Show>
|
rel="noopener noreferrer"
|
||||||
<Show when={resource.clusterName}>
|
onClick={(e) => e.stopPropagation()}
|
||||||
<span class="rounded px-2 py-0.5 text-[10px] font-medium whitespace-nowrap bg-blue-100 text-blue-700 dark:bg-blue-900/40 dark:text-blue-300">
|
class={`text-sm font-medium transition-colors duration-150 ${
|
||||||
{resource.clusterName}
|
resource.disabled
|
||||||
</span>
|
? 'text-gray-500 dark:text-gray-500'
|
||||||
</Show>
|
: 'text-gray-900 dark:text-gray-100 hover:text-sky-600 dark:hover:text-sky-400'
|
||||||
<Show when={resource.hasOverride || resource.disableConnectivity}>
|
}`}
|
||||||
<span class="text-xs px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded">
|
title={`Open ${resource.displayName || resource.name} web interface`}
|
||||||
Custom
|
>
|
||||||
</span>
|
{resource.type === 'node'
|
||||||
</Show>
|
? resource.name
|
||||||
</div>
|
: resource.displayName || resource.name}
|
||||||
</Show>
|
</a>
|
||||||
|
)}
|
||||||
|
</Show>
|
||||||
|
<Show when={resource.clusterName}>
|
||||||
|
<span class="rounded px-2 py-0.5 text-[10px] font-medium whitespace-nowrap bg-blue-100 text-blue-700 dark:bg-blue-900/40 dark:text-blue-300">
|
||||||
|
{resource.clusterName}
|
||||||
|
</span>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
<Show when={resource.subtitle}>
|
||||||
|
<span class="text-xs text-gray-500 dark:text-gray-400">
|
||||||
|
{resource.subtitle as string}
|
||||||
|
</span>
|
||||||
|
</Show>
|
||||||
|
<Show when={resource.hasOverride || resource.disableConnectivity}>
|
||||||
|
<span class="text-xs px-1.5 py-0.5 bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded">
|
||||||
|
Custom
|
||||||
|
</span>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
{/* Metric columns - dynamically rendered based on resource type */}
|
{/* Metric columns - dynamically rendered based on resource type */}
|
||||||
<For each={props.columns}>
|
<For each={props.columns}>
|
||||||
|
|
@ -1279,14 +1290,7 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
{/* Metric columns - dynamically rendered based on resource type */}
|
{/* Metric columns - dynamically rendered based on resource type */}
|
||||||
<For each={props.columns}>
|
<For each={props.columns}>
|
||||||
{(column) => {
|
{(column) => {
|
||||||
const metric = column
|
const metric = normalizeMetricKey(column);
|
||||||
.toLowerCase()
|
|
||||||
.replace(' %', '')
|
|
||||||
.replace(' mb/s', '')
|
|
||||||
.replace('disk r', 'diskRead')
|
|
||||||
.replace('disk w', 'diskWrite')
|
|
||||||
.replace('net in', 'networkIn')
|
|
||||||
.replace('net out', 'networkOut');
|
|
||||||
|
|
||||||
// Check if this metric applies to this resource type
|
// Check if this metric applies to this resource type
|
||||||
const showMetric = () => {
|
const showMetric = () => {
|
||||||
|
|
@ -1312,6 +1316,9 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
|
|
||||||
const openMetricEditor = (e: MouseEvent) => {
|
const openMetricEditor = (e: MouseEvent) => {
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
|
if (resource.editable === false) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
setActiveMetricInput({ resourceId: resource.id, metric });
|
setActiveMetricInput({ resourceId: resource.id, metric });
|
||||||
props.onEdit(
|
props.onEdit(
|
||||||
resource.id,
|
resource.id,
|
||||||
|
|
@ -1438,7 +1445,20 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
{/* Actions column */}
|
{/* Actions column */}
|
||||||
<td class="p-1 px-2">
|
<td class="p-1 px-2">
|
||||||
<div class="flex items-center justify-center gap-1">
|
<div class="flex items-center justify-center gap-1">
|
||||||
|
<Show when={typeof resource.toggleEnabled === 'function'}>
|
||||||
|
<StatusBadge
|
||||||
|
size="sm"
|
||||||
|
isEnabled={resource.isEnabled ?? true}
|
||||||
|
onToggle={resource.toggleEnabled}
|
||||||
|
titleEnabled={resource.toggleTitleEnabled}
|
||||||
|
titleDisabled={resource.toggleTitleDisabled}
|
||||||
|
/>
|
||||||
|
</Show>
|
||||||
<Show
|
<Show
|
||||||
|
when={resource.editable !== false && typeof props.onEdit === 'function'}
|
||||||
|
fallback={<span class="text-xs text-gray-400 dark:text-gray-600">—</span>}
|
||||||
|
>
|
||||||
|
<Show
|
||||||
when={!isEditing()}
|
when={!isEditing()}
|
||||||
fallback={
|
fallback={
|
||||||
<button
|
<button
|
||||||
|
|
@ -1466,43 +1486,18 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
</button>
|
</button>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<button
|
<div class="flex items-center gap-1">
|
||||||
type="button"
|
|
||||||
onClick={() =>
|
|
||||||
props.onEdit(
|
|
||||||
resource.id,
|
|
||||||
resource.thresholds ? { ...resource.thresholds } : {},
|
|
||||||
resource.defaults ? { ...resource.defaults } : {},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
class="p-1 text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300"
|
|
||||||
title="Edit thresholds"
|
|
||||||
>
|
|
||||||
<svg
|
|
||||||
class="w-4 h-4"
|
|
||||||
fill="none"
|
|
||||||
stroke="currentColor"
|
|
||||||
viewBox="0 0 24 24"
|
|
||||||
>
|
|
||||||
<path
|
|
||||||
stroke-linecap="round"
|
|
||||||
stroke-linejoin="round"
|
|
||||||
stroke-width="2"
|
|
||||||
d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"
|
|
||||||
/>
|
|
||||||
</svg>
|
|
||||||
</button>
|
|
||||||
<Show
|
|
||||||
when={
|
|
||||||
resource.hasOverride ||
|
|
||||||
(resource.type === 'node' && resource.disableConnectivity)
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => props.onRemoveOverride(resource.id)}
|
onClick={() =>
|
||||||
class="p-1 text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300"
|
props.onEdit(
|
||||||
title="Remove override"
|
resource.id,
|
||||||
|
resource.thresholds ? { ...resource.thresholds } : {},
|
||||||
|
resource.defaults ? { ...resource.defaults } : {},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
class="p-1 text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300"
|
||||||
|
title="Edit thresholds"
|
||||||
>
|
>
|
||||||
<svg
|
<svg
|
||||||
class="w-4 h-4"
|
class="w-4 h-4"
|
||||||
|
|
@ -1514,14 +1509,42 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||||
stroke-linecap="round"
|
stroke-linecap="round"
|
||||||
stroke-linejoin="round"
|
stroke-linejoin="round"
|
||||||
stroke-width="2"
|
stroke-width="2"
|
||||||
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
|
d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z"
|
||||||
/>
|
/>
|
||||||
</svg>
|
</svg>
|
||||||
</button>
|
</button>
|
||||||
</Show>
|
<Show
|
||||||
|
when={
|
||||||
|
resource.hasOverride ||
|
||||||
|
(resource.type === 'node' && resource.disableConnectivity)
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => props.onRemoveOverride(resource.id)}
|
||||||
|
class="p-1 text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300"
|
||||||
|
title="Remove override"
|
||||||
|
>
|
||||||
|
<svg
|
||||||
|
class="w-4 h-4"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
>
|
||||||
|
<path
|
||||||
|
stroke-linecap="round"
|
||||||
|
stroke-linejoin="round"
|
||||||
|
stroke-width="2"
|
||||||
|
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
</Show>
|
||||||
|
</div>
|
||||||
</Show>
|
</Show>
|
||||||
</div>
|
</Show>
|
||||||
</td>
|
</div>
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
);
|
);
|
||||||
}}
|
}}
|
||||||
|
|
|
||||||
|
|
@ -14,8 +14,12 @@ import type {
|
||||||
PMGInstance,
|
PMGInstance,
|
||||||
DockerHost,
|
DockerHost,
|
||||||
DockerContainer,
|
DockerContainer,
|
||||||
|
PVEBackups,
|
||||||
|
PBSBackup,
|
||||||
|
PMGBackup,
|
||||||
|
Backups,
|
||||||
} from '@/types/api';
|
} from '@/types/api';
|
||||||
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
|
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
|
||||||
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
|
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
|
||||||
type OverrideType =
|
type OverrideType =
|
||||||
| 'guest'
|
| 'guest'
|
||||||
|
|
@ -103,6 +107,11 @@ export const normalizeDockerIgnoredInput = (value: string): string[] =>
|
||||||
.map((entry) => entry.trim())
|
.map((entry) => entry.trim())
|
||||||
.filter((entry) => entry.length > 0);
|
.filter((entry) => entry.length > 0);
|
||||||
|
|
||||||
|
const DEFAULT_SNAPSHOT_WARNING = 30;
|
||||||
|
const DEFAULT_SNAPSHOT_CRITICAL = 45;
|
||||||
|
const DEFAULT_BACKUP_WARNING = 7;
|
||||||
|
const DEFAULT_BACKUP_CRITICAL = 14;
|
||||||
|
|
||||||
// Simple threshold object for the UI
|
// Simple threshold object for the UI
|
||||||
interface SimpleThresholds {
|
interface SimpleThresholds {
|
||||||
cpu?: number;
|
cpu?: number;
|
||||||
|
|
@ -127,6 +136,10 @@ interface ThresholdsTableProps {
|
||||||
dockerHosts: DockerHost[];
|
dockerHosts: DockerHost[];
|
||||||
pbsInstances?: PBSInstance[]; // PBS instances from state
|
pbsInstances?: PBSInstance[]; // PBS instances from state
|
||||||
pmgInstances?: PMGInstance[]; // PMG instances from state
|
pmgInstances?: PMGInstance[]; // PMG instances from state
|
||||||
|
backups?: Backups;
|
||||||
|
pveBackups?: PVEBackups;
|
||||||
|
pbsBackups?: PBSBackup[];
|
||||||
|
pmgBackups?: PMGBackup[];
|
||||||
pmgThresholds: () => PMGThresholdDefaults;
|
pmgThresholds: () => PMGThresholdDefaults;
|
||||||
setPMGThresholds: (
|
setPMGThresholds: (
|
||||||
value:
|
value:
|
||||||
|
|
@ -177,6 +190,14 @@ interface ThresholdsTableProps {
|
||||||
) => void;
|
) => void;
|
||||||
snapshotFactoryDefaults?: SnapshotAlertConfig;
|
snapshotFactoryDefaults?: SnapshotAlertConfig;
|
||||||
resetSnapshotDefaults?: () => void;
|
resetSnapshotDefaults?: () => void;
|
||||||
|
backupDefaults: () => BackupAlertConfig;
|
||||||
|
setBackupDefaults: (
|
||||||
|
value:
|
||||||
|
| BackupAlertConfig
|
||||||
|
| ((prev: BackupAlertConfig) => BackupAlertConfig),
|
||||||
|
) => void;
|
||||||
|
backupFactoryDefaults?: BackupAlertConfig;
|
||||||
|
resetBackupDefaults?: () => void;
|
||||||
setHasUnsavedChanges: (value: boolean) => void;
|
setHasUnsavedChanges: (value: boolean) => void;
|
||||||
activeAlerts?: Record<string, Alert>;
|
activeAlerts?: Record<string, Alert>;
|
||||||
removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
|
removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
|
||||||
|
|
@ -208,9 +229,6 @@ interface ThresholdsTableProps {
|
||||||
setDisableAllDockerHostsOffline: (value: boolean) => void;
|
setDisableAllDockerHostsOffline: (value: boolean) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_SNAPSHOT_WARNING = 30;
|
|
||||||
const DEFAULT_SNAPSHOT_CRITICAL = 45;
|
|
||||||
|
|
||||||
export function ThresholdsTable(props: ThresholdsTableProps) {
|
export function ThresholdsTable(props: ThresholdsTableProps) {
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
const location = useLocation();
|
const location = useLocation();
|
||||||
|
|
@ -803,26 +821,88 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshotFactoryDefaultsRecord = createMemo(() => {
|
const backupFactoryConfig = () =>
|
||||||
const factory = snapshotFactoryConfig();
|
props.backupFactoryDefaults ?? {
|
||||||
|
enabled: false,
|
||||||
|
warningDays: DEFAULT_BACKUP_WARNING,
|
||||||
|
criticalDays: DEFAULT_BACKUP_CRITICAL,
|
||||||
|
};
|
||||||
|
|
||||||
|
const sanitizeBackupConfig = (config: BackupAlertConfig): BackupAlertConfig => {
|
||||||
|
let warning = Math.max(0, Math.round(config.warningDays ?? 0));
|
||||||
|
let critical = Math.max(0, Math.round(config.criticalDays ?? 0));
|
||||||
|
|
||||||
|
if (critical > 0 && warning > critical) {
|
||||||
|
warning = critical;
|
||||||
|
}
|
||||||
|
if (critical === 0 && warning > 0) {
|
||||||
|
critical = warning;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'warning days': factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING,
|
enabled: !!config.enabled,
|
||||||
'critical days': factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL,
|
warningDays: warning,
|
||||||
|
criticalDays: critical,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const updateBackupDefaults = (
|
||||||
|
updater:
|
||||||
|
| BackupAlertConfig
|
||||||
|
| ((prev: BackupAlertConfig) => BackupAlertConfig),
|
||||||
|
) => {
|
||||||
|
props.setBackupDefaults((prev) => {
|
||||||
|
const next =
|
||||||
|
typeof updater === 'function'
|
||||||
|
? (updater as (prev: BackupAlertConfig) => BackupAlertConfig)(prev)
|
||||||
|
: { ...prev, ...updater };
|
||||||
|
return sanitizeBackupConfig(next);
|
||||||
|
});
|
||||||
|
props.setHasUnsavedChanges(true);
|
||||||
|
};
|
||||||
|
|
||||||
|
const backupDefaultsRecord = createMemo(() => {
|
||||||
|
const current = props.backupDefaults();
|
||||||
|
return {
|
||||||
|
'warning days': current.warningDays ?? 0,
|
||||||
|
'critical days': current.criticalDays ?? 0,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
const snapshotOverridesCount = createMemo(() => {
|
const backupFactoryDefaultsRecord = createMemo(() => {
|
||||||
const current = props.snapshotDefaults();
|
const factory = backupFactoryConfig();
|
||||||
const factory = snapshotFactoryConfig();
|
return {
|
||||||
return current.enabled !== factory.enabled ||
|
'warning days': factory.warningDays ?? DEFAULT_BACKUP_WARNING,
|
||||||
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !==
|
'critical days': factory.criticalDays ?? DEFAULT_BACKUP_CRITICAL,
|
||||||
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) ||
|
};
|
||||||
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !==
|
});
|
||||||
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
|
|
||||||
|
const snapshotOverridesCount = createMemo(() => {
|
||||||
|
const current = props.snapshotDefaults();
|
||||||
|
const factory = snapshotFactoryConfig();
|
||||||
|
return current.enabled !== factory.enabled ||
|
||||||
|
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !==
|
||||||
|
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) ||
|
||||||
|
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !==
|
||||||
|
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
|
||||||
|
? 1
|
||||||
|
: 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
const backupOverridesCount = createMemo(() => {
|
||||||
|
const backupCurrent = props.backupDefaults();
|
||||||
|
const backupFactory = backupFactoryConfig();
|
||||||
|
return backupCurrent.enabled !== backupFactory.enabled ||
|
||||||
|
(backupCurrent.warningDays ?? DEFAULT_BACKUP_WARNING) !==
|
||||||
|
(backupFactory.warningDays ?? DEFAULT_BACKUP_WARNING) ||
|
||||||
|
(backupCurrent.criticalDays ?? DEFAULT_BACKUP_CRITICAL) !==
|
||||||
|
(backupFactory.criticalDays ?? DEFAULT_BACKUP_CRITICAL)
|
||||||
? 1
|
? 1
|
||||||
: 0;
|
: 0;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
// Process guests with their overrides and group by node
|
// Process guests with their overrides and group by node
|
||||||
const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => {
|
const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => {
|
||||||
// If we're currently editing, return the previous value to avoid re-renders
|
// If we're currently editing, return the previous value to avoid re-renders
|
||||||
|
|
@ -1161,18 +1241,25 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'storage' as const,
|
key: 'storage' as const,
|
||||||
label: 'Storage',
|
label: 'Storage',
|
||||||
total: props.storage?.length ?? 0,
|
total: props.storage?.length ?? 0,
|
||||||
overrides: countOverrides(storageWithOverrides()),
|
overrides: countOverrides(storageWithOverrides()),
|
||||||
tab: 'proxmox' as const,
|
tab: 'proxmox' as const,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'snapshots' as const,
|
key: 'backups' as const,
|
||||||
label: 'Snapshots',
|
label: 'Backups',
|
||||||
total: 1,
|
total: 1,
|
||||||
overrides: snapshotOverridesCount(),
|
overrides: backupOverridesCount(),
|
||||||
tab: 'proxmox' as const,
|
tab: 'proxmox' as const,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: 'snapshots' as const,
|
||||||
|
label: 'Snapshot Age',
|
||||||
|
total: 1,
|
||||||
|
overrides: snapshotOverridesCount(),
|
||||||
|
tab: 'proxmox' as const,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
key: 'pbs' as const,
|
key: 'pbs' as const,
|
||||||
label: 'PBS Servers',
|
label: 'PBS Servers',
|
||||||
|
|
@ -1240,6 +1327,41 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||||
if (!resource) return;
|
if (!resource) return;
|
||||||
|
|
||||||
const editedThresholds = editingThresholds();
|
const editedThresholds = editingThresholds();
|
||||||
|
|
||||||
|
if (resource.editScope === 'backup') {
|
||||||
|
const currentBackupDefaults = props.backupDefaults();
|
||||||
|
const nextWarning =
|
||||||
|
editedThresholds['warning days'] ?? currentBackupDefaults.warningDays ?? DEFAULT_BACKUP_WARNING;
|
||||||
|
const nextCritical =
|
||||||
|
editedThresholds['critical days'] ?? currentBackupDefaults.criticalDays ?? DEFAULT_BACKUP_CRITICAL;
|
||||||
|
|
||||||
|
updateBackupDefaults({
|
||||||
|
enabled: currentBackupDefaults.enabled,
|
||||||
|
warningDays: nextWarning,
|
||||||
|
criticalDays: nextCritical,
|
||||||
|
});
|
||||||
|
|
||||||
|
cancelEdit();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resource.editScope === 'snapshot') {
|
||||||
|
const currentSnapshotDefaults = props.snapshotDefaults();
|
||||||
|
const nextWarning =
|
||||||
|
editedThresholds['warning days'] ?? currentSnapshotDefaults.warningDays ?? DEFAULT_SNAPSHOT_WARNING;
|
||||||
|
const nextCritical =
|
||||||
|
editedThresholds['critical days'] ?? currentSnapshotDefaults.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL;
|
||||||
|
|
||||||
|
updateSnapshotDefaults({
|
||||||
|
enabled: currentSnapshotDefaults.enabled,
|
||||||
|
warningDays: nextWarning,
|
||||||
|
criticalDays: nextCritical,
|
||||||
|
});
|
||||||
|
|
||||||
|
cancelEdit();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const defaultThresholds = (resource.defaults ?? {}) as Record<string, number | undefined>;
|
const defaultThresholds = (resource.defaults ?? {}) as Record<string, number | undefined>;
|
||||||
|
|
||||||
// Only include values that differ from defaults
|
// Only include values that differ from defaults
|
||||||
|
|
@ -1969,11 +2091,74 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||||
</div>
|
</div>
|
||||||
</Show>
|
</Show>
|
||||||
|
|
||||||
|
<Show when={hasSection('backups')}>
|
||||||
|
<div ref={registerSection('backups')} class="scroll-mt-24">
|
||||||
|
<ResourceTable
|
||||||
|
title="Backups"
|
||||||
|
resources={[{ id: "backups-defaults", name: "Global Defaults", thresholds: backupDefaultsRecord(), defaults: backupDefaultsRecord(), editable: true, editScope: "backup" }]}
|
||||||
|
columns={['Warning Days', 'Critical Days']}
|
||||||
|
activeAlerts={props.activeAlerts}
|
||||||
|
emptyMessage=""
|
||||||
|
onEdit={startEditing}
|
||||||
|
onSaveEdit={saveEdit}
|
||||||
|
onCancelEdit={cancelEdit}
|
||||||
|
onRemoveOverride={removeOverride}
|
||||||
|
showOfflineAlertsColumn={false}
|
||||||
|
editingId={editingId}
|
||||||
|
editingThresholds={editingThresholds}
|
||||||
|
setEditingThresholds={setEditingThresholds}
|
||||||
|
formatMetricValue={formatMetricValue}
|
||||||
|
hasActiveAlert={hasActiveAlert}
|
||||||
|
globalDefaults={backupDefaultsRecord()}
|
||||||
|
setGlobalDefaults={(value) => {
|
||||||
|
updateBackupDefaults((prev) => {
|
||||||
|
const currentRecord = {
|
||||||
|
'warning days': prev.warningDays ?? 0,
|
||||||
|
'critical days': prev.criticalDays ?? 0,
|
||||||
|
};
|
||||||
|
const nextRecord =
|
||||||
|
typeof value === 'function'
|
||||||
|
? value(currentRecord)
|
||||||
|
: { ...currentRecord, ...value };
|
||||||
|
return {
|
||||||
|
...prev,
|
||||||
|
warningDays:
|
||||||
|
typeof nextRecord['warning days'] === 'number'
|
||||||
|
? nextRecord['warning days']
|
||||||
|
: prev.warningDays,
|
||||||
|
criticalDays:
|
||||||
|
typeof nextRecord['critical days'] === 'number'
|
||||||
|
? nextRecord['critical days']
|
||||||
|
: prev.criticalDays,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||||
|
globalDisableFlag={() => !props.backupDefaults().enabled}
|
||||||
|
onToggleGlobalDisable={() =>
|
||||||
|
updateBackupDefaults((prev) => ({
|
||||||
|
...prev,
|
||||||
|
enabled: !prev.enabled,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
factoryDefaults={backupFactoryDefaultsRecord()}
|
||||||
|
onResetDefaults={() => {
|
||||||
|
if (props.resetBackupDefaults) {
|
||||||
|
props.resetBackupDefaults();
|
||||||
|
props.setHasUnsavedChanges(true);
|
||||||
|
} else {
|
||||||
|
updateBackupDefaults(backupFactoryConfig());
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
|
||||||
<Show when={hasSection('snapshots')}>
|
<Show when={hasSection('snapshots')}>
|
||||||
<div ref={registerSection('snapshots')} class="scroll-mt-24">
|
<div ref={registerSection('snapshots')} class="scroll-mt-24">
|
||||||
<ResourceTable
|
<ResourceTable
|
||||||
title="Snapshot Age"
|
title="Snapshot Age"
|
||||||
resources={[]}
|
resources={[{ id: "snapshots-defaults", name: "Global Defaults", thresholds: snapshotDefaultsRecord(), defaults: snapshotDefaultsRecord(), editable: true, editScope: "snapshot" }]}
|
||||||
columns={['Warning Days', 'Critical Days']}
|
columns={['Warning Days', 'Critical Days']}
|
||||||
activeAlerts={props.activeAlerts}
|
activeAlerts={props.activeAlerts}
|
||||||
emptyMessage=""
|
emptyMessage=""
|
||||||
|
|
@ -2019,18 +2204,15 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||||
enabled: !prev.enabled,
|
enabled: !prev.enabled,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
factoryDefaults={snapshotFactoryDefaultsRecord()}
|
factoryDefaults={snapshotFactoryConfig()}
|
||||||
onResetDefaults={
|
onResetDefaults={() => {
|
||||||
props.resetSnapshotDefaults
|
if (props.resetSnapshotDefaults) {
|
||||||
? () => {
|
props.resetSnapshotDefaults();
|
||||||
props.resetSnapshotDefaults?.();
|
props.setHasUnsavedChanges(true);
|
||||||
props.setHasUnsavedChanges(true);
|
} else {
|
||||||
}
|
updateSnapshotDefaults(snapshotFactoryConfig());
|
||||||
: () =>
|
}
|
||||||
updateSnapshotDefaults({
|
}}
|
||||||
...snapshotFactoryConfig(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Show>
|
</Show>
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import {
|
||||||
ThresholdsTable,
|
ThresholdsTable,
|
||||||
normalizeDockerIgnoredInput,
|
normalizeDockerIgnoredInput,
|
||||||
} from '../ThresholdsTable';
|
} from '../ThresholdsTable';
|
||||||
import type { PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
|
import type { PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
|
||||||
|
|
||||||
vi.mock('@solidjs/router', () => ({
|
vi.mock('@solidjs/router', () => ({
|
||||||
useNavigate: () => vi.fn(),
|
useNavigate: () => vi.fn(),
|
||||||
|
|
@ -87,6 +87,10 @@ const baseProps = () => ({
|
||||||
factoryNodeDefaults: {},
|
factoryNodeDefaults: {},
|
||||||
factoryDockerDefaults: {},
|
factoryDockerDefaults: {},
|
||||||
factoryStorageDefault: 85,
|
factoryStorageDefault: 85,
|
||||||
|
backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }),
|
||||||
|
setBackupDefaults: vi.fn(),
|
||||||
|
backupFactoryDefaults: { enabled: false, warningDays: 7, criticalDays: 14 } as BackupAlertConfig,
|
||||||
|
resetBackupDefaults: vi.fn(),
|
||||||
snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }),
|
snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }),
|
||||||
setSnapshotDefaults: vi.fn(),
|
setSnapshotDefaults: vi.fn(),
|
||||||
snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig,
|
snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig,
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,15 @@ const Backups: Component = () => {
|
||||||
<ProxmoxSectionNav current="backups" />
|
<ProxmoxSectionNav current="backups" />
|
||||||
|
|
||||||
{/* Loading State */}
|
{/* Loading State */}
|
||||||
<Show when={connected() && !state.pveBackups && !state.pbs}>
|
<Show
|
||||||
|
when={
|
||||||
|
connected() &&
|
||||||
|
!(state.backups?.pve?.guestSnapshots?.length ||
|
||||||
|
state.backups?.pve?.storageBackups?.length ||
|
||||||
|
state.backups?.pbs?.length ||
|
||||||
|
state.backups?.pmg?.length)
|
||||||
|
}
|
||||||
|
>
|
||||||
<Card padding="lg">
|
<Card padding="lg">
|
||||||
<EmptyState
|
<EmptyState
|
||||||
icon={
|
icon={
|
||||||
|
|
@ -72,7 +80,7 @@ const Backups: Component = () => {
|
||||||
</Show>
|
</Show>
|
||||||
|
|
||||||
{/* Main Content - Unified Backups View */}
|
{/* Main Content - Unified Backups View */}
|
||||||
<Show when={connected() && (state.pveBackups || state.pbs)}>
|
<Show when={connected() && (state.backups?.pve || state.backups?.pbs || state.pbs)}>
|
||||||
<UnifiedBackups />
|
<UnifiedBackups />
|
||||||
</Show>
|
</Show>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,9 @@ interface DateGroup {
|
||||||
|
|
||||||
const UnifiedBackups: Component = () => {
|
const UnifiedBackups: Component = () => {
|
||||||
const { state } = useWebSocket();
|
const { state } = useWebSocket();
|
||||||
|
const pveBackupsState = createMemo(() => state.backups?.pve ?? state.pveBackups);
|
||||||
|
const pbsBackupsState = createMemo(() => state.backups?.pbs ?? state.pbsBackups);
|
||||||
|
const pmgBackupsState = createMemo(() => state.backups?.pmg ?? state.pmgBackups);
|
||||||
const [searchTerm, setSearchTerm] = createSignal('');
|
const [searchTerm, setSearchTerm] = createSignal('');
|
||||||
const [selectedNode, setSelectedNode] = createSignal<string | null>(null);
|
const [selectedNode, setSelectedNode] = createSignal<string | null>(null);
|
||||||
const [typeFilter, setTypeFilter] = createSignal<'all' | FilterableGuestType>('all');
|
const [typeFilter, setTypeFilter] = createSignal<'all' | FilterableGuestType>('all');
|
||||||
|
|
@ -159,10 +162,14 @@ const UnifiedBackups: Component = () => {
|
||||||
|
|
||||||
// Check if we have any backup data yet
|
// Check if we have any backup data yet
|
||||||
const isLoading = createMemo(() => {
|
const isLoading = createMemo(() => {
|
||||||
|
const pve = pveBackupsState();
|
||||||
|
const pbs = pbsBackupsState();
|
||||||
|
const pmg = pmgBackupsState();
|
||||||
return (
|
return (
|
||||||
!state.pveBackups?.guestSnapshots &&
|
!(pve?.guestSnapshots?.length ?? 0) &&
|
||||||
!state.pveBackups?.storageBackups &&
|
!(pve?.storageBackups?.length ?? 0) &&
|
||||||
!state.pbsBackups?.length &&
|
!(pbs?.length ?? 0) &&
|
||||||
|
!(pmg?.length ?? 0) &&
|
||||||
!state.pbs?.length
|
!state.pbs?.length
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
@ -176,7 +183,7 @@ const UnifiedBackups: Component = () => {
|
||||||
const debugMode = typeof window !== 'undefined' && localStorage.getItem('debug-pmg') === 'true';
|
const debugMode = typeof window !== 'undefined' && localStorage.getItem('debug-pmg') === 'true';
|
||||||
|
|
||||||
// Normalize snapshots
|
// Normalize snapshots
|
||||||
state.pveBackups?.guestSnapshots?.forEach((snapshot) => {
|
pveBackupsState()?.guestSnapshots?.forEach((snapshot) => {
|
||||||
// Try to find the guest name by matching VMID and instance (not hostname)
|
// Try to find the guest name by matching VMID and instance (not hostname)
|
||||||
let guestName = '';
|
let guestName = '';
|
||||||
const vm = state.vms?.find((v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance);
|
const vm = state.vms?.find((v) => v.vmid === snapshot.vmid && v.instance === snapshot.instance);
|
||||||
|
|
@ -213,8 +220,8 @@ const UnifiedBackups: Component = () => {
|
||||||
// This ensures we have the complete PBS data with namespaces
|
// This ensures we have the complete PBS data with namespaces
|
||||||
// Filter by selected PBS instance if one is selected
|
// Filter by selected PBS instance if one is selected
|
||||||
const pbsBackupsToProcess = selectedPBSInstance()
|
const pbsBackupsToProcess = selectedPBSInstance()
|
||||||
? state.pbsBackups?.filter((b) => b.instance === selectedPBSInstance())
|
? pbsBackupsState()?.filter((b) => b.instance === selectedPBSInstance())
|
||||||
: state.pbsBackups;
|
: pbsBackupsState();
|
||||||
|
|
||||||
pbsBackupsToProcess?.forEach((backup) => {
|
pbsBackupsToProcess?.forEach((backup) => {
|
||||||
const backupDate = new Date(backup.backupTime);
|
const backupDate = new Date(backup.backupTime);
|
||||||
|
|
@ -302,8 +309,38 @@ const UnifiedBackups: Component = () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
pmgBackupsState()?.forEach((backup) => {
|
||||||
|
const backupTimeMs = backup.backupTime ? Date.parse(backup.backupTime) : Number.NaN;
|
||||||
|
const backupTimeSeconds = Number.isFinite(backupTimeMs) ? Math.floor(backupTimeMs / 1000) : 0;
|
||||||
|
const backupKey = backup.id || `${backup.instance}:${backup.node}:${backup.filename}`;
|
||||||
|
if (seenBackups.has(backupKey)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
seenBackups.add(backupKey);
|
||||||
|
|
||||||
|
const displayName = backup.node || backup.filename || 'PMG Host Backup';
|
||||||
|
unified.push({
|
||||||
|
backupType: 'local',
|
||||||
|
vmid: backup.node || backup.filename,
|
||||||
|
name: displayName,
|
||||||
|
type: 'Host',
|
||||||
|
node: backup.node || 'PMG',
|
||||||
|
instance: backup.instance || 'PMG',
|
||||||
|
backupTime: backupTimeSeconds,
|
||||||
|
backupName: backup.filename || displayName,
|
||||||
|
description: backup.filename || '',
|
||||||
|
status: 'ok',
|
||||||
|
size: backup.size || null,
|
||||||
|
storage: 'PMG',
|
||||||
|
datastore: null,
|
||||||
|
namespace: null,
|
||||||
|
verified: null,
|
||||||
|
protected: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Normalize local backups (including PBS through PVE storage)
|
// Normalize local backups (including PBS through PVE storage)
|
||||||
state.pveBackups?.storageBackups?.forEach((backup) => {
|
pveBackupsState()?.storageBackups?.forEach((backup) => {
|
||||||
// Skip templates and ISOs - they're not backups
|
// Skip templates and ISOs - they're not backups
|
||||||
if (backup.type === 'vztmpl' || backup.type === 'iso') {
|
if (backup.type === 'vztmpl' || backup.type === 'iso') {
|
||||||
return;
|
return;
|
||||||
|
|
@ -1199,7 +1236,7 @@ const UnifiedBackups: Component = () => {
|
||||||
|
|
||||||
// Count backups for this PBS instance
|
// Count backups for this PBS instance
|
||||||
const pbsBackups = () =>
|
const pbsBackups = () =>
|
||||||
state.pbsBackups?.filter((b) => b.instance === pbs.name).length || 0;
|
pbsBackupsState()?.filter((b) => b.instance === pbs.name).length || 0;
|
||||||
|
|
||||||
const isSelected = () => selectedPBSInstance() === pbs.name;
|
const isSelected = () => selectedPBSInstance() === pbs.name;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -317,6 +317,10 @@ const Settings: Component<SettingsProps> = (props) => {
|
||||||
description: 'Manage Pulse configuration.',
|
description: 'Manage Pulse configuration.',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const pveBackupsState = () => state.backups?.pve ?? state.pveBackups;
|
||||||
|
const pbsBackupsState = () => state.backups?.pbs ?? state.pbsBackups;
|
||||||
|
const pmgBackupsState = () => state.backups?.pmg ?? state.pmgBackups;
|
||||||
|
|
||||||
// Keep tab state in sync with URL and handle /settings redirect without flicker
|
// Keep tab state in sync with URL and handle /settings redirect without flicker
|
||||||
createEffect(
|
createEffect(
|
||||||
on(
|
on(
|
||||||
|
|
@ -5413,11 +5417,11 @@ const Settings: Component<SettingsProps> = (props) => {
|
||||||
storageCount: state.storage?.length || 0,
|
storageCount: state.storage?.length || 0,
|
||||||
physicalDisksCount: state.physicalDisks?.length || 0,
|
physicalDisksCount: state.physicalDisks?.length || 0,
|
||||||
pbsCount: state.pbs?.length || 0,
|
pbsCount: state.pbs?.length || 0,
|
||||||
pbsBackupsCount: state.pbsBackups?.length || 0,
|
pbsBackupsCount: pbsBackupsState()?.length || 0,
|
||||||
pveBackups: {
|
pveBackups: {
|
||||||
backupTasksCount: state.pveBackups?.backupTasks?.length || 0,
|
backupTasksCount: pveBackupsState()?.backupTasks?.length || 0,
|
||||||
storageBackupsCount: state.pveBackups?.storageBackups?.length || 0,
|
storageBackupsCount: pveBackupsState()?.storageBackups?.length || 0,
|
||||||
guestSnapshotsCount: state.pveBackups?.guestSnapshots?.length || 0,
|
guestSnapshotsCount: pveBackupsState()?.guestSnapshots?.length || 0,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
// Node status details
|
// Node status details
|
||||||
|
|
@ -5454,9 +5458,9 @@ const Settings: Component<SettingsProps> = (props) => {
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
hasBackups:
|
hasBackups:
|
||||||
(state.pveBackups?.storageBackups?.filter(
|
(pveBackupsState()?.storageBackups ?? []).filter(
|
||||||
(b) => b.storage === s.name,
|
(b) => b.storage === s.name,
|
||||||
).length || 0) > 0,
|
).length > 0,
|
||||||
})) || [],
|
})) || [],
|
||||||
// Physical disks - critical for troubleshooting
|
// Physical disks - critical for troubleshooting
|
||||||
physicalDisks:
|
physicalDisks:
|
||||||
|
|
@ -5472,10 +5476,10 @@ const Settings: Component<SettingsProps> = (props) => {
|
||||||
smart: d.smart ?? null,
|
smart: d.smart ?? null,
|
||||||
})) || [],
|
})) || [],
|
||||||
backups: {
|
backups: {
|
||||||
pveBackupTasks: state.pveBackups?.backupTasks?.slice(0, 10) || [],
|
pveBackupTasks: pveBackupsState()?.backupTasks?.slice(0, 10) || [],
|
||||||
pveStorageBackups:
|
pveStorageBackups:
|
||||||
state.pveBackups?.storageBackups?.slice(0, 10) || [],
|
pveBackupsState()?.storageBackups?.slice(0, 10) || [],
|
||||||
pbsBackups: state.pbsBackups?.slice(0, 10) || [],
|
pbsBackups: pbsBackupsState()?.slice(0, 10) || [],
|
||||||
},
|
},
|
||||||
connectionHealth: state.connectionHealth || {},
|
connectionHealth: state.connectionHealth || {},
|
||||||
performance: {
|
performance: {
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,8 @@ export const UnifiedNodeSelector: Component<UnifiedNodeSelectorProps> = (props)
|
||||||
// This allows users to select a node AND search within it
|
// This allows users to select a node AND search within it
|
||||||
|
|
||||||
// Calculate backup counts for nodes and PBS instances
|
// Calculate backup counts for nodes and PBS instances
|
||||||
|
const pveBackupsState = () => state.backups?.pve ?? state.pveBackups;
|
||||||
|
const pbsBackupsState = () => state.backups?.pbs ?? state.pbsBackups;
|
||||||
const backupCounts = createMemo(() => {
|
const backupCounts = createMemo(() => {
|
||||||
const counts: Record<string, number> = {};
|
const counts: Record<string, number> = {};
|
||||||
|
|
||||||
|
|
@ -54,15 +56,16 @@ export const UnifiedNodeSelector: Component<UnifiedNodeSelectorProps> = (props)
|
||||||
let count = 0;
|
let count = 0;
|
||||||
|
|
||||||
// Count storage backups (excluding PBS backups which are counted separately)
|
// Count storage backups (excluding PBS backups which are counted separately)
|
||||||
if (state.pveBackups?.storageBackups) {
|
const pveBackups = pveBackupsState();
|
||||||
count += state.pveBackups.storageBackups.filter(
|
if (pveBackups?.storageBackups) {
|
||||||
|
count += pveBackups.storageBackups.filter(
|
||||||
(b) => b.instance === node.instance && b.node === node.name && !b.isPBS,
|
(b) => b.instance === node.instance && b.node === node.name && !b.isPBS,
|
||||||
).length;
|
).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count snapshots
|
// Count snapshots
|
||||||
if (state.pveBackups?.guestSnapshots) {
|
if (pveBackups?.guestSnapshots) {
|
||||||
count += state.pveBackups.guestSnapshots.filter((s) => s.instance === node.instance && s.node === node.name).length;
|
count += pveBackups.guestSnapshots.filter((s) => s.instance === node.instance && s.node === node.name).length;
|
||||||
}
|
}
|
||||||
|
|
||||||
counts[node.id] = count;
|
counts[node.id] = count;
|
||||||
|
|
@ -70,9 +73,10 @@ export const UnifiedNodeSelector: Component<UnifiedNodeSelectorProps> = (props)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count PBS backups by instance
|
// Count PBS backups by instance
|
||||||
if (state.pbs && state.pbsBackups) {
|
const pbsBackups = pbsBackupsState();
|
||||||
|
if (state.pbs && pbsBackups) {
|
||||||
state.pbs.forEach((pbs) => {
|
state.pbs.forEach((pbs) => {
|
||||||
counts[pbs.name] = state.pbsBackups?.filter((b) => b.instance === pbs.name).length || 0;
|
counts[pbs.name] = pbsBackups.filter((b) => b.instance === pbs.name).length || 0;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import type { JSX } from 'solid-js';
|
||||||
import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect';
|
import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect';
|
||||||
import { WebhookConfig } from '@/components/Alerts/WebhookConfig';
|
import { WebhookConfig } from '@/components/Alerts/WebhookConfig';
|
||||||
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
|
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
|
||||||
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
|
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
|
||||||
import { Card } from '@/components/shared/Card';
|
import { Card } from '@/components/shared/Card';
|
||||||
import { SectionHeader } from '@/components/shared/SectionHeader';
|
import { SectionHeader } from '@/components/shared/SectionHeader';
|
||||||
import { SettingsPanel } from '@/components/shared/SettingsPanel';
|
import { SettingsPanel } from '@/components/shared/SettingsPanel';
|
||||||
|
|
@ -769,6 +769,25 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
setMetricTimeThresholds({});
|
setMetricTimeThresholds({});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load backup thresholds
|
||||||
|
if (config.backupDefaults) {
|
||||||
|
const enabled = Boolean(config.backupDefaults.enabled);
|
||||||
|
const rawWarning = config.backupDefaults.warningDays ?? FACTORY_BACKUP_DEFAULTS.warningDays;
|
||||||
|
const rawCritical = config.backupDefaults.criticalDays ?? FACTORY_BACKUP_DEFAULTS.criticalDays;
|
||||||
|
const safeCritical = Math.max(0, rawCritical);
|
||||||
|
const normalizedWarning = Math.max(0, rawWarning);
|
||||||
|
const warningDays =
|
||||||
|
safeCritical > 0 && normalizedWarning > safeCritical ? safeCritical : normalizedWarning;
|
||||||
|
const criticalDays = Math.max(safeCritical, warningDays);
|
||||||
|
setBackupDefaults({
|
||||||
|
enabled,
|
||||||
|
warningDays,
|
||||||
|
criticalDays,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
setBackupDefaults({ ...FACTORY_BACKUP_DEFAULTS });
|
||||||
|
}
|
||||||
|
|
||||||
// Load snapshot thresholds
|
// Load snapshot thresholds
|
||||||
if (config.snapshotDefaults) {
|
if (config.snapshotDefaults) {
|
||||||
const enabled = Boolean(config.snapshotDefaults.enabled);
|
const enabled = Boolean(config.snapshotDefaults.enabled);
|
||||||
|
|
@ -1058,6 +1077,11 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
warningDays: 30,
|
warningDays: 30,
|
||||||
criticalDays: 45,
|
criticalDays: 45,
|
||||||
};
|
};
|
||||||
|
const FACTORY_BACKUP_DEFAULTS: BackupAlertConfig = {
|
||||||
|
enabled: false,
|
||||||
|
warningDays: 7,
|
||||||
|
criticalDays: 14,
|
||||||
|
};
|
||||||
|
|
||||||
// Threshold states - using trigger values for display
|
// Threshold states - using trigger values for display
|
||||||
const [guestDefaults, setGuestDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_GUEST_DEFAULTS });
|
const [guestDefaults, setGuestDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_GUEST_DEFAULTS });
|
||||||
|
|
@ -1070,6 +1094,9 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal<string[]>([]);
|
const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal<string[]>([]);
|
||||||
|
|
||||||
const [storageDefault, setStorageDefault] = createSignal(FACTORY_STORAGE_DEFAULT);
|
const [storageDefault, setStorageDefault] = createSignal(FACTORY_STORAGE_DEFAULT);
|
||||||
|
const [backupDefaults, setBackupDefaults] = createSignal<BackupAlertConfig>({
|
||||||
|
...FACTORY_BACKUP_DEFAULTS,
|
||||||
|
});
|
||||||
|
|
||||||
// Reset functions
|
// Reset functions
|
||||||
const resetGuestDefaults = () => {
|
const resetGuestDefaults = () => {
|
||||||
|
|
@ -1096,6 +1123,10 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
setStorageDefault(FACTORY_STORAGE_DEFAULT);
|
setStorageDefault(FACTORY_STORAGE_DEFAULT);
|
||||||
setHasUnsavedChanges(true);
|
setHasUnsavedChanges(true);
|
||||||
};
|
};
|
||||||
|
const resetBackupDefaults = () => {
|
||||||
|
setBackupDefaults({ ...FACTORY_BACKUP_DEFAULTS });
|
||||||
|
setHasUnsavedChanges(true);
|
||||||
|
};
|
||||||
const resetSnapshotDefaults = () => {
|
const resetSnapshotDefaults = () => {
|
||||||
setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS });
|
setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS });
|
||||||
setHasUnsavedChanges(true);
|
setHasUnsavedChanges(true);
|
||||||
|
|
@ -1224,12 +1255,20 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
};
|
};
|
||||||
|
|
||||||
const snapshotConfig = snapshotDefaults();
|
const snapshotConfig = snapshotDefaults();
|
||||||
const normalizedWarningDays = Math.max(0, snapshotConfig.warningDays ?? 0);
|
const normalizedSnapshotWarning = Math.max(0, snapshotConfig.warningDays ?? 0);
|
||||||
const normalizedCriticalDays = Math.max(0, snapshotConfig.criticalDays ?? 0);
|
const normalizedSnapshotCritical = Math.max(0, snapshotConfig.criticalDays ?? 0);
|
||||||
const finalCriticalDays =
|
const finalSnapshotCritical =
|
||||||
normalizedCriticalDays > 0
|
normalizedSnapshotCritical > 0
|
||||||
? Math.max(normalizedCriticalDays, normalizedWarningDays)
|
? Math.max(normalizedSnapshotCritical, normalizedSnapshotWarning)
|
||||||
: normalizedWarningDays;
|
: normalizedSnapshotWarning;
|
||||||
|
|
||||||
|
const backupConfig = backupDefaults();
|
||||||
|
const normalizedBackupWarning = Math.max(0, backupConfig.warningDays ?? 0);
|
||||||
|
const normalizedBackupCritical = Math.max(0, backupConfig.criticalDays ?? 0);
|
||||||
|
const finalBackupCritical =
|
||||||
|
normalizedBackupCritical > 0
|
||||||
|
? Math.max(normalizedBackupCritical, normalizedBackupWarning)
|
||||||
|
: normalizedBackupWarning;
|
||||||
|
|
||||||
const alertConfig = {
|
const alertConfig = {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
|
|
@ -1284,8 +1323,13 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
|
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
|
||||||
snapshotDefaults: {
|
snapshotDefaults: {
|
||||||
enabled: snapshotConfig.enabled,
|
enabled: snapshotConfig.enabled,
|
||||||
warningDays: normalizedWarningDays,
|
warningDays: normalizedSnapshotWarning,
|
||||||
criticalDays: finalCriticalDays,
|
criticalDays: finalSnapshotCritical,
|
||||||
|
},
|
||||||
|
backupDefaults: {
|
||||||
|
enabled: backupConfig.enabled,
|
||||||
|
warningDays: normalizedBackupWarning,
|
||||||
|
criticalDays: finalBackupCritical,
|
||||||
},
|
},
|
||||||
pmgDefaults: pmgThresholds(),
|
pmgDefaults: pmgThresholds(),
|
||||||
// Use rawOverridesConfig which is already properly formatted with disabled flags
|
// Use rawOverridesConfig which is already properly formatted with disabled flags
|
||||||
|
|
@ -1491,14 +1535,18 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||||
resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes}
|
resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes}
|
||||||
resetStorageDefault={resetStorageDefault}
|
resetStorageDefault={resetStorageDefault}
|
||||||
resetSnapshotDefaults={resetSnapshotDefaults}
|
resetSnapshotDefaults={resetSnapshotDefaults}
|
||||||
|
resetBackupDefaults={resetBackupDefaults}
|
||||||
factoryGuestDefaults={FACTORY_GUEST_DEFAULTS}
|
factoryGuestDefaults={FACTORY_GUEST_DEFAULTS}
|
||||||
factoryNodeDefaults={FACTORY_NODE_DEFAULTS}
|
factoryNodeDefaults={FACTORY_NODE_DEFAULTS}
|
||||||
factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS}
|
factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS}
|
||||||
factoryStorageDefault={FACTORY_STORAGE_DEFAULT}
|
factoryStorageDefault={FACTORY_STORAGE_DEFAULT}
|
||||||
snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS}
|
snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS}
|
||||||
|
backupFactoryDefaults={FACTORY_BACKUP_DEFAULTS}
|
||||||
timeThresholds={timeThresholds}
|
timeThresholds={timeThresholds}
|
||||||
metricTimeThresholds={metricTimeThresholds}
|
metricTimeThresholds={metricTimeThresholds}
|
||||||
setMetricTimeThresholds={setMetricTimeThresholds}
|
setMetricTimeThresholds={setMetricTimeThresholds}
|
||||||
|
backupDefaults={backupDefaults}
|
||||||
|
setBackupDefaults={setBackupDefaults}
|
||||||
snapshotDefaults={snapshotDefaults}
|
snapshotDefaults={snapshotDefaults}
|
||||||
setSnapshotDefaults={setSnapshotDefaults}
|
setSnapshotDefaults={setSnapshotDefaults}
|
||||||
pmgThresholds={pmgThresholds}
|
pmgThresholds={pmgThresholds}
|
||||||
|
|
@ -2028,6 +2076,14 @@ interface ThresholdsTabProps {
|
||||||
) => void;
|
) => void;
|
||||||
snapshotFactoryDefaults: SnapshotAlertConfig;
|
snapshotFactoryDefaults: SnapshotAlertConfig;
|
||||||
resetSnapshotDefaults: () => void;
|
resetSnapshotDefaults: () => void;
|
||||||
|
backupDefaults: () => BackupAlertConfig;
|
||||||
|
setBackupDefaults: (
|
||||||
|
value:
|
||||||
|
| BackupAlertConfig
|
||||||
|
| ((prev: BackupAlertConfig) => BackupAlertConfig),
|
||||||
|
) => void;
|
||||||
|
backupFactoryDefaults: BackupAlertConfig;
|
||||||
|
resetBackupDefaults: () => void;
|
||||||
setOverrides: (value: Override[]) => void;
|
setOverrides: (value: Override[]) => void;
|
||||||
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
|
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
|
||||||
activeAlerts: Record<string, Alert>;
|
activeAlerts: Record<string, Alert>;
|
||||||
|
|
@ -2085,6 +2141,10 @@ function ThresholdsTab(props: ThresholdsTabProps) {
|
||||||
dockerHosts={props.state.dockerHosts || []}
|
dockerHosts={props.state.dockerHosts || []}
|
||||||
pbsInstances={props.state.pbs || []}
|
pbsInstances={props.state.pbs || []}
|
||||||
pmgInstances={props.state.pmg || []}
|
pmgInstances={props.state.pmg || []}
|
||||||
|
backups={props.state.backups}
|
||||||
|
pveBackups={props.state.pveBackups}
|
||||||
|
pbsBackups={props.state.pbsBackups}
|
||||||
|
pmgBackups={props.state.pmgBackups}
|
||||||
pmgThresholds={props.pmgThresholds}
|
pmgThresholds={props.pmgThresholds}
|
||||||
setPMGThresholds={props.setPMGThresholds}
|
setPMGThresholds={props.setPMGThresholds}
|
||||||
guestDefaults={props.guestDefaults()}
|
guestDefaults={props.guestDefaults()}
|
||||||
|
|
@ -2104,6 +2164,10 @@ function ThresholdsTab(props: ThresholdsTabProps) {
|
||||||
timeThresholds={props.timeThresholds}
|
timeThresholds={props.timeThresholds}
|
||||||
metricTimeThresholds={props.metricTimeThresholds}
|
metricTimeThresholds={props.metricTimeThresholds}
|
||||||
setMetricTimeThresholds={props.setMetricTimeThresholds}
|
setMetricTimeThresholds={props.setMetricTimeThresholds}
|
||||||
|
backupDefaults={props.backupDefaults}
|
||||||
|
setBackupDefaults={props.setBackupDefaults}
|
||||||
|
backupFactoryDefaults={props.backupFactoryDefaults}
|
||||||
|
resetBackupDefaults={props.resetBackupDefaults}
|
||||||
snapshotDefaults={props.snapshotDefaults}
|
snapshotDefaults={props.snapshotDefaults}
|
||||||
setSnapshotDefaults={props.setSnapshotDefaults}
|
setSnapshotDefaults={props.setSnapshotDefaults}
|
||||||
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
|
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,16 @@ export function createWebSocketStore(url: string) {
|
||||||
guestSnapshots: [],
|
guestSnapshots: [],
|
||||||
} as PVEBackups,
|
} as PVEBackups,
|
||||||
pbsBackups: [],
|
pbsBackups: [],
|
||||||
|
pmgBackups: [],
|
||||||
|
backups: {
|
||||||
|
pve: {
|
||||||
|
backupTasks: [],
|
||||||
|
storageBackups: [],
|
||||||
|
guestSnapshots: [],
|
||||||
|
},
|
||||||
|
pbs: [],
|
||||||
|
pmg: [],
|
||||||
|
},
|
||||||
performance: {
|
performance: {
|
||||||
apiCallDuration: {},
|
apiCallDuration: {},
|
||||||
lastPollDuration: 0,
|
lastPollDuration: 0,
|
||||||
|
|
@ -282,8 +292,19 @@ export function createWebSocketStore(url: string) {
|
||||||
setState('cephClusters', message.data.cephClusters);
|
setState('cephClusters', message.data.cephClusters);
|
||||||
if (message.data.pbs !== undefined) setState('pbs', message.data.pbs);
|
if (message.data.pbs !== undefined) setState('pbs', message.data.pbs);
|
||||||
if (message.data.pmg !== undefined) setState('pmg', message.data.pmg);
|
if (message.data.pmg !== undefined) setState('pmg', message.data.pmg);
|
||||||
|
if (message.data.backups !== undefined) {
|
||||||
|
setState('backups', message.data.backups);
|
||||||
|
if (message.data.backups.pve !== undefined)
|
||||||
|
setState('pveBackups', message.data.backups.pve);
|
||||||
|
if (message.data.backups.pbs !== undefined)
|
||||||
|
setState('pbsBackups', message.data.backups.pbs);
|
||||||
|
if (message.data.backups.pmg !== undefined)
|
||||||
|
setState('pmgBackups', message.data.backups.pmg);
|
||||||
|
}
|
||||||
if (message.data.pbsBackups !== undefined)
|
if (message.data.pbsBackups !== undefined)
|
||||||
setState('pbsBackups', message.data.pbsBackups);
|
setState('pbsBackups', message.data.pbsBackups);
|
||||||
|
if (message.data.pmgBackups !== undefined)
|
||||||
|
setState('pmgBackups', message.data.pmgBackups);
|
||||||
if (message.data.metrics !== undefined) setState('metrics', message.data.metrics);
|
if (message.data.metrics !== undefined) setState('metrics', message.data.metrics);
|
||||||
if (message.data.pveBackups !== undefined)
|
if (message.data.pveBackups !== undefined)
|
||||||
setState('pveBackups', message.data.pveBackups);
|
setState('pveBackups', message.data.pveBackups);
|
||||||
|
|
|
||||||
|
|
@ -91,6 +91,12 @@ export interface SnapshotAlertConfig {
|
||||||
criticalDays: number;
|
criticalDays: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BackupAlertConfig {
|
||||||
|
enabled: boolean;
|
||||||
|
warningDays: number;
|
||||||
|
criticalDays: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface AlertConfig {
|
export interface AlertConfig {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
guestDefaults: AlertThresholds;
|
guestDefaults: AlertThresholds;
|
||||||
|
|
@ -100,6 +106,7 @@ export interface AlertConfig {
|
||||||
dockerIgnoredContainerPrefixes?: string[];
|
dockerIgnoredContainerPrefixes?: string[];
|
||||||
pmgDefaults?: PMGThresholdDefaults;
|
pmgDefaults?: PMGThresholdDefaults;
|
||||||
snapshotDefaults?: SnapshotAlertConfig;
|
snapshotDefaults?: SnapshotAlertConfig;
|
||||||
|
backupDefaults?: BackupAlertConfig;
|
||||||
customRules?: CustomAlertRule[];
|
customRules?: CustomAlertRule[];
|
||||||
overrides: Record<string, RawOverrideConfig>; // key: resource ID
|
overrides: Record<string, RawOverrideConfig>; // key: resource ID
|
||||||
minimumDelta?: number;
|
minimumDelta?: number;
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,8 @@ export interface State {
|
||||||
pbs: PBSInstance[];
|
pbs: PBSInstance[];
|
||||||
pmg: PMGInstance[];
|
pmg: PMGInstance[];
|
||||||
pbsBackups: PBSBackup[];
|
pbsBackups: PBSBackup[];
|
||||||
|
pmgBackups: PMGBackup[];
|
||||||
|
backups: Backups;
|
||||||
metrics: Metric[];
|
metrics: Metric[];
|
||||||
pveBackups: PVEBackups;
|
pveBackups: PVEBackups;
|
||||||
performance: Performance;
|
performance: Performance;
|
||||||
|
|
@ -414,6 +416,21 @@ export interface PBSBackup {
|
||||||
owner?: string;
|
owner?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface PMGBackup {
|
||||||
|
id: string;
|
||||||
|
instance: string;
|
||||||
|
node: string;
|
||||||
|
filename: string;
|
||||||
|
backupTime: string;
|
||||||
|
size: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Backups {
|
||||||
|
pve: PVEBackups;
|
||||||
|
pbs: PBSBackup[];
|
||||||
|
pmg: PMGBackup[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface PBSBackupJob {
|
export interface PBSBackupJob {
|
||||||
id: string;
|
id: string;
|
||||||
store: string;
|
store: string;
|
||||||
|
|
|
||||||
|
|
@ -290,6 +290,22 @@ type SnapshotAlertConfig struct {
|
||||||
CriticalDays int `json:"criticalDays"`
|
CriticalDays int `json:"criticalDays"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BackupAlertConfig represents backup age alert configuration
|
||||||
|
type BackupAlertConfig struct {
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
WarningDays int `json:"warningDays"`
|
||||||
|
CriticalDays int `json:"criticalDays"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GuestLookup describes a guest identity used for snapshot/backup evaluations.
|
||||||
|
type GuestLookup struct {
|
||||||
|
Name string
|
||||||
|
Instance string
|
||||||
|
Node string
|
||||||
|
Type string
|
||||||
|
VMID int
|
||||||
|
}
|
||||||
|
|
||||||
// AlertConfig represents the complete alert configuration
|
// AlertConfig represents the complete alert configuration
|
||||||
type AlertConfig struct {
|
type AlertConfig struct {
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
|
|
@ -300,6 +316,7 @@ type AlertConfig struct {
|
||||||
DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"`
|
DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"`
|
||||||
PMGDefaults PMGThresholdConfig `json:"pmgDefaults"`
|
PMGDefaults PMGThresholdConfig `json:"pmgDefaults"`
|
||||||
SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"`
|
SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"`
|
||||||
|
BackupDefaults BackupAlertConfig `json:"backupDefaults"`
|
||||||
Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID
|
Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID
|
||||||
CustomRules []CustomAlertRule `json:"customRules,omitempty"`
|
CustomRules []CustomAlertRule `json:"customRules,omitempty"`
|
||||||
Schedule ScheduleConfig `json:"schedule"`
|
Schedule ScheduleConfig `json:"schedule"`
|
||||||
|
|
@ -486,6 +503,11 @@ func NewManager() *Manager {
|
||||||
WarningDays: 30,
|
WarningDays: 30,
|
||||||
CriticalDays: 45,
|
CriticalDays: 45,
|
||||||
},
|
},
|
||||||
|
BackupDefaults: BackupAlertConfig{
|
||||||
|
Enabled: false,
|
||||||
|
WarningDays: 7,
|
||||||
|
CriticalDays: 14,
|
||||||
|
},
|
||||||
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||||
MinimumDelta: 2.0, // 2% minimum change
|
MinimumDelta: 2.0, // 2% minimum change
|
||||||
SuppressionWindow: 5, // 5 minutes
|
SuppressionWindow: 5, // 5 minutes
|
||||||
|
|
@ -702,6 +724,15 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
||||||
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
||||||
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
||||||
}
|
}
|
||||||
|
if config.BackupDefaults.WarningDays < 0 {
|
||||||
|
config.BackupDefaults.WarningDays = 0
|
||||||
|
}
|
||||||
|
if config.BackupDefaults.CriticalDays < 0 {
|
||||||
|
config.BackupDefaults.CriticalDays = 0
|
||||||
|
}
|
||||||
|
if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays {
|
||||||
|
config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays
|
||||||
|
}
|
||||||
|
|
||||||
// Ensure minimums for other important fields
|
// Ensure minimums for other important fields
|
||||||
if config.MinimumDelta <= 0 {
|
if config.MinimumDelta <= 0 {
|
||||||
|
|
@ -764,6 +795,9 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
||||||
if !m.config.SnapshotDefaults.Enabled {
|
if !m.config.SnapshotDefaults.Enabled {
|
||||||
m.clearSnapshotAlertsForInstanceLocked("")
|
m.clearSnapshotAlertsForInstanceLocked("")
|
||||||
}
|
}
|
||||||
|
if !m.config.BackupDefaults.Enabled {
|
||||||
|
m.clearBackupAlertsLocked()
|
||||||
|
}
|
||||||
|
|
||||||
m.applyGlobalOfflineSettingsLocked()
|
m.applyGlobalOfflineSettingsLocked()
|
||||||
|
|
||||||
|
|
@ -2756,7 +2790,7 @@ func (m *Manager) CheckStorage(storage models.Storage) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildSnapshotGuestKey(instance, node string, vmid int) string {
|
func BuildGuestKey(instance, node string, vmid int) string {
|
||||||
instance = strings.TrimSpace(instance)
|
instance = strings.TrimSpace(instance)
|
||||||
node = strings.TrimSpace(node)
|
node = strings.TrimSpace(node)
|
||||||
if instance == "" {
|
if instance == "" {
|
||||||
|
|
@ -2817,7 +2851,7 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
|
||||||
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
|
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
|
||||||
validAlerts[alertID] = struct{}{}
|
validAlerts[alertID] = struct{}{}
|
||||||
|
|
||||||
guestKey := buildSnapshotGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
|
guestKey := BuildGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
|
||||||
guestName := strings.TrimSpace(guestNames[guestKey])
|
guestName := strings.TrimSpace(guestNames[guestKey])
|
||||||
|
|
||||||
guestType := "VM"
|
guestType := "VM"
|
||||||
|
|
@ -2966,6 +3000,356 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CheckBackups evaluates storage, PBS, and PMG backups for age-based alerts.
|
||||||
|
func (m *Manager) CheckBackups(
|
||||||
|
storageBackups []models.StorageBackup,
|
||||||
|
pbsBackups []models.PBSBackup,
|
||||||
|
pmgBackups []models.PMGBackup,
|
||||||
|
guestsByKey map[string]GuestLookup,
|
||||||
|
guestsByVMID map[string]GuestLookup,
|
||||||
|
) {
|
||||||
|
m.mu.RLock()
|
||||||
|
enabled := m.config.Enabled
|
||||||
|
backupCfg := m.config.BackupDefaults
|
||||||
|
m.mu.RUnlock()
|
||||||
|
|
||||||
|
if !enabled || !backupCfg.Enabled {
|
||||||
|
m.clearBackupAlerts()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if backupCfg.WarningDays <= 0 && backupCfg.CriticalDays <= 0 {
|
||||||
|
m.clearBackupAlerts()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
type backupRecord struct {
|
||||||
|
key string
|
||||||
|
lookup GuestLookup
|
||||||
|
fallbackName string
|
||||||
|
instance string
|
||||||
|
node string
|
||||||
|
source string
|
||||||
|
storage string
|
||||||
|
datastore string
|
||||||
|
backupType string
|
||||||
|
filename string
|
||||||
|
lastTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
records := make(map[string]*backupRecord)
|
||||||
|
|
||||||
|
updateRecord := func(key string, candidate backupRecord) {
|
||||||
|
if key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if existing, ok := records[key]; ok {
|
||||||
|
if candidate.lastTime.After(existing.lastTime) {
|
||||||
|
*existing = candidate
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
record := candidate
|
||||||
|
records[key] = &record
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
for _, backup := range storageBackups {
|
||||||
|
if backup.Time.IsZero() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
key := BuildGuestKey(backup.Instance, backup.Node, backup.VMID)
|
||||||
|
info := guestsByKey[key]
|
||||||
|
displayName := info.Name
|
||||||
|
if displayName == "" {
|
||||||
|
displayName = fmt.Sprintf("%s-%d", sanitizeAlertKey(backup.Node), backup.VMID)
|
||||||
|
}
|
||||||
|
|
||||||
|
updateRecord(key, backupRecord{
|
||||||
|
key: key,
|
||||||
|
lookup: info,
|
||||||
|
fallbackName: displayName,
|
||||||
|
instance: backup.Instance,
|
||||||
|
node: backup.Node,
|
||||||
|
source: "PVE storage",
|
||||||
|
storage: backup.Storage,
|
||||||
|
backupType: backup.Type,
|
||||||
|
lastTime: backup.Time,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, backup := range pbsBackups {
|
||||||
|
if backup.BackupTime.IsZero() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if backup.VMID == "0" {
|
||||||
|
// Host configuration backups - skip from age alerts
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
info, exists := guestsByVMID[backup.VMID]
|
||||||
|
var key string
|
||||||
|
var displayName string
|
||||||
|
var instance string
|
||||||
|
var node string
|
||||||
|
|
||||||
|
if exists && info.Instance != "" && info.Node != "" {
|
||||||
|
key = BuildGuestKey(info.Instance, info.Node, info.VMID)
|
||||||
|
displayName = info.Name
|
||||||
|
instance = info.Instance
|
||||||
|
node = info.Node
|
||||||
|
} else {
|
||||||
|
key = fmt.Sprintf("pbs:%s:%s:%s", backup.Instance, backup.BackupType, backup.VMID)
|
||||||
|
displayName = fmt.Sprintf("VMID %s", backup.VMID)
|
||||||
|
instance = fmt.Sprintf("PBS:%s", backup.Instance)
|
||||||
|
node = backup.Datastore
|
||||||
|
}
|
||||||
|
|
||||||
|
updateRecord(key, backupRecord{
|
||||||
|
key: key,
|
||||||
|
lookup: info,
|
||||||
|
fallbackName: displayName,
|
||||||
|
instance: instance,
|
||||||
|
node: node,
|
||||||
|
source: "PBS",
|
||||||
|
datastore: backup.Datastore,
|
||||||
|
backupType: backup.BackupType,
|
||||||
|
lastTime: backup.BackupTime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, backup := range pmgBackups {
|
||||||
|
if backup.BackupTime.IsZero() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
instanceLabel := strings.TrimSpace(backup.Instance)
|
||||||
|
if instanceLabel == "" {
|
||||||
|
instanceLabel = "PMG"
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeName := strings.TrimSpace(backup.Node)
|
||||||
|
keyComponent := nodeName
|
||||||
|
if keyComponent == "" {
|
||||||
|
keyComponent = strings.TrimSpace(backup.Filename)
|
||||||
|
}
|
||||||
|
if keyComponent == "" {
|
||||||
|
keyComponent = "unknown"
|
||||||
|
}
|
||||||
|
|
||||||
|
displayName := nodeName
|
||||||
|
if displayName == "" {
|
||||||
|
displayName = instanceLabel
|
||||||
|
}
|
||||||
|
if displayName == "" {
|
||||||
|
displayName = "PMG gateway"
|
||||||
|
} else {
|
||||||
|
displayName = fmt.Sprintf("PMG %s", displayName)
|
||||||
|
}
|
||||||
|
|
||||||
|
instanceField := fmt.Sprintf("PMG:%s", instanceLabel)
|
||||||
|
key := fmt.Sprintf("pmg:%s:%s", instanceLabel, keyComponent)
|
||||||
|
|
||||||
|
updateRecord(key, backupRecord{
|
||||||
|
key: key,
|
||||||
|
fallbackName: displayName,
|
||||||
|
instance: instanceField,
|
||||||
|
node: nodeName,
|
||||||
|
source: "PMG",
|
||||||
|
backupType: "pmg",
|
||||||
|
filename: backup.Filename,
|
||||||
|
lastTime: backup.BackupTime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(records) == 0 {
|
||||||
|
m.clearBackupAlerts()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
validAlerts := make(map[string]struct{})
|
||||||
|
|
||||||
|
for key, record := range records {
|
||||||
|
age := now.Sub(record.lastTime)
|
||||||
|
if age < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
ageDays := age.Hours() / 24
|
||||||
|
if ageDays < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ageDaysRounded := math.Round(ageDays*10) / 10
|
||||||
|
|
||||||
|
var level AlertLevel
|
||||||
|
var threshold int
|
||||||
|
switch {
|
||||||
|
case backupCfg.CriticalDays > 0 && ageDays >= float64(backupCfg.CriticalDays):
|
||||||
|
level = AlertLevelCritical
|
||||||
|
threshold = backupCfg.CriticalDays
|
||||||
|
case backupCfg.WarningDays > 0 && ageDays >= float64(backupCfg.WarningDays):
|
||||||
|
level = AlertLevelWarning
|
||||||
|
threshold = backupCfg.WarningDays
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
alertKey := sanitizeAlertKey(key)
|
||||||
|
alertID := fmt.Sprintf("backup-age-%s", alertKey)
|
||||||
|
validAlerts[alertID] = struct{}{}
|
||||||
|
|
||||||
|
displayName := record.lookup.Name
|
||||||
|
if displayName == "" {
|
||||||
|
displayName = record.fallbackName
|
||||||
|
}
|
||||||
|
if displayName == "" {
|
||||||
|
displayName = "Unknown guest"
|
||||||
|
}
|
||||||
|
|
||||||
|
node := record.node
|
||||||
|
if node == "" {
|
||||||
|
node = record.lookup.Node
|
||||||
|
}
|
||||||
|
instance := record.instance
|
||||||
|
if instance == "" {
|
||||||
|
instance = record.lookup.Instance
|
||||||
|
}
|
||||||
|
|
||||||
|
thresholdTime := record.lastTime.Add(time.Duration(threshold) * 24 * time.Hour)
|
||||||
|
if thresholdTime.After(now) {
|
||||||
|
thresholdTime = now
|
||||||
|
}
|
||||||
|
|
||||||
|
var sourceLabel string
|
||||||
|
switch record.source {
|
||||||
|
case "PBS":
|
||||||
|
sourceLabel = fmt.Sprintf("PBS datastore %s on %s", record.datastore, strings.TrimPrefix(instance, "PBS:"))
|
||||||
|
case "PMG":
|
||||||
|
if node != "" {
|
||||||
|
sourceLabel = fmt.Sprintf("PMG node %s", node)
|
||||||
|
} else {
|
||||||
|
sourceLabel = "PMG"
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
sourceLabel = fmt.Sprintf("storage %s on %s", record.storage, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
message := fmt.Sprintf(
|
||||||
|
"%s backup via %s is %.1f days old (threshold: %d days)",
|
||||||
|
displayName,
|
||||||
|
sourceLabel,
|
||||||
|
ageDaysRounded,
|
||||||
|
threshold,
|
||||||
|
)
|
||||||
|
|
||||||
|
metadata := map[string]interface{}{
|
||||||
|
"source": record.source,
|
||||||
|
"lastBackupTime": record.lastTime,
|
||||||
|
"ageDays": ageDays,
|
||||||
|
"thresholdDays": threshold,
|
||||||
|
}
|
||||||
|
if record.storage != "" {
|
||||||
|
metadata["storage"] = record.storage
|
||||||
|
}
|
||||||
|
if record.datastore != "" {
|
||||||
|
metadata["datastore"] = record.datastore
|
||||||
|
}
|
||||||
|
if record.backupType != "" {
|
||||||
|
metadata["backupType"] = record.backupType
|
||||||
|
}
|
||||||
|
if record.filename != "" {
|
||||||
|
metadata["filename"] = record.filename
|
||||||
|
}
|
||||||
|
|
||||||
|
m.mu.Lock()
|
||||||
|
if existing, exists := m.activeAlerts[alertID]; exists {
|
||||||
|
existing.LastSeen = now
|
||||||
|
existing.Level = level
|
||||||
|
existing.Value = ageDays
|
||||||
|
existing.Threshold = float64(threshold)
|
||||||
|
existing.Message = message
|
||||||
|
if existing.Metadata == nil {
|
||||||
|
existing.Metadata = make(map[string]interface{})
|
||||||
|
}
|
||||||
|
for k, v := range metadata {
|
||||||
|
existing.Metadata[k] = v
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
alert := &Alert{
|
||||||
|
ID: alertID,
|
||||||
|
Type: "backup-age",
|
||||||
|
Level: level,
|
||||||
|
ResourceID: alertKey,
|
||||||
|
ResourceName: fmt.Sprintf("%s backup", displayName),
|
||||||
|
Node: node,
|
||||||
|
Instance: instance,
|
||||||
|
Message: message,
|
||||||
|
Value: ageDays,
|
||||||
|
Threshold: float64(threshold),
|
||||||
|
StartTime: thresholdTime,
|
||||||
|
LastSeen: now,
|
||||||
|
Metadata: metadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
m.preserveAlertState(alertID, alert)
|
||||||
|
|
||||||
|
m.activeAlerts[alertID] = alert
|
||||||
|
m.recentAlerts[alertID] = alert
|
||||||
|
m.historyManager.AddAlert(*alert)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Error().Interface("panic", r).Msg("Panic in SaveActiveAlerts goroutine (backup)")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := m.SaveActiveAlerts(); err != nil {
|
||||||
|
log.Error().Err(err).Msg("Failed to save active alerts after backup alert creation")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if !m.checkRateLimit(alertID) {
|
||||||
|
m.mu.Unlock()
|
||||||
|
log.Debug().
|
||||||
|
Str("alertID", alertID).
|
||||||
|
Str("resource", displayName).
|
||||||
|
Msg("Backup alert suppressed due to rate limit")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.onAlert != nil {
|
||||||
|
notified := now
|
||||||
|
alert.LastNotified = ¬ified
|
||||||
|
if m.dispatchAlert(alert, true) {
|
||||||
|
log.Info().
|
||||||
|
Str("alertID", alertID).
|
||||||
|
Str("resource", displayName).
|
||||||
|
Msg("Backup age alert dispatched")
|
||||||
|
} else {
|
||||||
|
alert.LastNotified = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
m.mu.Lock()
|
||||||
|
for alertID, alert := range m.activeAlerts {
|
||||||
|
if alert == nil || alert.Type != "backup-age" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := validAlerts[alertID]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.clearAlertNoLock(alertID)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
// checkZFSPoolHealth checks ZFS pool for errors and degraded state
|
// checkZFSPoolHealth checks ZFS pool for errors and degraded state
|
||||||
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
|
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
|
||||||
pool := storage.ZFSPool
|
pool := storage.ZFSPool
|
||||||
|
|
@ -6463,3 +6847,18 @@ func (m *Manager) clearSnapshotAlertsForInstanceLocked(instance string) {
|
||||||
m.clearAlertNoLock(alertID)
|
m.clearAlertNoLock(alertID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Manager) clearBackupAlerts() {
|
||||||
|
m.mu.Lock()
|
||||||
|
m.clearBackupAlertsLocked()
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Manager) clearBackupAlertsLocked() {
|
||||||
|
for alertID, alert := range m.activeAlerts {
|
||||||
|
if alert == nil || alert.Type != "backup-age" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.clearAlertNoLock(alertID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ package alerts
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|
@ -273,6 +274,160 @@ func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCheckBackupsCreatesAndClearsAlerts(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.Enabled = true
|
||||||
|
m.config.BackupDefaults = BackupAlertConfig{
|
||||||
|
Enabled: true,
|
||||||
|
WarningDays: 7,
|
||||||
|
CriticalDays: 14,
|
||||||
|
}
|
||||||
|
m.config.TimeThreshold = 0
|
||||||
|
m.config.TimeThresholds = map[string]int{}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
storageBackups := []models.StorageBackup{
|
||||||
|
{
|
||||||
|
ID: "inst-node-100-backup",
|
||||||
|
Storage: "local",
|
||||||
|
Node: "node",
|
||||||
|
Instance: "inst",
|
||||||
|
Type: "qemu",
|
||||||
|
VMID: 100,
|
||||||
|
Time: now.Add(-15 * 24 * time.Hour),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
key := BuildGuestKey("inst", "node", 100)
|
||||||
|
guestsByKey := map[string]GuestLookup{
|
||||||
|
key: {
|
||||||
|
Name: "app-server",
|
||||||
|
Instance: "inst",
|
||||||
|
Node: "node",
|
||||||
|
Type: "qemu",
|
||||||
|
VMID: 100,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
guestsByVMID := map[string]GuestLookup{
|
||||||
|
"100": guestsByKey[key],
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID)
|
||||||
|
|
||||||
|
m.mu.RLock()
|
||||||
|
alert, exists := m.activeAlerts["backup-age-"+sanitizeAlertKey(key)]
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if !exists {
|
||||||
|
t.Fatalf("expected backup age alert to be created")
|
||||||
|
}
|
||||||
|
if alert.Level != AlertLevelCritical {
|
||||||
|
t.Fatalf("expected critical backup alert, got %s", alert.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recent backup clears alert
|
||||||
|
storageBackups[0].Time = now
|
||||||
|
m.CheckBackups(storageBackups, nil, nil, guestsByKey, guestsByVMID)
|
||||||
|
|
||||||
|
m.mu.RLock()
|
||||||
|
_, exists = m.activeAlerts["backup-age-"+sanitizeAlertKey(key)]
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if exists {
|
||||||
|
t.Fatalf("expected backup-age alert to clear after fresh backup")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckBackupsHandlesPbsOnlyGuests(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.Enabled = true
|
||||||
|
m.config.BackupDefaults = BackupAlertConfig{
|
||||||
|
Enabled: true,
|
||||||
|
WarningDays: 3,
|
||||||
|
CriticalDays: 5,
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
pbsBackups := []models.PBSBackup{
|
||||||
|
{
|
||||||
|
ID: "pbs-backup-999-0",
|
||||||
|
Instance: "pbs-main",
|
||||||
|
Datastore: "backup-store",
|
||||||
|
BackupType: "qemu",
|
||||||
|
VMID: "999",
|
||||||
|
BackupTime: now.Add(-6 * 24 * time.Hour),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckBackups(nil, pbsBackups, nil, map[string]GuestLookup{}, map[string]GuestLookup{})
|
||||||
|
|
||||||
|
m.mu.RLock()
|
||||||
|
found := false
|
||||||
|
for id, alert := range m.activeAlerts {
|
||||||
|
if strings.HasPrefix(id, "backup-age-") {
|
||||||
|
found = true
|
||||||
|
if alert.Level != AlertLevelCritical {
|
||||||
|
t.Fatalf("expected PBS backup alert to be critical")
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("expected PBS backup alert to be created")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckBackupsHandlesPmgBackups(t *testing.T) {
|
||||||
|
m := NewManager()
|
||||||
|
m.ClearActiveAlerts()
|
||||||
|
|
||||||
|
m.mu.Lock()
|
||||||
|
m.config.Enabled = true
|
||||||
|
m.config.BackupDefaults = BackupAlertConfig{
|
||||||
|
Enabled: true,
|
||||||
|
WarningDays: 5,
|
||||||
|
CriticalDays: 7,
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
pmgBackups := []models.PMGBackup{
|
||||||
|
{
|
||||||
|
ID: "pmg-backup-mail-01",
|
||||||
|
Instance: "mail",
|
||||||
|
Node: "mail-gateway",
|
||||||
|
Filename: "pmg-backup_2024-01-01.tgz",
|
||||||
|
BackupTime: now.Add(-8 * 24 * time.Hour),
|
||||||
|
Size: 123456,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
m.CheckBackups(nil, nil, pmgBackups, map[string]GuestLookup{}, map[string]GuestLookup{})
|
||||||
|
|
||||||
|
m.mu.RLock()
|
||||||
|
found := false
|
||||||
|
for id, alert := range m.activeAlerts {
|
||||||
|
if strings.HasPrefix(id, "backup-age-") {
|
||||||
|
found = true
|
||||||
|
if alert.Level != AlertLevelCritical {
|
||||||
|
t.Fatalf("expected PMG backup alert to be critical")
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("expected PMG backup alert to be created")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
|
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
|
||||||
m := NewManager()
|
m := NewManager()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2452,16 +2452,24 @@ func (r *Router) handleBackups(w http.ResponseWriter, req *http.Request) {
|
||||||
// Get current state
|
// Get current state
|
||||||
state := r.monitor.GetState()
|
state := r.monitor.GetState()
|
||||||
|
|
||||||
// Return backup data structure
|
|
||||||
backups := map[string]interface{}{
|
|
||||||
"backupTasks": state.PVEBackups.BackupTasks,
|
|
||||||
"storageBackups": state.PVEBackups.StorageBackups,
|
|
||||||
"guestSnapshots": state.PVEBackups.GuestSnapshots,
|
|
||||||
"pbsBackups": state.PBSBackups,
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
json.NewEncoder(w).Encode(backups)
|
json.NewEncoder(w).Encode(struct {
|
||||||
|
Backups models.Backups `json:"backups"`
|
||||||
|
PVEBackups models.PVEBackups `json:"pveBackups"`
|
||||||
|
PBSBackups []models.PBSBackup `json:"pbsBackups"`
|
||||||
|
PMGBackups []models.PMGBackup `json:"pmgBackups"`
|
||||||
|
BackupTasks []models.BackupTask `json:"backupTasks"`
|
||||||
|
Storage []models.StorageBackup `json:"storageBackups"`
|
||||||
|
GuestSnaps []models.GuestSnapshot `json:"guestSnapshots"`
|
||||||
|
}{
|
||||||
|
Backups: state.Backups,
|
||||||
|
PVEBackups: state.PVEBackups,
|
||||||
|
PBSBackups: state.PBSBackups,
|
||||||
|
PMGBackups: state.PMGBackups,
|
||||||
|
BackupTasks: state.PVEBackups.BackupTasks,
|
||||||
|
Storage: state.PVEBackups.StorageBackups,
|
||||||
|
GuestSnaps: state.PVEBackups.GuestSnapshots,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleBackupsPVE handles PVE backup requests
|
// handleBackupsPVE handles PVE backup requests
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,8 @@ type StateResponse struct {
|
||||||
PBSInstances []models.PBSInstance `json:"pbs"`
|
PBSInstances []models.PBSInstance `json:"pbs"`
|
||||||
PMGInstances []models.PMGInstance `json:"pmg"`
|
PMGInstances []models.PMGInstance `json:"pmg"`
|
||||||
PBSBackups []models.PBSBackup `json:"pbsBackups"`
|
PBSBackups []models.PBSBackup `json:"pbsBackups"`
|
||||||
|
PMGBackups []models.PMGBackup `json:"pmgBackups"`
|
||||||
|
Backups models.Backups `json:"backups"`
|
||||||
Metrics []models.Metric `json:"metrics"`
|
Metrics []models.Metric `json:"metrics"`
|
||||||
PVEBackups models.PVEBackups `json:"pveBackups"`
|
PVEBackups models.PVEBackups `json:"pveBackups"`
|
||||||
Performance models.Performance `json:"performance"`
|
Performance models.Performance `json:"performance"`
|
||||||
|
|
|
||||||
|
|
@ -194,6 +194,15 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
|
||||||
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
||||||
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
||||||
}
|
}
|
||||||
|
if config.BackupDefaults.WarningDays < 0 {
|
||||||
|
config.BackupDefaults.WarningDays = 0
|
||||||
|
}
|
||||||
|
if config.BackupDefaults.CriticalDays < 0 {
|
||||||
|
config.BackupDefaults.CriticalDays = 0
|
||||||
|
}
|
||||||
|
if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays {
|
||||||
|
config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays
|
||||||
|
}
|
||||||
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
||||||
|
|
||||||
data, err := json.MarshalIndent(config, "", " ")
|
data, err := json.MarshalIndent(config, "", " ")
|
||||||
|
|
@ -251,6 +260,11 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
|
||||||
WarningDays: 30,
|
WarningDays: 30,
|
||||||
CriticalDays: 45,
|
CriticalDays: 45,
|
||||||
},
|
},
|
||||||
|
BackupDefaults: alerts.BackupAlertConfig{
|
||||||
|
Enabled: false,
|
||||||
|
WarningDays: 7,
|
||||||
|
CriticalDays: 14,
|
||||||
|
},
|
||||||
Overrides: make(map[string]alerts.ThresholdConfig),
|
Overrides: make(map[string]alerts.ThresholdConfig),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
@ -310,6 +324,15 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
|
||||||
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
||||||
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
||||||
}
|
}
|
||||||
|
if config.BackupDefaults.WarningDays < 0 {
|
||||||
|
config.BackupDefaults.WarningDays = 0
|
||||||
|
}
|
||||||
|
if config.BackupDefaults.CriticalDays < 0 {
|
||||||
|
config.BackupDefaults.CriticalDays = 0
|
||||||
|
}
|
||||||
|
if config.BackupDefaults.CriticalDays > 0 && config.BackupDefaults.WarningDays > config.BackupDefaults.CriticalDays {
|
||||||
|
config.BackupDefaults.WarningDays = config.BackupDefaults.CriticalDays
|
||||||
|
}
|
||||||
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
|
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
|
||||||
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -140,6 +140,11 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
|
||||||
WarningDays: 20,
|
WarningDays: 20,
|
||||||
CriticalDays: 10,
|
CriticalDays: 10,
|
||||||
},
|
},
|
||||||
|
BackupDefaults: alerts.BackupAlertConfig{
|
||||||
|
Enabled: true,
|
||||||
|
WarningDays: 12,
|
||||||
|
CriticalDays: 8,
|
||||||
|
},
|
||||||
NodeDefaults: alerts.ThresholdConfig{
|
NodeDefaults: alerts.ThresholdConfig{
|
||||||
Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0},
|
Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0},
|
||||||
},
|
},
|
||||||
|
|
@ -174,6 +179,15 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
|
||||||
if loaded.NodeDefaults.Temperature.Trigger != 80 || loaded.NodeDefaults.Temperature.Clear != 75 {
|
if loaded.NodeDefaults.Temperature.Trigger != 80 || loaded.NodeDefaults.Temperature.Clear != 75 {
|
||||||
t.Fatalf("expected temperature defaults 80/75, got %+v", loaded.NodeDefaults.Temperature)
|
t.Fatalf("expected temperature defaults 80/75, got %+v", loaded.NodeDefaults.Temperature)
|
||||||
}
|
}
|
||||||
|
if !loaded.BackupDefaults.Enabled {
|
||||||
|
t.Fatalf("expected backup defaults to remain enabled")
|
||||||
|
}
|
||||||
|
if loaded.BackupDefaults.WarningDays != 8 {
|
||||||
|
t.Fatalf("expected backup warning normalized to 8, got %d", loaded.BackupDefaults.WarningDays)
|
||||||
|
}
|
||||||
|
if loaded.BackupDefaults.CriticalDays != 8 {
|
||||||
|
t.Fatalf("expected backup critical normalized to 8, got %d", loaded.BackupDefaults.CriticalDays)
|
||||||
|
}
|
||||||
expectedPrefixes := []string{"Runner"}
|
expectedPrefixes := []string{"Runner"}
|
||||||
if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) {
|
if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) {
|
||||||
t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes)
|
t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes)
|
||||||
|
|
|
||||||
|
|
@ -329,6 +329,12 @@ func GenerateMockData(config MockConfig) models.StateSnapshot {
|
||||||
StorageBackups: generateBackups(data.VMs, data.Containers),
|
StorageBackups: generateBackups(data.VMs, data.Containers),
|
||||||
GuestSnapshots: generateSnapshots(data.VMs, data.Containers),
|
GuestSnapshots: generateSnapshots(data.VMs, data.Containers),
|
||||||
}
|
}
|
||||||
|
data.PMGBackups = extractPMGBackups(data.PVEBackups.StorageBackups)
|
||||||
|
data.Backups = models.Backups{
|
||||||
|
PVE: data.PVEBackups,
|
||||||
|
PBS: append([]models.PBSBackup(nil), data.PBSBackups...),
|
||||||
|
PMG: append([]models.PMGBackup(nil), data.PMGBackups...),
|
||||||
|
}
|
||||||
|
|
||||||
// Calculate stats
|
// Calculate stats
|
||||||
data.Stats.StartTime = time.Now()
|
data.Stats.StartTime = time.Now()
|
||||||
|
|
@ -2100,6 +2106,47 @@ func generateBackups(vms []models.VM, containers []models.Container) []models.St
|
||||||
return backups
|
return backups
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func extractPMGBackups(storageBackups []models.StorageBackup) []models.PMGBackup {
|
||||||
|
pmgBackups := make([]models.PMGBackup, 0)
|
||||||
|
for _, backup := range storageBackups {
|
||||||
|
if !strings.EqualFold(backup.Type, "host") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
format := strings.ToLower(backup.Format)
|
||||||
|
notes := strings.ToLower(backup.Notes)
|
||||||
|
if !strings.Contains(format, "pmg") && !strings.Contains(notes, "pmg") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := backup.Volid
|
||||||
|
if filename == "" {
|
||||||
|
filename = backup.Notes
|
||||||
|
}
|
||||||
|
instance := backup.Instance
|
||||||
|
if instance == "" && backup.Node != "" {
|
||||||
|
instance = fmt.Sprintf("PMG:%s", backup.Node)
|
||||||
|
}
|
||||||
|
if instance == "" {
|
||||||
|
instance = "PMG:mock"
|
||||||
|
}
|
||||||
|
|
||||||
|
pmgBackups = append(pmgBackups, models.PMGBackup{
|
||||||
|
ID: backup.ID,
|
||||||
|
Instance: instance,
|
||||||
|
Node: backup.Node,
|
||||||
|
Filename: filename,
|
||||||
|
BackupTime: backup.Time,
|
||||||
|
Size: backup.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(pmgBackups, func(i, j int) bool {
|
||||||
|
return pmgBackups[i].BackupTime.After(pmgBackups[j].BackupTime)
|
||||||
|
})
|
||||||
|
|
||||||
|
return pmgBackups
|
||||||
|
}
|
||||||
|
|
||||||
// generatePBSInstances generates mock PBS instances
|
// generatePBSInstances generates mock PBS instances
|
||||||
func generatePBSInstances() []models.PBSInstance {
|
func generatePBSInstances() []models.PBSInstance {
|
||||||
pbsInstances := []models.PBSInstance{
|
pbsInstances := []models.PBSInstance{
|
||||||
|
|
|
||||||
|
|
@ -310,6 +310,7 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot {
|
||||||
PhysicalDisks: append([]models.PhysicalDisk(nil), state.PhysicalDisks...),
|
PhysicalDisks: append([]models.PhysicalDisk(nil), state.PhysicalDisks...),
|
||||||
PBSInstances: append([]models.PBSInstance(nil), state.PBSInstances...),
|
PBSInstances: append([]models.PBSInstance(nil), state.PBSInstances...),
|
||||||
PBSBackups: append([]models.PBSBackup(nil), state.PBSBackups...),
|
PBSBackups: append([]models.PBSBackup(nil), state.PBSBackups...),
|
||||||
|
PMGBackups: append([]models.PMGBackup(nil), state.PMGBackups...),
|
||||||
Metrics: append([]models.Metric(nil), state.Metrics...),
|
Metrics: append([]models.Metric(nil), state.Metrics...),
|
||||||
Performance: state.Performance,
|
Performance: state.Performance,
|
||||||
Stats: state.Stats,
|
Stats: state.Stats,
|
||||||
|
|
@ -324,6 +325,15 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot {
|
||||||
StorageBackups: append([]models.StorageBackup(nil), state.PVEBackups.StorageBackups...),
|
StorageBackups: append([]models.StorageBackup(nil), state.PVEBackups.StorageBackups...),
|
||||||
GuestSnapshots: append([]models.GuestSnapshot(nil), state.PVEBackups.GuestSnapshots...),
|
GuestSnapshots: append([]models.GuestSnapshot(nil), state.PVEBackups.GuestSnapshots...),
|
||||||
}
|
}
|
||||||
|
copyState.Backups = models.Backups{
|
||||||
|
PVE: models.PVEBackups{
|
||||||
|
BackupTasks: append([]models.BackupTask(nil), state.Backups.PVE.BackupTasks...),
|
||||||
|
StorageBackups: append([]models.StorageBackup(nil), state.Backups.PVE.StorageBackups...),
|
||||||
|
GuestSnapshots: append([]models.GuestSnapshot(nil), state.Backups.PVE.GuestSnapshots...),
|
||||||
|
},
|
||||||
|
PBS: append([]models.PBSBackup(nil), state.Backups.PBS...),
|
||||||
|
PMG: append([]models.PMGBackup(nil), state.Backups.PMG...),
|
||||||
|
}
|
||||||
|
|
||||||
for k, v := range state.ConnectionHealth {
|
for k, v := range state.ConnectionHealth {
|
||||||
copyState.ConnectionHealth[k] = v
|
copyState.ConnectionHealth[k] = v
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,8 @@ type State struct {
|
||||||
PBSInstances []PBSInstance `json:"pbs"`
|
PBSInstances []PBSInstance `json:"pbs"`
|
||||||
PMGInstances []PMGInstance `json:"pmg"`
|
PMGInstances []PMGInstance `json:"pmg"`
|
||||||
PBSBackups []PBSBackup `json:"pbsBackups"`
|
PBSBackups []PBSBackup `json:"pbsBackups"`
|
||||||
|
PMGBackups []PMGBackup `json:"pmgBackups"`
|
||||||
|
Backups Backups `json:"backups"`
|
||||||
Metrics []Metric `json:"metrics"`
|
Metrics []Metric `json:"metrics"`
|
||||||
PVEBackups PVEBackups `json:"pveBackups"`
|
PVEBackups PVEBackups `json:"pveBackups"`
|
||||||
Performance Performance `json:"performance"`
|
Performance Performance `json:"performance"`
|
||||||
|
|
@ -461,6 +463,23 @@ type PMGNodeStatus struct {
|
||||||
QueueStatus *PMGQueueStatus `json:"queueStatus,omitempty"` // Postfix queue status for this node
|
QueueStatus *PMGQueueStatus `json:"queueStatus,omitempty"` // Postfix queue status for this node
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PMGBackup represents a configuration backup generated by a PMG node.
|
||||||
|
type PMGBackup struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Instance string `json:"instance"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
BackupTime time.Time `json:"backupTime"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backups aggregates backup collections by source type.
|
||||||
|
type Backups struct {
|
||||||
|
PVE PVEBackups `json:"pve"`
|
||||||
|
PBS []PBSBackup `json:"pbs"`
|
||||||
|
PMG []PMGBackup `json:"pmg"`
|
||||||
|
}
|
||||||
|
|
||||||
// PMGMailStats summarizes aggregated mail statistics for a timeframe
|
// PMGMailStats summarizes aggregated mail statistics for a timeframe
|
||||||
type PMGMailStats struct {
|
type PMGMailStats struct {
|
||||||
Timeframe string `json:"timeframe"`
|
Timeframe string `json:"timeframe"`
|
||||||
|
|
@ -677,7 +696,13 @@ type Stats struct {
|
||||||
|
|
||||||
// NewState creates a new State instance
|
// NewState creates a new State instance
|
||||||
func NewState() *State {
|
func NewState() *State {
|
||||||
return &State{
|
pveBackups := PVEBackups{
|
||||||
|
BackupTasks: make([]BackupTask, 0),
|
||||||
|
StorageBackups: make([]StorageBackup, 0),
|
||||||
|
GuestSnapshots: make([]GuestSnapshot, 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
state := &State{
|
||||||
Nodes: make([]Node, 0),
|
Nodes: make([]Node, 0),
|
||||||
VMs: make([]VM, 0),
|
VMs: make([]VM, 0),
|
||||||
Containers: make([]Container, 0),
|
Containers: make([]Container, 0),
|
||||||
|
|
@ -687,17 +712,31 @@ func NewState() *State {
|
||||||
PBSInstances: make([]PBSInstance, 0),
|
PBSInstances: make([]PBSInstance, 0),
|
||||||
PMGInstances: make([]PMGInstance, 0),
|
PMGInstances: make([]PMGInstance, 0),
|
||||||
PBSBackups: make([]PBSBackup, 0),
|
PBSBackups: make([]PBSBackup, 0),
|
||||||
Metrics: make([]Metric, 0),
|
PMGBackups: make([]PMGBackup, 0),
|
||||||
PVEBackups: PVEBackups{
|
Backups: Backups{
|
||||||
BackupTasks: make([]BackupTask, 0),
|
PVE: pveBackups,
|
||||||
StorageBackups: make([]StorageBackup, 0),
|
PBS: make([]PBSBackup, 0),
|
||||||
GuestSnapshots: make([]GuestSnapshot, 0),
|
PMG: make([]PMGBackup, 0),
|
||||||
},
|
},
|
||||||
|
Metrics: make([]Metric, 0),
|
||||||
|
PVEBackups: pveBackups,
|
||||||
ConnectionHealth: make(map[string]bool),
|
ConnectionHealth: make(map[string]bool),
|
||||||
ActiveAlerts: make([]Alert, 0),
|
ActiveAlerts: make([]Alert, 0),
|
||||||
RecentlyResolved: make([]ResolvedAlert, 0),
|
RecentlyResolved: make([]ResolvedAlert, 0),
|
||||||
LastUpdate: time.Now(),
|
LastUpdate: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
state.syncBackupsLocked()
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncBackupsLocked updates the aggregated backups structure.
|
||||||
|
func (s *State) syncBackupsLocked() {
|
||||||
|
s.Backups = Backups{
|
||||||
|
PVE: s.PVEBackups,
|
||||||
|
PBS: append([]PBSBackup(nil), s.PBSBackups...),
|
||||||
|
PMG: append([]PMGBackup(nil), s.PMGBackups...),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateActiveAlerts updates the active alerts in the state
|
// UpdateActiveAlerts updates the active alerts in the state
|
||||||
|
|
@ -1211,6 +1250,7 @@ func (s *State) UpdateBackupTasksForInstance(instanceName string, tasks []Backup
|
||||||
})
|
})
|
||||||
|
|
||||||
s.PVEBackups.BackupTasks = newTasks
|
s.PVEBackups.BackupTasks = newTasks
|
||||||
|
s.syncBackupsLocked()
|
||||||
s.LastUpdate = time.Now()
|
s.LastUpdate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1245,6 +1285,7 @@ func (s *State) UpdateStorageBackupsForInstance(instanceName string, backups []S
|
||||||
})
|
})
|
||||||
|
|
||||||
s.PVEBackups.StorageBackups = newBackups
|
s.PVEBackups.StorageBackups = newBackups
|
||||||
|
s.syncBackupsLocked()
|
||||||
s.LastUpdate = time.Now()
|
s.LastUpdate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1279,6 +1320,7 @@ func (s *State) UpdateGuestSnapshotsForInstance(instanceName string, snapshots [
|
||||||
})
|
})
|
||||||
|
|
||||||
s.PVEBackups.GuestSnapshots = newSnapshots
|
s.PVEBackups.GuestSnapshots = newSnapshots
|
||||||
|
s.syncBackupsLocked()
|
||||||
s.LastUpdate = time.Now()
|
s.LastUpdate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1326,5 +1368,32 @@ func (s *State) UpdatePBSBackups(instanceName string, backups []PBSBackup) {
|
||||||
})
|
})
|
||||||
|
|
||||||
s.PBSBackups = newBackups
|
s.PBSBackups = newBackups
|
||||||
|
s.syncBackupsLocked()
|
||||||
|
s.LastUpdate = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePMGBackups updates PMG backups for a specific instance.
|
||||||
|
func (s *State) UpdatePMGBackups(instanceName string, backups []PMGBackup) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
combined := make([]PMGBackup, 0, len(s.PMGBackups)+len(backups))
|
||||||
|
for _, backup := range s.PMGBackups {
|
||||||
|
if backup.Instance != instanceName {
|
||||||
|
combined = append(combined, backup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(backups) > 0 {
|
||||||
|
combined = append(combined, backups...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(combined) > 1 {
|
||||||
|
sort.Slice(combined, func(i, j int) bool {
|
||||||
|
return combined[i].BackupTime.After(combined[j].BackupTime)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
s.PMGBackups = combined
|
||||||
|
s.syncBackupsLocked()
|
||||||
s.LastUpdate = time.Now()
|
s.LastUpdate = time.Now()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -230,6 +230,9 @@ type StateFrontend struct {
|
||||||
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
|
PhysicalDisks []PhysicalDisk `json:"physicalDisks"`
|
||||||
PBS []PBSInstance `json:"pbs"` // Keep as is
|
PBS []PBSInstance `json:"pbs"` // Keep as is
|
||||||
PMG []PMGInstance `json:"pmg"`
|
PMG []PMGInstance `json:"pmg"`
|
||||||
|
PBSBackups []PBSBackup `json:"pbsBackups"`
|
||||||
|
PMGBackups []PMGBackup `json:"pmgBackups"`
|
||||||
|
Backups Backups `json:"backups"`
|
||||||
ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts
|
ActiveAlerts []Alert `json:"activeAlerts"` // Active alerts
|
||||||
Metrics map[string]any `json:"metrics"` // Empty object for now
|
Metrics map[string]any `json:"metrics"` // Empty object for now
|
||||||
PVEBackups PVEBackups `json:"pveBackups"` // Keep as is
|
PVEBackups PVEBackups `json:"pveBackups"` // Keep as is
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,8 @@ type StateSnapshot struct {
|
||||||
PBSInstances []PBSInstance `json:"pbs"`
|
PBSInstances []PBSInstance `json:"pbs"`
|
||||||
PMGInstances []PMGInstance `json:"pmg"`
|
PMGInstances []PMGInstance `json:"pmg"`
|
||||||
PBSBackups []PBSBackup `json:"pbsBackups"`
|
PBSBackups []PBSBackup `json:"pbsBackups"`
|
||||||
|
PMGBackups []PMGBackup `json:"pmgBackups"`
|
||||||
|
Backups Backups `json:"backups"`
|
||||||
Metrics []Metric `json:"metrics"`
|
Metrics []Metric `json:"metrics"`
|
||||||
PVEBackups PVEBackups `json:"pveBackups"`
|
PVEBackups PVEBackups `json:"pveBackups"`
|
||||||
Performance Performance `json:"performance"`
|
Performance Performance `json:"performance"`
|
||||||
|
|
@ -29,6 +31,14 @@ func (s *State) GetSnapshot() StateSnapshot {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
|
pbsBackups := append([]PBSBackup{}, s.PBSBackups...)
|
||||||
|
pmgBackups := append([]PMGBackup{}, s.PMGBackups...)
|
||||||
|
pveBackups := PVEBackups{
|
||||||
|
BackupTasks: append([]BackupTask{}, s.PVEBackups.BackupTasks...),
|
||||||
|
StorageBackups: append([]StorageBackup{}, s.PVEBackups.StorageBackups...),
|
||||||
|
GuestSnapshots: append([]GuestSnapshot{}, s.PVEBackups.GuestSnapshots...),
|
||||||
|
}
|
||||||
|
|
||||||
// Create a snapshot without mutex
|
// Create a snapshot without mutex
|
||||||
snapshot := StateSnapshot{
|
snapshot := StateSnapshot{
|
||||||
Nodes: append([]Node{}, s.Nodes...),
|
Nodes: append([]Node{}, s.Nodes...),
|
||||||
|
|
@ -40,13 +50,15 @@ func (s *State) GetSnapshot() StateSnapshot {
|
||||||
PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...),
|
PhysicalDisks: append([]PhysicalDisk{}, s.PhysicalDisks...),
|
||||||
PBSInstances: append([]PBSInstance{}, s.PBSInstances...),
|
PBSInstances: append([]PBSInstance{}, s.PBSInstances...),
|
||||||
PMGInstances: append([]PMGInstance{}, s.PMGInstances...),
|
PMGInstances: append([]PMGInstance{}, s.PMGInstances...),
|
||||||
PBSBackups: append([]PBSBackup{}, s.PBSBackups...),
|
PBSBackups: pbsBackups,
|
||||||
Metrics: append([]Metric{}, s.Metrics...),
|
PMGBackups: pmgBackups,
|
||||||
PVEBackups: PVEBackups{
|
Backups: Backups{
|
||||||
BackupTasks: append([]BackupTask{}, s.PVEBackups.BackupTasks...),
|
PVE: pveBackups,
|
||||||
StorageBackups: append([]StorageBackup{}, s.PVEBackups.StorageBackups...),
|
PBS: pbsBackups,
|
||||||
GuestSnapshots: append([]GuestSnapshot{}, s.PVEBackups.GuestSnapshots...),
|
PMG: pmgBackups,
|
||||||
},
|
},
|
||||||
|
Metrics: append([]Metric{}, s.Metrics...),
|
||||||
|
PVEBackups: pveBackups,
|
||||||
Performance: s.Performance,
|
Performance: s.Performance,
|
||||||
ConnectionHealth: make(map[string]bool),
|
ConnectionHealth: make(map[string]bool),
|
||||||
Stats: s.Stats,
|
Stats: s.Stats,
|
||||||
|
|
@ -110,6 +122,9 @@ func (s StateSnapshot) ToFrontend() StateFrontend {
|
||||||
PhysicalDisks: s.PhysicalDisks,
|
PhysicalDisks: s.PhysicalDisks,
|
||||||
PBS: s.PBSInstances,
|
PBS: s.PBSInstances,
|
||||||
PMG: s.PMGInstances,
|
PMG: s.PMGInstances,
|
||||||
|
PBSBackups: s.PBSBackups,
|
||||||
|
PMGBackups: s.PMGBackups,
|
||||||
|
Backups: s.Backups,
|
||||||
ActiveAlerts: s.ActiveAlerts,
|
ActiveAlerts: s.ActiveAlerts,
|
||||||
Metrics: make(map[string]any),
|
Metrics: make(map[string]any),
|
||||||
PVEBackups: s.PVEBackups,
|
PVEBackups: s.PVEBackups,
|
||||||
|
|
|
||||||
|
|
@ -1707,7 +1707,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||||
Int("vms", len(state.VMs)).
|
Int("vms", len(state.VMs)).
|
||||||
Int("containers", len(state.Containers)).
|
Int("containers", len(state.Containers)).
|
||||||
Int("pbs", len(state.PBSInstances)).
|
Int("pbs", len(state.PBSInstances)).
|
||||||
Int("pbsBackups", len(state.PBSBackups)).
|
Int("pbsBackups", len(state.Backups.PBS)).
|
||||||
Int("physicalDisks", len(state.PhysicalDisks)).
|
Int("physicalDisks", len(state.PhysicalDisks)).
|
||||||
Msg("Broadcasting state update (ticker)")
|
Msg("Broadcasting state update (ticker)")
|
||||||
// Convert to frontend format before broadcasting (converts time.Time to int64, etc.)
|
// Convert to frontend format before broadcasting (converts time.Time to int64, etc.)
|
||||||
|
|
@ -5167,7 +5167,11 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
|
||||||
cluster, err := client.GetClusterStatus(ctx, true)
|
cluster, err := client.GetClusterStatus(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Err(err).Str("instance", instanceName).Msg("Failed to retrieve PMG cluster status")
|
log.Debug().Err(err).Str("instance", instanceName).Msg("Failed to retrieve PMG cluster status")
|
||||||
} else if len(cluster) > 0 {
|
}
|
||||||
|
|
||||||
|
backupNodes := make(map[string]struct{})
|
||||||
|
|
||||||
|
if len(cluster) > 0 {
|
||||||
nodes := make([]models.PMGNodeStatus, 0, len(cluster))
|
nodes := make([]models.PMGNodeStatus, 0, len(cluster))
|
||||||
for _, entry := range cluster {
|
for _, entry := range cluster {
|
||||||
status := strings.ToLower(strings.TrimSpace(entry.Type))
|
status := strings.ToLower(strings.TrimSpace(entry.Type))
|
||||||
|
|
@ -5180,6 +5184,8 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
|
||||||
Role: entry.Type,
|
Role: entry.Type,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
backupNodes[entry.Name] = struct{}{}
|
||||||
|
|
||||||
// Fetch queue status for this node
|
// Fetch queue status for this node
|
||||||
if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil {
|
if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil {
|
||||||
log.Debug().Err(qErr).
|
log.Debug().Err(qErr).
|
||||||
|
|
@ -5204,6 +5210,53 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
|
||||||
pmgInst.Nodes = nodes
|
pmgInst.Nodes = nodes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(backupNodes) == 0 {
|
||||||
|
trimmed := strings.TrimSpace(instanceName)
|
||||||
|
if trimmed != "" {
|
||||||
|
backupNodes[trimmed] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pmgBackups := make([]models.PMGBackup, 0)
|
||||||
|
seenBackupIDs := make(map[string]struct{})
|
||||||
|
|
||||||
|
for nodeName := range backupNodes {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
backups, backupErr := client.ListBackups(ctx, nodeName)
|
||||||
|
if backupErr != nil {
|
||||||
|
log.Debug().Err(backupErr).
|
||||||
|
Str("instance", instanceName).
|
||||||
|
Str("node", nodeName).
|
||||||
|
Msg("Failed to list PMG configuration backups")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range backups {
|
||||||
|
backupTime := time.Unix(b.Timestamp, 0)
|
||||||
|
id := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, b.Timestamp)
|
||||||
|
if _, exists := seenBackupIDs[id]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seenBackupIDs[id] = struct{}{}
|
||||||
|
pmgBackups = append(pmgBackups, models.PMGBackup{
|
||||||
|
ID: id,
|
||||||
|
Instance: instanceName,
|
||||||
|
Node: nodeName,
|
||||||
|
Filename: b.Filename,
|
||||||
|
BackupTime: backupTime,
|
||||||
|
Size: b.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug().
|
||||||
|
Str("instance", instanceName).
|
||||||
|
Int("backupCount", len(pmgBackups)).
|
||||||
|
Msg("PMG backups polled")
|
||||||
|
|
||||||
if stats, err := client.GetMailStatistics(ctx, "day"); err != nil {
|
if stats, err := client.GetMailStatistics(ctx, "day"); err != nil {
|
||||||
log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to fetch PMG mail statistics")
|
log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to fetch PMG mail statistics")
|
||||||
} else if stats != nil {
|
} else if stats != nil {
|
||||||
|
|
@ -5279,6 +5332,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
|
||||||
}
|
}
|
||||||
pmgInst.Quarantine = &quarantine
|
pmgInst.Quarantine = &quarantine
|
||||||
|
|
||||||
|
m.state.UpdatePMGBackups(instanceName, pmgBackups)
|
||||||
m.state.UpdatePMGInstance(pmgInst)
|
m.state.UpdatePMGInstance(pmgInst)
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("instance", instanceName).
|
Str("instance", instanceName).
|
||||||
|
|
@ -5620,6 +5674,24 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
|
||||||
// Update state with storage backups for this instance
|
// Update state with storage backups for this instance
|
||||||
m.state.UpdateStorageBackupsForInstance(instanceName, allBackups)
|
m.state.UpdateStorageBackupsForInstance(instanceName, allBackups)
|
||||||
|
|
||||||
|
if m.alertManager != nil {
|
||||||
|
snapshot := m.state.GetSnapshot()
|
||||||
|
guestsByKey, guestsByVMID := buildGuestLookups(snapshot)
|
||||||
|
pveStorage := snapshot.Backups.PVE.StorageBackups
|
||||||
|
if len(pveStorage) == 0 && len(snapshot.PVEBackups.StorageBackups) > 0 {
|
||||||
|
pveStorage = snapshot.PVEBackups.StorageBackups
|
||||||
|
}
|
||||||
|
pbsBackups := snapshot.Backups.PBS
|
||||||
|
if len(pbsBackups) == 0 && len(snapshot.PBSBackups) > 0 {
|
||||||
|
pbsBackups = snapshot.PBSBackups
|
||||||
|
}
|
||||||
|
pmgBackups := snapshot.Backups.PMG
|
||||||
|
if len(pmgBackups) == 0 && len(snapshot.PMGBackups) > 0 {
|
||||||
|
pmgBackups = snapshot.PMGBackups
|
||||||
|
}
|
||||||
|
m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID)
|
||||||
|
}
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("instance", instanceName).
|
Str("instance", instanceName).
|
||||||
Int("count", len(allBackups)).
|
Int("count", len(allBackups)).
|
||||||
|
|
@ -5636,6 +5708,49 @@ func shouldPreserveBackups(nodeCount int, hadSuccessfulNode bool, storagesWithBa
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildGuestLookups(snapshot models.StateSnapshot) (map[string]alerts.GuestLookup, map[string]alerts.GuestLookup) {
|
||||||
|
byKey := make(map[string]alerts.GuestLookup)
|
||||||
|
byVMID := make(map[string]alerts.GuestLookup)
|
||||||
|
|
||||||
|
for _, vm := range snapshot.VMs {
|
||||||
|
info := alerts.GuestLookup{
|
||||||
|
Name: vm.Name,
|
||||||
|
Instance: vm.Instance,
|
||||||
|
Node: vm.Node,
|
||||||
|
Type: vm.Type,
|
||||||
|
VMID: vm.VMID,
|
||||||
|
}
|
||||||
|
key := alerts.BuildGuestKey(vm.Instance, vm.Node, vm.VMID)
|
||||||
|
byKey[key] = info
|
||||||
|
|
||||||
|
vmidKey := fmt.Sprintf("%d", vm.VMID)
|
||||||
|
if _, exists := byVMID[vmidKey]; !exists {
|
||||||
|
byVMID[vmidKey] = info
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ct := range snapshot.Containers {
|
||||||
|
info := alerts.GuestLookup{
|
||||||
|
Name: ct.Name,
|
||||||
|
Instance: ct.Instance,
|
||||||
|
Node: ct.Node,
|
||||||
|
Type: ct.Type,
|
||||||
|
VMID: int(ct.VMID),
|
||||||
|
}
|
||||||
|
key := alerts.BuildGuestKey(ct.Instance, ct.Node, int(ct.VMID))
|
||||||
|
if _, exists := byKey[key]; !exists {
|
||||||
|
byKey[key] = info
|
||||||
|
}
|
||||||
|
|
||||||
|
vmidKey := fmt.Sprintf("%d", ct.VMID)
|
||||||
|
if _, exists := byVMID[vmidKey]; !exists {
|
||||||
|
byVMID[vmidKey] = info
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return byKey, byVMID
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Monitor) calculateBackupOperationTimeout(instanceName string) time.Duration {
|
func (m *Monitor) calculateBackupOperationTimeout(instanceName string) time.Duration {
|
||||||
const (
|
const (
|
||||||
minTimeout = 2 * time.Minute
|
minTimeout = 2 * time.Minute
|
||||||
|
|
@ -6042,6 +6157,7 @@ func (m *Monitor) removeFailedPMGInstance(instanceName string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
m.state.UpdatePMGInstances(updated)
|
m.state.UpdatePMGInstances(updated)
|
||||||
|
m.state.UpdatePMGBackups(instanceName, nil)
|
||||||
m.state.SetConnectionHealth("pmg-"+instanceName, false)
|
m.state.SetConnectionHealth("pmg-"+instanceName, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -6149,6 +6265,24 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
|
||||||
|
|
||||||
// Update state
|
// Update state
|
||||||
m.state.UpdatePBSBackups(instanceName, allBackups)
|
m.state.UpdatePBSBackups(instanceName, allBackups)
|
||||||
|
|
||||||
|
if m.alertManager != nil {
|
||||||
|
snapshot := m.state.GetSnapshot()
|
||||||
|
guestsByKey, guestsByVMID := buildGuestLookups(snapshot)
|
||||||
|
pveStorage := snapshot.Backups.PVE.StorageBackups
|
||||||
|
if len(pveStorage) == 0 && len(snapshot.PVEBackups.StorageBackups) > 0 {
|
||||||
|
pveStorage = snapshot.PVEBackups.StorageBackups
|
||||||
|
}
|
||||||
|
pbsBackups := snapshot.Backups.PBS
|
||||||
|
if len(pbsBackups) == 0 && len(snapshot.PBSBackups) > 0 {
|
||||||
|
pbsBackups = snapshot.PBSBackups
|
||||||
|
}
|
||||||
|
pmgBackups := snapshot.Backups.PMG
|
||||||
|
if len(pmgBackups) == 0 && len(snapshot.PMGBackups) > 0 {
|
||||||
|
pmgBackups = snapshot.PMGBackups
|
||||||
|
}
|
||||||
|
m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkMockAlerts checks alerts for mock data
|
// checkMockAlerts checks alerts for mock data
|
||||||
|
|
@ -6188,6 +6322,21 @@ func (m *Monitor) checkMockAlerts() {
|
||||||
Msg("Collecting resources for alert cleanup in mock mode")
|
Msg("Collecting resources for alert cleanup in mock mode")
|
||||||
m.alertManager.CleanupAlertsForNodes(existingNodes)
|
m.alertManager.CleanupAlertsForNodes(existingNodes)
|
||||||
|
|
||||||
|
guestsByKey, guestsByVMID := buildGuestLookups(state)
|
||||||
|
pveStorage := state.Backups.PVE.StorageBackups
|
||||||
|
if len(pveStorage) == 0 && len(state.PVEBackups.StorageBackups) > 0 {
|
||||||
|
pveStorage = state.PVEBackups.StorageBackups
|
||||||
|
}
|
||||||
|
pbsBackups := state.Backups.PBS
|
||||||
|
if len(pbsBackups) == 0 && len(state.PBSBackups) > 0 {
|
||||||
|
pbsBackups = state.PBSBackups
|
||||||
|
}
|
||||||
|
pmgBackups := state.Backups.PMG
|
||||||
|
if len(pmgBackups) == 0 && len(state.PMGBackups) > 0 {
|
||||||
|
pmgBackups = state.PMGBackups
|
||||||
|
}
|
||||||
|
m.alertManager.CheckBackups(pveStorage, pbsBackups, pmgBackups, guestsByKey, guestsByVMID)
|
||||||
|
|
||||||
// Limit how many guests we check per cycle to prevent blocking with large datasets
|
// Limit how many guests we check per cycle to prevent blocking with large datasets
|
||||||
const maxGuestsPerCycle = 50
|
const maxGuestsPerCycle = 50
|
||||||
guestsChecked := 0
|
guestsChecked := 0
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,11 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`)
|
fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`)
|
||||||
|
|
||||||
|
case "/api2/json/nodes/mail-gateway/backup":
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
timestamp := time.Now().Add(-8 * 24 * time.Hour).Unix()
|
||||||
|
fmt.Fprintf(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":%d}]}`, timestamp)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
t.Fatalf("unexpected path: %s", r.URL.Path)
|
||||||
}
|
}
|
||||||
|
|
@ -144,6 +149,15 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {
|
||||||
if failures := mon.authFailures["pmg-primary"]; failures != 0 {
|
if failures := mon.authFailures["pmg-primary"]; failures != 0 {
|
||||||
t.Fatalf("expected no auth failures tracked, got %d", failures)
|
t.Fatalf("expected no auth failures tracked, got %d", failures)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(snapshot.PMGBackups) != 1 {
|
||||||
|
t.Fatalf("expected 1 PMG backup in state, got %d", len(snapshot.PMGBackups))
|
||||||
|
}
|
||||||
|
|
||||||
|
pmgBackup := snapshot.PMGBackups[0]
|
||||||
|
if pmgBackup.Node != "mail-gateway" {
|
||||||
|
t.Fatalf("expected PMG backup node mail-gateway, got %s", pmgBackup.Node)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPollPMGInstanceRecordsAuthFailures(t *testing.T) {
|
func TestPollPMGInstanceRecordsAuthFailures(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -119,6 +119,13 @@ type QueueStatusEntry struct {
|
||||||
OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds
|
OldestAge int64 `json:"oldest_age,omitempty"` // Age of oldest message in seconds
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BackupEntry represents a PMG configuration backup stored on a node.
|
||||||
|
type BackupEntry struct {
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
Timestamp int64 `json:"timestamp"`
|
||||||
|
}
|
||||||
|
|
||||||
func NewClient(cfg ClientConfig) (*Client, error) {
|
func NewClient(cfg ClientConfig) (*Client, error) {
|
||||||
if cfg.Timeout <= 0 {
|
if cfg.Timeout <= 0 {
|
||||||
cfg.Timeout = 60 * time.Second
|
cfg.Timeout = 60 * time.Second
|
||||||
|
|
@ -459,3 +466,13 @@ func (c *Client) GetQueueStatus(ctx context.Context, node string) (*QueueStatusE
|
||||||
}
|
}
|
||||||
return &resp.Data, nil
|
return &resp.Data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListBackups returns configuration backup archives available on a PMG node.
|
||||||
|
func (c *Client) ListBackups(ctx context.Context, node string) ([]BackupEntry, error) {
|
||||||
|
path := fmt.Sprintf("/nodes/%s/backup", url.PathEscape(node))
|
||||||
|
var resp apiResponse[[]BackupEntry]
|
||||||
|
if err := c.getJSON(ctx, path, nil, &resp); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return resp.Data, nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -231,3 +231,55 @@ func TestClientUsesTokenAuthorizationHeader(t *testing.T) {
|
||||||
t.Fatalf("expected authorization header %q, got %q", expected, authHeader)
|
t.Fatalf("expected authorization header %q, got %q", expected, authHeader)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListBackups(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/api2/json/access/ticket":
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
t.Fatalf("expected POST for auth, got %s", r.Method)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{"data":{"ticket":"ticket789","CSRFPreventionToken":"csrf789"}}`)
|
||||||
|
case "/api2/json/nodes/node1/backup":
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
t.Fatalf("expected GET for backup listing, got %s", r.Method)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprint(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":1704096000}]}`)
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected request path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
client, err := NewClient(ClientConfig{
|
||||||
|
Host: server.URL,
|
||||||
|
User: "api@pmg",
|
||||||
|
Password: "secret",
|
||||||
|
VerifySSL: false,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error creating client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
backups, err := client.ListBackups(context.Background(), "node1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListBackups failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(backups) != 1 {
|
||||||
|
t.Fatalf("expected 1 backup, got %d", len(backups))
|
||||||
|
}
|
||||||
|
backup := backups[0]
|
||||||
|
if backup.Filename != "pmg-backup_2024-01-01.tgz" {
|
||||||
|
t.Fatalf("unexpected filename: %s", backup.Filename)
|
||||||
|
}
|
||||||
|
if backup.Size != 123456 {
|
||||||
|
t.Fatalf("unexpected size: %d", backup.Size)
|
||||||
|
}
|
||||||
|
if backup.Timestamp != 1704096000 {
|
||||||
|
t.Fatalf("unexpected timestamp: %d", backup.Timestamp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue