perf: reduce polling allocations and guest metadata load

This commit is contained in:
rcourtman 2025-10-25 13:12:47 +00:00
parent 138d8facd2
commit d643dcf0bc
22 changed files with 1980 additions and 392 deletions

View file

@ -3,7 +3,10 @@
The Pulse host agent extends monitoring to standalone servers that do not expose The Pulse host agent extends monitoring to standalone servers that do not expose
Proxmox or Docker APIs. With it you can surface uptime, OS metadata, CPU load, Proxmox or Docker APIs. With it you can surface uptime, OS metadata, CPU load,
memory/disk utilisation, and connection health for any Linux, macOS, or Windows memory/disk utilisation, and connection health for any Linux, macOS, or Windows
machine alongside the rest of your infrastructure. machine alongside the rest of your infrastructure. Beginning with the upcoming
release the installer now handshakes with Pulse in real time so you can confirm
registration directly from the UI and receive host-agent alerts alongside your
existing Docker/Proxmox notifications.
## Prerequisites ## Prerequisites
@ -13,6 +16,11 @@ machine alongside the rest of your infrastructure.
> The agent only initiates outbound connections; no inbound firewall rules are required. > The agent only initiates outbound connections; no inbound firewall rules are required.
If your Pulse instance does not require API tokens (e.g. during an on-premises
lab install) you can still generate commands without embedding a credential.
Confirm the warning in **Settings → Agents → Host agents** and the script will
prompt for a token instead of hard-coding one.
## Quick Start ## Quick Start
> Replace `<api-token>` with a Pulse API token limited to the `host-agent:report` scope. Tokens generated from **Settings → Agents → Host agents** already apply this scope. > Replace `<api-token>` with a Pulse API token limited to the `host-agent:report` scope. Tokens generated from **Settings → Agents → Host agents** already apply this scope.
@ -151,6 +159,17 @@ Run `pulse-host-agent --help` for the full list.
- **Settings → Agents → Host agents** lists every reporting host and provides ready-made install commands. - **Settings → Agents → Host agents** lists every reporting host and provides ready-made install commands.
- The **Servers** tab surfaces host telemetry alongside Proxmox/Docker data in the main dashboard. - The **Servers** tab surfaces host telemetry alongside Proxmox/Docker data in the main dashboard.
### Checking installation status
- Click **Check status** under **Settings → Agents → Host agents** and enter the host ID or hostname you just installed.
- Pulse hits `/api/agents/host/lookup`, highlights the matching row for 10seconds, and refreshes the connection badge, last-seen timestamp, and agent version in-line.
- If the host has not checked in yet, the UI returns a friendly "Host has not registered" message so you can retry without re-running the script.
### Alerts and notifications
- Host agents now participate in the main alert engine. Offline detection, metric thresholds, and override scopes (global or per-host) live in **Settings → Alerts → Thresholds** beside your Docker and Proxmox rules.
- Alert notifications, webhooks, and quiet-hours behaviour reuse the existing pipelines—no extra setup is required once you enable host-agent monitoring.
## Updating ## Updating
Since the agent is a single static binary, updates are as simple as replacing the file and restarting your launchd/systemd unit. The Settings pane always links to the latest release artefacts. Since the agent is a single static binary, updates are as simple as replacing the file and restarting your launchd/systemd unit. The Settings pane always links to the latest release artefacts.

12
docs/RELEASE_NOTES.md Normal file
View file

@ -0,0 +1,12 @@
# Upcoming Release Highlights
## Improvements
- Host agent installs now report back to Pulse in real time. The **Settings → Agents → Host agents** view surfaces the lookup status, highlights the matching row, and refreshes the connection badge without reloading.
- Host agents participate in the alert system alongside Proxmox and Docker sources. Offline detection, thresholds, and overrides live in **Settings → Alerts** and flow through the existing notification channels.
## Documentation
- Updated the host agent guide with the new lookup workflow, token-free commands, and alert integration notes.
_(Replace this stub with per-version notes when tagging the release.)_

View file

@ -8,6 +8,7 @@ import type {
VM, VM,
Container, Container,
Node, Node,
Host,
Alert, Alert,
Storage, Storage,
PBSInstance, PBSInstance,
@ -27,7 +28,15 @@ import type {
} from '@/types/alerts'; } from '@/types/alerts';
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable'; import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
import { useAlertsActivation } from '@/stores/alertsActivation'; import { useAlertsActivation } from '@/stores/alertsActivation';
type OverrideType = 'guest' | 'node' | 'storage' | 'pbs' | 'pmg' | 'dockerHost' | 'dockerContainer'; type OverrideType =
| 'guest'
| 'node'
| 'hostAgent'
| 'storage'
| 'pbs'
| 'pmg'
| 'dockerHost'
| 'dockerContainer';
type OfflineState = 'off' | 'warning' | 'critical'; type OfflineState = 'off' | 'warning' | 'critical';
@ -133,6 +142,7 @@ interface ThresholdsTableProps {
setRawOverridesConfig: (config: Record<string, RawOverrideConfig>) => void; setRawOverridesConfig: (config: Record<string, RawOverrideConfig>) => void;
allGuests: () => (VM | Container)[]; allGuests: () => (VM | Container)[];
nodes: Node[]; nodes: Node[];
hosts: Host[];
storage: Storage[]; storage: Storage[];
dockerHosts: DockerHost[]; dockerHosts: DockerHost[];
pbsInstances?: PBSInstance[]; // PBS instances from state pbsInstances?: PBSInstance[]; // PBS instances from state
@ -156,11 +166,17 @@ interface ThresholdsTableProps {
guestPoweredOffSeverity: () => 'warning' | 'critical'; guestPoweredOffSeverity: () => 'warning' | 'critical';
setGuestPoweredOffSeverity: (value: 'warning' | 'critical') => void; setGuestPoweredOffSeverity: (value: 'warning' | 'critical') => void;
nodeDefaults: SimpleThresholds; nodeDefaults: SimpleThresholds;
hostDefaults: SimpleThresholds;
setNodeDefaults: ( setNodeDefaults: (
value: value:
| Record<string, number | undefined> | Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>), | ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void; ) => void;
setHostDefaults: (
value:
| Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void;
dockerDefaults: { dockerDefaults: {
cpu: number; cpu: number;
memory: number; memory: number;
@ -201,11 +217,13 @@ interface ThresholdsTableProps {
setStorageDefault: (value: number) => void; setStorageDefault: (value: number) => void;
resetGuestDefaults?: () => void; resetGuestDefaults?: () => void;
resetNodeDefaults?: () => void; resetNodeDefaults?: () => void;
resetHostDefaults?: () => void;
resetDockerDefaults?: () => void; resetDockerDefaults?: () => void;
resetDockerIgnoredPrefixes?: () => void; resetDockerIgnoredPrefixes?: () => void;
resetStorageDefault?: () => void; resetStorageDefault?: () => void;
factoryGuestDefaults?: Record<string, number | undefined>; factoryGuestDefaults?: Record<string, number | undefined>;
factoryNodeDefaults?: Record<string, number | undefined>; factoryNodeDefaults?: Record<string, number | undefined>;
factoryHostDefaults?: Record<string, number | undefined>;
factoryDockerDefaults?: Record<string, number | undefined>; factoryDockerDefaults?: Record<string, number | undefined>;
factoryStorageDefault?: number; factoryStorageDefault?: number;
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number }; timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
@ -235,6 +253,8 @@ interface ThresholdsTableProps {
setDisableAllNodes: (value: boolean) => void; setDisableAllNodes: (value: boolean) => void;
disableAllGuests: () => boolean; disableAllGuests: () => boolean;
setDisableAllGuests: (value: boolean) => void; setDisableAllGuests: (value: boolean) => void;
disableAllHosts: () => boolean;
setDisableAllHosts: (value: boolean) => void;
disableAllStorage: () => boolean; disableAllStorage: () => boolean;
setDisableAllStorage: (value: boolean) => void; setDisableAllStorage: (value: boolean) => void;
disableAllPBS: () => boolean; disableAllPBS: () => boolean;
@ -250,6 +270,8 @@ interface ThresholdsTableProps {
setDisableAllNodesOffline: (value: boolean) => void; setDisableAllNodesOffline: (value: boolean) => void;
disableAllGuestsOffline: () => boolean; disableAllGuestsOffline: () => boolean;
setDisableAllGuestsOffline: (value: boolean) => void; setDisableAllGuestsOffline: (value: boolean) => void;
disableAllHostsOffline: () => boolean;
setDisableAllHostsOffline: (value: boolean) => void;
disableAllPBSOffline: () => boolean; disableAllPBSOffline: () => boolean;
setDisableAllPBSOffline: (value: boolean) => void; setDisableAllPBSOffline: (value: boolean) => void;
disableAllPMGOffline: () => boolean; disableAllPMGOffline: () => boolean;
@ -568,7 +590,79 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
if (search) { if (search) {
return nodes.filter((n) => n.name.toLowerCase().includes(search)); return nodes.filter((n) => n.name.toLowerCase().includes(search));
} }
return nodes; return nodes;
}, []);
const hostAgentsWithOverrides = createMemo<Resource[]>((prev = []) => {
if (editingId()) {
return prev;
}
const search = searchTerm().toLowerCase();
const overridesMap = new Map((props.overrides() ?? []).map((o) => [o.id, o]));
const seen = new Set<string>();
const hosts = (props.hosts ?? []).map((host) => {
const override = overridesMap.get(host.id);
const hasCustomThresholds =
override?.thresholds &&
Object.keys(override.thresholds).some((key) => {
const k = key as keyof typeof override.thresholds;
return (
override.thresholds[k] !== undefined &&
override.thresholds[k] !== (props.hostDefaults as any)[k]
);
});
const displayName = host.displayName?.trim() || host.hostname || host.id;
const status = host.status || (host.lastSeen ? 'online' : 'offline');
seen.add(host.id);
return {
id: host.id,
name: displayName,
displayName,
rawName: host.hostname || host.id,
type: 'hostAgent' as const,
resourceType: 'Host Agent',
node: host.hostname,
instance: host.platform || host.osName || '',
status,
hasOverride:
hasCustomThresholds ||
Boolean(override?.disabled) ||
Boolean(override?.disableConnectivity),
disableConnectivity: override?.disableConnectivity || false,
thresholds: override?.thresholds || {},
defaults: props.hostDefaults,
} satisfies Resource;
});
(props.overrides() ?? [])
.filter((override) => override.type === 'hostAgent' && !seen.has(override.id))
.forEach((override) => {
const name = override.name?.trim() || override.id;
hosts.push({
id: override.id,
name,
displayName: name,
rawName: name,
type: 'hostAgent',
resourceType: 'Host Agent',
status: 'unknown',
hasOverride: true,
disableConnectivity: override.disableConnectivity || false,
thresholds: override.thresholds || {},
defaults: props.hostDefaults,
} satisfies Resource);
});
if (search) {
return hosts.filter((host) => host.name.toLowerCase().includes(search));
}
return hosts;
}, []); }, []);
// Process Docker hosts with their overrides (primarily for connectivity toggles) // Process Docker hosts with their overrides (primarily for connectivity toggles)
@ -1318,6 +1412,13 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
overrides: countOverrides(dockerHostsWithOverrides()), overrides: countOverrides(dockerHostsWithOverrides()),
tab: 'docker' as const, tab: 'docker' as const,
}, },
{
key: 'hostAgents' as const,
label: 'Host Agents',
total: props.hosts?.length ?? 0,
overrides: countOverrides(hostAgentsWithOverrides()),
tab: 'docker' as const,
},
{ {
key: 'storage' as const, key: 'storage' as const,
label: 'Storage', label: 'Storage',
@ -1396,6 +1497,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
const allDockerContainers = dockerContainersFlat(); const allDockerContainers = dockerContainersFlat();
const allResources = [ const allResources = [
...nodesWithOverrides(), ...nodesWithOverrides(),
...hostAgentsWithOverrides(),
...dockerHostsWithOverrides(), ...dockerHostsWithOverrides(),
...allGuests, ...allGuests,
...allDockerContainers, ...allDockerContainers,
@ -1753,8 +1855,9 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
const nodes = nodesWithOverrides(); const nodes = nodesWithOverrides();
const pbsServers = pbsServersWithOverrides(); const pbsServers = pbsServersWithOverrides();
const guests = guestsFlat(); const guests = guestsFlat();
const hostAgents = hostAgentsWithOverrides();
const dockerHosts = dockerHostsWithOverrides(); const dockerHosts = dockerHostsWithOverrides();
const resource = [...nodes, ...pbsServers, ...guests, ...dockerHosts].find( const resource = [...nodes, ...pbsServers, ...guests, ...hostAgents, ...dockerHosts].find(
(r) => r.id === resourceId, (r) => r.id === resourceId,
); );
if ( if (
@ -1762,6 +1865,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
(resource.type !== 'node' && (resource.type !== 'node' &&
resource.type !== 'pbs' && resource.type !== 'pbs' &&
resource.type !== 'guest' && resource.type !== 'guest' &&
resource.type !== 'hostAgent' &&
resource.type !== 'dockerHost') resource.type !== 'dockerHost')
) )
return; return;
@ -2507,6 +2611,42 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
</Show> </Show>
<Show when={activeTab() === 'docker'}> <Show when={activeTab() === 'docker'}>
<Show when={hasSection('hostAgents')}>
<div ref={registerSection('hostAgents')} class="scroll-mt-24">
<ResourceTable
title="Host Agents"
resources={hostAgentsWithOverrides()}
columns={['CPU %', 'Memory %', 'Disk %']}
activeAlerts={props.activeAlerts}
emptyMessage="No host agents match the current filters."
onEdit={startEditing}
onSaveEdit={saveEdit}
onCancelEdit={cancelEdit}
onRemoveOverride={removeOverride}
onToggleDisabled={toggleDisabled}
onToggleNodeConnectivity={toggleNodeConnectivity}
showOfflineAlertsColumn={true}
editingId={editingId}
editingThresholds={editingThresholds}
setEditingThresholds={setEditingThresholds}
formatMetricValue={formatMetricValue}
hasActiveAlert={hasActiveAlert}
globalDefaults={props.hostDefaults}
setGlobalDefaults={props.setHostDefaults}
setHasUnsavedChanges={props.setHasUnsavedChanges}
globalDisableFlag={props.disableAllHosts}
onToggleGlobalDisable={() =>
props.setDisableAllHosts(!props.disableAllHosts())
}
globalDisableOfflineFlag={props.disableAllHostsOffline}
onToggleGlobalDisableOffline={() =>
props.setDisableAllHostsOffline(!props.disableAllHostsOffline())
}
factoryDefaults={props.factoryHostDefaults}
onResetDefaults={props.resetHostDefaults}
/>
</div>
</Show>
<div class="mb-6 rounded-lg border border-gray-200 bg-white p-5 shadow-sm dark:border-gray-700 dark:bg-gray-900"> <div class="mb-6 rounded-lg border border-gray-200 bg-white p-5 shadow-sm dark:border-gray-700 dark:bg-gray-900">
<div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between"> <div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
<div> <div>

View file

@ -57,6 +57,7 @@ const baseProps = () => ({
setRawOverridesConfig: vi.fn(), setRawOverridesConfig: vi.fn(),
allGuests: () => [], allGuests: () => [],
nodes: [], nodes: [],
hosts: [],
storage: [], storage: [],
dockerHosts: [], dockerHosts: [],
pbsInstances: [], pbsInstances: [],
@ -71,17 +72,21 @@ const baseProps = () => ({
setGuestPoweredOffSeverity: vi.fn(), setGuestPoweredOffSeverity: vi.fn(),
nodeDefaults: {}, nodeDefaults: {},
setNodeDefaults: vi.fn(), setNodeDefaults: vi.fn(),
hostDefaults: { cpu: 80, memory: 85, disk: 90 },
setHostDefaults: vi.fn(),
dockerDefaults: DEFAULT_DOCKER_DEFAULTS, dockerDefaults: DEFAULT_DOCKER_DEFAULTS,
setDockerDefaults: vi.fn(), setDockerDefaults: vi.fn(),
storageDefault: () => 85, storageDefault: () => 85,
setStorageDefault: vi.fn(), setStorageDefault: vi.fn(),
resetGuestDefaults: vi.fn(), resetGuestDefaults: vi.fn(),
resetNodeDefaults: vi.fn(), resetNodeDefaults: vi.fn(),
resetHostDefaults: vi.fn(),
resetDockerDefaults: vi.fn(), resetDockerDefaults: vi.fn(),
resetDockerIgnoredPrefixes: undefined as (() => void) | undefined, resetDockerIgnoredPrefixes: undefined as (() => void) | undefined,
resetStorageDefault: vi.fn(), resetStorageDefault: vi.fn(),
factoryGuestDefaults: {}, factoryGuestDefaults: {},
factoryNodeDefaults: {}, factoryNodeDefaults: {},
factoryHostDefaults: { cpu: 80, memory: 85, disk: 90 },
factoryDockerDefaults: {}, factoryDockerDefaults: {},
factoryStorageDefault: 85, factoryStorageDefault: 85,
backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }), backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }),
@ -113,6 +118,8 @@ const baseProps = () => ({
setDisableAllNodes: vi.fn(), setDisableAllNodes: vi.fn(),
disableAllGuests: () => false, disableAllGuests: () => false,
setDisableAllGuests: vi.fn(), setDisableAllGuests: vi.fn(),
disableAllHosts: () => false,
setDisableAllHosts: vi.fn(),
disableAllStorage: () => false, disableAllStorage: () => false,
setDisableAllStorage: vi.fn(), setDisableAllStorage: vi.fn(),
disableAllPBS: () => false, disableAllPBS: () => false,
@ -127,6 +134,8 @@ const baseProps = () => ({
setDisableAllNodesOffline: vi.fn(), setDisableAllNodesOffline: vi.fn(),
disableAllGuestsOffline: () => false, disableAllGuestsOffline: () => false,
setDisableAllGuestsOffline: vi.fn(), setDisableAllGuestsOffline: vi.fn(),
disableAllHostsOffline: () => false,
setDisableAllHostsOffline: vi.fn(),
disableAllPBSOffline: () => false, disableAllPBSOffline: () => false,
setDisableAllPBSOffline: vi.fn(), setDisableAllPBSOffline: vi.fn(),
disableAllPMGOffline: () => false, disableAllPMGOffline: () => false,

View file

@ -755,7 +755,7 @@ sudo systemctl daemon-reload`;
}; };
return ( return (
<tr class={`${rowClass} ${highlightClass}`}> <tr data-host-id={host.id} class={`${rowClass} ${highlightClass}`}>
<td class="py-3 px-4"> <td class="py-3 px-4">
<div class="font-medium text-gray-900 dark:text-gray-100"> <div class="font-medium text-gray-900 dark:text-gray-100">
{host.displayName || host.hostname || host.id} {host.displayName || host.hostname || host.id}

View file

@ -0,0 +1,206 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { render, fireEvent, screen, waitFor, cleanup } from '@solidjs/testing-library';
import { createStore } from 'solid-js/store';
import { HostAgents } from '../HostAgents';
import type { Host } from '@/types/api';
let mockWsStore: {
state: { hosts: Host[]; connectionHealth?: Record<string, boolean> };
connected: () => boolean;
reconnecting: () => boolean;
activeAlerts: unknown[];
};
const lookupMock = vi.fn();
const createTokenMock = vi.fn();
vi.mock('@/App', () => ({
useWebSocket: () => mockWsStore,
}));
vi.mock('@/api/monitoring', () => ({
MonitoringAPI: {
lookupHost: (...args: unknown[]) => lookupMock(...args),
},
}));
vi.mock('@/api/security', () => ({
SecurityAPI: {
createToken: (...args: unknown[]) => createTokenMock(...args),
},
}));
const createHost = (overrides?: Partial<Host>): Host => ({
id: 'host-1',
hostname: 'host-1.local',
displayName: 'Host One',
platform: 'linux',
osName: 'Ubuntu',
osVersion: '22.04',
kernelVersion: '6.5.0',
architecture: 'x86_64',
cpuCount: 4,
cpuUsage: 12.5,
memory: {
total: 8 * 1024 * 1024 * 1024,
used: 4 * 1024 * 1024 * 1024,
free: 4 * 1024 * 1024 * 1024,
usage: 50,
balloon: 0,
swapUsed: 0,
swapTotal: 0,
},
loadAverage: [],
disks: [],
networkInterfaces: [],
sensors: {
temperatureCelsius: {},
fanRpm: {},
additional: {},
},
status: 'online',
uptimeSeconds: 12_345,
intervalSeconds: 30,
lastSeen: Date.now(),
agentVersion: '1.2.3',
tags: ['prod'],
...overrides,
});
const stubFetchSuccess = vi.fn();
beforeEach(() => {
lookupMock.mockReset();
createTokenMock.mockReset();
stubFetchSuccess.mockImplementation(
async () =>
new Response(JSON.stringify({ requiresAuth: true, apiTokenConfigured: false }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
);
vi.stubGlobal('fetch', stubFetchSuccess);
});
afterEach(() => {
cleanup();
vi.unstubAllGlobals();
});
describe('HostAgents lookup flow', () => {
const setupComponent = (hosts: Host[]) => {
const [state] = createStore({
hosts,
connectionHealth: {},
});
mockWsStore = {
state,
connected: () => true,
reconnecting: () => false,
activeAlerts: [],
};
return render(() => <HostAgents />);
};
it('highlights a host after a successful lookup and clears highlight after timeout', async () => {
const host = createHost();
setupComponent([host]);
createTokenMock.mockResolvedValue({
token: 'token-123',
record: {
id: 'token-record',
name: 'Test Token',
prefix: 'abcdef',
suffix: '1234',
createdAt: new Date().toISOString(),
},
});
const generateButton = screen.getByRole('button', { name: 'Generate token' });
fireEvent.click(generateButton);
await waitFor(() => expect(createTokenMock).toHaveBeenCalled(), { interval: 0 });
await waitFor(() => expect(screen.getByRole('button', { name: 'Check status' })).toBeEnabled(), {
interval: 0,
});
lookupMock.mockResolvedValue({
success: true,
host: {
id: host.id,
hostname: host.hostname,
displayName: host.displayName,
status: host.status,
connected: true,
lastSeen: host.lastSeen,
agentVersion: host.agentVersion,
},
});
const input = screen.getByPlaceholderText('Hostname or host ID') as HTMLInputElement;
fireEvent.input(input, { target: { value: host.id } });
const button = screen.getByRole('button', { name: 'Check status' });
fireEvent.click(button);
await waitFor(() => expect(lookupMock).toHaveBeenCalled(), { interval: 0 });
const [lookupArgs] = lookupMock.mock.calls.at(-1) ?? [];
expect(lookupArgs).toEqual({ id: host.id, hostname: host.id });
await waitFor(
() =>
expect(
screen.getByText('Connected', { selector: 'span' }),
).toBeInTheDocument(),
{ interval: 0 },
);
const statusBadges = screen.getAllByText('online', { selector: 'span' });
expect(statusBadges.length).toBeGreaterThan(0);
expect(screen.getByText('Agent version 1.2.3')).toBeInTheDocument();
});
it('shows an error when lookup returns no host and does not highlight rows', async () => {
const host = createHost();
const { container } = setupComponent([host]);
createTokenMock.mockResolvedValue({
token: 'token-456',
record: {
id: 'token-record-2',
name: 'Test Token 2',
prefix: 'ghijkl',
suffix: '5678',
createdAt: new Date().toISOString(),
},
});
const generateButton = screen.getByRole('button', { name: 'Generate token' });
fireEvent.click(generateButton);
await waitFor(() => expect(createTokenMock).toHaveBeenCalled(), { interval: 0 });
await waitFor(() => expect(screen.getByRole('button', { name: 'Check status' })).toBeEnabled(), {
interval: 0,
});
lookupMock.mockResolvedValue(null);
const query = 'missing-host';
const input = screen.getByPlaceholderText('Hostname or host ID') as HTMLInputElement;
fireEvent.input(input, { target: { value: query } });
const button = screen.getByRole('button', { name: 'Check status' });
fireEvent.click(button);
await waitFor(
() =>
expect(
screen.getByText(`No host has reported with "${query}" yet. Try again in a few seconds.`),
).toBeInTheDocument(),
{ interval: 0 },
);
const row = container.querySelector(`tr[data-host-id="${host.id}"]`) as HTMLTableRowElement;
expect(row.classList.contains('ring-2')).toBe(false);
});
});

View file

@ -18,7 +18,7 @@ import { AlertsAPI } from '@/api/alerts';
import { NotificationsAPI, Webhook } from '@/api/notifications'; import { NotificationsAPI, Webhook } from '@/api/notifications';
import type { EmailConfig, AppriseConfig } from '@/api/notifications'; import type { EmailConfig, AppriseConfig } from '@/api/notifications';
import type { HysteresisThreshold } from '@/types/alerts'; import type { HysteresisThreshold } from '@/types/alerts';
import type { Alert, State, VM, Container, DockerHost, DockerContainer } from '@/types/api'; import type { Alert, State, VM, Container, DockerHost, DockerContainer, Host } from '@/types/api';
import { useNavigate, useLocation } from '@solidjs/router'; import { useNavigate, useLocation } from '@solidjs/router';
import { useAlertsActivation } from '@/stores/alertsActivation'; import { useAlertsActivation } from '@/stores/alertsActivation';
import LayoutDashboard from 'lucide-solid/icons/layout-dashboard'; import LayoutDashboard from 'lucide-solid/icons/layout-dashboard';
@ -806,6 +806,16 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
}); });
} }
if (config.hostDefaults) {
setHostDefaults({
cpu: getTriggerValue(config.hostDefaults.cpu) ?? 80,
memory: getTriggerValue(config.hostDefaults.memory) ?? 85,
disk: getTriggerValue(config.hostDefaults.disk) ?? 90,
});
} else {
setHostDefaults({ ...FACTORY_HOST_DEFAULTS });
}
if (config.dockerDefaults) { if (config.dockerDefaults) {
setDockerDefaults({ setDockerDefaults({
cpu: getTriggerValue(config.dockerDefaults.cpu) ?? 80, cpu: getTriggerValue(config.dockerDefaults.cpu) ?? 80,
@ -913,6 +923,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
// Load global disable flags // Load global disable flags
setDisableAllNodes(config.disableAllNodes ?? false); setDisableAllNodes(config.disableAllNodes ?? false);
setDisableAllGuests(config.disableAllGuests ?? false); setDisableAllGuests(config.disableAllGuests ?? false);
setDisableAllHosts(config.disableAllHosts ?? false);
setDisableAllStorage(config.disableAllStorage ?? false); setDisableAllStorage(config.disableAllStorage ?? false);
setDisableAllPBS(config.disableAllPBS ?? false); setDisableAllPBS(config.disableAllPBS ?? false);
setDisableAllPMG(config.disableAllPMG ?? false); setDisableAllPMG(config.disableAllPMG ?? false);
@ -922,6 +933,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
// Load global disable offline alerts flags // Load global disable offline alerts flags
setDisableAllNodesOffline(config.disableAllNodesOffline ?? false); setDisableAllNodesOffline(config.disableAllNodesOffline ?? false);
setDisableAllGuestsOffline(config.disableAllGuestsOffline ?? false); setDisableAllGuestsOffline(config.disableAllGuestsOffline ?? false);
setDisableAllHostsOffline(config.disableAllHostsOffline ?? false);
setDisableAllPBSOffline(config.disableAllPBSOffline ?? false); setDisableAllPBSOffline(config.disableAllPBSOffline ?? false);
setDisableAllPMGOffline(config.disableAllPMGOffline ?? false); setDisableAllPMGOffline(config.disableAllPMGOffline ?? false);
setDisableAllDockerHostsOffline(config.disableAllDockerHostsOffline ?? false); setDisableAllDockerHostsOffline(config.disableAllDockerHostsOffline ?? false);
@ -1151,6 +1163,12 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
temperature: 80, temperature: 80,
}; };
const FACTORY_HOST_DEFAULTS = {
cpu: 80,
memory: 85,
disk: 90,
};
const FACTORY_DOCKER_DEFAULTS = { const FACTORY_DOCKER_DEFAULTS = {
cpu: 80, cpu: 80,
memory: 85, memory: 85,
@ -1178,6 +1196,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
const [guestPoweredOffSeverity, setGuestPoweredOffSeverity] = createSignal<'warning' | 'critical'>('warning'); const [guestPoweredOffSeverity, setGuestPoweredOffSeverity] = createSignal<'warning' | 'critical'>('warning');
const [nodeDefaults, setNodeDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_NODE_DEFAULTS }); const [nodeDefaults, setNodeDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_NODE_DEFAULTS });
const [hostDefaults, setHostDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_HOST_DEFAULTS });
const [dockerDefaults, setDockerDefaults] = createSignal({ ...FACTORY_DOCKER_DEFAULTS }); const [dockerDefaults, setDockerDefaults] = createSignal({ ...FACTORY_DOCKER_DEFAULTS });
const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal<string[]>([]); const [dockerIgnoredPrefixes, setDockerIgnoredPrefixes] = createSignal<string[]>([]);
@ -1198,6 +1217,11 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setHasUnsavedChanges(true); setHasUnsavedChanges(true);
}; };
const resetHostDefaults = () => {
setHostDefaults({ ...FACTORY_HOST_DEFAULTS });
setHasUnsavedChanges(true);
};
const resetDockerDefaults = () => { const resetDockerDefaults = () => {
setDockerDefaults({ ...FACTORY_DOCKER_DEFAULTS }); setDockerDefaults({ ...FACTORY_DOCKER_DEFAULTS });
setHasUnsavedChanges(true); setHasUnsavedChanges(true);
@ -1255,6 +1279,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
// Global disable flags per resource type // Global disable flags per resource type
const [disableAllNodes, setDisableAllNodes] = createSignal(false); const [disableAllNodes, setDisableAllNodes] = createSignal(false);
const [disableAllGuests, setDisableAllGuests] = createSignal(false); const [disableAllGuests, setDisableAllGuests] = createSignal(false);
const [disableAllHosts, setDisableAllHosts] = createSignal(false);
const [disableAllStorage, setDisableAllStorage] = createSignal(false); const [disableAllStorage, setDisableAllStorage] = createSignal(false);
const [disableAllPBS, setDisableAllPBS] = createSignal(false); const [disableAllPBS, setDisableAllPBS] = createSignal(false);
const [disableAllPMG, setDisableAllPMG] = createSignal(false); const [disableAllPMG, setDisableAllPMG] = createSignal(false);
@ -1264,6 +1289,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
// Global disable offline alerts flags // Global disable offline alerts flags
const [disableAllNodesOffline, setDisableAllNodesOffline] = createSignal(false); const [disableAllNodesOffline, setDisableAllNodesOffline] = createSignal(false);
const [disableAllGuestsOffline, setDisableAllGuestsOffline] = createSignal(false); const [disableAllGuestsOffline, setDisableAllGuestsOffline] = createSignal(false);
const [disableAllHostsOffline, setDisableAllHostsOffline] = createSignal(false);
const [disableAllPBSOffline, setDisableAllPBSOffline] = createSignal(false); const [disableAllPBSOffline, setDisableAllPBSOffline] = createSignal(false);
const [disableAllPMGOffline, setDisableAllPMGOffline] = createSignal(false); const [disableAllPMGOffline, setDisableAllPMGOffline] = createSignal(false);
const [disableAllDockerHostsOffline, setDisableAllDockerHostsOffline] = createSignal(false); const [disableAllDockerHostsOffline, setDisableAllDockerHostsOffline] = createSignal(false);
@ -1404,6 +1430,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
// Global disable flags per resource type // Global disable flags per resource type
disableAllNodes: disableAllNodes(), disableAllNodes: disableAllNodes(),
disableAllGuests: disableAllGuests(), disableAllGuests: disableAllGuests(),
disableAllHosts: disableAllHosts(),
disableAllStorage: disableAllStorage(), disableAllStorage: disableAllStorage(),
disableAllPBS: disableAllPBS(), disableAllPBS: disableAllPBS(),
disableAllPMG: disableAllPMG(), disableAllPMG: disableAllPMG(),
@ -1413,6 +1440,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
disableAllNodesOffline: disableAllNodesOffline(), disableAllNodesOffline: disableAllNodesOffline(),
disableAllGuestsOffline: disableAllGuestsOffline(), disableAllGuestsOffline: disableAllGuestsOffline(),
disableAllPBSOffline: disableAllPBSOffline(), disableAllPBSOffline: disableAllPBSOffline(),
disableAllHostsOffline: disableAllHostsOffline(),
disableAllPMGOffline: disableAllPMGOffline(), disableAllPMGOffline: disableAllPMGOffline(),
disableAllDockerHostsOffline: disableAllDockerHostsOffline(), disableAllDockerHostsOffline: disableAllDockerHostsOffline(),
guestDefaults: { guestDefaults: {
@ -1432,6 +1460,11 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
disk: createHysteresisThreshold(nodeDefaults().disk), disk: createHysteresisThreshold(nodeDefaults().disk),
temperature: createHysteresisThreshold(nodeDefaults().temperature), temperature: createHysteresisThreshold(nodeDefaults().temperature),
}, },
hostDefaults: {
cpu: createHysteresisThreshold(hostDefaults().cpu),
memory: createHysteresisThreshold(hostDefaults().memory),
disk: createHysteresisThreshold(hostDefaults().disk),
},
dockerDefaults: { dockerDefaults: {
cpu: createHysteresisThreshold(dockerDefaults().cpu), cpu: createHysteresisThreshold(dockerDefaults().cpu),
memory: createHysteresisThreshold(dockerDefaults().memory), memory: createHysteresisThreshold(dockerDefaults().memory),
@ -1677,6 +1710,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setRawOverridesConfig={setRawOverridesConfig} setRawOverridesConfig={setRawOverridesConfig}
allGuests={allGuests} allGuests={allGuests}
state={state} state={state}
hosts={state.hosts || []}
guestDefaults={guestDefaults} guestDefaults={guestDefaults}
guestDisableConnectivity={guestDisableConnectivity} guestDisableConnectivity={guestDisableConnectivity}
setGuestDefaults={setGuestDefaults} setGuestDefaults={setGuestDefaults}
@ -1685,6 +1719,8 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setGuestPoweredOffSeverity={setGuestPoweredOffSeverity} setGuestPoweredOffSeverity={setGuestPoweredOffSeverity}
nodeDefaults={nodeDefaults} nodeDefaults={nodeDefaults}
setNodeDefaults={setNodeDefaults} setNodeDefaults={setNodeDefaults}
hostDefaults={hostDefaults}
setHostDefaults={setHostDefaults}
dockerDefaults={dockerDefaults} dockerDefaults={dockerDefaults}
setDockerDefaults={setDockerDefaults} setDockerDefaults={setDockerDefaults}
dockerIgnoredPrefixes={dockerIgnoredPrefixes} dockerIgnoredPrefixes={dockerIgnoredPrefixes}
@ -1693,6 +1729,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setStorageDefault={setStorageDefault} setStorageDefault={setStorageDefault}
resetGuestDefaults={resetGuestDefaults} resetGuestDefaults={resetGuestDefaults}
resetNodeDefaults={resetNodeDefaults} resetNodeDefaults={resetNodeDefaults}
resetHostDefaults={resetHostDefaults}
resetDockerDefaults={resetDockerDefaults} resetDockerDefaults={resetDockerDefaults}
resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes} resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes}
resetStorageDefault={resetStorageDefault} resetStorageDefault={resetStorageDefault}
@ -1700,6 +1737,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
resetBackupDefaults={resetBackupDefaults} resetBackupDefaults={resetBackupDefaults}
factoryGuestDefaults={FACTORY_GUEST_DEFAULTS} factoryGuestDefaults={FACTORY_GUEST_DEFAULTS}
factoryNodeDefaults={FACTORY_NODE_DEFAULTS} factoryNodeDefaults={FACTORY_NODE_DEFAULTS}
factoryHostDefaults={FACTORY_HOST_DEFAULTS}
factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS} factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS}
factoryStorageDefault={FACTORY_STORAGE_DEFAULT} factoryStorageDefault={FACTORY_STORAGE_DEFAULT}
snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS} snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS}
@ -1721,6 +1759,8 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setDisableAllNodes={setDisableAllNodes} setDisableAllNodes={setDisableAllNodes}
disableAllGuests={disableAllGuests} disableAllGuests={disableAllGuests}
setDisableAllGuests={setDisableAllGuests} setDisableAllGuests={setDisableAllGuests}
disableAllHosts={disableAllHosts}
setDisableAllHosts={setDisableAllHosts}
disableAllStorage={disableAllStorage} disableAllStorage={disableAllStorage}
setDisableAllStorage={setDisableAllStorage} setDisableAllStorage={setDisableAllStorage}
disableAllPBS={disableAllPBS} disableAllPBS={disableAllPBS}
@ -1735,6 +1775,8 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
setDisableAllNodesOffline={setDisableAllNodesOffline} setDisableAllNodesOffline={setDisableAllNodesOffline}
disableAllGuestsOffline={disableAllGuestsOffline} disableAllGuestsOffline={disableAllGuestsOffline}
setDisableAllGuestsOffline={setDisableAllGuestsOffline} setDisableAllGuestsOffline={setDisableAllGuestsOffline}
disableAllHostsOffline={disableAllHostsOffline}
setDisableAllHostsOffline={setDisableAllHostsOffline}
disableAllPBSOffline={disableAllPBSOffline} disableAllPBSOffline={disableAllPBSOffline}
setDisableAllPBSOffline={setDisableAllPBSOffline} setDisableAllPBSOffline={setDisableAllPBSOffline}
disableAllPMGOffline={disableAllPMGOffline} disableAllPMGOffline={disableAllPMGOffline}
@ -2192,8 +2234,10 @@ function OverviewTab(props: {
interface ThresholdsTabProps { interface ThresholdsTabProps {
allGuests: () => (VM | Container)[]; allGuests: () => (VM | Container)[];
state: State; state: State;
hosts: Host[];
guestDefaults: () => Record<string, number | undefined>; guestDefaults: () => Record<string, number | undefined>;
nodeDefaults: () => Record<string, number | undefined>; nodeDefaults: () => Record<string, number | undefined>;
hostDefaults: () => Record<string, number | undefined>;
dockerDefaults: () => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }; dockerDefaults: () => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number };
dockerIgnoredPrefixes: () => string[]; dockerIgnoredPrefixes: () => string[];
storageDefault: () => number; storageDefault: () => number;
@ -2221,6 +2265,11 @@ interface ThresholdsTabProps {
| Record<string, number | undefined> | Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>), | ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void; ) => void;
setHostDefaults: (
value:
| Record<string, number | undefined>
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
) => void;
setDockerDefaults: ( setDockerDefaults: (
value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }), value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }),
) => void; ) => void;
@ -2258,6 +2307,8 @@ interface ThresholdsTabProps {
setDisableAllNodes: (value: boolean) => void; setDisableAllNodes: (value: boolean) => void;
disableAllGuests: () => boolean; disableAllGuests: () => boolean;
setDisableAllGuests: (value: boolean) => void; setDisableAllGuests: (value: boolean) => void;
disableAllHosts: () => boolean;
setDisableAllHosts: (value: boolean) => void;
disableAllStorage: () => boolean; disableAllStorage: () => boolean;
setDisableAllStorage: (value: boolean) => void; setDisableAllStorage: (value: boolean) => void;
disableAllPBS: () => boolean; disableAllPBS: () => boolean;
@ -2273,6 +2324,8 @@ interface ThresholdsTabProps {
setDisableAllNodesOffline: (value: boolean) => void; setDisableAllNodesOffline: (value: boolean) => void;
disableAllGuestsOffline: () => boolean; disableAllGuestsOffline: () => boolean;
setDisableAllGuestsOffline: (value: boolean) => void; setDisableAllGuestsOffline: (value: boolean) => void;
disableAllHostsOffline: () => boolean;
setDisableAllHostsOffline: (value: boolean) => void;
disableAllPBSOffline: () => boolean; disableAllPBSOffline: () => boolean;
setDisableAllPBSOffline: (value: boolean) => void; setDisableAllPBSOffline: (value: boolean) => void;
disableAllPMGOffline: () => boolean; disableAllPMGOffline: () => boolean;
@ -2282,11 +2335,13 @@ interface ThresholdsTabProps {
// Reset functions and factory defaults // Reset functions and factory defaults
resetGuestDefaults?: () => void; resetGuestDefaults?: () => void;
resetNodeDefaults?: () => void; resetNodeDefaults?: () => void;
resetHostDefaults?: () => void;
resetDockerDefaults?: () => void; resetDockerDefaults?: () => void;
resetDockerIgnoredPrefixes?: () => void; resetDockerIgnoredPrefixes?: () => void;
resetStorageDefault?: () => void; resetStorageDefault?: () => void;
factoryGuestDefaults?: Record<string, number | undefined>; factoryGuestDefaults?: Record<string, number | undefined>;
factoryNodeDefaults?: Record<string, number | undefined>; factoryNodeDefaults?: Record<string, number | undefined>;
factoryHostDefaults?: Record<string, number | undefined>;
factoryDockerDefaults?: Record<string, number | undefined>; factoryDockerDefaults?: Record<string, number | undefined>;
factoryStorageDefault?: number; factoryStorageDefault?: number;
} }
@ -2300,6 +2355,7 @@ function ThresholdsTab(props: ThresholdsTabProps) {
setRawOverridesConfig={props.setRawOverridesConfig} setRawOverridesConfig={props.setRawOverridesConfig}
allGuests={props.allGuests} allGuests={props.allGuests}
nodes={props.state.nodes || []} nodes={props.state.nodes || []}
hosts={props.hosts}
storage={props.state.storage || []} storage={props.state.storage || []}
dockerHosts={props.state.dockerHosts || []} dockerHosts={props.state.dockerHosts || []}
pbsInstances={props.state.pbs || []} pbsInstances={props.state.pbs || []}
@ -2317,7 +2373,9 @@ function ThresholdsTab(props: ThresholdsTabProps) {
guestPoweredOffSeverity={props.guestPoweredOffSeverity} guestPoweredOffSeverity={props.guestPoweredOffSeverity}
setGuestPoweredOffSeverity={props.setGuestPoweredOffSeverity} setGuestPoweredOffSeverity={props.setGuestPoweredOffSeverity}
nodeDefaults={props.nodeDefaults()} nodeDefaults={props.nodeDefaults()}
hostDefaults={props.hostDefaults()}
setNodeDefaults={props.setNodeDefaults} setNodeDefaults={props.setNodeDefaults}
setHostDefaults={props.setHostDefaults}
dockerDefaults={props.dockerDefaults()} dockerDefaults={props.dockerDefaults()}
setDockerDefaults={props.setDockerDefaults} setDockerDefaults={props.setDockerDefaults}
dockerIgnoredPrefixes={props.dockerIgnoredPrefixes} dockerIgnoredPrefixes={props.dockerIgnoredPrefixes}
@ -2342,6 +2400,8 @@ function ThresholdsTab(props: ThresholdsTabProps) {
setDisableAllNodes={props.setDisableAllNodes} setDisableAllNodes={props.setDisableAllNodes}
disableAllGuests={props.disableAllGuests} disableAllGuests={props.disableAllGuests}
setDisableAllGuests={props.setDisableAllGuests} setDisableAllGuests={props.setDisableAllGuests}
disableAllHosts={props.disableAllHosts}
setDisableAllHosts={props.setDisableAllHosts}
disableAllStorage={props.disableAllStorage} disableAllStorage={props.disableAllStorage}
setDisableAllStorage={props.setDisableAllStorage} setDisableAllStorage={props.setDisableAllStorage}
disableAllPBS={props.disableAllPBS} disableAllPBS={props.disableAllPBS}
@ -2356,6 +2416,8 @@ function ThresholdsTab(props: ThresholdsTabProps) {
setDisableAllNodesOffline={props.setDisableAllNodesOffline} setDisableAllNodesOffline={props.setDisableAllNodesOffline}
disableAllGuestsOffline={props.disableAllGuestsOffline} disableAllGuestsOffline={props.disableAllGuestsOffline}
setDisableAllGuestsOffline={props.setDisableAllGuestsOffline} setDisableAllGuestsOffline={props.setDisableAllGuestsOffline}
disableAllHostsOffline={props.disableAllHostsOffline}
setDisableAllHostsOffline={props.setDisableAllHostsOffline}
disableAllPBSOffline={props.disableAllPBSOffline} disableAllPBSOffline={props.disableAllPBSOffline}
setDisableAllPBSOffline={props.setDisableAllPBSOffline} setDisableAllPBSOffline={props.setDisableAllPBSOffline}
disableAllPMGOffline={props.disableAllPMGOffline} disableAllPMGOffline={props.disableAllPMGOffline}
@ -2364,11 +2426,13 @@ function ThresholdsTab(props: ThresholdsTabProps) {
setDisableAllDockerHostsOffline={props.setDisableAllDockerHostsOffline} setDisableAllDockerHostsOffline={props.setDisableAllDockerHostsOffline}
resetGuestDefaults={props.resetGuestDefaults} resetGuestDefaults={props.resetGuestDefaults}
resetNodeDefaults={props.resetNodeDefaults} resetNodeDefaults={props.resetNodeDefaults}
resetHostDefaults={props.resetHostDefaults}
resetDockerDefaults={props.resetDockerDefaults} resetDockerDefaults={props.resetDockerDefaults}
resetDockerIgnoredPrefixes={props.resetDockerIgnoredPrefixes} resetDockerIgnoredPrefixes={props.resetDockerIgnoredPrefixes}
resetStorageDefault={props.resetStorageDefault} resetStorageDefault={props.resetStorageDefault}
factoryGuestDefaults={props.factoryGuestDefaults} factoryGuestDefaults={props.factoryGuestDefaults}
factoryNodeDefaults={props.factoryNodeDefaults} factoryNodeDefaults={props.factoryNodeDefaults}
factoryHostDefaults={props.factoryHostDefaults}
factoryDockerDefaults={props.factoryDockerDefaults} factoryDockerDefaults={props.factoryDockerDefaults}
factoryStorageDefault={props.factoryStorageDefault} factoryStorageDefault={props.factoryStorageDefault}
/> />

View file

@ -108,6 +108,7 @@ export interface AlertConfig {
activationTime?: string; activationTime?: string;
guestDefaults: AlertThresholds; guestDefaults: AlertThresholds;
nodeDefaults: AlertThresholds; nodeDefaults: AlertThresholds;
hostDefaults?: AlertThresholds;
storageDefault: HysteresisThreshold; storageDefault: HysteresisThreshold;
dockerDefaults?: DockerThresholdConfig; dockerDefaults?: DockerThresholdConfig;
dockerIgnoredContainerPrefixes?: string[]; dockerIgnoredContainerPrefixes?: string[];
@ -194,12 +195,14 @@ export interface AlertConfig {
disableAllStorage?: boolean; disableAllStorage?: boolean;
disableAllPBS?: boolean; disableAllPBS?: boolean;
disableAllPMG?: boolean; disableAllPMG?: boolean;
disableAllHosts?: boolean;
disableAllDockerHosts?: boolean; disableAllDockerHosts?: boolean;
disableAllDockerContainers?: boolean; disableAllDockerContainers?: boolean;
disableAllNodesOffline?: boolean; disableAllNodesOffline?: boolean;
disableAllGuestsOffline?: boolean; disableAllGuestsOffline?: boolean;
disableAllPBSOffline?: boolean; disableAllPBSOffline?: boolean;
disableAllPMGOffline?: boolean; disableAllPMGOffline?: boolean;
disableAllHostsOffline?: boolean;
disableAllDockerHostsOffline?: boolean; disableAllDockerHostsOffline?: boolean;
} }

View file

@ -325,6 +325,7 @@ type AlertConfig struct {
ActivationTime *time.Time `json:"activationTime,omitempty"` ActivationTime *time.Time `json:"activationTime,omitempty"`
GuestDefaults ThresholdConfig `json:"guestDefaults"` GuestDefaults ThresholdConfig `json:"guestDefaults"`
NodeDefaults ThresholdConfig `json:"nodeDefaults"` NodeDefaults ThresholdConfig `json:"nodeDefaults"`
HostDefaults ThresholdConfig `json:"hostDefaults"`
StorageDefault HysteresisThreshold `json:"storageDefault"` StorageDefault HysteresisThreshold `json:"storageDefault"`
DockerDefaults DockerThresholdConfig `json:"dockerDefaults"` DockerDefaults DockerThresholdConfig `json:"dockerDefaults"`
DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"` DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"`
@ -337,6 +338,7 @@ type AlertConfig struct {
// Global disable flags per resource type // Global disable flags per resource type
DisableAllNodes bool `json:"disableAllNodes"` // Disable all alerts for Proxmox nodes DisableAllNodes bool `json:"disableAllNodes"` // Disable all alerts for Proxmox nodes
DisableAllGuests bool `json:"disableAllGuests"` // Disable all alerts for VMs/containers DisableAllGuests bool `json:"disableAllGuests"` // Disable all alerts for VMs/containers
DisableAllHosts bool `json:"disableAllHosts"` // Disable all alerts for Pulse host agents
DisableAllStorage bool `json:"disableAllStorage"` // Disable all alerts for storage DisableAllStorage bool `json:"disableAllStorage"` // Disable all alerts for storage
DisableAllPBS bool `json:"disableAllPBS"` // Disable all alerts for PBS servers DisableAllPBS bool `json:"disableAllPBS"` // Disable all alerts for PBS servers
DisableAllPMG bool `json:"disableAllPMG"` // Disable all alerts for PMG instances DisableAllPMG bool `json:"disableAllPMG"` // Disable all alerts for PMG instances
@ -344,6 +346,7 @@ type AlertConfig struct {
DisableAllDockerContainers bool `json:"disableAllDockerContainers"` // Disable all alerts for Docker containers DisableAllDockerContainers bool `json:"disableAllDockerContainers"` // Disable all alerts for Docker containers
DisableAllNodesOffline bool `json:"disableAllNodesOffline"` // Disable node offline/connectivity alerts globally DisableAllNodesOffline bool `json:"disableAllNodesOffline"` // Disable node offline/connectivity alerts globally
DisableAllGuestsOffline bool `json:"disableAllGuestsOffline"` // Disable guest powered-off alerts globally DisableAllGuestsOffline bool `json:"disableAllGuestsOffline"` // Disable guest powered-off alerts globally
DisableAllHostsOffline bool `json:"disableAllHostsOffline"` // Disable host agent offline alerts globally
DisableAllPBSOffline bool `json:"disableAllPBSOffline"` // Disable PBS offline alerts globally DisableAllPBSOffline bool `json:"disableAllPBSOffline"` // Disable PBS offline alerts globally
DisableAllPMGOffline bool `json:"disableAllPMGOffline"` // Disable PMG offline alerts globally DisableAllPMGOffline bool `json:"disableAllPMGOffline"` // Disable PMG offline alerts globally
DisableAllDockerHostsOffline bool `json:"disableAllDockerHostsOffline"` // Disable Docker host offline alerts globally DisableAllDockerHostsOffline bool `json:"disableAllDockerHostsOffline"` // Disable Docker host offline alerts globally
@ -488,6 +491,11 @@ func NewManager() *Manager {
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85}, Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
Temperature: &HysteresisThreshold{Trigger: 80, Clear: 75}, // Warning at 80°C, clear at 75°C Temperature: &HysteresisThreshold{Trigger: 80, Clear: 75}, // Warning at 80°C, clear at 75°C
}, },
HostDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
},
DockerDefaults: DockerThresholdConfig{ DockerDefaults: DockerThresholdConfig{
CPU: HysteresisThreshold{Trigger: 80, Clear: 75}, CPU: HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: HysteresisThreshold{Trigger: 85, Clear: 80}, Memory: HysteresisThreshold{Trigger: 85, Clear: 80},
@ -886,6 +894,32 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
} }
} }
// Ensure host agent defaults exist
if config.HostDefaults.CPU == nil || config.HostDefaults.CPU.Trigger <= 0 {
config.HostDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 75}
} else if config.HostDefaults.CPU.Clear <= 0 {
config.HostDefaults.CPU.Clear = config.HostDefaults.CPU.Trigger - 5
if config.HostDefaults.CPU.Clear <= 0 {
config.HostDefaults.CPU.Clear = 75
}
}
if config.HostDefaults.Memory == nil || config.HostDefaults.Memory.Trigger <= 0 {
config.HostDefaults.Memory = &HysteresisThreshold{Trigger: 85, Clear: 80}
} else if config.HostDefaults.Memory.Clear <= 0 {
config.HostDefaults.Memory.Clear = config.HostDefaults.Memory.Trigger - 5
if config.HostDefaults.Memory.Clear <= 0 {
config.HostDefaults.Memory.Clear = 80
}
}
if config.HostDefaults.Disk == nil || config.HostDefaults.Disk.Trigger <= 0 {
config.HostDefaults.Disk = &HysteresisThreshold{Trigger: 90, Clear: 85}
} else if config.HostDefaults.Disk.Clear <= 0 {
config.HostDefaults.Disk.Clear = config.HostDefaults.Disk.Trigger - 5
if config.HostDefaults.Disk.Clear <= 0 {
config.HostDefaults.Disk.Clear = 85
}
}
// Normalize any metric-level delay overrides // Normalize any metric-level delay overrides
config.MetricTimeThresholds = normalizeMetricTimeThresholds(config.MetricTimeThresholds) config.MetricTimeThresholds = normalizeMetricTimeThresholds(config.MetricTimeThresholds)
@ -1879,6 +1913,373 @@ func (m *Manager) CheckNode(node models.Node) {
} }
} }
func hostResourceID(hostID string) string {
trimmed := strings.TrimSpace(hostID)
if trimmed == "" {
return "host:unknown"
}
return fmt.Sprintf("host:%s", trimmed)
}
func hostDisplayName(host models.Host) string {
if name := strings.TrimSpace(host.DisplayName); name != "" {
return name
}
if name := strings.TrimSpace(host.Hostname); name != "" {
return name
}
if host.ID != "" {
return host.ID
}
return "Host"
}
func hostInstanceName(host models.Host) string {
if platform := strings.TrimSpace(host.Platform); platform != "" {
return platform
}
if osName := strings.TrimSpace(host.OSName); osName != "" {
return osName
}
return "Host Agent"
}
func sanitizeHostComponent(value string) string {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {
return "unknown"
}
var builder strings.Builder
lastHyphen := false
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
builder.WriteRune(r)
lastHyphen = false
case r >= '0' && r <= '9':
builder.WriteRune(r)
lastHyphen = false
default:
if !lastHyphen {
builder.WriteRune('-')
lastHyphen = true
}
}
}
sanitized := strings.Trim(builder.String(), "-")
if sanitized == "" {
return "unknown"
}
return sanitized
}
func hostDiskResourceID(host models.Host, disk models.Disk) (string, string) {
label := strings.TrimSpace(disk.Mountpoint)
if label == "" {
label = strings.TrimSpace(disk.Device)
}
if label == "" {
label = "disk"
}
resourceID := fmt.Sprintf("%s/disk:%s", hostResourceID(host.ID), sanitizeHostComponent(label))
resourceName := fmt.Sprintf("%s (%s)", hostDisplayName(host), label)
return resourceID, resourceName
}
// CheckHost evaluates host agent telemetry for alerts.
func (m *Manager) CheckHost(host models.Host) {
if host.ID == "" {
return
}
// Fresh telemetry marks the host as online and clears offline tracking.
m.HandleHostOnline(host)
m.mu.RLock()
alertsEnabled := m.config.Enabled
disableAllHosts := m.config.DisableAllHosts
thresholds := m.config.HostDefaults
override, hasOverride := m.config.Overrides[host.ID]
m.mu.RUnlock()
if !alertsEnabled || disableAllHosts {
return
}
if hasOverride {
thresholds = m.applyThresholdOverride(thresholds, override)
if thresholds.Disabled {
m.clearHostMetricAlerts(host.ID)
m.clearHostDiskAlerts(host.ID)
return
}
}
resourceID := hostResourceID(host.ID)
resourceName := hostDisplayName(host)
nodeName := strings.TrimSpace(host.Hostname)
instanceName := hostInstanceName(host)
baseMetadata := map[string]interface{}{
"resourceType": "Host",
"hostId": host.ID,
"hostname": host.Hostname,
"displayName": host.DisplayName,
"platform": host.Platform,
"osName": host.OSName,
"osVersion": host.OSVersion,
"agentVersion": host.AgentVersion,
"architecture": host.Architecture,
}
if len(host.Tags) > 0 {
baseMetadata["tags"] = append([]string(nil), host.Tags...)
}
if thresholds.CPU != nil {
cpuMetadata := cloneMetadata(baseMetadata)
cpuMetadata["metric"] = "cpu"
cpuMetadata["cpuUsagePercent"] = host.CPUUsage
if host.CPUCount > 0 {
cpuMetadata["cpuCount"] = host.CPUCount
}
m.checkMetric(resourceID, resourceName, nodeName, instanceName, "Host", "cpu", host.CPUUsage, thresholds.CPU, &metricOptions{Metadata: cpuMetadata})
} else {
m.clearHostMetricAlerts(host.ID, "cpu")
}
if thresholds.Memory != nil {
memMetadata := cloneMetadata(baseMetadata)
memMetadata["metric"] = "memory"
memMetadata["memoryUsagePercent"] = host.Memory.Usage
if host.Memory.Total > 0 {
memMetadata["memoryTotalBytes"] = host.Memory.Total
memMetadata["memoryUsedBytes"] = host.Memory.Used
memMetadata["memoryFreeBytes"] = host.Memory.Free
}
m.checkMetric(resourceID, resourceName, nodeName, instanceName, "Host", "memory", host.Memory.Usage, thresholds.Memory, &metricOptions{Metadata: memMetadata})
} else {
m.clearHostMetricAlerts(host.ID, "memory")
}
seenDisks := make(map[string]struct{}, len(host.Disks))
if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 {
for _, disk := range host.Disks {
diskResourceID, diskName := hostDiskResourceID(host, disk)
seenDisks[diskResourceID] = struct{}{}
diskMetadata := cloneMetadata(baseMetadata)
diskMetadata["metric"] = "disk"
diskMetadata["mountpoint"] = disk.Mountpoint
diskMetadata["device"] = disk.Device
diskMetadata["diskType"] = disk.Type
diskMetadata["diskUsagePercent"] = disk.Usage
if disk.Total > 0 {
diskMetadata["diskTotalBytes"] = disk.Total
diskMetadata["diskUsedBytes"] = disk.Used
diskMetadata["diskFreeBytes"] = disk.Free
}
m.checkMetric(diskResourceID, diskName, nodeName, instanceName, "Host Disk", "disk", disk.Usage, thresholds.Disk, &metricOptions{Metadata: diskMetadata})
}
} else {
m.clearHostDiskAlerts(host.ID)
}
m.cleanupHostDiskAlerts(host, seenDisks)
}
// HandleHostOnline clears offline tracking and alerts for a host agent.
func (m *Manager) HandleHostOnline(host models.Host) {
if host.ID == "" {
return
}
alertID := fmt.Sprintf("host-offline-%s", host.ID)
resourceKey := hostResourceID(host.ID)
m.mu.Lock()
delete(m.offlineConfirmations, resourceKey)
_, exists := m.activeAlerts[alertID]
m.mu.Unlock()
if exists {
m.clearAlert(alertID)
}
}
// HandleHostRemoved clears alerts and tracking when a host agent is deleted.
func (m *Manager) HandleHostRemoved(host models.Host) {
if host.ID == "" {
return
}
m.HandleHostOnline(host)
m.clearHostMetricAlerts(host.ID)
m.clearHostDiskAlerts(host.ID)
}
// HandleHostOffline raises an alert when a host agent stops reporting.
func (m *Manager) HandleHostOffline(host models.Host) {
if host.ID == "" {
return
}
m.mu.RLock()
if !m.config.Enabled {
m.mu.RUnlock()
return
}
disableHostsOffline := m.config.DisableAllHostsOffline
m.mu.RUnlock()
alertID := fmt.Sprintf("host-offline-%s", host.ID)
resourceKey := hostResourceID(host.ID)
resourceName := hostDisplayName(host)
nodeName := strings.TrimSpace(host.Hostname)
instanceName := hostInstanceName(host)
if disableHostsOffline {
m.mu.Lock()
delete(m.offlineConfirmations, resourceKey)
m.mu.Unlock()
m.clearAlert(alertID)
return
}
var disableConnectivity bool
m.mu.RLock()
if override, exists := m.config.Overrides[host.ID]; exists {
disableConnectivity = override.DisableConnectivity || override.Disabled
}
m.mu.RUnlock()
if disableConnectivity {
m.clearAlert(alertID)
m.mu.Lock()
delete(m.offlineConfirmations, resourceKey)
m.mu.Unlock()
return
}
m.mu.Lock()
if alert, exists := m.activeAlerts[alertID]; exists && alert != nil {
alert.LastSeen = time.Now()
m.activeAlerts[alertID] = alert
m.mu.Unlock()
return
}
m.offlineConfirmations[resourceKey]++
const requiredConfirmations = 3
if confirmations := m.offlineConfirmations[resourceKey]; confirmations < requiredConfirmations {
m.mu.Unlock()
log.Debug().
Str("host", resourceName).
Str("hostID", host.ID).
Int("confirmations", confirmations).
Int("required", requiredConfirmations).
Msg("Host agent appears offline, awaiting confirmation")
return
}
alert := &Alert{
ID: alertID,
Type: "host-offline",
Level: AlertLevelCritical,
ResourceID: resourceKey,
ResourceName: resourceName,
Node: nodeName,
Instance: instanceName,
Message: fmt.Sprintf("Host '%s' is offline", resourceName),
Value: 0,
Threshold: 0,
StartTime: time.Now(),
LastSeen: time.Now(),
Metadata: map[string]interface{}{
"resourceType": "Host",
"hostId": host.ID,
"hostname": host.Hostname,
"displayName": host.DisplayName,
"platform": host.Platform,
"osName": host.OSName,
"osVersion": host.OSVersion,
},
}
m.preserveAlertState(alertID, alert)
m.activeAlerts[alertID] = alert
m.recentAlerts[alertID] = alert
m.historyManager.AddAlert(*alert)
m.dispatchAlert(alert, false)
m.mu.Unlock()
log.Error().
Str("host", resourceName).
Str("hostID", host.ID).
Str("hostname", host.Hostname).
Msg("CRITICAL: Host agent is offline")
}
func (m *Manager) clearHostMetricAlerts(hostID string, metrics ...string) {
if hostID == "" {
return
}
resourceID := hostResourceID(hostID)
if len(metrics) == 0 {
metrics = []string{"cpu", "memory"}
}
for _, metric := range metrics {
m.clearAlert(fmt.Sprintf("%s-%s", resourceID, metric))
}
}
func (m *Manager) clearHostDiskAlerts(hostID string) {
if hostID == "" {
return
}
prefix := fmt.Sprintf("%s/disk:", hostResourceID(hostID))
m.mu.Lock()
defer m.mu.Unlock()
for alertID, alert := range m.activeAlerts {
if alert == nil {
continue
}
if !strings.HasPrefix(alert.ResourceID, prefix) {
continue
}
m.clearAlertNoLock(alertID)
}
}
func (m *Manager) cleanupHostDiskAlerts(host models.Host, seen map[string]struct{}) {
if host.ID == "" {
return
}
prefix := fmt.Sprintf("%s/disk:", hostResourceID(host.ID))
m.mu.Lock()
defer m.mu.Unlock()
for alertID, alert := range m.activeAlerts {
if alert == nil {
continue
}
if !strings.HasPrefix(alert.ResourceID, prefix) {
continue
}
if _, exists := seen[alert.ResourceID]; exists {
continue
}
m.clearAlertNoLock(alertID)
}
}
// CheckPBS checks PBS instance metrics against thresholds // CheckPBS checks PBS instance metrics against thresholds
func (m *Manager) CheckPBS(pbs models.PBSInstance) { func (m *Manager) CheckPBS(pbs models.PBSInstance) {
m.mu.RLock() m.mu.RLock()
@ -6728,7 +7129,7 @@ func (m *Manager) SaveActiveAlerts() error {
alerts = append(alerts, alert) alerts = append(alerts, alert)
} }
data, err := json.MarshalIndent(alerts, "", " ") data, err := json.Marshal(alerts)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal active alerts: %w", err) return fmt.Errorf("failed to marshal active alerts: %w", err)
} }

View file

@ -213,6 +213,167 @@ func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
} }
} }
func TestCheckHostGeneratesMetricAlerts(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
host := models.Host{
ID: "host-1",
DisplayName: "Test Host",
Hostname: "host-1.example",
Platform: "linux",
OSName: "ubuntu",
CPUUsage: 95,
CPUCount: 8,
Memory: models.Memory{
Usage: 92,
Total: 16384,
Used: 15000,
Free: 1384,
},
Disks: []models.Disk{
{
Mountpoint: "/",
Usage: 93,
Total: 100,
Used: 93,
Free: 7,
},
},
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now(),
Tags: []string{"prod"},
}
m.CheckHost(host)
m.mu.RLock()
defer m.mu.RUnlock()
cpuAlertID := fmt.Sprintf("%s-cpu", hostResourceID(host.ID))
if _, exists := m.activeAlerts[cpuAlertID]; !exists {
t.Fatalf("expected CPU alert %q to be active", cpuAlertID)
}
memAlertID := fmt.Sprintf("%s-memory", hostResourceID(host.ID))
if _, exists := m.activeAlerts[memAlertID]; !exists {
t.Fatalf("expected memory alert %q to be active", memAlertID)
}
diskResourceID, _ := hostDiskResourceID(host, host.Disks[0])
diskAlertID := fmt.Sprintf("%s-disk", diskResourceID)
if _, exists := m.activeAlerts[diskAlertID]; !exists {
t.Fatalf("expected disk alert %q to be active", diskAlertID)
}
}
func TestHandleHostOfflineRequiresConfirmations(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
host := models.Host{ID: "host-2", DisplayName: "Second Host", Hostname: "host-two"}
alertID := fmt.Sprintf("host-offline-%s", host.ID)
resourceKey := hostResourceID(host.ID)
m.HandleHostOffline(host)
m.mu.RLock()
if _, exists := m.activeAlerts[alertID]; exists {
t.Fatalf("expected no alert after first offline detection")
}
if count := m.offlineConfirmations[resourceKey]; count != 1 {
t.Fatalf("expected confirmation count to be 1, got %d", count)
}
m.mu.RUnlock()
m.HandleHostOffline(host)
m.mu.RLock()
if _, exists := m.activeAlerts[alertID]; exists {
t.Fatalf("expected no alert after second offline detection")
}
if count := m.offlineConfirmations[resourceKey]; count != 2 {
t.Fatalf("expected confirmation count to be 2, got %d", count)
}
m.mu.RUnlock()
m.HandleHostOffline(host)
m.mu.RLock()
if _, exists := m.activeAlerts[alertID]; !exists {
t.Fatalf("expected alert %q after third offline detection", alertID)
}
m.mu.RUnlock()
m.HandleHostOnline(host)
m.mu.RLock()
if _, exists := m.activeAlerts[alertID]; exists {
t.Fatalf("expected offline alert %q to be cleared after host online", alertID)
}
if _, exists := m.offlineConfirmations[resourceKey]; exists {
t.Fatalf("expected offline confirmations to be cleared when host online")
}
m.mu.RUnlock()
}
func TestCheckHostDisabledOverrideClearsAlerts(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
host := models.Host{
ID: "host-3",
DisplayName: "Override Host",
Hostname: "override.example",
CPUUsage: 90,
Memory: models.Memory{
Usage: 91,
Total: 16000,
Used: 14560,
Free: 1440,
},
Disks: []models.Disk{
{Mountpoint: "/data", Usage: 92, Total: 200, Used: 184, Free: 16},
},
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now(),
}
m.CheckHost(host)
m.mu.RLock()
if len(m.activeAlerts) == 0 {
m.mu.RUnlock()
t.Fatalf("expected active alerts prior to disabling host overrides")
}
m.mu.RUnlock()
cfg := m.GetConfig()
cfg.Overrides = map[string]ThresholdConfig{
host.ID: {
Disabled: true,
},
}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
m.CheckHost(host)
m.mu.RLock()
defer m.mu.RUnlock()
if len(m.activeAlerts) != 0 {
t.Fatalf("expected all host alerts to be cleared after disabling override, got %d", len(m.activeAlerts))
}
}
func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) { func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
m := NewManager() m := NewManager()
m.ClearActiveAlerts() m.ClearActiveAlerts()

View file

@ -174,7 +174,7 @@ func (hm *HistoryManager) saveHistory() error {
copy(snapshot, hm.history) copy(snapshot, hm.history)
hm.mu.RUnlock() hm.mu.RUnlock()
data, err := json.MarshalIndent(snapshot, "", " ") data, err := json.Marshal(snapshot)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal history: %w", err) return fmt.Errorf("failed to marshal history: %w", err)

View file

@ -207,7 +207,7 @@ func (c *CSRFTokenStore) saveUnsafe() {
} }
// Marshal tokens // Marshal tokens
jsonData, err := json.MarshalIndent(data, "", " ") jsonData, err := json.Marshal(data)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed to marshal CSRF tokens") log.Error().Err(err).Msg("Failed to marshal CSRF tokens")
return return

View file

@ -0,0 +1,136 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"time"
"unsafe"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
)
func TestHandleLookupByIDSuccess(t *testing.T) {
t.Parallel()
hostID := "host-123"
tokenID := "token-abc"
lastSeen := time.Now().UTC()
handler := newHostAgentHandlerForTests(t, models.Host{
ID: hostID,
Hostname: "host.local",
DisplayName:"Host Local",
Status: "online",
TokenID: tokenID,
LastSeen: lastSeen,
})
req := httptest.NewRequest(http.MethodGet, "/api/agents/host/lookup?id="+hostID, nil)
attachAPITokenRecord(req, &config.APITokenRecord{ID: tokenID})
rec := httptest.NewRecorder()
handler.HandleLookup(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code)
}
var resp struct {
Success bool `json:"success"`
Host struct {
ID string `json:"id"`
Hostname string `json:"hostname"`
Status string `json:"status"`
Connected bool `json:"connected"`
LastSeen time.Time `json:"lastSeen"`
} `json:"host"`
}
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
t.Fatalf("failed to decode response: %v", err)
}
if !resp.Success {
t.Fatalf("expected success=true")
}
if resp.Host.ID != hostID {
t.Fatalf("unexpected host id %q", resp.Host.ID)
}
if !resp.Host.Connected {
t.Fatalf("expected connected host")
}
if !resp.Host.LastSeen.Equal(lastSeen) {
t.Fatalf("expected lastSeen %v, got %v", lastSeen, resp.Host.LastSeen)
}
}
func TestHandleLookupForbiddenOnTokenMismatch(t *testing.T) {
t.Parallel()
hostID := "host-456"
handler := newHostAgentHandlerForTests(t, models.Host{
ID: hostID,
Hostname: "mismatch.local",
Status: "online",
TokenID: "token-correct",
})
req := httptest.NewRequest(http.MethodGet, "/api/agents/host/lookup?id="+hostID, nil)
attachAPITokenRecord(req, &config.APITokenRecord{ID: "token-wrong"})
rec := httptest.NewRecorder()
handler.HandleLookup(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("expected status %d, got %d", http.StatusForbidden, rec.Code)
}
}
func TestHandleLookupNotFound(t *testing.T) {
t.Parallel()
handler := newHostAgentHandlerForTests(t)
req := httptest.NewRequest(http.MethodGet, "/api/agents/host/lookup?id=missing", nil)
rec := httptest.NewRecorder()
handler.HandleLookup(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("expected status %d, got %d", http.StatusNotFound, rec.Code)
}
}
func newHostAgentHandlerForTests(t *testing.T, hosts ...models.Host) *HostAgentHandlers {
t.Helper()
monitor := &monitoring.Monitor{}
state := models.NewState()
for _, host := range hosts {
state.UpsertHost(host)
}
setUnexportedField(t, monitor, "state", state)
return &HostAgentHandlers{
monitor: monitor,
}
}
func setUnexportedField(t *testing.T, target interface{}, field string, value interface{}) {
t.Helper()
v := reflect.ValueOf(target).Elem()
f := v.FieldByName(field)
if !f.IsValid() {
t.Fatalf("field %q not found", field)
}
ptr := unsafe.Pointer(f.UnsafeAddr())
reflect.NewAt(f.Type(), ptr).Elem().Set(reflect.ValueOf(value))
}

View file

@ -218,7 +218,7 @@ func (r *RecoveryTokenStore) saveUnsafe() {
} }
// Marshal tokens // Marshal tokens
data, err := json.MarshalIndent(r.tokens, "", " ") data, err := json.Marshal(r.tokens)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed to marshal recovery tokens") log.Error().Err(err).Msg("Failed to marshal recovery tokens")
return return

View file

@ -162,7 +162,7 @@ func (s *SessionStore) saveUnsafe() {
} }
// Marshal sessions // Marshal sessions
data, err := json.MarshalIndent(s.sessions, "", " ") data, err := json.Marshal(s.sessions)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed to marshal sessions") log.Error().Err(err).Msg("Failed to marshal sessions")
return return

View file

@ -28,6 +28,13 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
const (
DefaultGuestMetadataMinRefresh = 2 * time.Minute
DefaultGuestMetadataRefreshJitter = 45 * time.Second
DefaultGuestMetadataRetryBackoff = 30 * time.Second
DefaultGuestMetadataMaxConcurrent = 4
)
// IsPasswordHashed checks if a string looks like a bcrypt hash // IsPasswordHashed checks if a string looks like a bcrypt hash
func IsPasswordHashed(password string) bool { func IsPasswordHashed(password string) bool {
// Bcrypt hashes start with $2a$, $2b$, or $2y$ and are 60 characters long // Bcrypt hashes start with $2a$, $2b$, or $2y$ and are 60 characters long
@ -76,19 +83,23 @@ type Config struct {
// Monitoring settings // Monitoring settings
// Note: PVE polling is hardcoded to 10s since Proxmox cluster/resources endpoint only updates every 10s // Note: PVE polling is hardcoded to 10s since Proxmox cluster/resources endpoint only updates every 10s
PBSPollingInterval time.Duration `envconfig:"PBS_POLLING_INTERVAL"` // PBS polling interval (60s default) PBSPollingInterval time.Duration `envconfig:"PBS_POLLING_INTERVAL"` // PBS polling interval (60s default)
PMGPollingInterval time.Duration `envconfig:"PMG_POLLING_INTERVAL"` // PMG polling interval (60s default) PMGPollingInterval time.Duration `envconfig:"PMG_POLLING_INTERVAL"` // PMG polling interval (60s default)
ConcurrentPolling bool `envconfig:"CONCURRENT_POLLING" default:"true"` ConcurrentPolling bool `envconfig:"CONCURRENT_POLLING" default:"true"`
ConnectionTimeout time.Duration `envconfig:"CONNECTION_TIMEOUT" default:"45s"` // Increased for slow storage operations ConnectionTimeout time.Duration `envconfig:"CONNECTION_TIMEOUT" default:"45s"` // Increased for slow storage operations
MetricsRetentionDays int `envconfig:"METRICS_RETENTION_DAYS" default:"7"` MetricsRetentionDays int `envconfig:"METRICS_RETENTION_DAYS" default:"7"`
BackupPollingCycles int `envconfig:"BACKUP_POLLING_CYCLES" default:"10"` BackupPollingCycles int `envconfig:"BACKUP_POLLING_CYCLES" default:"10"`
BackupPollingInterval time.Duration `envconfig:"BACKUP_POLLING_INTERVAL"` BackupPollingInterval time.Duration `envconfig:"BACKUP_POLLING_INTERVAL"`
EnableBackupPolling bool `envconfig:"ENABLE_BACKUP_POLLING" default:"true"` EnableBackupPolling bool `envconfig:"ENABLE_BACKUP_POLLING" default:"true"`
WebhookBatchDelay time.Duration `envconfig:"WEBHOOK_BATCH_DELAY" default:"10s"` WebhookBatchDelay time.Duration `envconfig:"WEBHOOK_BATCH_DELAY" default:"10s"`
AdaptivePollingEnabled bool `envconfig:"ADAPTIVE_POLLING_ENABLED" default:"false"` AdaptivePollingEnabled bool `envconfig:"ADAPTIVE_POLLING_ENABLED" default:"false"`
AdaptivePollingBaseInterval time.Duration `envconfig:"ADAPTIVE_POLLING_BASE_INTERVAL" default:"10s"` AdaptivePollingBaseInterval time.Duration `envconfig:"ADAPTIVE_POLLING_BASE_INTERVAL" default:"10s"`
AdaptivePollingMinInterval time.Duration `envconfig:"ADAPTIVE_POLLING_MIN_INTERVAL" default:"5s"` AdaptivePollingMinInterval time.Duration `envconfig:"ADAPTIVE_POLLING_MIN_INTERVAL" default:"5s"`
AdaptivePollingMaxInterval time.Duration `envconfig:"ADAPTIVE_POLLING_MAX_INTERVAL" default:"5m"` AdaptivePollingMaxInterval time.Duration `envconfig:"ADAPTIVE_POLLING_MAX_INTERVAL" default:"5m"`
GuestMetadataMinRefreshInterval time.Duration `envconfig:"GUEST_METADATA_MIN_REFRESH_INTERVAL" default:"2m" json:"guestMetadataMinRefreshInterval"`
GuestMetadataRefreshJitter time.Duration `envconfig:"GUEST_METADATA_REFRESH_JITTER" default:"45s" json:"guestMetadataRefreshJitter"`
GuestMetadataRetryBackoff time.Duration `envconfig:"GUEST_METADATA_RETRY_BACKOFF" default:"30s" json:"guestMetadataRetryBackoff"`
GuestMetadataMaxConcurrent int `envconfig:"GUEST_METADATA_MAX_CONCURRENT" default:"4" json:"guestMetadataMaxConcurrent"`
// Logging settings // Logging settings
LogLevel string `envconfig:"LOG_LEVEL" default:"info"` LogLevel string `envconfig:"LOG_LEVEL" default:"info"`
@ -485,36 +496,40 @@ func Load() (*Config, error) {
// Initialize config with defaults // Initialize config with defaults
cfg := &Config{ cfg := &Config{
BackendHost: "0.0.0.0", BackendHost: "0.0.0.0",
BackendPort: 3000, BackendPort: 3000,
FrontendHost: "0.0.0.0", FrontendHost: "0.0.0.0",
FrontendPort: 7655, FrontendPort: 7655,
ConfigPath: dataDir, ConfigPath: dataDir,
DataPath: dataDir, DataPath: dataDir,
ConcurrentPolling: true, ConcurrentPolling: true,
ConnectionTimeout: 60 * time.Second, ConnectionTimeout: 60 * time.Second,
MetricsRetentionDays: 7, MetricsRetentionDays: 7,
BackupPollingCycles: 10, BackupPollingCycles: 10,
BackupPollingInterval: 0, BackupPollingInterval: 0,
EnableBackupPolling: true, EnableBackupPolling: true,
WebhookBatchDelay: 10 * time.Second, WebhookBatchDelay: 10 * time.Second,
AdaptivePollingEnabled: false, AdaptivePollingEnabled: false,
AdaptivePollingBaseInterval: 10 * time.Second, AdaptivePollingBaseInterval: 10 * time.Second,
AdaptivePollingMinInterval: 5 * time.Second, AdaptivePollingMinInterval: 5 * time.Second,
AdaptivePollingMaxInterval: 5 * time.Minute, AdaptivePollingMaxInterval: 5 * time.Minute,
LogLevel: "info", GuestMetadataMinRefreshInterval: DefaultGuestMetadataMinRefresh,
LogFormat: "auto", GuestMetadataRefreshJitter: DefaultGuestMetadataRefreshJitter,
LogMaxSize: 100, GuestMetadataRetryBackoff: DefaultGuestMetadataRetryBackoff,
LogMaxAge: 30, GuestMetadataMaxConcurrent: DefaultGuestMetadataMaxConcurrent,
LogCompress: true, LogLevel: "info",
AllowedOrigins: "", // Empty means no CORS headers (same-origin only) LogFormat: "auto",
IframeEmbeddingAllow: "SAMEORIGIN", LogMaxSize: 100,
PBSPollingInterval: 60 * time.Second, // Default PBS polling (slower) LogMaxAge: 30,
PMGPollingInterval: 60 * time.Second, // Default PMG polling (aggregated stats) LogCompress: true,
DiscoveryEnabled: false, AllowedOrigins: "", // Empty means no CORS headers (same-origin only)
DiscoverySubnet: "auto", IframeEmbeddingAllow: "SAMEORIGIN",
EnvOverrides: make(map[string]bool), PBSPollingInterval: 60 * time.Second, // Default PBS polling (slower)
OIDC: NewOIDCConfig(), PMGPollingInterval: 60 * time.Second, // Default PMG polling (aggregated stats)
DiscoveryEnabled: false,
DiscoverySubnet: "auto",
EnvOverrides: make(map[string]bool),
OIDC: NewOIDCConfig(),
} }
cfg.Discovery = DefaultDiscoveryConfig() cfg.Discovery = DefaultDiscoveryConfig()
@ -725,6 +740,62 @@ func Load() (*Config, error) {
} }
} }
if minRefresh := strings.TrimSpace(os.Getenv("GUEST_METADATA_MIN_REFRESH_INTERVAL")); minRefresh != "" {
if dur, err := time.ParseDuration(minRefresh); err == nil {
if dur <= 0 {
log.Warn().Str("value", minRefresh).Msg("Ignoring non-positive GUEST_METADATA_MIN_REFRESH_INTERVAL from environment")
} else {
cfg.GuestMetadataMinRefreshInterval = dur
cfg.EnvOverrides["GUEST_METADATA_MIN_REFRESH_INTERVAL"] = true
log.Info().Dur("interval", dur).Msg("Guest metadata min refresh interval overridden by environment")
}
} else {
log.Warn().Str("value", minRefresh).Msg("Invalid GUEST_METADATA_MIN_REFRESH_INTERVAL value, expected duration string")
}
}
if jitter := strings.TrimSpace(os.Getenv("GUEST_METADATA_REFRESH_JITTER")); jitter != "" {
if dur, err := time.ParseDuration(jitter); err == nil {
if dur < 0 {
log.Warn().Str("value", jitter).Msg("Ignoring negative GUEST_METADATA_REFRESH_JITTER from environment")
} else {
cfg.GuestMetadataRefreshJitter = dur
cfg.EnvOverrides["GUEST_METADATA_REFRESH_JITTER"] = true
log.Info().Dur("jitter", dur).Msg("Guest metadata refresh jitter overridden by environment")
}
} else {
log.Warn().Str("value", jitter).Msg("Invalid GUEST_METADATA_REFRESH_JITTER value, expected duration string")
}
}
if backoff := strings.TrimSpace(os.Getenv("GUEST_METADATA_RETRY_BACKOFF")); backoff != "" {
if dur, err := time.ParseDuration(backoff); err == nil {
if dur <= 0 {
log.Warn().Str("value", backoff).Msg("Ignoring non-positive GUEST_METADATA_RETRY_BACKOFF from environment")
} else {
cfg.GuestMetadataRetryBackoff = dur
cfg.EnvOverrides["GUEST_METADATA_RETRY_BACKOFF"] = true
log.Info().Dur("backoff", dur).Msg("Guest metadata retry backoff overridden by environment")
}
} else {
log.Warn().Str("value", backoff).Msg("Invalid GUEST_METADATA_RETRY_BACKOFF value, expected duration string")
}
}
if concurrent := strings.TrimSpace(os.Getenv("GUEST_METADATA_MAX_CONCURRENT")); concurrent != "" {
if val, err := strconv.Atoi(concurrent); err == nil {
if val <= 0 {
log.Warn().Str("value", concurrent).Msg("Ignoring non-positive GUEST_METADATA_MAX_CONCURRENT from environment")
} else {
cfg.GuestMetadataMaxConcurrent = val
cfg.EnvOverrides["GUEST_METADATA_MAX_CONCURRENT"] = true
log.Info().Int("maxConcurrent", val).Msg("Guest metadata max concurrency overridden by environment")
}
} else {
log.Warn().Str("value", concurrent).Msg("Invalid GUEST_METADATA_MAX_CONCURRENT value, expected integer")
}
}
// Support both FRONTEND_PORT (preferred) and PORT (legacy) env vars // Support both FRONTEND_PORT (preferred) and PORT (legacy) env vars
if frontendPort := os.Getenv("FRONTEND_PORT"); frontendPort != "" { if frontendPort := os.Getenv("FRONTEND_PORT"); frontendPort != "" {
if p, err := strconv.Atoi(frontendPort); err == nil { if p, err := strconv.Atoi(frontendPort); err == nil {

View file

@ -150,7 +150,7 @@ func (s *GuestMetadataStore) save() error {
log.Debug().Str("path", filePath).Msg("Saving guest metadata to disk") log.Debug().Str("path", filePath).Msg("Saving guest metadata to disk")
data, err := json.MarshalIndent(s.metadata, "", " ") data, err := json.Marshal(s.metadata)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err) return fmt.Errorf("failed to marshal metadata: %w", err)
} }

View file

@ -156,7 +156,7 @@ func (c *ConfigPersistence) SaveAPITokens(tokens []APITokenRecord) error {
sanitized[i] = record sanitized[i] = record
} }
data, err := json.MarshalIndent(sanitized, "", " ") data, err := json.Marshal(sanitized)
if err != nil { if err != nil {
return err return err
} }
@ -198,6 +198,32 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
if config.HysteresisMargin <= 0 { if config.HysteresisMargin <= 0 {
config.HysteresisMargin = 5.0 config.HysteresisMargin = 5.0
} }
if config.HostDefaults.CPU == nil || config.HostDefaults.CPU.Trigger <= 0 {
config.HostDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
} else if config.HostDefaults.CPU.Clear <= 0 {
config.HostDefaults.CPU.Clear = config.HostDefaults.CPU.Trigger - 5
if config.HostDefaults.CPU.Clear <= 0 {
config.HostDefaults.CPU.Clear = 75
}
}
if config.HostDefaults.Memory == nil || config.HostDefaults.Memory.Trigger <= 0 {
config.HostDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}
} else if config.HostDefaults.Memory.Clear <= 0 {
config.HostDefaults.Memory.Clear = config.HostDefaults.Memory.Trigger - 5
if config.HostDefaults.Memory.Clear <= 0 {
config.HostDefaults.Memory.Clear = 80
}
}
if config.HostDefaults.Disk == nil || config.HostDefaults.Disk.Trigger <= 0 {
config.HostDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}
} else if config.HostDefaults.Disk.Clear <= 0 {
config.HostDefaults.Disk.Clear = config.HostDefaults.Disk.Trigger - 5
if config.HostDefaults.Disk.Clear <= 0 {
config.HostDefaults.Disk.Clear = 85
}
}
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds) config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
if config.TimeThreshold <= 0 { if config.TimeThreshold <= 0 {
config.TimeThreshold = 5 config.TimeThreshold = 5
@ -252,7 +278,7 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
} }
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes) config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
data, err := json.MarshalIndent(config, "", " ") data, err := json.Marshal(config)
if err != nil { if err != nil {
return err return err
} }
@ -291,6 +317,11 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}, Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
Temperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}, Temperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
}, },
HostDefaults: alerts.ThresholdConfig{
CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
},
StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
TimeThreshold: 5, TimeThreshold: 5,
TimeThresholds: map[string]int{ TimeThresholds: map[string]int{
@ -346,6 +377,30 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 { if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 {
config.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} config.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
} }
if config.HostDefaults.CPU == nil || config.HostDefaults.CPU.Trigger <= 0 {
config.HostDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
} else if config.HostDefaults.CPU.Clear <= 0 {
config.HostDefaults.CPU.Clear = config.HostDefaults.CPU.Trigger - 5
if config.HostDefaults.CPU.Clear <= 0 {
config.HostDefaults.CPU.Clear = 75
}
}
if config.HostDefaults.Memory == nil || config.HostDefaults.Memory.Trigger <= 0 {
config.HostDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}
} else if config.HostDefaults.Memory.Clear <= 0 {
config.HostDefaults.Memory.Clear = config.HostDefaults.Memory.Trigger - 5
if config.HostDefaults.Memory.Clear <= 0 {
config.HostDefaults.Memory.Clear = 80
}
}
if config.HostDefaults.Disk == nil || config.HostDefaults.Disk.Trigger <= 0 {
config.HostDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}
} else if config.HostDefaults.Disk.Clear <= 0 {
config.HostDefaults.Disk.Clear = config.HostDefaults.Disk.Trigger - 5
if config.HostDefaults.Disk.Clear <= 0 {
config.HostDefaults.Disk.Clear = 85
}
}
if config.TimeThreshold <= 0 { if config.TimeThreshold <= 0 {
config.TimeThreshold = 5 config.TimeThreshold = 5
} }
@ -425,7 +480,7 @@ func (c *ConfigPersistence) SaveEmailConfig(config notifications.EmailConfig) er
defer c.mu.Unlock() defer c.mu.Unlock()
// Marshal to JSON first // Marshal to JSON first
data, err := json.MarshalIndent(config, "", " ") data, err := json.Marshal(config)
if err != nil { if err != nil {
return err return err
} }
@ -502,7 +557,7 @@ func (c *ConfigPersistence) SaveAppriseConfig(config notifications.AppriseConfig
config = notifications.NormalizeAppriseConfig(config) config = notifications.NormalizeAppriseConfig(config)
data, err := json.MarshalIndent(config, "", " ") data, err := json.Marshal(config)
if err != nil { if err != nil {
return err return err
} }
@ -578,7 +633,7 @@ func (c *ConfigPersistence) SaveWebhooks(webhooks []notifications.WebhookConfig)
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
data, err := json.MarshalIndent(webhooks, "", " ") data, err := json.Marshal(webhooks)
if err != nil { if err != nil {
return err return err
} }
@ -811,7 +866,7 @@ func (c *ConfigPersistence) saveNodesConfig(pveInstances []PVEInstance, pbsInsta
PMGInstances: pmgInstances, PMGInstances: pmgInstances,
} }
data, err := json.MarshalIndent(config, "", " ") data, err := json.Marshal(config)
if err != nil { if err != nil {
return err return err
} }
@ -1123,7 +1178,7 @@ func (c *ConfigPersistence) SaveSystemSettings(settings SystemSettings) error {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
data, err := json.MarshalIndent(settings, "", " ") data, err := json.Marshal(settings)
if err != nil { if err != nil {
return err return err
} }
@ -1159,7 +1214,7 @@ func (c *ConfigPersistence) SaveOIDCConfig(settings OIDCConfig) error {
// Do not persist runtime-only flags. // Do not persist runtime-only flags.
settings.EnvOverrides = nil settings.EnvOverrides = nil
data, err := json.MarshalIndent(settings, "", " ") data, err := json.Marshal(settings)
if err != nil { if err != nil {
return err return err
} }

View file

@ -2,43 +2,53 @@ package monitoring
import ( import (
stdErrors "errors" stdErrors "errors"
"fmt"
"strings" "strings"
"sync" "sync"
"time" "time"
internalerrors "github.com/rcourtman/pulse-go-rewrite/internal/errors"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
internalerrors "github.com/rcourtman/pulse-go-rewrite/internal/errors"
) )
// PollMetrics manages Prometheus instrumentation for polling activity. // PollMetrics manages Prometheus instrumentation for polling activity.
type PollMetrics struct { type PollMetrics struct {
pollDuration *prometheus.HistogramVec pollDuration *prometheus.HistogramVec
pollResults *prometheus.CounterVec pollResults *prometheus.CounterVec
pollErrors *prometheus.CounterVec pollErrors *prometheus.CounterVec
lastSuccess *prometheus.GaugeVec lastSuccess *prometheus.GaugeVec
staleness *prometheus.GaugeVec staleness *prometheus.GaugeVec
queueDepth prometheus.Gauge queueDepth prometheus.Gauge
inflight *prometheus.GaugeVec inflight *prometheus.GaugeVec
nodePollDuration *prometheus.HistogramVec nodePollDuration *prometheus.HistogramVec
nodePollResults *prometheus.CounterVec nodePollResults *prometheus.CounterVec
nodePollErrors *prometheus.CounterVec nodePollErrors *prometheus.CounterVec
nodeLastSuccess *prometheus.GaugeVec nodeLastSuccess *prometheus.GaugeVec
nodeStaleness *prometheus.GaugeVec nodeStaleness *prometheus.GaugeVec
schedulerQueueReady prometheus.Gauge schedulerQueueReady prometheus.Gauge
schedulerQueueDepthByType *prometheus.GaugeVec schedulerQueueDepthByType *prometheus.GaugeVec
schedulerQueueWait *prometheus.HistogramVec schedulerQueueWait *prometheus.HistogramVec
schedulerDeadLetterDepth *prometheus.GaugeVec schedulerDeadLetterDepth *prometheus.GaugeVec
schedulerBreakerState *prometheus.GaugeVec schedulerBreakerState *prometheus.GaugeVec
schedulerBreakerFailureCount *prometheus.GaugeVec schedulerBreakerFailureCount *prometheus.GaugeVec
schedulerBreakerRetrySeconds *prometheus.GaugeVec schedulerBreakerRetrySeconds *prometheus.GaugeVec
mu sync.RWMutex mu sync.RWMutex
lastSuccessByKey map[string]time.Time lastSuccessByKey map[metricKey]time.Time
nodeLastSuccessByKey map[string]time.Time nodeLastSuccessByKey map[nodeMetricKey]time.Time
lastQueueTypeKeys map[string]struct{} lastQueueTypeKeys map[string]struct{}
lastDLQKeys map[string]struct{} lastDLQKeys map[string]struct{}
pending int pending int
}
type metricKey struct {
instanceType string
instance string
}
type nodeMetricKey struct {
instanceType string
instance string
node string
} }
var ( var (
@ -227,10 +237,10 @@ func newPollMetrics() *PollMetrics {
}, },
[]string{"instance_type", "instance"}, []string{"instance_type", "instance"},
), ),
lastSuccessByKey: make(map[string]time.Time), lastSuccessByKey: make(map[metricKey]time.Time),
nodeLastSuccessByKey: make(map[string]time.Time), nodeLastSuccessByKey: make(map[nodeMetricKey]time.Time),
lastQueueTypeKeys: make(map[string]struct{}), lastQueueTypeKeys: make(map[string]struct{}),
lastDLQKeys: make(map[string]struct{}), lastDLQKeys: make(map[string]struct{}),
} }
prometheus.MustRegister( prometheus.MustRegister(
@ -260,13 +270,13 @@ func newPollMetrics() *PollMetrics {
// NodePollResult captures timing and outcome for a specific node within a poll cycle. // NodePollResult captures timing and outcome for a specific node within a poll cycle.
type NodePollResult struct { type NodePollResult struct {
InstanceName string InstanceName string
InstanceType string InstanceType string
NodeName string NodeName string
Success bool Success bool
Error error Error error
StartTime time.Time StartTime time.Time
EndTime time.Time EndTime time.Time
} }
// RecordNodeResult records metrics for an individual node poll. // RecordNodeResult records metrics for an individual node poll.
@ -275,173 +285,136 @@ func (pm *PollMetrics) RecordNodeResult(result NodePollResult) {
return return
} }
nodeLabel := strings.TrimSpace(result.NodeName) instType, inst := sanitizeInstanceLabels(result.InstanceType, result.InstanceName)
if nodeLabel == "" { nodeLabel := normalizeNodeLabel(result.NodeName)
nodeLabel = "unknown-node"
}
labels := prometheus.Labels{ duration := result.EndTime.Sub(result.StartTime).Seconds()
"instance_type": result.InstanceType, if duration < 0 {
"instance": result.InstanceName, duration = 0
"node": nodeLabel, }
} pm.nodePollDuration.WithLabelValues(instType, inst, nodeLabel).Observe(duration)
duration := result.EndTime.Sub(result.StartTime).Seconds() resultValue := "success"
if duration < 0 { if !result.Success {
duration = 0 resultValue = "error"
} }
pm.nodePollDuration.With(labels).Observe(duration) pm.nodePollResults.WithLabelValues(instType, inst, nodeLabel, resultValue).Inc()
resultValue := "success" if result.Success {
if !result.Success { pm.nodeLastSuccess.WithLabelValues(instType, inst, nodeLabel).Set(float64(result.EndTime.Unix()))
resultValue = "error" pm.storeNodeLastSuccess(instType, inst, nodeLabel, result.EndTime)
} pm.updateNodeStaleness(instType, inst, nodeLabel, 0)
pm.nodePollResults.With(prometheus.Labels{ return
"instance_type": result.InstanceType, }
"instance": result.InstanceName,
"node": nodeLabel,
"result": resultValue,
}).Inc()
if result.Success { errType := pm.classifyError(result.Error)
pm.nodeLastSuccess.With(labels).Set(float64(result.EndTime.Unix())) pm.nodePollErrors.WithLabelValues(instType, inst, nodeLabel, errType).Inc()
pm.storeNodeLastSuccess(result.InstanceType, result.InstanceName, nodeLabel, result.EndTime)
pm.updateNodeStaleness(result.InstanceType, result.InstanceName, nodeLabel, 0)
return
}
errType := pm.classifyError(result.Error) if last, ok := pm.lastNodeSuccessFor(instType, inst, nodeLabel); ok && !last.IsZero() {
pm.nodePollErrors.With(prometheus.Labels{ staleness := result.EndTime.Sub(last).Seconds()
"instance_type": result.InstanceType, if staleness < 0 {
"instance": result.InstanceName, staleness = 0
"node": nodeLabel, }
"error_type": errType, pm.updateNodeStaleness(instType, inst, nodeLabel, staleness)
}).Inc() } else {
pm.updateNodeStaleness(instType, inst, nodeLabel, -1)
if last, ok := pm.lastNodeSuccessFor(result.InstanceType, result.InstanceName, nodeLabel); ok && !last.IsZero() {
staleness := result.EndTime.Sub(last).Seconds()
if staleness < 0 {
staleness = 0
}
pm.updateNodeStaleness(result.InstanceType, result.InstanceName, nodeLabel, staleness)
} else {
pm.updateNodeStaleness(result.InstanceType, result.InstanceName, nodeLabel, -1)
} }
} }
// RecordQueueWait observes the time a task spent waiting in the scheduler queue. // RecordQueueWait observes the time a task spent waiting in the scheduler queue.
func (pm *PollMetrics) RecordQueueWait(instanceType string, wait time.Duration) { func (pm *PollMetrics) RecordQueueWait(instanceType string, wait time.Duration) {
if pm == nil { if pm == nil {
return return
} }
if wait < 0 { if wait < 0 {
wait = 0 wait = 0
} }
instanceType = strings.TrimSpace(instanceType) label := normalizeLabel(instanceType)
if instanceType == "" { pm.schedulerQueueWait.WithLabelValues(label).Observe(wait.Seconds())
instanceType = "unknown"
}
pm.schedulerQueueWait.WithLabelValues(instanceType).Observe(wait.Seconds())
} }
// UpdateQueueSnapshot updates scheduler queue depth metrics. // UpdateQueueSnapshot updates scheduler queue depth metrics.
func (pm *PollMetrics) UpdateQueueSnapshot(snapshot QueueSnapshot) { func (pm *PollMetrics) UpdateQueueSnapshot(snapshot QueueSnapshot) {
if pm == nil { if pm == nil {
return return
} }
pm.schedulerQueueReady.Set(float64(snapshot.DueWithinSeconds)) pm.schedulerQueueReady.Set(float64(snapshot.DueWithinSeconds))
current := make(map[string]struct{}, len(snapshot.PerType)) current := make(map[string]struct{}, len(snapshot.PerType))
for instanceType, depth := range snapshot.PerType { for instanceType, depth := range snapshot.PerType {
key := strings.TrimSpace(instanceType) key := normalizeLabel(instanceType)
if key == "" { pm.schedulerQueueDepthByType.WithLabelValues(key).Set(float64(depth))
key = "unknown" current[key] = struct{}{}
} }
pm.schedulerQueueDepthByType.WithLabelValues(key).Set(float64(depth))
current[key] = struct{}{}
}
pm.mu.Lock() pm.mu.Lock()
for key := range pm.lastQueueTypeKeys { for key := range pm.lastQueueTypeKeys {
if _, ok := current[key]; !ok { if _, ok := current[key]; !ok {
pm.schedulerQueueDepthByType.WithLabelValues(key).Set(0) pm.schedulerQueueDepthByType.WithLabelValues(key).Set(0)
} }
} }
pm.lastQueueTypeKeys = current pm.lastQueueTypeKeys = current
pm.mu.Unlock() pm.mu.Unlock()
} }
// UpdateDeadLetterCounts refreshes dead-letter queue gauges based on the provided tasks. // UpdateDeadLetterCounts refreshes dead-letter queue gauges based on the provided tasks.
func (pm *PollMetrics) UpdateDeadLetterCounts(tasks []DeadLetterTask) { func (pm *PollMetrics) UpdateDeadLetterCounts(tasks []DeadLetterTask) {
if pm == nil { if pm == nil {
return return
} }
current := make(map[string]float64) current := make(map[string]float64)
for _, task := range tasks { for _, task := range tasks {
instType := strings.TrimSpace(task.Type) instType := normalizeLabel(task.Type)
if instType == "" { inst := normalizeLabel(task.Instance)
instType = "unknown" key := instType + "::" + inst
} current[key] = current[key] + 1
inst := strings.TrimSpace(task.Instance) }
if inst == "" {
inst = "unknown"
}
key := instType + "::" + inst
current[key] = current[key] + 1
}
pm.mu.Lock() pm.mu.Lock()
prev := pm.lastDLQKeys prev := pm.lastDLQKeys
pm.lastDLQKeys = make(map[string]struct{}, len(current)) pm.lastDLQKeys = make(map[string]struct{}, len(current))
pm.mu.Unlock() pm.mu.Unlock()
for key, count := range current { for key, count := range current {
instType, inst := splitInstanceKey(key) instType, inst := splitInstanceKey(key)
pm.schedulerDeadLetterDepth.WithLabelValues(instType, inst).Set(count) pm.schedulerDeadLetterDepth.WithLabelValues(instType, inst).Set(count)
} }
pm.mu.Lock() pm.mu.Lock()
for key := range current { for key := range current {
pm.lastDLQKeys[key] = struct{}{} pm.lastDLQKeys[key] = struct{}{}
} }
for key := range prev { for key := range prev {
if _, ok := current[key]; !ok { if _, ok := current[key]; !ok {
instType, inst := splitInstanceKey(key) instType, inst := splitInstanceKey(key)
pm.schedulerDeadLetterDepth.WithLabelValues(instType, inst).Set(0) pm.schedulerDeadLetterDepth.WithLabelValues(instType, inst).Set(0)
} }
} }
pm.mu.Unlock() pm.mu.Unlock()
} }
// SetBreakerState updates circuit breaker metrics for a specific instance. // SetBreakerState updates circuit breaker metrics for a specific instance.
func (pm *PollMetrics) SetBreakerState(instanceType, instance, state string, failures int, retryAt time.Time) { func (pm *PollMetrics) SetBreakerState(instanceType, instance, state string, failures int, retryAt time.Time) {
if pm == nil { if pm == nil {
return return
} }
instType := strings.TrimSpace(instanceType) instType, inst := sanitizeInstanceLabels(instanceType, instance)
if instType == "" {
instType = "unknown"
}
inst := strings.TrimSpace(instance)
if inst == "" {
inst = "unknown"
}
value := breakerStateToValue(state) value := breakerStateToValue(state)
pm.schedulerBreakerState.WithLabelValues(instType, inst).Set(value) pm.schedulerBreakerState.WithLabelValues(instType, inst).Set(value)
pm.schedulerBreakerFailureCount.WithLabelValues(instType, inst).Set(float64(failures)) pm.schedulerBreakerFailureCount.WithLabelValues(instType, inst).Set(float64(failures))
retrySeconds := 0.0 retrySeconds := 0.0
if !retryAt.IsZero() { if !retryAt.IsZero() {
retrySeconds = retryAt.Sub(time.Now()).Seconds() retrySeconds = retryAt.Sub(time.Now()).Seconds()
if retrySeconds < 0 { if retrySeconds < 0 {
retrySeconds = 0 retrySeconds = 0
} }
} }
pm.schedulerBreakerRetrySeconds.WithLabelValues(instType, inst).Set(retrySeconds) pm.schedulerBreakerRetrySeconds.WithLabelValues(instType, inst).Set(retrySeconds)
} }
// RecordResult records metrics for a polling result. // RecordResult records metrics for a polling result.
@ -450,47 +423,36 @@ func (pm *PollMetrics) RecordResult(result PollResult) {
return return
} }
labels := prometheus.Labels{ instType, inst := sanitizeInstanceLabels(result.InstanceType, result.InstanceName)
"instance_type": result.InstanceType,
"instance": result.InstanceName,
}
duration := result.EndTime.Sub(result.StartTime).Seconds() duration := result.EndTime.Sub(result.StartTime).Seconds()
if duration < 0 { if duration < 0 {
duration = 0 duration = 0
} }
pm.pollDuration.With(labels).Observe(duration) pm.pollDuration.WithLabelValues(instType, inst).Observe(duration)
resultValue := "success" resultValue := "success"
if !result.Success { if !result.Success {
resultValue = "error" resultValue = "error"
} }
pm.pollResults.With(prometheus.Labels{ pm.pollResults.WithLabelValues(instType, inst, resultValue).Inc()
"instance_type": result.InstanceType,
"instance": result.InstanceName,
"result": resultValue,
}).Inc()
if result.Success { if result.Success {
pm.lastSuccess.With(labels).Set(float64(result.EndTime.Unix())) pm.lastSuccess.WithLabelValues(instType, inst).Set(float64(result.EndTime.Unix()))
pm.storeLastSuccess(result.InstanceType, result.InstanceName, result.EndTime) pm.storeLastSuccess(instType, inst, result.EndTime)
pm.updateStaleness(result.InstanceType, result.InstanceName, 0) pm.updateStaleness(instType, inst, 0)
} else { } else {
errType := pm.classifyError(result.Error) errType := pm.classifyError(result.Error)
pm.pollErrors.With(prometheus.Labels{ pm.pollErrors.WithLabelValues(instType, inst, errType).Inc()
"instance_type": result.InstanceType,
"instance": result.InstanceName,
"error_type": errType,
}).Inc()
if last, ok := pm.lastSuccessFor(result.InstanceType, result.InstanceName); ok && !last.IsZero() { if last, ok := pm.lastSuccessFor(instType, inst); ok && !last.IsZero() {
staleness := result.EndTime.Sub(last).Seconds() staleness := result.EndTime.Sub(last).Seconds()
if staleness < 0 { if staleness < 0 {
staleness = 0 staleness = 0
} }
pm.updateStaleness(result.InstanceType, result.InstanceName, staleness) pm.updateStaleness(instType, inst, staleness)
} else { } else {
pm.updateStaleness(result.InstanceType, result.InstanceName, -1) pm.updateStaleness(instType, inst, -1)
} }
} }
@ -555,75 +517,101 @@ func (pm *PollMetrics) decrementPending() {
} }
func (pm *PollMetrics) storeLastSuccess(instanceType, instance string, ts time.Time) { func (pm *PollMetrics) storeLastSuccess(instanceType, instance string, ts time.Time) {
pm.mu.Lock() pm.mu.Lock()
pm.lastSuccessByKey[pm.key(instanceType, instance)] = ts pm.lastSuccessByKey[makeMetricKey(instanceType, instance)] = ts
pm.mu.Unlock() pm.mu.Unlock()
} }
func (pm *PollMetrics) lastSuccessFor(instanceType, instance string) (time.Time, bool) { func (pm *PollMetrics) lastSuccessFor(instanceType, instance string) (time.Time, bool) {
pm.mu.RLock() pm.mu.RLock()
ts, ok := pm.lastSuccessByKey[pm.key(instanceType, instance)] ts, ok := pm.lastSuccessByKey[makeMetricKey(instanceType, instance)]
pm.mu.RUnlock() pm.mu.RUnlock()
return ts, ok return ts, ok
} }
func (pm *PollMetrics) updateStaleness(instanceType, instance string, value float64) { func (pm *PollMetrics) updateStaleness(instanceType, instance string, value float64) {
pm.staleness.WithLabelValues(instanceType, instance).Set(value) instType, inst := sanitizeInstanceLabels(instanceType, instance)
} pm.staleness.WithLabelValues(instType, inst).Set(value)
func (pm *PollMetrics) key(instanceType, instance string) string {
return fmt.Sprintf("%s::%s", instanceType, instance)
} }
func (pm *PollMetrics) storeNodeLastSuccess(instanceType, instance, node string, ts time.Time) { func (pm *PollMetrics) storeNodeLastSuccess(instanceType, instance, node string, ts time.Time) {
pm.mu.Lock() pm.mu.Lock()
pm.nodeLastSuccessByKey[pm.nodeKey(instanceType, instance, node)] = ts pm.nodeLastSuccessByKey[makeNodeMetricKey(instanceType, instance, node)] = ts
pm.mu.Unlock() pm.mu.Unlock()
} }
func (pm *PollMetrics) lastNodeSuccessFor(instanceType, instance, node string) (time.Time, bool) { func (pm *PollMetrics) lastNodeSuccessFor(instanceType, instance, node string) (time.Time, bool) {
pm.mu.RLock() pm.mu.RLock()
ts, ok := pm.nodeLastSuccessByKey[pm.nodeKey(instanceType, instance, node)] ts, ok := pm.nodeLastSuccessByKey[makeNodeMetricKey(instanceType, instance, node)]
pm.mu.RUnlock() pm.mu.RUnlock()
return ts, ok return ts, ok
} }
func (pm *PollMetrics) updateNodeStaleness(instanceType, instance, node string, value float64) { func (pm *PollMetrics) updateNodeStaleness(instanceType, instance, node string, value float64) {
pm.nodeStaleness.WithLabelValues(instanceType, instance, node).Set(value) instType, inst := sanitizeInstanceLabels(instanceType, instance)
} nodeLabel := normalizeLabel(node)
pm.nodeStaleness.WithLabelValues(instType, inst, nodeLabel).Set(value)
func (pm *PollMetrics) nodeKey(instanceType, instance, node string) string {
return fmt.Sprintf("%s::%s::%s", instanceType, instance, node)
} }
func splitInstanceKey(key string) (string, string) { func splitInstanceKey(key string) (string, string) {
parts := strings.SplitN(key, "::", 2) parts := strings.SplitN(key, "::", 2)
if len(parts) == 2 { if len(parts) == 2 {
if parts[0] == "" { return normalizeLabel(parts[0]), normalizeLabel(parts[1])
parts[0] = "unknown" }
} if key == "" {
if parts[1] == "" { return "unknown", "unknown"
parts[1] = "unknown" }
} return "unknown", normalizeLabel(key)
return parts[0], parts[1]
}
if key == "" {
return "unknown", "unknown"
}
return "unknown", key
} }
func breakerStateToValue(state string) float64 { func breakerStateToValue(state string) float64 {
switch strings.ToLower(state) { switch strings.ToLower(state) {
case "closed": case "closed":
return 0 return 0
case "half_open", "half-open": case "half_open", "half-open":
return 1 return 1
case "open": case "open":
return 2 return 2
default: default:
return -1 return -1
} }
}
func sanitizeInstanceLabels(instanceType, instance string) (string, string) {
return normalizeLabel(instanceType), normalizeLabel(instance)
}
func makeMetricKey(instanceType, instance string) metricKey {
instType, inst := sanitizeInstanceLabels(instanceType, instance)
return metricKey{
instanceType: instType,
instance: inst,
}
}
func makeNodeMetricKey(instanceType, instance, node string) nodeMetricKey {
instType, inst := sanitizeInstanceLabels(instanceType, instance)
return nodeMetricKey{
instanceType: instType,
instance: inst,
node: normalizeLabel(node),
}
}
func normalizeLabel(value string) string {
v := strings.TrimSpace(value)
if v == "" {
return "unknown"
}
return v
}
func normalizeNodeLabel(value string) string {
label := normalizeLabel(value)
if label == "unknown" {
return "unknown-node"
}
return label
} }
func (pm *PollMetrics) classifyError(err error) string { func (pm *PollMetrics) classifyError(err error) string {

View file

@ -421,60 +421,67 @@ func timePtr(t time.Time) *time.Time {
// Monitor handles all monitoring operations // Monitor handles all monitoring operations
type Monitor struct { type Monitor struct {
config *config.Config config *config.Config
state *models.State state *models.State
pveClients map[string]PVEClientInterface pveClients map[string]PVEClientInterface
pbsClients map[string]*pbs.Client pbsClients map[string]*pbs.Client
pmgClients map[string]*pmg.Client pmgClients map[string]*pmg.Client
pollMetrics *PollMetrics pollMetrics *PollMetrics
scheduler *AdaptiveScheduler scheduler *AdaptiveScheduler
stalenessTracker *StalenessTracker stalenessTracker *StalenessTracker
taskQueue *TaskQueue taskQueue *TaskQueue
circuitBreakers map[string]*circuitBreaker circuitBreakers map[string]*circuitBreaker
deadLetterQueue *TaskQueue deadLetterQueue *TaskQueue
failureCounts map[string]int failureCounts map[string]int
lastOutcome map[string]taskOutcome lastOutcome map[string]taskOutcome
backoffCfg backoffConfig backoffCfg backoffConfig
rng *rand.Rand rng *rand.Rand
maxRetryAttempts int maxRetryAttempts int
tempCollector *TemperatureCollector // SSH-based temperature collector tempCollector *TemperatureCollector // SSH-based temperature collector
mu sync.RWMutex mu sync.RWMutex
startTime time.Time startTime time.Time
rateTracker *RateTracker rateTracker *RateTracker
metricsHistory *MetricsHistory metricsHistory *MetricsHistory
alertManager *alerts.Manager alertManager *alerts.Manager
notificationMgr *notifications.NotificationManager notificationMgr *notifications.NotificationManager
configPersist *config.ConfigPersistence configPersist *config.ConfigPersistence
discoveryService *discovery.Service // Background discovery service discoveryService *discovery.Service // Background discovery service
activePollCount int32 // Number of active polling operations activePollCount int32 // Number of active polling operations
pollCounter int64 // Counter for polling cycles pollCounter int64 // Counter for polling cycles
authFailures map[string]int // Track consecutive auth failures per node authFailures map[string]int // Track consecutive auth failures per node
lastAuthAttempt map[string]time.Time // Track last auth attempt time lastAuthAttempt map[string]time.Time // Track last auth attempt time
lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes
lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance
lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance
lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instance lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instance
persistence *config.ConfigPersistence // Add persistence for saving updated configs persistence *config.ConfigPersistence // Add persistence for saving updated configs
pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance
runtimeCtx context.Context // Context used while monitor is running runtimeCtx context.Context // Context used while monitor is running
wsHub *websocket.Hub // Hub used for broadcasting state wsHub *websocket.Hub // Hub used for broadcasting state
diagMu sync.RWMutex // Protects diagnostic snapshot maps diagMu sync.RWMutex // Protects diagnostic snapshot maps
nodeSnapshots map[string]NodeMemorySnapshot nodeSnapshots map[string]NodeMemorySnapshot
guestSnapshots map[string]GuestMemorySnapshot guestSnapshots map[string]GuestMemorySnapshot
rrdCacheMu sync.RWMutex // Protects RRD memavailable cache rrdCacheMu sync.RWMutex // Protects RRD memavailable cache
nodeRRDMemCache map[string]rrdMemCacheEntry nodeRRDMemCache map[string]rrdMemCacheEntry
removedDockerHosts map[string]time.Time // Track deliberately removed Docker hosts (ID -> removal time) removedDockerHosts map[string]time.Time // Track deliberately removed Docker hosts (ID -> removal time)
dockerCommands map[string]*dockerHostCommand dockerCommands map[string]*dockerHostCommand
dockerCommandIndex map[string]string dockerCommandIndex map[string]string
guestMetadataMu sync.RWMutex guestMetadataMu sync.RWMutex
guestMetadataCache map[string]guestMetadataCacheEntry guestMetadataCache map[string]guestMetadataCacheEntry
executor PollExecutor guestMetadataLimiterMu sync.Mutex
breakerBaseRetry time.Duration guestMetadataLimiter map[string]time.Time
breakerMaxDelay time.Duration guestMetadataSlots chan struct{}
breakerHalfOpenWindow time.Duration guestMetadataMinRefresh time.Duration
instanceInfoCache map[string]*instanceInfo guestMetadataRefreshJitter time.Duration
pollStatusMap map[string]*pollStatus guestMetadataRetryBackoff time.Duration
dlqInsightMap map[string]*dlqInsight guestMetadataHoldDuration time.Duration
executor PollExecutor
breakerBaseRetry time.Duration
breakerMaxDelay time.Duration
breakerHalfOpenWindow time.Duration
instanceInfoCache map[string]*instanceInfo
pollStatusMap map[string]*pollStatus
dlqInsightMap map[string]*dlqInsight
} }
type rrdMemCacheEntry struct { type rrdMemCacheEntry struct {
@ -565,9 +572,13 @@ const (
dockerOfflineGraceMultiplier = 4 dockerOfflineGraceMultiplier = 4
dockerMinimumHealthWindow = 30 * time.Second dockerMinimumHealthWindow = 30 * time.Second
dockerMaximumHealthWindow = 10 * time.Minute dockerMaximumHealthWindow = 10 * time.Minute
hostOfflineGraceMultiplier = 4
hostMinimumHealthWindow = 30 * time.Second
hostMaximumHealthWindow = 10 * time.Minute
nodeRRDCacheTTL = 30 * time.Second nodeRRDCacheTTL = 30 * time.Second
nodeRRDRequestTimeout = 2 * time.Second nodeRRDRequestTimeout = 2 * time.Second
guestMetadataCacheTTL = 5 * time.Minute guestMetadataCacheTTL = 5 * time.Minute
defaultGuestMetadataHold = 15 * time.Second
) )
type guestMetadataCacheEntry struct { type guestMetadataCacheEntry struct {
@ -728,6 +739,10 @@ func (m *Monitor) RemoveHostAgent(hostID string) (models.Host, error) {
Bool("removed", removed). Bool("removed", removed).
Msg("Host agent removed from monitoring") Msg("Host agent removed from monitoring")
if m.alertManager != nil {
m.alertManager.HandleHostRemoved(host)
}
return host, nil return host, nil
} }
@ -1471,6 +1486,10 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
m.state.UpsertHost(host) m.state.UpsertHost(host)
m.state.SetConnectionHealth(hostConnectionPrefix+host.ID, true) m.state.SetConnectionHealth(hostConnectionPrefix+host.ID, true)
if m.alertManager != nil {
m.alertManager.CheckHost(host)
}
return host, nil return host, nil
} }
@ -1530,6 +1549,43 @@ func (m *Monitor) evaluateDockerAgents(now time.Time) {
} }
} }
// evaluateHostAgents updates health for host agents based on last report time.
func (m *Monitor) evaluateHostAgents(now time.Time) {
hosts := m.state.GetHosts()
for _, host := range hosts {
interval := host.IntervalSeconds
if interval <= 0 {
interval = int(hostMinimumHealthWindow / time.Second)
}
window := time.Duration(interval) * time.Second * hostOfflineGraceMultiplier
if window < hostMinimumHealthWindow {
window = hostMinimumHealthWindow
} else if window > hostMaximumHealthWindow {
window = hostMaximumHealthWindow
}
healthy := !host.LastSeen.IsZero() && now.Sub(host.LastSeen) <= window
key := hostConnectionPrefix + host.ID
m.state.SetConnectionHealth(key, healthy)
hostCopy := host
if healthy {
hostCopy.Status = "online"
m.state.SetHostStatus(host.ID, "online")
if m.alertManager != nil {
m.alertManager.HandleHostOnline(hostCopy)
}
} else {
hostCopy.Status = "offline"
m.state.SetHostStatus(host.ID, "offline")
if m.alertManager != nil {
m.alertManager.HandleHostOffline(hostCopy)
}
}
}
}
// sortContent sorts comma-separated content values for consistent display // sortContent sorts comma-separated content values for consistent display
func sortContent(content string) string { func sortContent(content string) string {
if content == "" { if content == "" {
@ -1540,6 +1596,76 @@ func sortContent(content string) string {
return strings.Join(parts, ",") return strings.Join(parts, ",")
} }
func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {
if m == nil {
return false
}
m.guestMetadataLimiterMu.Lock()
defer m.guestMetadataLimiterMu.Unlock()
if next, ok := m.guestMetadataLimiter[key]; ok && now.Before(next) {
return false
}
hold := m.guestMetadataHoldDuration
if hold <= 0 {
hold = defaultGuestMetadataHold
}
m.guestMetadataLimiter[key] = now.Add(hold)
return true
}
func (m *Monitor) scheduleNextGuestMetadataFetch(key string, now time.Time) {
if m == nil {
return
}
interval := m.guestMetadataMinRefresh
if interval <= 0 {
interval = config.DefaultGuestMetadataMinRefresh
}
jitter := m.guestMetadataRefreshJitter
if jitter > 0 && m.rng != nil {
interval += time.Duration(m.rng.Int63n(int64(jitter)))
}
m.guestMetadataLimiterMu.Lock()
m.guestMetadataLimiter[key] = now.Add(interval)
m.guestMetadataLimiterMu.Unlock()
}
func (m *Monitor) deferGuestMetadataRetry(key string, now time.Time) {
if m == nil {
return
}
backoff := m.guestMetadataRetryBackoff
if backoff <= 0 {
backoff = config.DefaultGuestMetadataRetryBackoff
}
m.guestMetadataLimiterMu.Lock()
m.guestMetadataLimiter[key] = now.Add(backoff)
m.guestMetadataLimiterMu.Unlock()
}
func (m *Monitor) acquireGuestMetadataSlot(ctx context.Context) bool {
if m == nil || m.guestMetadataSlots == nil {
return true
}
select {
case m.guestMetadataSlots <- struct{}{}:
return true
case <-ctx.Done():
return false
}
}
func (m *Monitor) releaseGuestMetadataSlot() {
if m == nil || m.guestMetadataSlots == nil {
return
}
select {
case <-m.guestMetadataSlots:
default:
}
}
func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus) ([]string, []models.GuestNetworkInterface, string, string, string) { func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus) ([]string, []models.GuestNetworkInterface, string, string, string) {
if vmStatus == nil || client == nil { if vmStatus == nil || client == nil {
m.clearGuestMetadataCache(instanceName, nodeName, vmid) m.clearGuestMetadataCache(instanceName, nodeName, vmid)
@ -1562,6 +1688,19 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion
} }
needsFetch := !ok || now.Sub(cached.fetchedAt) >= guestMetadataCacheTTL
if !needsFetch {
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion
}
reserved := m.tryReserveGuestMetadataFetch(key, now)
if !reserved && ok {
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion
}
if !reserved && !ok {
reserved = true
}
// Start with cached values as fallback in case new calls fail // Start with cached values as fallback in case new calls fail
ipAddresses := cloneStringSlice(cached.ipAddresses) ipAddresses := cloneStringSlice(cached.ipAddresses)
networkIfaces := cloneGuestNetworkInterfaces(cached.networkInterfaces) networkIfaces := cloneGuestNetworkInterfaces(cached.networkInterfaces)
@ -1569,6 +1708,17 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
osVersion := cached.osVersion osVersion := cached.osVersion
agentVersion := cached.agentVersion agentVersion := cached.agentVersion
if reserved {
if !m.acquireGuestMetadataSlot(ctx) {
m.deferGuestMetadataRetry(key, time.Now())
return ipAddresses, networkIfaces, osName, osVersion, agentVersion
}
defer m.releaseGuestMetadataSlot()
defer func() {
m.scheduleNextGuestMetadataFetch(key, time.Now())
}()
}
ifaceCtx, cancelIface := context.WithTimeout(ctx, 5*time.Second) ifaceCtx, cancelIface := context.WithTimeout(ctx, 5*time.Second)
interfaces, err := client.GetVMNetworkInterfaces(ifaceCtx, nodeName, vmid) interfaces, err := client.GetVMNetworkInterfaces(ifaceCtx, nodeName, vmid)
cancelIface() cancelIface()
@ -2584,49 +2734,72 @@ func New(cfg *config.Config) (*Monitor, error) {
}, stalenessTracker, nil, nil) }, stalenessTracker, nil, nil)
} }
minRefresh := cfg.GuestMetadataMinRefreshInterval
if minRefresh <= 0 {
minRefresh = config.DefaultGuestMetadataMinRefresh
}
jitter := cfg.GuestMetadataRefreshJitter
if jitter < 0 {
jitter = 0
}
retryBackoff := cfg.GuestMetadataRetryBackoff
if retryBackoff <= 0 {
retryBackoff = config.DefaultGuestMetadataRetryBackoff
}
concurrency := cfg.GuestMetadataMaxConcurrent
if concurrency <= 0 {
concurrency = config.DefaultGuestMetadataMaxConcurrent
}
holdDuration := defaultGuestMetadataHold
m := &Monitor{ m := &Monitor{
config: cfg, config: cfg,
state: models.NewState(), state: models.NewState(),
pveClients: make(map[string]PVEClientInterface), pveClients: make(map[string]PVEClientInterface),
pbsClients: make(map[string]*pbs.Client), pbsClients: make(map[string]*pbs.Client),
pmgClients: make(map[string]*pmg.Client), pmgClients: make(map[string]*pmg.Client),
pollMetrics: getPollMetrics(), pollMetrics: getPollMetrics(),
scheduler: scheduler, scheduler: scheduler,
stalenessTracker: stalenessTracker, stalenessTracker: stalenessTracker,
taskQueue: taskQueue, taskQueue: taskQueue,
deadLetterQueue: deadLetterQueue, deadLetterQueue: deadLetterQueue,
circuitBreakers: breakers, circuitBreakers: breakers,
failureCounts: failureCounts, failureCounts: failureCounts,
lastOutcome: lastOutcome, lastOutcome: lastOutcome,
backoffCfg: backoff, backoffCfg: backoff,
rng: rand.New(rand.NewSource(time.Now().UnixNano())), rng: rand.New(rand.NewSource(time.Now().UnixNano())),
maxRetryAttempts: 5, maxRetryAttempts: 5,
tempCollector: tempCollector, tempCollector: tempCollector,
startTime: time.Now(), startTime: time.Now(),
rateTracker: NewRateTracker(), rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points or 24 hours
alertManager: alerts.NewManager(), alertManager: alerts.NewManager(),
notificationMgr: notifications.NewNotificationManager(cfg.PublicURL), notificationMgr: notifications.NewNotificationManager(cfg.PublicURL),
configPersist: config.NewConfigPersistence(cfg.DataPath), configPersist: config.NewConfigPersistence(cfg.DataPath),
discoveryService: nil, // Will be initialized in Start() discoveryService: nil, // Will be initialized in Start()
authFailures: make(map[string]int), authFailures: make(map[string]int),
lastAuthAttempt: make(map[string]time.Time), lastAuthAttempt: make(map[string]time.Time),
lastClusterCheck: make(map[string]time.Time), lastClusterCheck: make(map[string]time.Time),
lastPhysicalDiskPoll: make(map[string]time.Time), lastPhysicalDiskPoll: make(map[string]time.Time),
lastPVEBackupPoll: make(map[string]time.Time), lastPVEBackupPoll: make(map[string]time.Time),
lastPBSBackupPoll: make(map[string]time.Time), lastPBSBackupPoll: make(map[string]time.Time),
persistence: config.NewConfigPersistence(cfg.DataPath), persistence: config.NewConfigPersistence(cfg.DataPath),
pbsBackupPollers: make(map[string]bool), pbsBackupPollers: make(map[string]bool),
nodeSnapshots: make(map[string]NodeMemorySnapshot), nodeSnapshots: make(map[string]NodeMemorySnapshot),
guestSnapshots: make(map[string]GuestMemorySnapshot), guestSnapshots: make(map[string]GuestMemorySnapshot),
nodeRRDMemCache: make(map[string]rrdMemCacheEntry), nodeRRDMemCache: make(map[string]rrdMemCacheEntry),
removedDockerHosts: make(map[string]time.Time), removedDockerHosts: make(map[string]time.Time),
dockerCommands: make(map[string]*dockerHostCommand), dockerCommands: make(map[string]*dockerHostCommand),
dockerCommandIndex: make(map[string]string), dockerCommandIndex: make(map[string]string),
guestMetadataCache: make(map[string]guestMetadataCacheEntry), guestMetadataCache: make(map[string]guestMetadataCacheEntry),
instanceInfoCache: make(map[string]*instanceInfo), guestMetadataLimiter: make(map[string]time.Time),
pollStatusMap: make(map[string]*pollStatus), guestMetadataMinRefresh: minRefresh,
dlqInsightMap: make(map[string]*dlqInsight), guestMetadataRefreshJitter: jitter,
guestMetadataRetryBackoff: retryBackoff,
guestMetadataHoldDuration: holdDuration,
instanceInfoCache: make(map[string]*instanceInfo),
pollStatusMap: make(map[string]*pollStatus),
dlqInsightMap: make(map[string]*dlqInsight),
} }
m.breakerBaseRetry = 5 * time.Second m.breakerBaseRetry = 5 * time.Second
@ -2672,6 +2845,10 @@ func New(cfg *config.Config) (*Monitor, error) {
log.Warn().Err(err).Msg("Failed to load email configuration") log.Warn().Err(err).Msg("Failed to load email configuration")
} }
if concurrency > 0 {
m.guestMetadataSlots = make(chan struct{}, concurrency)
}
if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil { if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil {
m.notificationMgr.SetAppriseConfig(*appriseConfig) m.notificationMgr.SetAppriseConfig(*appriseConfig)
} else { } else {
@ -3103,6 +3280,7 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
case <-pollTicker.C: case <-pollTicker.C:
now := time.Now() now := time.Now()
m.evaluateDockerAgents(now) m.evaluateDockerAgents(now)
m.evaluateHostAgents(now)
m.cleanupRemovedDockerHosts(now) m.cleanupRemovedDockerHosts(now)
if mock.IsMockEnabled() { if mock.IsMockEnabled() {
// In mock mode, keep synthetic alerts fresh // In mock mode, keep synthetic alerts fresh

View file

@ -0,0 +1,112 @@
package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestEvaluateHostAgentsTriggersOfflineAlert(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-offline"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "offline.local",
DisplayName: "Offline Host",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now().Add(-10 * time.Minute),
})
now := time.Now()
for i := 0; i < 3; i++ {
monitor.evaluateHostAgents(now.Add(time.Duration(i) * time.Second))
}
snapshot := monitor.state.GetSnapshot()
statusUpdated := false
for _, host := range snapshot.Hosts {
if host.ID == hostID {
statusUpdated = true
if got := host.Status; got != "offline" {
t.Fatalf("expected host status offline, got %q", got)
}
}
}
if !statusUpdated {
t.Fatalf("host %q not found in state snapshot", hostID)
}
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || healthy {
t.Fatalf("expected connection health false, got %v (exists=%v)", healthy, ok)
}
alerts := monitor.alertManager.GetActiveAlerts()
found := false
for _, alert := range alerts {
if alert.ID == "host-offline-"+hostID {
found = true
break
}
}
if !found {
t.Fatalf("expected host offline alert to remain active")
}
}
func TestEvaluateHostAgentsClearsAlertWhenHostReturns(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-recover"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "recover.local",
DisplayName: "Recover Host",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now().Add(-10 * time.Minute),
})
for i := 0; i < 3; i++ {
monitor.evaluateHostAgents(time.Now().Add(time.Duration(i) * time.Second))
}
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "recover.local",
DisplayName: "Recover Host",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now(),
})
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true after recovery, got %v (exists=%v)", healthy, ok)
}
for _, alert := range monitor.alertManager.GetActiveAlerts() {
if alert.ID == "host-offline-"+hostID {
t.Fatalf("offline alert still active after recovery")
}
}
}

View file

@ -104,6 +104,39 @@ func (q *TaskQueue) Remove(instanceType InstanceType, instance string) {
// WaitNext blocks until a task is due or context is cancelled. // WaitNext blocks until a task is due or context is cancelled.
func (q *TaskQueue) WaitNext(ctx context.Context) (ScheduledTask, bool) { func (q *TaskQueue) WaitNext(ctx context.Context) (ScheduledTask, bool) {
var (
timer *time.Timer
resetTimer = func(d time.Duration) <-chan time.Time {
if d <= 0 {
d = time.Millisecond
}
if timer == nil {
timer = time.NewTimer(d)
return timer.C
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
timer.Reset(d)
return timer.C
}
stopTimer = func() {
if timer == nil {
return
}
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
}
)
defer stopTimer()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -114,10 +147,11 @@ func (q *TaskQueue) WaitNext(ctx context.Context) (ScheduledTask, bool) {
q.mu.Lock() q.mu.Lock()
if len(q.heap) == 0 { if len(q.heap) == 0 {
q.mu.Unlock() q.mu.Unlock()
timerCh := resetTimer(100 * time.Millisecond)
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ScheduledTask{}, false return ScheduledTask{}, false
case <-time.After(100 * time.Millisecond): case <-timerCh:
continue continue
} }
} }
@ -136,12 +170,11 @@ func (q *TaskQueue) WaitNext(ctx context.Context) (ScheduledTask, bool) {
if delay > 250*time.Millisecond { if delay > 250*time.Millisecond {
delay = 250 * time.Millisecond delay = 250 * time.Millisecond
} }
timer := time.NewTimer(delay) timerCh := resetTimer(delay)
select { select {
case <-ctx.Done(): case <-ctx.Done():
timer.Stop()
return ScheduledTask{}, false return ScheduledTask{}, false
case <-timer.C: case <-timerCh:
} }
} }
} }