refactor: simplify singleton pattern
This commit is contained in:
parent
23a2a168be
commit
af4ac1c39c
12 changed files with 266 additions and 270 deletions
|
|
@ -11,6 +11,7 @@
|
||||||
!**/components.json
|
!**/components.json
|
||||||
|
|
||||||
!app/**
|
!app/**
|
||||||
|
!apps/agent/**
|
||||||
!packages/**
|
!packages/**
|
||||||
!public/**
|
!public/**
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -64,6 +64,8 @@ COPY --from=deps /deps/shoutrrr /usr/local/bin/shoutrrr
|
||||||
|
|
||||||
COPY ./package.json ./bun.lock ./
|
COPY ./package.json ./bun.lock ./
|
||||||
COPY ./packages/core/package.json ./packages/core/package.json
|
COPY ./packages/core/package.json ./packages/core/package.json
|
||||||
|
COPY ./packages/contracts/package.json ./packages/contracts/package.json
|
||||||
|
COPY ./apps/agent/package.json ./apps/agent/package.json
|
||||||
|
|
||||||
RUN bun install --frozen-lockfile --ignore-scripts
|
RUN bun install --frozen-lockfile --ignore-scripts
|
||||||
|
|
||||||
|
|
@ -86,6 +88,8 @@ WORKDIR /app
|
||||||
|
|
||||||
COPY ./package.json ./bun.lock ./
|
COPY ./package.json ./bun.lock ./
|
||||||
COPY ./packages/core/package.json ./packages/core/package.json
|
COPY ./packages/core/package.json ./packages/core/package.json
|
||||||
|
COPY ./packages/contracts/package.json ./packages/contracts/package.json
|
||||||
|
COPY ./apps/agent/package.json ./apps/agent/package.json
|
||||||
RUN bun install --frozen-lockfile
|
RUN bun install --frozen-lockfile
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
import { afterEach, describe, expect, mock, test } from "bun:test";
|
|
||||||
|
|
||||||
const loadAgentsManagerModule = async () => {
|
|
||||||
const moduleUrl = new URL("../agents-manager.ts", import.meta.url);
|
|
||||||
moduleUrl.searchParams.set("test", crypto.randomUUID());
|
|
||||||
return import(moduleUrl.href);
|
|
||||||
};
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
delete (globalThis as Record<string, unknown>).__agentManager;
|
|
||||||
mock.restore();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("agents-manager module", () => {
|
|
||||||
test("reuses the existing global agent manager without stopping it", async () => {
|
|
||||||
const existingManager = {
|
|
||||||
start: mock(() => {}),
|
|
||||||
sendBackup: mock(() => false),
|
|
||||||
cancelBackup: mock(() => false),
|
|
||||||
setBackupEventHandlers: mock(() => {}),
|
|
||||||
getBackupEventHandlers: mock(() => ({})),
|
|
||||||
stop: mock(() => {}),
|
|
||||||
};
|
|
||||||
|
|
||||||
(globalThis as Record<string, unknown>).__agentManager = existingManager;
|
|
||||||
|
|
||||||
const { agentManager } = await loadAgentsManagerModule();
|
|
||||||
|
|
||||||
expect(agentManager).toBe(existingManager);
|
|
||||||
expect(existingManager.stop).not.toHaveBeenCalled();
|
|
||||||
expect((globalThis as Record<string, unknown>).__agentManager).toBe(existingManager);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -14,14 +14,11 @@ import type {
|
||||||
} from "@zerobyte/contracts/agent-protocol";
|
} from "@zerobyte/contracts/agent-protocol";
|
||||||
import { config } from "../../core/config";
|
import { config } from "../../core/config";
|
||||||
import { validateAgentToken, deriveLocalAgentToken } from "./agent-tokens";
|
import { validateAgentToken, deriveLocalAgentToken } from "./agent-tokens";
|
||||||
import { createControllerAgentSession, type ControllerAgentSession } from "./controller-agent-session";
|
import {
|
||||||
|
createControllerAgentSession,
|
||||||
type AgentConnectionData = {
|
type AgentConnectionData,
|
||||||
id: string;
|
type ControllerAgentSession,
|
||||||
agentId: string;
|
} from "./controller-agent-session";
|
||||||
organizationId: string | null;
|
|
||||||
agentName: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type AgentBackupEventContext = {
|
type AgentBackupEventContext = {
|
||||||
agentId: string;
|
agentId: string;
|
||||||
|
|
@ -42,12 +39,38 @@ export type AgentBackupEventHandlers = {
|
||||||
onBackupCancelled?: (context: AgentBackupEventContext & { payload: BackupCancelledPayload }) => void;
|
onBackupCancelled?: (context: AgentBackupEventContext & { payload: BackupCancelledPayload }) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const spawnLocalAgent = async () => {
|
type AgentManagerRuntime = ReturnType<typeof createAgentManagerRuntime>;
|
||||||
const previousAgent = (globalThis as Record<string, unknown>).__localAgent as ChildProcess | undefined;
|
type AgentRuntimeState = {
|
||||||
if (previousAgent) {
|
agentManager: AgentManagerRuntime;
|
||||||
previousAgent.kill();
|
localAgent: ChildProcess | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ProcessWithAgentRuntime = NodeJS.Process & {
|
||||||
|
__zerobyteAgentRuntime?: AgentRuntimeState;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getAgentRuntimeState = () => {
|
||||||
|
const runtimeProcess = process as ProcessWithAgentRuntime;
|
||||||
|
const existingRuntime = runtimeProcess.__zerobyteAgentRuntime;
|
||||||
|
|
||||||
|
if (existingRuntime) {
|
||||||
|
return existingRuntime;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const runtime = {
|
||||||
|
agentManager: createAgentManagerRuntime(),
|
||||||
|
localAgent: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
runtimeProcess.__zerobyteAgentRuntime = runtime;
|
||||||
|
return runtime;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getAgentManagerRuntime = () => getAgentRuntimeState().agentManager;
|
||||||
|
|
||||||
|
export const spawnLocalAgent = async () => {
|
||||||
|
await stopLocalAgent();
|
||||||
|
|
||||||
const sourceEntryPoint = path.join(process.cwd(), "apps", "agent", "src", "index.ts");
|
const sourceEntryPoint = path.join(process.cwd(), "apps", "agent", "src", "index.ts");
|
||||||
const productionEntryPoint = path.join(process.cwd(), ".output", "agent", "index.mjs");
|
const productionEntryPoint = path.join(process.cwd(), ".output", "agent", "index.mjs");
|
||||||
|
|
||||||
|
|
@ -59,7 +82,8 @@ export const spawnLocalAgent = async () => {
|
||||||
const agentToken = await deriveLocalAgentToken();
|
const agentToken = await deriveLocalAgentToken();
|
||||||
const args = config.__prod__ ? ["run", agentEntryPoint] : ["run", "--watch", agentEntryPoint];
|
const args = config.__prod__ ? ["run", agentEntryPoint] : ["run", "--watch", agentEntryPoint];
|
||||||
|
|
||||||
const localAgent = spawn("bun", args, {
|
const runtime = getAgentRuntimeState();
|
||||||
|
const agentProcess = spawn("bun", args, {
|
||||||
env: {
|
env: {
|
||||||
...process.env,
|
...process.env,
|
||||||
ZEROBYTE_CONTROLLER_URL: "ws://localhost:3001",
|
ZEROBYTE_CONTROLLER_URL: "ws://localhost:3001",
|
||||||
|
|
@ -68,23 +92,49 @@ export const spawnLocalAgent = async () => {
|
||||||
stdio: ["ignore", "pipe", "pipe"],
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
});
|
});
|
||||||
|
|
||||||
(globalThis as Record<string, unknown>).__localAgent = localAgent;
|
runtime.localAgent = agentProcess;
|
||||||
|
|
||||||
localAgent.stdout?.on("data", (data: Buffer) => {
|
agentProcess.stdout?.on("data", (data: Buffer) => {
|
||||||
const line = data.toString().trim();
|
const line = data.toString().trim();
|
||||||
if (line) logger.info(`[agent] ${line}`);
|
if (line) logger.info(`[agent] ${line}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
localAgent.stderr?.on("data", (data: Buffer) => {
|
agentProcess.stderr?.on("data", (data: Buffer) => {
|
||||||
const line = data.toString().trim();
|
const line = data.toString().trim();
|
||||||
if (line) logger.error(`[agent] ${line}`);
|
if (line) logger.error(`[agent] ${line}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
localAgent.on("exit", (code, signal) => {
|
agentProcess.on("exit", (code, signal) => {
|
||||||
|
if (runtime.localAgent === agentProcess) {
|
||||||
|
runtime.localAgent = null;
|
||||||
|
}
|
||||||
logger.info(`Agent process exited with code ${code} and signal ${signal}`);
|
logger.info(`Agent process exited with code ${code} and signal ${signal}`);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const stopLocalAgent = async () => {
|
||||||
|
const runtime = getAgentRuntimeState();
|
||||||
|
if (!runtime.localAgent) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const agentProcess = runtime.localAgent;
|
||||||
|
runtime.localAgent = null;
|
||||||
|
|
||||||
|
if (agentProcess.exitCode !== null || agentProcess.signalCode !== null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const exited = new Promise<void>((resolve) => {
|
||||||
|
agentProcess.once("exit", () => {
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
agentProcess.kill();
|
||||||
|
await exited;
|
||||||
|
};
|
||||||
|
|
||||||
const createAgentManagerRuntime = () => {
|
const createAgentManagerRuntime = () => {
|
||||||
const sessionsRef = Effect.runSync(Ref.make<Map<string, ControllerAgentSession>>(new Map()));
|
const sessionsRef = Effect.runSync(Ref.make<Map<string, ControllerAgentSession>>(new Map()));
|
||||||
const backupHandlersRef = Effect.runSync(Ref.make<AgentBackupEventHandlers>({}));
|
const backupHandlersRef = Effect.runSync(Ref.make<AgentBackupEventHandlers>({}));
|
||||||
|
|
@ -290,9 +340,9 @@ const createAgentManagerRuntime = () => {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const previous = (globalThis as Record<string, unknown>).__agentManager as
|
export const agentManager = getAgentManagerRuntime();
|
||||||
| ReturnType<typeof createAgentManagerRuntime>
|
|
||||||
| undefined;
|
|
||||||
|
|
||||||
export const agentManager = previous ?? createAgentManagerRuntime();
|
export const stopAgentRuntime = async () => {
|
||||||
(globalThis as Record<string, unknown>).__agentManager = agentManager;
|
getAgentManagerRuntime().stop();
|
||||||
|
await stopLocalAgent();
|
||||||
|
};
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ import {
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { toMessage } from "@zerobyte/core/utils";
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
|
||||||
type AgentConnectionData = {
|
export type AgentConnectionData = {
|
||||||
id: string;
|
id: string;
|
||||||
agentId: string;
|
agentId: string;
|
||||||
organizationId: string | null;
|
organizationId: string | null;
|
||||||
|
|
@ -123,14 +123,15 @@ export const createControllerAgentSession = (
|
||||||
);
|
);
|
||||||
|
|
||||||
const handleAgentMessage = (message: AgentMessage) => {
|
const handleAgentMessage = (message: AgentMessage) => {
|
||||||
|
updateState((current) => ({ ...current, lastSeenAt: Date.now() }));
|
||||||
|
|
||||||
switch (message.type) {
|
switch (message.type) {
|
||||||
case "agent.ready": {
|
case "agent.ready": {
|
||||||
updateState((current) => ({ ...current, isReady: true, lastSeenAt: Date.now() }));
|
updateState((current) => ({ ...current, isReady: true }));
|
||||||
logger.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
|
logger.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "backup.started": {
|
case "backup.started": {
|
||||||
updateState((current) => ({ ...current, lastSeenAt: Date.now() }));
|
|
||||||
setActiveBackupJob(message.payload.jobId, message.payload.scheduleId);
|
setActiveBackupJob(message.payload.jobId, message.payload.scheduleId);
|
||||||
logger.info(
|
logger.info(
|
||||||
`Backup ${message.payload.jobId} started on agent ${socket.data.agentId} for schedule ${message.payload.scheduleId}`,
|
`Backup ${message.payload.jobId} started on agent ${socket.data.agentId} for schedule ${message.payload.scheduleId}`,
|
||||||
|
|
@ -139,34 +140,26 @@ export const createControllerAgentSession = (
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "backup.progress": {
|
case "backup.progress": {
|
||||||
updateState((current) => ({ ...current, lastSeenAt: Date.now() }));
|
|
||||||
handlers.onBackupProgress?.(message.payload);
|
handlers.onBackupProgress?.(message.payload);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "backup.completed": {
|
case "backup.completed": {
|
||||||
updateState((current) => ({ ...current, lastSeenAt: Date.now() }));
|
|
||||||
deleteActiveBackupJob(message.payload.jobId);
|
deleteActiveBackupJob(message.payload.jobId);
|
||||||
handlers.onBackupCompleted?.(message.payload);
|
handlers.onBackupCompleted?.(message.payload);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "backup.failed": {
|
case "backup.failed": {
|
||||||
updateState((current) => ({ ...current, lastSeenAt: Date.now() }));
|
|
||||||
deleteActiveBackupJob(message.payload.jobId);
|
deleteActiveBackupJob(message.payload.jobId);
|
||||||
handlers.onBackupFailed?.(message.payload);
|
handlers.onBackupFailed?.(message.payload);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "backup.cancelled": {
|
case "backup.cancelled": {
|
||||||
updateState((current) => ({ ...current, lastSeenAt: Date.now() }));
|
|
||||||
deleteActiveBackupJob(message.payload.jobId);
|
deleteActiveBackupJob(message.payload.jobId);
|
||||||
handlers.onBackupCancelled?.(message.payload);
|
handlers.onBackupCancelled?.(message.payload);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "heartbeat.pong": {
|
case "heartbeat.pong": {
|
||||||
updateState((current) => ({
|
updateState((current) => ({ ...current, lastPongAt: message.payload.sentAt }));
|
||||||
...current,
|
|
||||||
lastSeenAt: Date.now(),
|
|
||||||
lastPongAt: message.payload.sentAt,
|
|
||||||
}));
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -190,19 +183,11 @@ export const createControllerAgentSession = (
|
||||||
handleAgentMessage(parsed.data);
|
handleAgentMessage(parsed.data);
|
||||||
},
|
},
|
||||||
sendBackup: (payload) => {
|
sendBackup: (payload) => {
|
||||||
offerOutbound(
|
offerOutbound(createControllerMessage("backup.run", payload));
|
||||||
createControllerMessage("backup.run", {
|
|
||||||
...payload,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
return payload.jobId;
|
return payload.jobId;
|
||||||
},
|
},
|
||||||
sendBackupCancel: (payload) => {
|
sendBackupCancel: (payload) => {
|
||||||
offerOutbound(
|
offerOutbound(createControllerMessage("backup.cancel", payload));
|
||||||
createControllerMessage("backup.cancel", {
|
|
||||||
...payload,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
},
|
},
|
||||||
isReady: () => Effect.runSync(Ref.get(state)).isReady,
|
isReady: () => Effect.runSync(Ref.get(state)).isReady,
|
||||||
close: () => {
|
close: () => {
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ import { NotFoundError, BadRequestError } from "http-errors-enhanced";
|
||||||
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
||||||
import { repoMutex } from "~/server/core/repository-mutex";
|
import { repoMutex } from "~/server/core/repository-mutex";
|
||||||
import { agentManager } from "~/server/modules/agents/agents-manager";
|
import { agentManager } from "~/server/modules/agents/agents-manager";
|
||||||
import { fromAny } from "@total-typescript/shoehorn";
|
import { createAgentBackupMocks } from "~/test/helpers/agent-mock";
|
||||||
|
|
||||||
const setup = () => {
|
const setup = () => {
|
||||||
const resticBackupMock = vi.fn((_: SafeSpawnParams) =>
|
const resticBackupMock = vi.fn((_: SafeSpawnParams) =>
|
||||||
|
|
@ -24,94 +24,7 @@ const setup = () => {
|
||||||
);
|
);
|
||||||
const resticForgetMock = vi.fn(() => Promise.resolve({ success: true, data: null }));
|
const resticForgetMock = vi.fn(() => Promise.resolve({ success: true, data: null }));
|
||||||
const resticCopyMock = vi.fn(() => Promise.resolve({ success: true, output: "" }));
|
const resticCopyMock = vi.fn(() => Promise.resolve({ success: true, output: "" }));
|
||||||
const runningJobs = new Map<string, { scheduleId: string; cancelled: boolean }>();
|
const { sendBackupMock, cancelBackupMock } = createAgentBackupMocks(resticBackupMock);
|
||||||
const sendBackupMock = vi.fn((_agentId: string, payload: { jobId: string; scheduleId: string }) => {
|
|
||||||
const handlers = agentManager.getBackupEventHandlers();
|
|
||||||
|
|
||||||
runningJobs.set(payload.jobId, { scheduleId: payload.scheduleId, cancelled: false });
|
|
||||||
|
|
||||||
handlers.onBackupStarted?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: { jobId: payload.jobId, scheduleId: payload.scheduleId },
|
|
||||||
});
|
|
||||||
|
|
||||||
void (async () => {
|
|
||||||
const stderrLines: string[] = [];
|
|
||||||
const result = await resticBackupMock(
|
|
||||||
fromAny({
|
|
||||||
onStderr: (line: string) => {
|
|
||||||
stderrLines.push(line);
|
|
||||||
},
|
|
||||||
} satisfies Partial<SafeSpawnParams>),
|
|
||||||
);
|
|
||||||
const running = runningJobs.get(payload.jobId);
|
|
||||||
if (!running || running.cancelled) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.exitCode === 0 || result.exitCode === 3) {
|
|
||||||
let parsedResult: Record<string, unknown> | null = null;
|
|
||||||
if (result.summary) {
|
|
||||||
try {
|
|
||||||
parsedResult = JSON.parse(result.summary) as Record<string, unknown>;
|
|
||||||
} catch {
|
|
||||||
parsedResult = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
handlers.onBackupCompleted?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: {
|
|
||||||
jobId: payload.jobId,
|
|
||||||
scheduleId: payload.scheduleId,
|
|
||||||
exitCode: result.exitCode,
|
|
||||||
result: fromAny(parsedResult),
|
|
||||||
warningDetails: stderrLines.join("\n") || undefined,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
const resultWithStderr = result as typeof result & { stderr?: string };
|
|
||||||
const errorDetails = stderrLines.join("\n") || resultWithStderr.stderr || result.error;
|
|
||||||
|
|
||||||
handlers.onBackupFailed?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: {
|
|
||||||
jobId: payload.jobId,
|
|
||||||
scheduleId: payload.scheduleId,
|
|
||||||
error: result.error || `Backup failed with code ${result.exitCode}`,
|
|
||||||
errorDetails,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
runningJobs.delete(payload.jobId);
|
|
||||||
})().catch(() => {});
|
|
||||||
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
const cancelBackupMock = vi.fn((_agentId: string, payload: { jobId: string; scheduleId: string }) => {
|
|
||||||
const running = runningJobs.get(payload.jobId);
|
|
||||||
if (!running) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
running.cancelled = true;
|
|
||||||
const handlers = agentManager.getBackupEventHandlers();
|
|
||||||
handlers.onBackupCancelled?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: {
|
|
||||||
jobId: payload.jobId,
|
|
||||||
scheduleId: payload.scheduleId,
|
|
||||||
message: "Backup was stopped by user",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
runningJobs.delete(payload.jobId);
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
const refreshStatsMock = vi.fn(() =>
|
const refreshStatsMock = vi.fn(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
total_size: 0,
|
total_size: 0,
|
||||||
|
|
|
||||||
|
|
@ -15,98 +15,11 @@ import * as context from "~/server/core/request-context";
|
||||||
import { backupsExecutionService } from "../backups.execution";
|
import { backupsExecutionService } from "../backups.execution";
|
||||||
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
||||||
import { agentManager } from "~/server/modules/agents/agents-manager";
|
import { agentManager } from "~/server/modules/agents/agents-manager";
|
||||||
import { fromAny } from "@total-typescript/shoehorn";
|
import { createAgentBackupMocks } from "~/test/helpers/agent-mock";
|
||||||
|
|
||||||
const setup = () => {
|
const setup = () => {
|
||||||
const resticBackupMock = vi.fn((_: unknown) => Promise.resolve({ exitCode: 0, summary: "", error: "" }));
|
const resticBackupMock = vi.fn((_: unknown) => Promise.resolve({ exitCode: 0, summary: "", error: "" }));
|
||||||
const runningJobs = new Map<string, { scheduleId: string; cancelled: boolean }>();
|
const { sendBackupMock, cancelBackupMock } = createAgentBackupMocks(resticBackupMock);
|
||||||
const sendBackupMock = vi.fn((_agentId: string, payload: { jobId: string; scheduleId: string }) => {
|
|
||||||
const handlers = agentManager.getBackupEventHandlers();
|
|
||||||
|
|
||||||
runningJobs.set(payload.jobId, { scheduleId: payload.scheduleId, cancelled: false });
|
|
||||||
|
|
||||||
handlers.onBackupStarted?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: { jobId: payload.jobId, scheduleId: payload.scheduleId },
|
|
||||||
});
|
|
||||||
|
|
||||||
void (async () => {
|
|
||||||
const stderrLines: string[] = [];
|
|
||||||
const result = await resticBackupMock(
|
|
||||||
fromAny({
|
|
||||||
onStderr: (line: string) => {
|
|
||||||
stderrLines.push(line);
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
const running = runningJobs.get(payload.jobId);
|
|
||||||
if (!running || running.cancelled) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.exitCode === 0 || result.exitCode === 3) {
|
|
||||||
let parsedResult: Record<string, unknown> | null = null;
|
|
||||||
if (result.summary) {
|
|
||||||
try {
|
|
||||||
parsedResult = JSON.parse(result.summary) as Record<string, unknown>;
|
|
||||||
} catch {
|
|
||||||
parsedResult = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
handlers.onBackupCompleted?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: {
|
|
||||||
jobId: payload.jobId,
|
|
||||||
scheduleId: payload.scheduleId,
|
|
||||||
exitCode: result.exitCode,
|
|
||||||
result: fromAny(parsedResult),
|
|
||||||
warningDetails: stderrLines.join("\n") || undefined,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
const resultWithStderr = result as typeof result & { stderr?: string };
|
|
||||||
const errorDetails = stderrLines.join("\n") || resultWithStderr.stderr || result.error;
|
|
||||||
|
|
||||||
handlers.onBackupFailed?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: {
|
|
||||||
jobId: payload.jobId,
|
|
||||||
scheduleId: payload.scheduleId,
|
|
||||||
error: result.error || `Backup failed with code ${result.exitCode}`,
|
|
||||||
errorDetails,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
runningJobs.delete(payload.jobId);
|
|
||||||
})().catch(() => {});
|
|
||||||
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
const cancelBackupMock = vi.fn((_agentId: string, payload: { jobId: string; scheduleId: string }) => {
|
|
||||||
const running = runningJobs.get(payload.jobId);
|
|
||||||
if (!running) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
running.cancelled = true;
|
|
||||||
const handlers = agentManager.getBackupEventHandlers();
|
|
||||||
handlers.onBackupCancelled?.({
|
|
||||||
agentId: "local",
|
|
||||||
agentName: "local",
|
|
||||||
payload: {
|
|
||||||
jobId: payload.jobId,
|
|
||||||
scheduleId: payload.scheduleId,
|
|
||||||
message: "Backup was stopped by user",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
runningJobs.delete(payload.jobId);
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
const refreshStatsMock = vi.fn(() =>
|
const refreshStatsMock = vi.fn(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
total_size: 0,
|
total_size: 0,
|
||||||
|
|
|
||||||
58
app/server/modules/lifecycle/__tests__/shutdown.test.ts
Normal file
58
app/server/modules/lifecycle/__tests__/shutdown.test.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
import { afterEach, describe, expect, mock, spyOn, test } from "bun:test";
|
||||||
|
import { Scheduler } from "../../../core/scheduler";
|
||||||
|
import * as agentsManagerModule from "../../agents/agents-manager";
|
||||||
|
import * as backendModule from "../../backends/backend";
|
||||||
|
import type { VolumeBackend } from "../../backends/backend";
|
||||||
|
import { createTestVolume } from "~/test/helpers/volume";
|
||||||
|
|
||||||
|
const loadShutdownModule = async () => {
|
||||||
|
const moduleUrl = new URL("../shutdown.ts", import.meta.url);
|
||||||
|
moduleUrl.searchParams.set("test", crypto.randomUUID());
|
||||||
|
return import(moduleUrl.href);
|
||||||
|
};
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
mock.restore();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("shutdown", () => {
|
||||||
|
test("stops the agent runtime before unmounting mounted volumes", async () => {
|
||||||
|
const events: string[] = [];
|
||||||
|
const stopScheduler = mock(async () => {
|
||||||
|
events.push("scheduler.stop");
|
||||||
|
});
|
||||||
|
const stopAgentRuntime = mock(async () => {
|
||||||
|
events.push("agents.stop");
|
||||||
|
});
|
||||||
|
const unmountVolume = mock(async () => {
|
||||||
|
events.push("backend.unmount");
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
});
|
||||||
|
|
||||||
|
await createTestVolume({
|
||||||
|
name: "Shutdown test volume",
|
||||||
|
config: {
|
||||||
|
backend: "directory",
|
||||||
|
path: "/Applications",
|
||||||
|
},
|
||||||
|
status: "mounted",
|
||||||
|
});
|
||||||
|
|
||||||
|
spyOn(Scheduler, "stop").mockImplementation(stopScheduler);
|
||||||
|
spyOn(agentsManagerModule, "stopAgentRuntime").mockImplementation(stopAgentRuntime);
|
||||||
|
spyOn(backendModule, "createVolumeBackend").mockImplementation(
|
||||||
|
() =>
|
||||||
|
({
|
||||||
|
mount: async () => ({ status: "mounted" as const }),
|
||||||
|
unmount: unmountVolume,
|
||||||
|
checkHealth: async () => ({ status: "mounted" as const }),
|
||||||
|
}) satisfies VolumeBackend,
|
||||||
|
);
|
||||||
|
|
||||||
|
const { shutdown } = await loadShutdownModule();
|
||||||
|
|
||||||
|
await shutdown();
|
||||||
|
|
||||||
|
expect(events).toEqual(["scheduler.stop", "agents.stop", "backend.unmount"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -2,9 +2,11 @@ import { Scheduler } from "../../core/scheduler";
|
||||||
import { db } from "../../db/db";
|
import { db } from "../../db/db";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { createVolumeBackend } from "../backends/backend";
|
import { createVolumeBackend } from "../backends/backend";
|
||||||
|
import { stopAgentRuntime } from "../agents/agents-manager";
|
||||||
|
|
||||||
export const shutdown = async () => {
|
export const shutdown = async () => {
|
||||||
await Scheduler.stop();
|
await Scheduler.stop();
|
||||||
|
await stopAgentRuntime();
|
||||||
|
|
||||||
const volumes = await db.query.volumesTable.findMany({
|
const volumes = await db.query.volumesTable.findMany({
|
||||||
where: { status: "mounted" },
|
where: { status: "mounted" },
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,20 @@ import { definePlugin } from "nitro";
|
||||||
import { bootstrapApplication } from "../modules/lifecycle/bootstrap";
|
import { bootstrapApplication } from "../modules/lifecycle/bootstrap";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { toMessage } from "../utils/errors";
|
import { toMessage } from "../utils/errors";
|
||||||
|
import { stopAgentRuntime } from "../modules/agents/agents-manager";
|
||||||
|
|
||||||
|
type ProcessWithAgentCloseHook = NodeJS.Process & {
|
||||||
|
__zerobyteAgentRuntimeCloseHookRegistered?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export default definePlugin(async (nitroApp) => {
|
||||||
|
const runtimeProcess = process as ProcessWithAgentCloseHook;
|
||||||
|
|
||||||
|
if (!runtimeProcess.__zerobyteAgentRuntimeCloseHookRegistered) {
|
||||||
|
nitroApp.hooks.hook("close", stopAgentRuntime);
|
||||||
|
runtimeProcess.__zerobyteAgentRuntimeCloseHookRegistered = true;
|
||||||
|
}
|
||||||
|
|
||||||
export default definePlugin(async () => {
|
|
||||||
await bootstrapApplication().catch((err) => {
|
await bootstrapApplication().catch((err) => {
|
||||||
logger.error(`Bootstrap failed: ${toMessage(err)}`);
|
logger.error(`Bootstrap failed: ${toMessage(err)}`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|
|
||||||
105
app/test/helpers/agent-mock.ts
Normal file
105
app/test/helpers/agent-mock.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
import { mock } from "bun:test";
|
||||||
|
import { fromAny } from "@total-typescript/shoehorn";
|
||||||
|
import { agentManager } from "~/server/modules/agents/agents-manager";
|
||||||
|
|
||||||
|
export const createAgentBackupMocks = (
|
||||||
|
resticBackupMock: (params: never) => Promise<{
|
||||||
|
exitCode: number;
|
||||||
|
summary: string;
|
||||||
|
error: string;
|
||||||
|
stderr?: string;
|
||||||
|
}>,
|
||||||
|
) => {
|
||||||
|
const runningJobs = new Map<string, { scheduleId: string; cancelled: boolean }>();
|
||||||
|
|
||||||
|
const sendBackupMock = mock((_agentId: string, payload: { jobId: string; scheduleId: string }) => {
|
||||||
|
const handlers = agentManager.getBackupEventHandlers();
|
||||||
|
|
||||||
|
runningJobs.set(payload.jobId, { scheduleId: payload.scheduleId, cancelled: false });
|
||||||
|
|
||||||
|
handlers.onBackupStarted?.({
|
||||||
|
agentId: "local",
|
||||||
|
agentName: "local",
|
||||||
|
payload: { jobId: payload.jobId, scheduleId: payload.scheduleId },
|
||||||
|
});
|
||||||
|
|
||||||
|
void (async () => {
|
||||||
|
const stderrLines: string[] = [];
|
||||||
|
const result = await resticBackupMock(
|
||||||
|
fromAny({
|
||||||
|
onStderr: (line: string) => {
|
||||||
|
stderrLines.push(line);
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const running = runningJobs.get(payload.jobId);
|
||||||
|
if (!running || running.cancelled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.exitCode === 0 || result.exitCode === 3) {
|
||||||
|
let parsedResult: Record<string, unknown> | null = null;
|
||||||
|
if (result.summary) {
|
||||||
|
try {
|
||||||
|
parsedResult = JSON.parse(result.summary) as Record<string, unknown>;
|
||||||
|
} catch {
|
||||||
|
parsedResult = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handlers.onBackupCompleted?.({
|
||||||
|
agentId: "local",
|
||||||
|
agentName: "local",
|
||||||
|
payload: {
|
||||||
|
jobId: payload.jobId,
|
||||||
|
scheduleId: payload.scheduleId,
|
||||||
|
exitCode: result.exitCode,
|
||||||
|
result: fromAny(parsedResult),
|
||||||
|
warningDetails: stderrLines.join("\n") || undefined,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
const resultWithStderr = result as typeof result & { stderr?: string };
|
||||||
|
const errorDetails = stderrLines.join("\n") || resultWithStderr.stderr || result.error;
|
||||||
|
|
||||||
|
handlers.onBackupFailed?.({
|
||||||
|
agentId: "local",
|
||||||
|
agentName: "local",
|
||||||
|
payload: {
|
||||||
|
jobId: payload.jobId,
|
||||||
|
scheduleId: payload.scheduleId,
|
||||||
|
error: result.error || `Backup failed with code ${result.exitCode}`,
|
||||||
|
errorDetails,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
runningJobs.delete(payload.jobId);
|
||||||
|
})().catch(() => {});
|
||||||
|
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
const cancelBackupMock = mock((_agentId: string, payload: { jobId: string; scheduleId: string }) => {
|
||||||
|
const running = runningJobs.get(payload.jobId);
|
||||||
|
if (!running) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
running.cancelled = true;
|
||||||
|
const handlers = agentManager.getBackupEventHandlers();
|
||||||
|
handlers.onBackupCancelled?.({
|
||||||
|
agentId: "local",
|
||||||
|
agentName: "local",
|
||||||
|
payload: {
|
||||||
|
jobId: payload.jobId,
|
||||||
|
scheduleId: payload.scheduleId,
|
||||||
|
message: "Backup was stopped by user",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
runningJobs.delete(payload.jobId);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
return { sendBackupMock, cancelBackupMock };
|
||||||
|
};
|
||||||
|
|
@ -168,10 +168,6 @@ type Brand<TValue, TBrand extends string> = TValue & {
|
||||||
readonly __brand: TBrand;
|
readonly __brand: TBrand;
|
||||||
};
|
};
|
||||||
|
|
||||||
type MessageSender = {
|
|
||||||
send(message: string): unknown;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ControllerWireMessage = Brand<string, "ControllerWireMessage">;
|
export type ControllerWireMessage = Brand<string, "ControllerWireMessage">;
|
||||||
export type AgentWireMessage = Brand<string, "AgentWireMessage">;
|
export type AgentWireMessage = Brand<string, "AgentWireMessage">;
|
||||||
|
|
||||||
|
|
@ -221,13 +217,3 @@ export const createAgentMessage = <TType extends AgentMessage["type"]>(
|
||||||
payload,
|
payload,
|
||||||
}),
|
}),
|
||||||
) as AgentWireMessage;
|
) as AgentWireMessage;
|
||||||
|
|
||||||
export const sendControllerMessage = (target: MessageSender, message: ControllerWireMessage) => {
|
|
||||||
target.send(message);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const sendAgentMessage = (target: MessageSender, message: AgentWireMessage) => {
|
|
||||||
target.send(message);
|
|
||||||
};
|
|
||||||
|
|
||||||
export type ControllerData = MessageEvent<ControllerWireMessage>;
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue