diff --git a/apps/agent/src/commands/backup-run.ts b/apps/agent/src/commands/backup-run.ts index 43eb64c8..0b00d7ae 100644 --- a/apps/agent/src/commands/backup-run.ts +++ b/apps/agent/src/commands/backup-run.ts @@ -66,33 +66,36 @@ export const handleBackupRunCommand = (context: ControllerCommandContext, payloa }, webhooks: payload.webhooks, signal: abortController.signal, - runBackup: () => - restic.backup(payload.repositoryConfig, payload.sourcePath, { - organizationId: payload.organizationId, - ...payload.options, - signal: abortController.signal, - onProgress: (progress) => { - void Runtime.runPromise( - runtime, - context.offerOutbound( - createAgentMessage("backup.progress", { - jobId: payload.jobId, - scheduleId: payload.scheduleId, - progress, - }), - ), - ).catch((error) => { - logger.error(`Failed to send backup progress update: ${toMessage(error)}`); - }); - }, - }).pipe( - Effect.map((result) => ({ - status: "completed" as const, - exitCode: result.exitCode, - result: result.result, - warningDetails: result.warningDetails, - })), - ), + runBackup: () => { + return restic + .backup(payload.repositoryConfig, payload.sourcePath, { + organizationId: payload.organizationId, + ...payload.options, + signal: abortController.signal, + onProgress: (progress) => { + void Runtime.runPromise( + runtime, + context.offerOutbound( + createAgentMessage("backup.progress", { + jobId: payload.jobId, + scheduleId: payload.scheduleId, + progress, + }), + ), + ).catch((error) => { + logger.error(`Failed to send backup progress update: ${toMessage(error)}`); + }); + }, + }) + .pipe( + Effect.map((result) => ({ + status: "completed" as const, + exitCode: result.exitCode, + result: result.result, + warningDetails: result.warningDetails, + })), + ); + }, }); switch (backupResult.status) { diff --git a/packages/core/src/backup-hooks/__tests__/hooks.test.ts b/packages/core/src/backup-hooks/__tests__/hooks.test.ts new file mode 100644 index 00000000..2eb937b8 --- /dev/null +++ b/packages/core/src/backup-hooks/__tests__/hooks.test.ts @@ -0,0 +1,337 @@ +import { Effect } from "effect"; +import { HttpResponse, http } from "msw"; +import { setupServer } from "msw/node"; +import { afterAll, afterEach, beforeAll, expect, test } from "vitest"; +import { runBackupWithWebhooks } from "../index.js"; + +const server = setupServer(); + +beforeAll(() => { + server.listen({ onUnhandledRequest: "error" }); +}); + +afterEach(() => { + server.resetHandlers(); +}); + +afterAll(() => { + server.close(); +}); + +const metadata = { + jobId: "job-1", + scheduleId: "schedule-1", + organizationId: "org-1", + sourcePath: "/tmp/source", +}; + +type WebhookBody = { + phase?: string; + event?: string; + jobId?: string; + scheduleId?: string; + organizationId?: string; + sourcePath?: string; + status?: string; + error?: string; +}; + +test("runs pre and post webhooks around a successful backup", async () => { + const events: string[] = []; + let preBody: WebhookBody | undefined; + let postBody: WebhookBody | undefined; + + server.use( + http.post("http://localhost:8080/pre", async ({ request }) => { + events.push("pre"); + preBody = (await request.json()) as WebhookBody; + return new HttpResponse(null, { status: 204 }); + }), + http.post("http://localhost:8080/post", async ({ request }) => { + events.push("post"); + postBody = (await request.json()) as WebhookBody; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: { url: "http://localhost:8080/pre" }, + post: { url: "http://localhost:8080/post" }, + }, + signal: new AbortController().signal, + runBackup: () => + Effect.sync(() => { + events.push("backup"); + return { status: "completed" as const, exitCode: 0, result: "snapshot-1", warningDetails: null }; + }), + }), + ); + + expect(events).toEqual(["pre", "backup", "post"]); + expect(preBody).toMatchObject({ ...metadata, phase: "pre", event: "backup.pre" }); + expect(postBody).toMatchObject({ ...metadata, phase: "post", event: "backup.post", status: "success" }); + expect(result).toEqual({ status: "completed", exitCode: 0, result: "snapshot-1", warningDetails: null }); +}); + +test("sends warning details to the post-backup webhook for a non-zero completed backup", async () => { + let postBody: WebhookBody | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + postBody = (await request.json()) as WebhookBody; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: null, + post: { url: "http://localhost:8080/post" }, + }, + signal: new AbortController().signal, + runBackup: () => + Effect.succeed({ + status: "completed" as const, + exitCode: 3, + result: "snapshot-1", + warningDetails: "some files could not be read", + }), + }), + ); + + expect(postBody).toMatchObject({ status: "warning", error: "some files could not be read" }); + expect(result).toEqual({ + status: "completed", + exitCode: 3, + result: "snapshot-1", + warningDetails: "some files could not be read", + }); +}); + +test("sends error details to the post-backup webhook when the backup fails", async () => { + let postBody: WebhookBody | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + postBody = (await request.json()) as WebhookBody; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: null, + post: { url: "http://localhost:8080/post" }, + }, + signal: new AbortController().signal, + runBackup: () => Effect.succeed({ status: "failed" as const, error: new Error("restic failed") }), + }), + ); + + expect(postBody).toMatchObject({ status: "error", error: "restic failed" }); + expect(result).toEqual({ status: "failed", error: "restic failed" }); +}); + +test("fails without running the backup or post webhook when the pre-backup webhook fails", async () => { + let backupRan = false; + let postRan = false; + + server.use( + http.post("http://localhost:8080/pre", () => { + return new HttpResponse("stop failed", { status: 500 }); + }), + http.post("http://localhost:8080/post", () => { + postRan = true; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: { url: "http://localhost:8080/pre" }, + post: { url: "http://localhost:8080/post" }, + }, + signal: new AbortController().signal, + runBackup: () => + Effect.sync(() => { + backupRan = true; + return { status: "completed" as const, exitCode: 0, result: null, warningDetails: null }; + }), + }), + ); + + expect(backupRan).toBe(false); + expect(postRan).toBe(false); + expect(result.status).toBe("failed"); + if (result.status === "failed") { + expect(result.error).toContain("Pre-backup webhook returned HTTP 500: stop failed"); + } +}); + +test("sends configured webhook headers and body without replacing them", async () => { + let body: string | undefined; + let authorization: string | null | undefined; + let contentType: string | null | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + body = await request.text(); + authorization = request.headers.get("authorization"); + contentType = request.headers.get("content-type"); + return new HttpResponse(null, { status: 204 }); + }), + ); + + await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: null, + post: { + url: "http://localhost:8080/post", + headers: { authorization: "Bearer post-token" }, + body: "start-container", + }, + }, + signal: new AbortController().signal, + runBackup: () => + Effect.succeed({ status: "completed" as const, exitCode: 0, result: null, warningDetails: null }), + }), + ); + + expect(body).toBe("start-container"); + expect(authorization).toBe("Bearer post-token"); + expect(contentType).toBeNull(); +}); + +test("runs the post-backup webhook after cancellation without using the cancelled signal", async () => { + const abortController = new AbortController(); + let postBody: { status?: string; error?: string } | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + postBody = (await request.json()) as { status?: string; error?: string }; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: null, + post: { url: "http://localhost:8080/post" }, + }, + signal: abortController.signal, + runBackup: () => + Effect.sync(() => { + abortController.abort(new Error("Backup was cancelled")); + return { status: "failed" as const, error: new Error("restic cancelled") }; + }), + }), + ); + + expect(postBody).toMatchObject({ status: "cancelled", error: "restic cancelled" }); + expect(result).toEqual({ status: "cancelled", message: "Backup was cancelled" }); +}); + +test("runs the post-backup webhook when cancellation returns a completed backup result", async () => { + const abortController = new AbortController(); + let postBody: { status?: string; error?: string } | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + postBody = (await request.json()) as { status?: string; error?: string }; + return new HttpResponse(null, { status: 204 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: null, + post: { url: "http://localhost:8080/post" }, + }, + signal: abortController.signal, + runBackup: () => + Effect.sync(() => { + abortController.abort(new Error("Backup was cancelled")); + return { status: "completed" as const, exitCode: 0, result: null, warningDetails: null }; + }), + }), + ); + + expect(postBody).toMatchObject({ status: "cancelled" }); + expect(result).toEqual({ status: "cancelled", message: "Backup was cancelled" }); +}); + +test("includes post-backup webhook failure details after cancellation", async () => { + const abortController = new AbortController(); + let postBody: { status?: string; error?: string } | undefined; + + server.use( + http.post("http://localhost:8080/post", async ({ request }) => { + postBody = (await request.json()) as { status?: string; error?: string }; + return new HttpResponse("start failed", { status: 500 }); + }), + ); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: null, + post: { url: "http://localhost:8080/post" }, + }, + signal: abortController.signal, + runBackup: () => + Effect.sync(() => { + abortController.abort(new Error("Backup was cancelled")); + return { status: "failed" as const, error: new Error("restic cancelled") }; + }), + }), + ); + + expect(postBody).toMatchObject({ status: "cancelled", error: "restic cancelled" }); + expect(result.status).toBe("cancelled"); + if (result.status === "cancelled") { + expect(result.message).toContain("Backup was cancelled"); + expect(result.message).toContain("Post-backup webhook returned HTTP 500: start failed"); + } +}); + +test("cancels before the pre-backup webhook without running the backup", async () => { + const abortController = new AbortController(); + let backupRan = false; + + abortController.abort(new Error("Backup was cancelled")); + + const result = await Effect.runPromise( + runBackupWithWebhooks({ + metadata, + webhooks: { + pre: { url: "http://localhost:8080/pre" }, + post: { url: "http://localhost:8080/post" }, + }, + signal: abortController.signal, + runBackup: () => + Effect.sync(() => { + backupRan = true; + return { status: "completed" as const, exitCode: 0, result: null, warningDetails: null }; + }), + }), + ); + + expect(backupRan).toBe(false); + expect(result).toEqual({ status: "cancelled", message: "Backup was cancelled" }); +}); diff --git a/packages/core/src/backup-hooks/index.ts b/packages/core/src/backup-hooks/index.ts index 235c9464..6fa6d1f7 100644 --- a/packages/core/src/backup-hooks/index.ts +++ b/packages/core/src/backup-hooks/index.ts @@ -1,4 +1,4 @@ -import { Effect } from "effect"; +import { Data, Effect } from "effect"; import { z } from "zod"; import { toErrorDetails, toMessage } from "../utils/index.js"; @@ -40,29 +40,13 @@ export type BackupWebhookContext = { }; export type BackupOperationResult = - | { status: "completed"; exitCode: number; - result: TResult; - warningDetails: string | null; - } - | { status: "failed"; - error: unknown; - }; + | { status: "completed"; exitCode: number; result: TResult; warningDetails: string | null } + | { status: "failed"; error: unknown }; export type BackupHookedExecutionResult = - | { - status: "completed"; - exitCode: number; - result: TResult; - warningDetails: string | null; - } - | { - status: "failed"; - error: string; - } - | { - status: "cancelled"; - message?: string; - }; + | { status: "completed"; exitCode: number; result: TResult; warningDetails: string | null } + | { status: "failed"; error: string } + | { status: "cancelled"; message?: string }; export type BackupHookedExecutionOptions = { metadata: BackupWebhookMetadata; @@ -73,11 +57,10 @@ export type BackupHookedExecutionOptions = { formatErrorMessage?: (error: unknown) => string; }; -export class BackupWebhookError extends Error { - override name = "BackupWebhookError"; -} - -const getPhaseLabel = (phase: BackupWebhookPhase) => (phase === "pre" ? "Pre-backup" : "Post-backup"); +export class BackupWebhookError extends Data.TaggedError("BackupWebhookError")<{ + cause: unknown; + message: string; +}> {} export const createBackupWebhooks = ( pre: BackupWebhookConfig | null | undefined, @@ -171,7 +154,6 @@ export const runBackupWebhook = async ( context: BackupWebhookContext, options: { signal?: AbortSignal; timeoutMs?: number } = {}, ) => { - const phaseLabel = getPhaseLabel(context.phase); const timeoutMs = options.timeoutMs ?? DEFAULT_BACKUP_WEBHOOK_TIMEOUT_MS; const controller = createAbortController(timeoutMs, options.signal); @@ -187,16 +169,17 @@ export const runBackupWebhook = async ( if (!response.ok) { const responseText = await response.text().catch(() => ""); const details = responseText.trim().slice(0, 500); - throw new Error( - `${phaseLabel} webhook returned HTTP ${response.status}${details ? `: ${details}` : ""}`, - ); + throw new Error(`${context.phase} webhook returned HTTP ${response.status}${details ? `: ${details}` : ""}`); } } catch (error) { if (controller.signal.aborted && controller.signal.reason instanceof Error) { - throw new BackupWebhookError(`${phaseLabel} webhook failed: ${controller.signal.reason.message}`); + throw new BackupWebhookError({ + cause: controller.signal.reason, + message: `${context.phase} webhook failed: ${controller.signal.reason.message}`, + }); } - throw new BackupWebhookError(`${phaseLabel} webhook failed: ${toMessage(error)}`); + throw new BackupWebhookError({ cause: error, message: `${context.phase} webhook failed: ${toMessage(error)}` }); } finally { controller.cleanup(); } @@ -214,7 +197,13 @@ const runConfiguredWebhook = ( return Effect.tryPromise({ try: () => runBackupWebhook(config, context, { signal }), - catch: (error) => error, + catch: (error) => { + if (error instanceof BackupWebhookError) { + return error; + } + + return new BackupWebhookError({ cause: error, message: toMessage(error) }); + }, }).pipe( Effect.as(null), Effect.catchAll((error) => Effect.succeed(formatErrorDetails(error))),