fix(hooks): run post when cancelled

apps/agent/src/commands/backup-run.ts

@@ -66,33 +66,36 @@ export const handleBackupRunCommand = (context: ControllerCommandContext, payloa
 					},
 					webhooks: payload.webhooks,
 					signal: abortController.signal,
-					runBackup: () =>
+					runBackup: () => {
-						restic.backup(payload.repositoryConfig, payload.sourcePath, {
+						return restic
-							organizationId: payload.organizationId,
+							.backup(payload.repositoryConfig, payload.sourcePath, {
-							...payload.options,
+								organizationId: payload.organizationId,
-							signal: abortController.signal,
+								...payload.options,
-							onProgress: (progress) => {
+								signal: abortController.signal,
-								void Runtime.runPromise(
+								onProgress: (progress) => {
-									runtime,
+									void Runtime.runPromise(
-									context.offerOutbound(
+										runtime,
-										createAgentMessage("backup.progress", {
+										context.offerOutbound(
-											jobId: payload.jobId,
+											createAgentMessage("backup.progress", {
-											scheduleId: payload.scheduleId,
+												jobId: payload.jobId,
-											progress,
+												scheduleId: payload.scheduleId,
-										}),
+												progress,
-									),
+											}),
-								).catch((error) => {
+										),
-									logger.error(`Failed to send backup progress update: ${toMessage(error)}`);
+									).catch((error) => {
-								});
+										logger.error(`Failed to send backup progress update: ${toMessage(error)}`);
-							},
+									});
-						}).pipe(
+								},
-							Effect.map((result) => ({
+							})
-								status: "completed" as const,
+							.pipe(
-								exitCode: result.exitCode,
+								Effect.map((result) => ({
-								result: result.result,
+									status: "completed" as const,
-								warningDetails: result.warningDetails,
+									exitCode: result.exitCode,
-							})),
+									result: result.result,
-						),
+									warningDetails: result.warningDetails,
+								})),
+							);
+					},
 				});
 
 				switch (backupResult.status) {

packages/core/src/backup-hooks/__tests__/hooks.test.ts

@@ -0,0 +1,337 @@
+import { Effect } from "effect";
+import { HttpResponse, http } from "msw";
+import { setupServer } from "msw/node";
+import { afterAll, afterEach, beforeAll, expect, test } from "vitest";
+import { runBackupWithWebhooks } from "../index.js";
+
+const server = setupServer();
+
+beforeAll(() => {
+	server.listen({ onUnhandledRequest: "error" });
+});
+
+afterEach(() => {
+	server.resetHandlers();
+});
+
+afterAll(() => {
+	server.close();
+});
+
+const metadata = {
+	jobId: "job-1",
+	scheduleId: "schedule-1",
+	organizationId: "org-1",
+	sourcePath: "/tmp/source",
+};
+
+type WebhookBody = {
+	phase?: string;
+	event?: string;
+	jobId?: string;
+	scheduleId?: string;
+	organizationId?: string;
+	sourcePath?: string;
+	status?: string;
+	error?: string;
+};
+
+test("runs pre and post webhooks around a successful backup", async () => {
+	const events: string[] = [];
+	let preBody: WebhookBody | undefined;
+	let postBody: WebhookBody | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/pre", async ({ request }) => {
+			events.push("pre");
+			preBody = (await request.json()) as WebhookBody;
+			return new HttpResponse(null, { status: 204 });
+		}),
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			events.push("post");
+			postBody = (await request.json()) as WebhookBody;
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: { url: "http://localhost:8080/pre" },
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: new AbortController().signal,
+			runBackup: () =>
+				Effect.sync(() => {
+					events.push("backup");
+					return { status: "completed" as const, exitCode: 0, result: "snapshot-1", warningDetails: null };
+				}),
+		}),
+	);
+
+	expect(events).toEqual(["pre", "backup", "post"]);
+	expect(preBody).toMatchObject({ ...metadata, phase: "pre", event: "backup.pre" });
+	expect(postBody).toMatchObject({ ...metadata, phase: "post", event: "backup.post", status: "success" });
+	expect(result).toEqual({ status: "completed", exitCode: 0, result: "snapshot-1", warningDetails: null });
+});
+
+test("sends warning details to the post-backup webhook for a non-zero completed backup", async () => {
+	let postBody: WebhookBody | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			postBody = (await request.json()) as WebhookBody;
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: null,
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: new AbortController().signal,
+			runBackup: () =>
+				Effect.succeed({
+					status: "completed" as const,
+					exitCode: 3,
+					result: "snapshot-1",
+					warningDetails: "some files could not be read",
+				}),
+		}),
+	);
+
+	expect(postBody).toMatchObject({ status: "warning", error: "some files could not be read" });
+	expect(result).toEqual({
+		status: "completed",
+		exitCode: 3,
+		result: "snapshot-1",
+		warningDetails: "some files could not be read",
+	});
+});
+
+test("sends error details to the post-backup webhook when the backup fails", async () => {
+	let postBody: WebhookBody | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			postBody = (await request.json()) as WebhookBody;
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: null,
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: new AbortController().signal,
+			runBackup: () => Effect.succeed({ status: "failed" as const, error: new Error("restic failed") }),
+		}),
+	);
+
+	expect(postBody).toMatchObject({ status: "error", error: "restic failed" });
+	expect(result).toEqual({ status: "failed", error: "restic failed" });
+});
+
+test("fails without running the backup or post webhook when the pre-backup webhook fails", async () => {
+	let backupRan = false;
+	let postRan = false;
+
+	server.use(
+		http.post("http://localhost:8080/pre", () => {
+			return new HttpResponse("stop failed", { status: 500 });
+		}),
+		http.post("http://localhost:8080/post", () => {
+			postRan = true;
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: { url: "http://localhost:8080/pre" },
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: new AbortController().signal,
+			runBackup: () =>
+				Effect.sync(() => {
+					backupRan = true;
+					return { status: "completed" as const, exitCode: 0, result: null, warningDetails: null };
+				}),
+		}),
+	);
+
+	expect(backupRan).toBe(false);
+	expect(postRan).toBe(false);
+	expect(result.status).toBe("failed");
+	if (result.status === "failed") {
+		expect(result.error).toContain("Pre-backup webhook returned HTTP 500: stop failed");
+	}
+});
+
+test("sends configured webhook headers and body without replacing them", async () => {
+	let body: string | undefined;
+	let authorization: string | null | undefined;
+	let contentType: string | null | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			body = await request.text();
+			authorization = request.headers.get("authorization");
+			contentType = request.headers.get("content-type");
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: null,
+				post: {
+					url: "http://localhost:8080/post",
+					headers: { authorization: "Bearer post-token" },
+					body: "start-container",
+				},
+			},
+			signal: new AbortController().signal,
+			runBackup: () =>
+				Effect.succeed({ status: "completed" as const, exitCode: 0, result: null, warningDetails: null }),
+		}),
+	);
+
+	expect(body).toBe("start-container");
+	expect(authorization).toBe("Bearer post-token");
+	expect(contentType).toBeNull();
+});
+
+test("runs the post-backup webhook after cancellation without using the cancelled signal", async () => {
+	const abortController = new AbortController();
+	let postBody: { status?: string; error?: string } | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			postBody = (await request.json()) as { status?: string; error?: string };
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: null,
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: abortController.signal,
+			runBackup: () =>
+				Effect.sync(() => {
+					abortController.abort(new Error("Backup was cancelled"));
+					return { status: "failed" as const, error: new Error("restic cancelled") };
+				}),
+		}),
+	);
+
+	expect(postBody).toMatchObject({ status: "cancelled", error: "restic cancelled" });
+	expect(result).toEqual({ status: "cancelled", message: "Backup was cancelled" });
+});
+
+test("runs the post-backup webhook when cancellation returns a completed backup result", async () => {
+	const abortController = new AbortController();
+	let postBody: { status?: string; error?: string } | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			postBody = (await request.json()) as { status?: string; error?: string };
+			return new HttpResponse(null, { status: 204 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: null,
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: abortController.signal,
+			runBackup: () =>
+				Effect.sync(() => {
+					abortController.abort(new Error("Backup was cancelled"));
+					return { status: "completed" as const, exitCode: 0, result: null, warningDetails: null };
+				}),
+		}),
+	);
+
+	expect(postBody).toMatchObject({ status: "cancelled" });
+	expect(result).toEqual({ status: "cancelled", message: "Backup was cancelled" });
+});
+
+test("includes post-backup webhook failure details after cancellation", async () => {
+	const abortController = new AbortController();
+	let postBody: { status?: string; error?: string } | undefined;
+
+	server.use(
+		http.post("http://localhost:8080/post", async ({ request }) => {
+			postBody = (await request.json()) as { status?: string; error?: string };
+			return new HttpResponse("start failed", { status: 500 });
+		}),
+	);
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: null,
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: abortController.signal,
+			runBackup: () =>
+				Effect.sync(() => {
+					abortController.abort(new Error("Backup was cancelled"));
+					return { status: "failed" as const, error: new Error("restic cancelled") };
+				}),
+		}),
+	);
+
+	expect(postBody).toMatchObject({ status: "cancelled", error: "restic cancelled" });
+	expect(result.status).toBe("cancelled");
+	if (result.status === "cancelled") {
+		expect(result.message).toContain("Backup was cancelled");
+		expect(result.message).toContain("Post-backup webhook returned HTTP 500: start failed");
+	}
+});
+
+test("cancels before the pre-backup webhook without running the backup", async () => {
+	const abortController = new AbortController();
+	let backupRan = false;
+
+	abortController.abort(new Error("Backup was cancelled"));
+
+	const result = await Effect.runPromise(
+		runBackupWithWebhooks({
+			metadata,
+			webhooks: {
+				pre: { url: "http://localhost:8080/pre" },
+				post: { url: "http://localhost:8080/post" },
+			},
+			signal: abortController.signal,
+			runBackup: () =>
+				Effect.sync(() => {
+					backupRan = true;
+					return { status: "completed" as const, exitCode: 0, result: null, warningDetails: null };
+				}),
+		}),
+	);
+
+	expect(backupRan).toBe(false);
+	expect(result).toEqual({ status: "cancelled", message: "Backup was cancelled" });
+});

packages/core/src/backup-hooks/index.ts

@@ -1,4 +1,4 @@
-import { Effect } from "effect";
+import { Data, Effect } from "effect";
 import { z } from "zod";
 import { toErrorDetails, toMessage } from "../utils/index.js";
 
@@ -40,29 +40,13 @@ export type BackupWebhookContext = {
 };
 
 export type BackupOperationResult<TResult> =
-	| { status: "completed"; exitCode: number;
+	| { status: "completed"; exitCode: number; result: TResult; warningDetails: string | null }
-			result: TResult;
+	| { status: "failed"; error: unknown };
-			warningDetails: string | null;
-	  }
-	| { status: "failed";
-			error: unknown;
-	  };
 
 export type BackupHookedExecutionResult<TResult> =
-	| {
+	| { status: "completed"; exitCode: number; result: TResult; warningDetails: string | null }
-			status: "completed";
+	| { status: "failed"; error: string }
-			exitCode: number;
+	| { status: "cancelled"; message?: string };
-			result: TResult;
-			warningDetails: string | null;
-	  }
-	| {
-			status: "failed";
-			error: string;
-	  }
-	| {
-			status: "cancelled";
-			message?: string;
-	  };
 
 export type BackupHookedExecutionOptions<TResult, R = never> = {
 	metadata: BackupWebhookMetadata;
@@ -73,11 +57,10 @@ export type BackupHookedExecutionOptions<TResult, R = never> = {
 	formatErrorMessage?: (error: unknown) => string;
 };
 
-export class BackupWebhookError extends Error {
+export class BackupWebhookError extends Data.TaggedError("BackupWebhookError")<{
-	override name = "BackupWebhookError";
+	cause: unknown;
-}
+	message: string;
-
+}> {}
-const getPhaseLabel = (phase: BackupWebhookPhase) => (phase === "pre" ? "Pre-backup" : "Post-backup");
 
 export const createBackupWebhooks = (
 	pre: BackupWebhookConfig | null | undefined,
@@ -171,7 +154,6 @@ export const runBackupWebhook = async (
 	context: BackupWebhookContext,
 	options: { signal?: AbortSignal; timeoutMs?: number } = {},
 ) => {
-	const phaseLabel = getPhaseLabel(context.phase);
 	const timeoutMs = options.timeoutMs ?? DEFAULT_BACKUP_WEBHOOK_TIMEOUT_MS;
 	const controller = createAbortController(timeoutMs, options.signal);
 
@@ -187,16 +169,17 @@ export const runBackupWebhook = async (
 		if (!response.ok) {
 			const responseText = await response.text().catch(() => "");
 			const details = responseText.trim().slice(0, 500);
-			throw new Error(
+			throw new Error(`${context.phase} webhook returned HTTP ${response.status}${details ? `: ${details}` : ""}`);
-				`${phaseLabel} webhook returned HTTP ${response.status}${details ? `: ${details}` : ""}`,
-			);
 		}
 	} catch (error) {
 		if (controller.signal.aborted && controller.signal.reason instanceof Error) {
-			throw new BackupWebhookError(`${phaseLabel} webhook failed: ${controller.signal.reason.message}`);
+			throw new BackupWebhookError({
+				cause: controller.signal.reason,
+				message: `${context.phase} webhook failed: ${controller.signal.reason.message}`,
+			});
 		}
 
-		throw new BackupWebhookError(`${phaseLabel} webhook failed: ${toMessage(error)}`);
+		throw new BackupWebhookError({ cause: error, message: `${context.phase} webhook failed: ${toMessage(error)}` });
 	} finally {
 		controller.cleanup();
 	}
@@ -214,7 +197,13 @@ const runConfiguredWebhook = (
 
 	return Effect.tryPromise({
 		try: () => runBackupWebhook(config, context, { signal }),
-		catch: (error) => error,
+		catch: (error) => {
+			if (error instanceof BackupWebhookError) {
+				return error;
+			}
+
+			return new BackupWebhookError({ cause: error, message: toMessage(error) });
+		},
 	}).pipe(
 		Effect.as(null),
 		Effect.catchAll((error) => Effect.succeed(formatErrorDetails(error))),