refactor(restic): auto try to unlock and remove stale locks (#926)

app/server/modules/backups/__tests__/backups.service.execution.test.ts

@@ -377,6 +377,47 @@ describe("backup execution - validation failures", () => {
 		expect(updatedSchedule.lastBackupStatus).toBe("error");
 		expect(updatedSchedule.lastBackupError).toBe("Local backup agent is not connected");
 	});
+
+	test("removes stale locks and retries once when the local backup fallback hits a restic lock", async () => {
+		const { resticBackupMock, runBackupMock } = setup();
+		config.flags.enableLocalAgent = false;
+		const safeExecMock = vi.spyOn(spawnModule, "safeExec").mockResolvedValue({
+			exitCode: 0,
+			stdout: "",
+			stderr: "",
+			timedOut: false,
+		});
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+		});
+
+		runBackupMock.mockResolvedValueOnce({
+			status: "unavailable",
+			error: new Error("Local backup agent is not connected"),
+		});
+		resticBackupMock
+			.mockImplementationOnce((params: SafeSpawnParams) => {
+				params.onStderr?.("unable to create lock in backend: repository is already locked");
+				return Promise.resolve({
+					exitCode: 11,
+					summary: "",
+					error: "unable to create lock in backend: repository is already locked",
+				});
+			})
+			.mockImplementationOnce(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }));
+
+		await backupsService.executeBackup(schedule.id);
+
+		const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
+		expect(updatedSchedule.lastBackupStatus).toBe("success");
+		expect(resticBackupMock).toHaveBeenCalledTimes(2);
+		const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock"));
+		expect(unlockCalls).toHaveLength(1);
+		expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all");
+	});
 });
 
 describe("backup execution - routing", () => {

app/server/utils/errors.ts

@@ -1,6 +1,6 @@
 import { HttpError } from "http-errors-enhanced";
 import { sanitizeSensitiveData } from "@zerobyte/core/node";
-import { ResticError } from "@zerobyte/core/restic";
+import { isResticError } from "@zerobyte/core/restic";
 import { toErrorDetails as getErrorDetails, toMessage as getMessage } from "@zerobyte/core/utils";
 import { Cause, Effect, Exit, Option } from "effect";
 
@@ -31,7 +31,7 @@ export const handleServiceError = (error: unknown) => {
 		return { message: sanitizeSensitiveData(error.message), status: error.statusCode };
 	}
 
-	if (error instanceof ResticError) {
+	if (isResticError(error)) {
 		return {
 			message: sanitizeSensitiveData(error.summary),
 			details: error.details ? sanitizeSensitiveData(error.details) : undefined,

apps/docs/content/docs/guides/repository-maintenance.mdx

@@ -22,7 +22,7 @@ Repository status is stored metadata inside Zerobyte. It is not a continuous liv
 | Action | What it does | What it does **not** do |
 | --- | --- | --- |
 | `Run doctor` | Runs `unlock`, then `check`, then `repair-index` if restic says it is needed. Saves step output into the Doctor Report. | It does **not** run a full `check --read-data` scrub, it does **not** prune unused data, and it does **not** refresh the stats card. |
-| `Unlock` | Runs `restic unlock --remove-all` to clear stale repository locks. | It does **not** validate repository integrity and it does **not** change the stored health status by itself. |
+| `Unlock` | Runs `restic unlock` to clear stale repository locks. | It does **not** validate repository integrity and it does **not** change the stored health status by itself. |
 | Stats refresh button | Runs `restic stats --mode raw-data` and updates stored size and snapshot metrics. | It does **not** change repository health status, **Last Checked**, or the Doctor Report. |
 
 ## Run doctor in detail

packages/core/src/restic/__tests__/server-lock-recovery.test.ts

@@ -0,0 +1,169 @@
+import { Effect } from "effect";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import * as nodeModule from "../../node";
+import * as cleanupModule from "../helpers/cleanup-temporary-keys";
+import { createRestic } from "../server";
+import type { ResticDeps } from "../types";
+
+const mockDeps: ResticDeps = {
+	resolveSecret: async (s) => s,
+	getOrganizationResticPassword: async () => "org-restic-password",
+	resticCacheDir: "/tmp/restic-cache",
+	resticPassFile: "/tmp/restic.pass",
+	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
+};
+
+const repositoryConfig = {
+	backend: "local" as const,
+	path: "/tmp/restic-repo",
+	isExistingRepository: true,
+	customPassword: "custom-password",
+};
+
+const sourceRepositoryConfig = {
+	backend: "local" as const,
+	path: "/tmp/source-restic-repo",
+	isExistingRepository: true,
+	customPassword: "source-password",
+};
+
+const destinationRepositoryConfig = {
+	backend: "local" as const,
+	path: "/tmp/destination-restic-repo",
+	isExistingRepository: true,
+	customPassword: "destination-password",
+};
+
+const backupSummary = JSON.stringify({
+	message_type: "summary",
+	files_new: 10,
+	files_changed: 5,
+	files_unmodified: 85,
+	dirs_new: 2,
+	dirs_changed: 1,
+	dirs_unmodified: 17,
+	data_blobs: 20,
+	tree_blobs: 5,
+	data_added: 1_048_576,
+	total_files_processed: 100,
+	total_bytes_processed: 2_097_152,
+	total_duration: 12.34,
+	snapshot_id: "abcd1234",
+});
+
+afterEach(() => {
+	vi.restoreAllMocks();
+});
+
+describe("restic command lock recovery", () => {
+	test("runs stale-only unlock and retries once after a lock failure", async () => {
+		vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
+		const safeExecMock = vi.spyOn(nodeModule, "safeExec").mockResolvedValue({
+			exitCode: 0,
+			stdout: "",
+			stderr: "",
+			timedOut: false,
+		});
+		const safeSpawnMock = vi
+			.spyOn(nodeModule, "safeSpawn")
+			.mockImplementationOnce(async ({ onStderr }) => {
+				onStderr?.("unable to create lock in backend: repository is already locked");
+				return {
+					exitCode: 11,
+					summary: "",
+					error: "unable to create lock in backend: repository is already locked",
+					stderr: "",
+				};
+			})
+			.mockResolvedValueOnce({
+				exitCode: 0,
+				summary: backupSummary,
+				error: "",
+				stderr: "",
+			});
+
+		const restic = createRestic(mockDeps);
+		const result = await Effect.runPromise(
+			restic.backup(repositoryConfig, "/tmp/source", { organizationId: "org-1" }),
+		);
+
+		expect(result.exitCode).toBe(0);
+		expect(safeSpawnMock).toHaveBeenCalledTimes(2);
+		const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock"));
+		expect(unlockCalls).toHaveLength(1);
+		expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all");
+	});
+
+	test("propagates the retry error when the retry is still locked", async () => {
+		vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
+		const safeExecMock = vi.spyOn(nodeModule, "safeExec").mockResolvedValue({
+			exitCode: 0,
+			stdout: "",
+			stderr: "",
+			timedOut: false,
+		});
+		vi.spyOn(nodeModule, "safeSpawn").mockImplementation(async ({ onStderr }) => {
+			onStderr?.("unable to create lock in backend: repository is already locked");
+			return {
+				exitCode: 11,
+				summary: "",
+				error: "unable to create lock in backend: repository is already locked",
+				stderr: "",
+			};
+		});
+
+		const restic = createRestic(mockDeps);
+
+		await expect(
+			Effect.runPromise(restic.backup(repositoryConfig, "/tmp/source", { organizationId: "org-1" })),
+		).rejects.toThrow("unable to create lock in backend: repository is already locked");
+		const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock"));
+		expect(unlockCalls).toHaveLength(1);
+		expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all");
+	});
+
+	test("runs stale-only unlock for both repositories before retrying copy", async () => {
+		vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
+		let copyCalls = 0;
+		const safeExecMock = vi.spyOn(nodeModule, "safeExec").mockImplementation(async ({ args }) => {
+			if (args?.includes("copy")) {
+				copyCalls += 1;
+				return copyCalls === 1
+					? {
+							exitCode: 11,
+							stdout: "",
+							stderr: "unable to create lock in backend: repository is already locked",
+							timedOut: false,
+						}
+					: {
+							exitCode: 0,
+							stdout: "copied",
+							stderr: "",
+							timedOut: false,
+						};
+			}
+
+			return {
+				exitCode: 0,
+				stdout: "",
+				stderr: "",
+				timedOut: false,
+			};
+		});
+
+		const restic = createRestic(mockDeps);
+		await Effect.runPromise(
+			restic.copy(sourceRepositoryConfig, destinationRepositoryConfig, {
+				organizationId: "org-1",
+			}),
+		);
+
+		const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock"));
+		expect(copyCalls).toBe(2);
+		expect(unlockCalls).toHaveLength(2);
+		for (const [params] of unlockCalls) {
+			expect(params.args).not.toContain("--remove-all");
+		}
+	});
+});

packages/core/src/restic/commands/__tests__/unlock.test.ts

@@ -0,0 +1,73 @@
+import { Effect } from "effect";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import * as nodeModule from "../../../node";
+import { ResticLockError } from "../../error";
+import * as cleanupModule from "../../helpers/cleanup-temporary-keys";
+import type { ResticDeps } from "../../types";
+import { unlock } from "../unlock";
+
+const mockDeps: ResticDeps = {
+	resolveSecret: async (s) => s,
+	getOrganizationResticPassword: async () => "org-restic-password",
+	resticCacheDir: "/tmp/restic-cache",
+	resticPassFile: "/tmp/restic.pass",
+	defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"],
+	rcloneConfigFile: "/root/.config/rclone/rclone.conf",
+};
+
+const config = {
+	backend: "local" as const,
+	path: "/tmp/restic-repo",
+	isExistingRepository: true,
+	customPassword: "custom-password",
+};
+
+const setup = () => {
+	let capturedArgs: string[] = [];
+
+	vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve());
+	vi.spyOn(nodeModule, "safeExec").mockImplementation(async ({ args }) => {
+		capturedArgs = args ?? [];
+		return { exitCode: 0, stdout: "", stderr: "", timedOut: false };
+	});
+
+	return {
+		getArgs: () => capturedArgs,
+	};
+};
+
+afterEach(() => {
+	vi.restoreAllMocks();
+});
+
+describe("unlock command", () => {
+	test("removes only stale locks by default", async () => {
+		const { getArgs } = setup();
+
+		await Effect.runPromise(unlock(config, { organizationId: "org-1" }, mockDeps));
+
+		expect(getArgs()).toContain("unlock");
+		expect(getArgs()).not.toContain("--remove-all");
+	});
+
+	test("can opt in to removing all locks", async () => {
+		const { getArgs } = setup();
+
+		await Effect.runPromise(unlock(config, { organizationId: "org-1", removeAll: true }, mockDeps));
+
+		expect(getArgs()).toContain("--remove-all");
+	});
+
+	test("returns a typed lock error for restic lock failures", async () => {
+		setup();
+		vi.spyOn(nodeModule, "safeExec").mockResolvedValueOnce({
+			exitCode: 11,
+			stdout: "",
+			stderr: "unable to create lock in backend: repository is already locked",
+			timedOut: false,
+		});
+
+		const error = await Effect.runPromise(Effect.flip(unlock(config, { organizationId: "org-1" }, mockDeps)));
+		expect(error).toBeInstanceOf(ResticLockError);
+	});
+});

packages/core/src/restic/commands/backup.ts

@@ -10,7 +10,7 @@ import { buildEnv } from "../helpers/build-env";
 import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import { validateCustomResticParams } from "../helpers/validate-custom-params";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { logger, safeSpawn } from "../../node";
 import type { ResticDeps } from "../types";
 import { toMessage } from "../../utils";
@@ -190,7 +190,7 @@ export const backup = (
 				logger.error(`Restic backup failed: ${res.error}`);
 				logger.error(`Command executed: restic ${args.join(" ")}`);
 
-				throw new ResticError(res.exitCode, stderrLines.join("\n") || res.stderr || res.error);
+				throw createResticError(res.exitCode, stderrLines.join("\n") || res.stderr || res.error);
 			}
 
 			const lastLine = res.summary.trim();
@@ -221,7 +221,7 @@ export const backup = (
 			};
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/check.ts

@@ -6,7 +6,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
 import type { ResticDeps } from "../types";
-import { ResticError } from "../error";
+import { isResticError } from "../error";
 import { toMessage } from "../../utils";
 
 class ResticCheckCommandError extends Data.TaggedError("ResticCheckCommandError")<{
@@ -77,7 +77,7 @@ export const check = (
 			};
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/copy.ts

@@ -4,7 +4,7 @@ import { buildEnv } from "../helpers/build-env";
 import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { logger, safeExec } from "../../node";
 import type { ResticDeps } from "../types";
 import { Data, Effect } from "effect";
@@ -72,7 +72,7 @@ export const copy = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic copy failed: ${stderr}`);
-				throw new ResticError(res.exitCode, stderr);
+				throw createResticError(res.exitCode, stderr);
 			}
 
 			logger.info(`Restic copy completed from ${sourceRepoUrl} to ${destRepoUrl}`);
@@ -82,7 +82,7 @@ export const copy = (
 			};
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/delete-snapshots.ts

@@ -1,6 +1,6 @@
 import { Data, Effect } from "effect";
 import { logger, safeExec } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { addCommonArgs } from "../helpers/add-common-args";
 import { buildEnv } from "../helpers/build-env";
 import { buildRepoUrl } from "../helpers/build-repo-url";
@@ -38,13 +38,13 @@ export const deleteSnapshots = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic snapshot deletion failed: ${res.stderr}`);
-				throw new ResticError(res.exitCode, res.stderr);
+				throw createResticError(res.exitCode, res.stderr);
 			}
 
 			return { success: true };
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/dump.ts

@@ -7,7 +7,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
 import { logger, safeSpawn } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { Data, Effect } from "effect";
 import { toMessage } from "../../utils";
 
@@ -94,7 +94,7 @@ export const dump = (
 
 					const stderr = stderrTail.trim() || result.error;
 					logger.error(`Restic dump failed: ${stderr}`);
-					throw new ResticError(result.exitCode, stderr);
+					throw createResticError(result.exitCode, stderr);
 				})
 				.finally(async () => {
 					abortController = null;
@@ -120,7 +120,7 @@ export const dump = (
 			};
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/forget.ts

@@ -6,7 +6,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
 import { logger, safeExec } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { toMessage } from "../../utils";
 import { Data, Effect } from "effect";
 
@@ -65,7 +65,7 @@ export const forget = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic forget failed: ${res.stderr}`);
-				throw new ResticError(res.exitCode, res.stderr);
+				throw createResticError(res.exitCode, res.stderr);
 			}
 
 			const lines = res.stdout.split("\n").filter((line) => line.trim());
@@ -74,7 +74,7 @@ export const forget = (
 			return { success: true, data: result };
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/init.ts

@@ -7,7 +7,7 @@ import type { RepositoryConfig } from "../schemas";
 import { logger, safeExec } from "../../node";
 import type { ResticDeps } from "../types";
 import { Data, Effect } from "effect";
-import { ResticError } from "../error";
+import { isResticError } from "../error";
 import { toMessage } from "../../utils";
 
 class ResticInitCommandError extends Data.TaggedError("ResticInitCommandError")<{
@@ -70,7 +70,7 @@ export const init = (
 			return { success: true, error: null };
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/key-add.ts

@@ -7,7 +7,7 @@ import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
 import type { ResticDeps } from "../types";
 import { toMessage } from "../../utils";
-import { ResticError } from "../error";
+import { isResticError } from "../error";
 
 class ResticKeyAddCommandError extends Data.TaggedError("ResticKeyAddCommandError")<{
 	cause: unknown;
@@ -52,7 +52,7 @@ export const keyAdd = (
 			return { success: true, error: null };
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/ls.ts

@@ -5,7 +5,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
 import { logger, safeSpawn } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import type { ResticDeps } from "../types";
 import { Data, Effect } from "effect";
 import { toMessage } from "../../utils";
@@ -134,7 +134,7 @@ export const ls = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic ls failed: ${res.error}`);
-				throw new ResticError(res.exitCode, res.stderr || res.error);
+				throw createResticError(res.exitCode, res.stderr || res.error);
 			}
 
 			if (totalNodes > offset + limit) {
@@ -153,7 +153,7 @@ export const ls = (
 			} as ResticLsResult;
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/repair-index.ts

@@ -1,6 +1,6 @@
 import { Data, Effect } from "effect";
 import { logger, safeExec } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { addCommonArgs } from "../helpers/add-common-args";
 import { buildEnv } from "../helpers/build-env";
 import { buildRepoUrl } from "../helpers/build-repo-url";
@@ -44,7 +44,7 @@ export const repairIndex = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic repair index failed: ${stderr}`);
-				throw new ResticError(res.exitCode, stderr);
+				throw createResticError(res.exitCode, stderr);
 			}
 
 			logger.info(`Restic repair index completed for repository: ${repoUrl}`);
@@ -55,7 +55,7 @@ export const repairIndex = (
 			};
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/restore.ts

@@ -8,7 +8,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import { type RepositoryConfig, type OverwriteMode } from "../schemas";
 import { logger, safeSpawn } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { resticRestoreOutputSchema, type ResticRestoreOutputDto } from "../restic-dto";
 import type { ResticDeps } from "../types";
 import { Data, Effect } from "effect";
@@ -147,7 +147,7 @@ export const restore = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic restore failed: ${res.error}`);
-				throw new ResticError(res.exitCode, res.stderr || res.error);
+				throw createResticError(res.exitCode, res.stderr || res.error);
 			}
 
 			const lastLine = res.summary.trim();
@@ -183,7 +183,7 @@ export const restore = (
 			return result.data;
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/snapshots.ts

@@ -9,7 +9,7 @@ import { logger, safeSpawn } from "../../node";
 import type { ResticDeps } from "../types";
 import { Data, Effect } from "effect";
 import { toMessage } from "../../utils";
-import { ResticError } from "../error";
+import { isResticError } from "../error";
 
 class ResticSnapshotsCommandError extends Data.TaggedError("ResticSnapshotsCommandError")<{
 	cause: unknown;
@@ -81,7 +81,7 @@ export const snapshots = (
 		},
 
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/stats.ts

@@ -5,7 +5,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url";
 import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys";
 import type { RepositoryConfig } from "../schemas";
 import { logger, safeExec } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { resticStatsSchema } from "../restic-dto";
 import type { ResticDeps } from "../types";
 import { Data, Effect } from "effect";
@@ -30,7 +30,7 @@ export const stats = (config: RepositoryConfig, options: { organizationId: strin
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic stats retrieval failed: ${res.stderr}`);
-				throw new ResticError(res.exitCode, res.stderr);
+				throw createResticError(res.exitCode, res.stderr);
 			}
 
 			const parsedJson = safeJsonParse<unknown>(res.stdout);
@@ -44,7 +44,7 @@ export const stats = (config: RepositoryConfig, options: { organizationId: strin
 			return result.data;
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/tag-snapshots.ts

@@ -1,6 +1,6 @@
 import { Data, Effect } from "effect";
 import { logger, safeExec } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { addCommonArgs } from "../helpers/add-common-args";
 import { buildEnv } from "../helpers/build-env";
 import { buildRepoUrl } from "../helpers/build-repo-url";
@@ -58,14 +58,14 @@ export const tagSnapshots = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic snapshot tagging failed: ${res.stderr}`);
-				throw new ResticError(res.exitCode, res.stderr);
+				throw createResticError(res.exitCode, res.stderr);
 			}
 
 			return { success: true };
 		},
 
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/commands/unlock.ts

@@ -1,6 +1,6 @@
 import { Data, Effect } from "effect";
 import { logger, safeExec } from "../../node";
-import { ResticError } from "../error";
+import { createResticError, isResticError } from "../error";
 import { addCommonArgs } from "../helpers/add-common-args";
 import { buildEnv } from "../helpers/build-env";
 import { buildRepoUrl } from "../helpers/build-repo-url";
@@ -16,7 +16,7 @@ class ResticUnlockCommandError extends Data.TaggedError("ResticUnlockCommandErro
 
 export const unlock = (
 	config: RepositoryConfig,
-	options: { signal?: AbortSignal; organizationId: string },
+	options: { signal?: AbortSignal; organizationId: string; removeAll?: boolean },
 	deps: ResticDeps,
 ) => {
 	return Effect.tryPromise({
@@ -24,7 +24,10 @@ export const unlock = (
 			const repoUrl = buildRepoUrl(config);
 			const env = await buildEnv(config, options.organizationId, deps);
 
-			const args = ["unlock", "--repo", repoUrl, "--remove-all"];
+			const args = ["unlock", "--repo", repoUrl];
+			if (options.removeAll) {
+				args.push("--remove-all");
+			}
 			addCommonArgs(args, env, config);
 
 			const res = await safeExec({
@@ -42,14 +45,14 @@ export const unlock = (
 
 			if (res.exitCode !== 0) {
 				logger.error(`Restic unlock failed: ${res.stderr}`);
-				throw new ResticError(res.exitCode, res.stderr);
+				throw createResticError(res.exitCode, res.stderr);
 			}
 
 			logger.info(`Restic unlock succeeded for repository: ${repoUrl}`);
 			return { success: true, message: "Repository unlocked successfully" };
 		},
 		catch: (error) => {
-			if (error instanceof ResticError) {
+			if (isResticError(error)) {
 				return error;
 			}
 

packages/core/src/restic/error.ts

@@ -1,3 +1,5 @@
+import { Data } from "effect";
+
 const resticErrorCodes: Record<number, string> = {
 	1: "Command failed: An error occurred while executing the command.",
 	2: "Go runtime error: A runtime error occurred in the Go program.",
@@ -23,3 +25,34 @@ export class ResticError extends Error {
 		this.name = "ResticError";
 	}
 }
+
+export class ResticLockError extends Data.TaggedError("ResticLockError")<{
+	code: number;
+	summary: string;
+	details: string;
+	message: string;
+}> {
+	constructor(details: string) {
+		const summary = resticErrorCodes[11]!;
+		super({
+			code: 11,
+			summary,
+			details,
+			message: details ? `${summary}\n${details}` : summary,
+		});
+		this.name = "ResticLockError";
+	}
+}
+
+export type AnyResticError = ResticError | ResticLockError;
+
+export const isResticError = (error: unknown): error is AnyResticError =>
+	error instanceof ResticError || error instanceof ResticLockError;
+
+export const createResticError = (code: number, details: string) => {
+	if (code === 11) {
+		return new ResticLockError(details);
+	}
+
+	return new ResticError(code, details);
+};

packages/core/src/restic/index.ts

@@ -1,6 +1,6 @@
 export * from "./schemas";
 export * from "./restic-dto";
-export { ResticError } from "./error";
+export { isResticError, ResticError, ResticLockError } from "./error";
 
 export type { RestoreProgress } from "./commands/restore";
 export type {

packages/core/src/restic/lock-diagnostics.ts

@@ -1,6 +1,6 @@
 import { logger, safeExec } from "../node";
 import { toMessage } from "../utils";
-import { ResticError } from "./error";
+import { ResticLockError } from "./error";
 import { addCommonArgs } from "./helpers/add-common-args";
 import { buildEnv } from "./helpers/build-env";
 import { buildRepoUrl } from "./helpers/build-repo-url";
@@ -26,7 +26,7 @@ const LOCK_ERROR_PATTERNS = [
 ];
 
 export const isResticLockFailure = (error: unknown) => {
-	if (error instanceof ResticError && error.code === 11) {
+	if (error instanceof ResticLockError) {
 		return true;
 	}
 

packages/core/src/restic/server.ts

@@ -1,4 +1,5 @@
 import { Effect } from "effect";
+import { logger } from "../node";
 import { backup } from "./commands/backup";
 import { check } from "./commands/check";
 import { copy } from "./commands/copy";
@@ -14,6 +15,7 @@ import { snapshots } from "./commands/snapshots";
 import { stats } from "./commands/stats";
 import { tagSnapshots } from "./commands/tag-snapshots";
 import { unlock } from "./commands/unlock";
+import { ResticLockError } from "./error";
 import { logResticLockFailureDiagnostics } from "./lock-diagnostics";
 import type { RepositoryConfig } from "./schemas";
 import type { ResticDeps } from "./types";
@@ -24,7 +26,7 @@ export { buildRepoUrl } from "./helpers/build-repo-url";
 export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
 export { validateCustomResticParams } from "./helpers/validate-custom-params";
 export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics";
-export { ResticError } from "./error";
+export { isResticError, ResticError, ResticLockError } from "./error";
 
 type LockDiagnosticCommandContext = {
 	repositoryConfig: RepositoryConfig;
@@ -34,6 +36,12 @@ type LockDiagnosticCommandContext = {
 
 type ResticCommandOptions = { organizationId: string };
 type ResticCommandResult = { error?: unknown };
+type ResticCommandFailure<Failure> = Failure | ResticLockError;
+type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
+	Success,
+	ResticCommandFailure<Failure>,
+	Requirements
+>;
 
 const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => {
 	const firstRepositoryConfig = args[0] as RepositoryConfig;
@@ -68,15 +76,47 @@ const logLockFailure = async (
 		relatedRepositoryConfigs: context.relatedRepositoryConfigs,
 	});
 
+const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect<void, Error> =>
+	Effect.gen(function* () {
+		for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) {
+			yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
+		}
+	});
+
+const recoverFromLockFailure = <Success, Failure, Requirements>(
+	error: ResticLockError,
+	operation: string,
+	context: LockDiagnosticCommandContext,
+	deps: ResticDeps,
+	runCommand: RunResticCommand<Success, Failure, Requirements>,
+): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
+	Effect.gen(function* () {
+		yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
+		yield* Effect.promise(() => logLockFailure(error, operation, context, deps));
+		yield* unlockStaleLocks(context, deps);
+
+		const retryResult = yield* runCommand();
+		yield* logger.effect.info("Restic lock failure recovered by removing stale locks and retrying once.");
+
+		return retryResult;
+	});
+
 function withDeps<Args extends unknown[], Success, Failure, Requirements>(
 	operation: string,
-	command: (...args: [...Args, ResticDeps]) => Effect.Effect<Success, Failure, Requirements>,
+	command: (...args: [...Args, ResticDeps]) => Effect.Effect<Success, ResticCommandFailure<Failure>, Requirements>,
 	deps: ResticDeps,
-): (...args: Args) => Effect.Effect<Success, Failure, Requirements> {
+): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
 	return (...args: Args) => {
 		const context = getCommandContext(operation, args);
-		return command(...args, deps).pipe(
+		const runCommand = () => command(...args, deps);
-			Effect.tapError((error) => Effect.promise(() => logLockFailure(error, operation, context, deps))),
+
+		const commandEffect = runCommand().pipe(
+			Effect.catchTag("ResticLockError", (error) =>
+				recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand),
+			),
+		);
+
+		return commandEffect.pipe(
 			Effect.tap((result) => {
 				const { error } = result as ResticCommandResult;
 				return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void;

packages/core/src/utils/errors.ts

@@ -1,4 +1,4 @@
-import { ResticError } from "../restic/error.js";
+import { isResticError } from "../restic/error.js";
 
 export const toMessage = (error: unknown) => {
 	if (error instanceof Error) {
@@ -9,7 +9,7 @@ export const toMessage = (error: unknown) => {
 };
 
 export const toErrorDetails = (error: unknown) => {
-	if (error instanceof ResticError) {
+	if (isResticError(error)) {
 		return error.details || error.summary;
 	}