diff --git a/app/server/modules/backups/__tests__/backups.service.execution.test.ts b/app/server/modules/backups/__tests__/backups.service.execution.test.ts index b8582c39..4150244d 100644 --- a/app/server/modules/backups/__tests__/backups.service.execution.test.ts +++ b/app/server/modules/backups/__tests__/backups.service.execution.test.ts @@ -377,6 +377,47 @@ describe("backup execution - validation failures", () => { expect(updatedSchedule.lastBackupStatus).toBe("error"); expect(updatedSchedule.lastBackupError).toBe("Local backup agent is not connected"); }); + + test("removes stale locks and retries once when the local backup fallback hits a restic lock", async () => { + const { resticBackupMock, runBackupMock } = setup(); + config.flags.enableLocalAgent = false; + const safeExecMock = vi.spyOn(spawnModule, "safeExec").mockResolvedValue({ + exitCode: 0, + stdout: "", + stderr: "", + timedOut: false, + }); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + + runBackupMock.mockResolvedValueOnce({ + status: "unavailable", + error: new Error("Local backup agent is not connected"), + }); + resticBackupMock + .mockImplementationOnce((params: SafeSpawnParams) => { + params.onStderr?.("unable to create lock in backend: repository is already locked"); + return Promise.resolve({ + exitCode: 11, + summary: "", + error: "unable to create lock in backend: repository is already locked", + }); + }) + .mockImplementationOnce(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" })); + + await backupsService.executeBackup(schedule.id); + + const updatedSchedule = await getScheduleByIdOrShortId(schedule.id); + expect(updatedSchedule.lastBackupStatus).toBe("success"); + expect(resticBackupMock).toHaveBeenCalledTimes(2); + const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock")); + expect(unlockCalls).toHaveLength(1); + expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all"); + }); }); describe("backup execution - routing", () => { diff --git a/app/server/utils/errors.ts b/app/server/utils/errors.ts index a424ea7a..9a15397f 100644 --- a/app/server/utils/errors.ts +++ b/app/server/utils/errors.ts @@ -1,6 +1,6 @@ import { HttpError } from "http-errors-enhanced"; import { sanitizeSensitiveData } from "@zerobyte/core/node"; -import { ResticError } from "@zerobyte/core/restic"; +import { isResticError } from "@zerobyte/core/restic"; import { toErrorDetails as getErrorDetails, toMessage as getMessage } from "@zerobyte/core/utils"; import { Cause, Effect, Exit, Option } from "effect"; @@ -31,7 +31,7 @@ export const handleServiceError = (error: unknown) => { return { message: sanitizeSensitiveData(error.message), status: error.statusCode }; } - if (error instanceof ResticError) { + if (isResticError(error)) { return { message: sanitizeSensitiveData(error.summary), details: error.details ? sanitizeSensitiveData(error.details) : undefined, diff --git a/apps/docs/content/docs/guides/repository-maintenance.mdx b/apps/docs/content/docs/guides/repository-maintenance.mdx index c9f00cb4..cb2d45d4 100644 --- a/apps/docs/content/docs/guides/repository-maintenance.mdx +++ b/apps/docs/content/docs/guides/repository-maintenance.mdx @@ -22,7 +22,7 @@ Repository status is stored metadata inside Zerobyte. It is not a continuous liv | Action | What it does | What it does **not** do | | --- | --- | --- | | `Run doctor` | Runs `unlock`, then `check`, then `repair-index` if restic says it is needed. Saves step output into the Doctor Report. | It does **not** run a full `check --read-data` scrub, it does **not** prune unused data, and it does **not** refresh the stats card. | -| `Unlock` | Runs `restic unlock --remove-all` to clear stale repository locks. | It does **not** validate repository integrity and it does **not** change the stored health status by itself. | +| `Unlock` | Runs `restic unlock` to clear stale repository locks. | It does **not** validate repository integrity and it does **not** change the stored health status by itself. | | Stats refresh button | Runs `restic stats --mode raw-data` and updates stored size and snapshot metrics. | It does **not** change repository health status, **Last Checked**, or the Doctor Report. | ## Run doctor in detail diff --git a/packages/core/src/restic/__tests__/server-lock-recovery.test.ts b/packages/core/src/restic/__tests__/server-lock-recovery.test.ts new file mode 100644 index 00000000..d21d5cfe --- /dev/null +++ b/packages/core/src/restic/__tests__/server-lock-recovery.test.ts @@ -0,0 +1,169 @@ +import { Effect } from "effect"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import * as nodeModule from "../../node"; +import * as cleanupModule from "../helpers/cleanup-temporary-keys"; +import { createRestic } from "../server"; +import type { ResticDeps } from "../types"; + +const mockDeps: ResticDeps = { + resolveSecret: async (s) => s, + getOrganizationResticPassword: async () => "org-restic-password", + resticCacheDir: "/tmp/restic-cache", + resticPassFile: "/tmp/restic.pass", + defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", +}; + +const repositoryConfig = { + backend: "local" as const, + path: "/tmp/restic-repo", + isExistingRepository: true, + customPassword: "custom-password", +}; + +const sourceRepositoryConfig = { + backend: "local" as const, + path: "/tmp/source-restic-repo", + isExistingRepository: true, + customPassword: "source-password", +}; + +const destinationRepositoryConfig = { + backend: "local" as const, + path: "/tmp/destination-restic-repo", + isExistingRepository: true, + customPassword: "destination-password", +}; + +const backupSummary = JSON.stringify({ + message_type: "summary", + files_new: 10, + files_changed: 5, + files_unmodified: 85, + dirs_new: 2, + dirs_changed: 1, + dirs_unmodified: 17, + data_blobs: 20, + tree_blobs: 5, + data_added: 1_048_576, + total_files_processed: 100, + total_bytes_processed: 2_097_152, + total_duration: 12.34, + snapshot_id: "abcd1234", +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("restic command lock recovery", () => { + test("runs stale-only unlock and retries once after a lock failure", async () => { + vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve()); + const safeExecMock = vi.spyOn(nodeModule, "safeExec").mockResolvedValue({ + exitCode: 0, + stdout: "", + stderr: "", + timedOut: false, + }); + const safeSpawnMock = vi + .spyOn(nodeModule, "safeSpawn") + .mockImplementationOnce(async ({ onStderr }) => { + onStderr?.("unable to create lock in backend: repository is already locked"); + return { + exitCode: 11, + summary: "", + error: "unable to create lock in backend: repository is already locked", + stderr: "", + }; + }) + .mockResolvedValueOnce({ + exitCode: 0, + summary: backupSummary, + error: "", + stderr: "", + }); + + const restic = createRestic(mockDeps); + const result = await Effect.runPromise( + restic.backup(repositoryConfig, "/tmp/source", { organizationId: "org-1" }), + ); + + expect(result.exitCode).toBe(0); + expect(safeSpawnMock).toHaveBeenCalledTimes(2); + const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock")); + expect(unlockCalls).toHaveLength(1); + expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all"); + }); + + test("propagates the retry error when the retry is still locked", async () => { + vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve()); + const safeExecMock = vi.spyOn(nodeModule, "safeExec").mockResolvedValue({ + exitCode: 0, + stdout: "", + stderr: "", + timedOut: false, + }); + vi.spyOn(nodeModule, "safeSpawn").mockImplementation(async ({ onStderr }) => { + onStderr?.("unable to create lock in backend: repository is already locked"); + return { + exitCode: 11, + summary: "", + error: "unable to create lock in backend: repository is already locked", + stderr: "", + }; + }); + + const restic = createRestic(mockDeps); + + await expect( + Effect.runPromise(restic.backup(repositoryConfig, "/tmp/source", { organizationId: "org-1" })), + ).rejects.toThrow("unable to create lock in backend: repository is already locked"); + const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock")); + expect(unlockCalls).toHaveLength(1); + expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all"); + }); + + test("runs stale-only unlock for both repositories before retrying copy", async () => { + vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve()); + let copyCalls = 0; + const safeExecMock = vi.spyOn(nodeModule, "safeExec").mockImplementation(async ({ args }) => { + if (args?.includes("copy")) { + copyCalls += 1; + return copyCalls === 1 + ? { + exitCode: 11, + stdout: "", + stderr: "unable to create lock in backend: repository is already locked", + timedOut: false, + } + : { + exitCode: 0, + stdout: "copied", + stderr: "", + timedOut: false, + }; + } + + return { + exitCode: 0, + stdout: "", + stderr: "", + timedOut: false, + }; + }); + + const restic = createRestic(mockDeps); + await Effect.runPromise( + restic.copy(sourceRepositoryConfig, destinationRepositoryConfig, { + organizationId: "org-1", + }), + ); + + const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock")); + expect(copyCalls).toBe(2); + expect(unlockCalls).toHaveLength(2); + for (const [params] of unlockCalls) { + expect(params.args).not.toContain("--remove-all"); + } + }); +}); diff --git a/packages/core/src/restic/commands/__tests__/unlock.test.ts b/packages/core/src/restic/commands/__tests__/unlock.test.ts new file mode 100644 index 00000000..29a361ca --- /dev/null +++ b/packages/core/src/restic/commands/__tests__/unlock.test.ts @@ -0,0 +1,73 @@ +import { Effect } from "effect"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import * as nodeModule from "../../../node"; +import { ResticLockError } from "../../error"; +import * as cleanupModule from "../../helpers/cleanup-temporary-keys"; +import type { ResticDeps } from "../../types"; +import { unlock } from "../unlock"; + +const mockDeps: ResticDeps = { + resolveSecret: async (s) => s, + getOrganizationResticPassword: async () => "org-restic-password", + resticCacheDir: "/tmp/restic-cache", + resticPassFile: "/tmp/restic.pass", + defaultExcludes: ["/tmp/restic.pass", "/var/lib/zerobyte/repositories"], + rcloneConfigFile: "/root/.config/rclone/rclone.conf", +}; + +const config = { + backend: "local" as const, + path: "/tmp/restic-repo", + isExistingRepository: true, + customPassword: "custom-password", +}; + +const setup = () => { + let capturedArgs: string[] = []; + + vi.spyOn(cleanupModule, "cleanupTemporaryKeys").mockImplementation(() => Promise.resolve()); + vi.spyOn(nodeModule, "safeExec").mockImplementation(async ({ args }) => { + capturedArgs = args ?? []; + return { exitCode: 0, stdout: "", stderr: "", timedOut: false }; + }); + + return { + getArgs: () => capturedArgs, + }; +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("unlock command", () => { + test("removes only stale locks by default", async () => { + const { getArgs } = setup(); + + await Effect.runPromise(unlock(config, { organizationId: "org-1" }, mockDeps)); + + expect(getArgs()).toContain("unlock"); + expect(getArgs()).not.toContain("--remove-all"); + }); + + test("can opt in to removing all locks", async () => { + const { getArgs } = setup(); + + await Effect.runPromise(unlock(config, { organizationId: "org-1", removeAll: true }, mockDeps)); + + expect(getArgs()).toContain("--remove-all"); + }); + + test("returns a typed lock error for restic lock failures", async () => { + setup(); + vi.spyOn(nodeModule, "safeExec").mockResolvedValueOnce({ + exitCode: 11, + stdout: "", + stderr: "unable to create lock in backend: repository is already locked", + timedOut: false, + }); + + const error = await Effect.runPromise(Effect.flip(unlock(config, { organizationId: "org-1" }, mockDeps))); + expect(error).toBeInstanceOf(ResticLockError); + }); +}); diff --git a/packages/core/src/restic/commands/backup.ts b/packages/core/src/restic/commands/backup.ts index e6c03119..6bb9bad2 100644 --- a/packages/core/src/restic/commands/backup.ts +++ b/packages/core/src/restic/commands/backup.ts @@ -10,7 +10,7 @@ import { buildEnv } from "../helpers/build-env"; import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import { validateCustomResticParams } from "../helpers/validate-custom-params"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { logger, safeSpawn } from "../../node"; import type { ResticDeps } from "../types"; import { toMessage } from "../../utils"; @@ -190,7 +190,7 @@ export const backup = ( logger.error(`Restic backup failed: ${res.error}`); logger.error(`Command executed: restic ${args.join(" ")}`); - throw new ResticError(res.exitCode, stderrLines.join("\n") || res.stderr || res.error); + throw createResticError(res.exitCode, stderrLines.join("\n") || res.stderr || res.error); } const lastLine = res.summary.trim(); @@ -221,7 +221,7 @@ export const backup = ( }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/check.ts b/packages/core/src/restic/commands/check.ts index 48743da1..770826a7 100644 --- a/packages/core/src/restic/commands/check.ts +++ b/packages/core/src/restic/commands/check.ts @@ -6,7 +6,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; import type { ResticDeps } from "../types"; -import { ResticError } from "../error"; +import { isResticError } from "../error"; import { toMessage } from "../../utils"; class ResticCheckCommandError extends Data.TaggedError("ResticCheckCommandError")<{ @@ -77,7 +77,7 @@ export const check = ( }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/copy.ts b/packages/core/src/restic/commands/copy.ts index 03cc739f..88a21f3c 100644 --- a/packages/core/src/restic/commands/copy.ts +++ b/packages/core/src/restic/commands/copy.ts @@ -4,7 +4,7 @@ import { buildEnv } from "../helpers/build-env"; import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { logger, safeExec } from "../../node"; import type { ResticDeps } from "../types"; import { Data, Effect } from "effect"; @@ -72,7 +72,7 @@ export const copy = ( if (res.exitCode !== 0) { logger.error(`Restic copy failed: ${stderr}`); - throw new ResticError(res.exitCode, stderr); + throw createResticError(res.exitCode, stderr); } logger.info(`Restic copy completed from ${sourceRepoUrl} to ${destRepoUrl}`); @@ -82,7 +82,7 @@ export const copy = ( }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/delete-snapshots.ts b/packages/core/src/restic/commands/delete-snapshots.ts index d6a521b7..523524e9 100644 --- a/packages/core/src/restic/commands/delete-snapshots.ts +++ b/packages/core/src/restic/commands/delete-snapshots.ts @@ -1,6 +1,6 @@ import { Data, Effect } from "effect"; import { logger, safeExec } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { addCommonArgs } from "../helpers/add-common-args"; import { buildEnv } from "../helpers/build-env"; import { buildRepoUrl } from "../helpers/build-repo-url"; @@ -38,13 +38,13 @@ export const deleteSnapshots = ( if (res.exitCode !== 0) { logger.error(`Restic snapshot deletion failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + throw createResticError(res.exitCode, res.stderr); } return { success: true }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/dump.ts b/packages/core/src/restic/commands/dump.ts index 99406362..0dd0c0b6 100644 --- a/packages/core/src/restic/commands/dump.ts +++ b/packages/core/src/restic/commands/dump.ts @@ -7,7 +7,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; import { logger, safeSpawn } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { Data, Effect } from "effect"; import { toMessage } from "../../utils"; @@ -94,7 +94,7 @@ export const dump = ( const stderr = stderrTail.trim() || result.error; logger.error(`Restic dump failed: ${stderr}`); - throw new ResticError(result.exitCode, stderr); + throw createResticError(result.exitCode, stderr); }) .finally(async () => { abortController = null; @@ -120,7 +120,7 @@ export const dump = ( }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/forget.ts b/packages/core/src/restic/commands/forget.ts index 72c9c0eb..f2fef598 100644 --- a/packages/core/src/restic/commands/forget.ts +++ b/packages/core/src/restic/commands/forget.ts @@ -6,7 +6,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; import { logger, safeExec } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { toMessage } from "../../utils"; import { Data, Effect } from "effect"; @@ -65,7 +65,7 @@ export const forget = ( if (res.exitCode !== 0) { logger.error(`Restic forget failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + throw createResticError(res.exitCode, res.stderr); } const lines = res.stdout.split("\n").filter((line) => line.trim()); @@ -74,7 +74,7 @@ export const forget = ( return { success: true, data: result }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/init.ts b/packages/core/src/restic/commands/init.ts index 58ffbb9b..65703d86 100644 --- a/packages/core/src/restic/commands/init.ts +++ b/packages/core/src/restic/commands/init.ts @@ -7,7 +7,7 @@ import type { RepositoryConfig } from "../schemas"; import { logger, safeExec } from "../../node"; import type { ResticDeps } from "../types"; import { Data, Effect } from "effect"; -import { ResticError } from "../error"; +import { isResticError } from "../error"; import { toMessage } from "../../utils"; class ResticInitCommandError extends Data.TaggedError("ResticInitCommandError")<{ @@ -70,7 +70,7 @@ export const init = ( return { success: true, error: null }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/key-add.ts b/packages/core/src/restic/commands/key-add.ts index e4b80796..c2b9de66 100644 --- a/packages/core/src/restic/commands/key-add.ts +++ b/packages/core/src/restic/commands/key-add.ts @@ -7,7 +7,7 @@ import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; import type { ResticDeps } from "../types"; import { toMessage } from "../../utils"; -import { ResticError } from "../error"; +import { isResticError } from "../error"; class ResticKeyAddCommandError extends Data.TaggedError("ResticKeyAddCommandError")<{ cause: unknown; @@ -52,7 +52,7 @@ export const keyAdd = ( return { success: true, error: null }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/ls.ts b/packages/core/src/restic/commands/ls.ts index 68a4bd52..f8db1202 100644 --- a/packages/core/src/restic/commands/ls.ts +++ b/packages/core/src/restic/commands/ls.ts @@ -5,7 +5,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; import { logger, safeSpawn } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import type { ResticDeps } from "../types"; import { Data, Effect } from "effect"; import { toMessage } from "../../utils"; @@ -134,7 +134,7 @@ export const ls = ( if (res.exitCode !== 0) { logger.error(`Restic ls failed: ${res.error}`); - throw new ResticError(res.exitCode, res.stderr || res.error); + throw createResticError(res.exitCode, res.stderr || res.error); } if (totalNodes > offset + limit) { @@ -153,7 +153,7 @@ export const ls = ( } as ResticLsResult; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/repair-index.ts b/packages/core/src/restic/commands/repair-index.ts index 6d1906db..8ce2f349 100644 --- a/packages/core/src/restic/commands/repair-index.ts +++ b/packages/core/src/restic/commands/repair-index.ts @@ -1,6 +1,6 @@ import { Data, Effect } from "effect"; import { logger, safeExec } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { addCommonArgs } from "../helpers/add-common-args"; import { buildEnv } from "../helpers/build-env"; import { buildRepoUrl } from "../helpers/build-repo-url"; @@ -44,7 +44,7 @@ export const repairIndex = ( if (res.exitCode !== 0) { logger.error(`Restic repair index failed: ${stderr}`); - throw new ResticError(res.exitCode, stderr); + throw createResticError(res.exitCode, stderr); } logger.info(`Restic repair index completed for repository: ${repoUrl}`); @@ -55,7 +55,7 @@ export const repairIndex = ( }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/restore.ts b/packages/core/src/restic/commands/restore.ts index 01553382..24cc72d9 100644 --- a/packages/core/src/restic/commands/restore.ts +++ b/packages/core/src/restic/commands/restore.ts @@ -8,7 +8,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import { type RepositoryConfig, type OverwriteMode } from "../schemas"; import { logger, safeSpawn } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { resticRestoreOutputSchema, type ResticRestoreOutputDto } from "../restic-dto"; import type { ResticDeps } from "../types"; import { Data, Effect } from "effect"; @@ -147,7 +147,7 @@ export const restore = ( if (res.exitCode !== 0) { logger.error(`Restic restore failed: ${res.error}`); - throw new ResticError(res.exitCode, res.stderr || res.error); + throw createResticError(res.exitCode, res.stderr || res.error); } const lastLine = res.summary.trim(); @@ -183,7 +183,7 @@ export const restore = ( return result.data; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/snapshots.ts b/packages/core/src/restic/commands/snapshots.ts index 2f14e80d..1c6a8d97 100644 --- a/packages/core/src/restic/commands/snapshots.ts +++ b/packages/core/src/restic/commands/snapshots.ts @@ -9,7 +9,7 @@ import { logger, safeSpawn } from "../../node"; import type { ResticDeps } from "../types"; import { Data, Effect } from "effect"; import { toMessage } from "../../utils"; -import { ResticError } from "../error"; +import { isResticError } from "../error"; class ResticSnapshotsCommandError extends Data.TaggedError("ResticSnapshotsCommandError")<{ cause: unknown; @@ -81,7 +81,7 @@ export const snapshots = ( }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/stats.ts b/packages/core/src/restic/commands/stats.ts index c49178fc..b50bb125 100644 --- a/packages/core/src/restic/commands/stats.ts +++ b/packages/core/src/restic/commands/stats.ts @@ -5,7 +5,7 @@ import { buildRepoUrl } from "../helpers/build-repo-url"; import { cleanupTemporaryKeys } from "../helpers/cleanup-temporary-keys"; import type { RepositoryConfig } from "../schemas"; import { logger, safeExec } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { resticStatsSchema } from "../restic-dto"; import type { ResticDeps } from "../types"; import { Data, Effect } from "effect"; @@ -30,7 +30,7 @@ export const stats = (config: RepositoryConfig, options: { organizationId: strin if (res.exitCode !== 0) { logger.error(`Restic stats retrieval failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + throw createResticError(res.exitCode, res.stderr); } const parsedJson = safeJsonParse(res.stdout); @@ -44,7 +44,7 @@ export const stats = (config: RepositoryConfig, options: { organizationId: strin return result.data; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/tag-snapshots.ts b/packages/core/src/restic/commands/tag-snapshots.ts index fdf1b9ff..9c2422b4 100644 --- a/packages/core/src/restic/commands/tag-snapshots.ts +++ b/packages/core/src/restic/commands/tag-snapshots.ts @@ -1,6 +1,6 @@ import { Data, Effect } from "effect"; import { logger, safeExec } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { addCommonArgs } from "../helpers/add-common-args"; import { buildEnv } from "../helpers/build-env"; import { buildRepoUrl } from "../helpers/build-repo-url"; @@ -58,14 +58,14 @@ export const tagSnapshots = ( if (res.exitCode !== 0) { logger.error(`Restic snapshot tagging failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + throw createResticError(res.exitCode, res.stderr); } return { success: true }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/commands/unlock.ts b/packages/core/src/restic/commands/unlock.ts index f8afa3ec..c7a581fc 100644 --- a/packages/core/src/restic/commands/unlock.ts +++ b/packages/core/src/restic/commands/unlock.ts @@ -1,6 +1,6 @@ import { Data, Effect } from "effect"; import { logger, safeExec } from "../../node"; -import { ResticError } from "../error"; +import { createResticError, isResticError } from "../error"; import { addCommonArgs } from "../helpers/add-common-args"; import { buildEnv } from "../helpers/build-env"; import { buildRepoUrl } from "../helpers/build-repo-url"; @@ -16,7 +16,7 @@ class ResticUnlockCommandError extends Data.TaggedError("ResticUnlockCommandErro export const unlock = ( config: RepositoryConfig, - options: { signal?: AbortSignal; organizationId: string }, + options: { signal?: AbortSignal; organizationId: string; removeAll?: boolean }, deps: ResticDeps, ) => { return Effect.tryPromise({ @@ -24,7 +24,10 @@ export const unlock = ( const repoUrl = buildRepoUrl(config); const env = await buildEnv(config, options.organizationId, deps); - const args = ["unlock", "--repo", repoUrl, "--remove-all"]; + const args = ["unlock", "--repo", repoUrl]; + if (options.removeAll) { + args.push("--remove-all"); + } addCommonArgs(args, env, config); const res = await safeExec({ @@ -42,14 +45,14 @@ export const unlock = ( if (res.exitCode !== 0) { logger.error(`Restic unlock failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + throw createResticError(res.exitCode, res.stderr); } logger.info(`Restic unlock succeeded for repository: ${repoUrl}`); return { success: true, message: "Repository unlocked successfully" }; }, catch: (error) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error; } diff --git a/packages/core/src/restic/error.ts b/packages/core/src/restic/error.ts index 7f631543..fe11b041 100644 --- a/packages/core/src/restic/error.ts +++ b/packages/core/src/restic/error.ts @@ -1,3 +1,5 @@ +import { Data } from "effect"; + const resticErrorCodes: Record = { 1: "Command failed: An error occurred while executing the command.", 2: "Go runtime error: A runtime error occurred in the Go program.", @@ -23,3 +25,34 @@ export class ResticError extends Error { this.name = "ResticError"; } } + +export class ResticLockError extends Data.TaggedError("ResticLockError")<{ + code: number; + summary: string; + details: string; + message: string; +}> { + constructor(details: string) { + const summary = resticErrorCodes[11]!; + super({ + code: 11, + summary, + details, + message: details ? `${summary}\n${details}` : summary, + }); + this.name = "ResticLockError"; + } +} + +export type AnyResticError = ResticError | ResticLockError; + +export const isResticError = (error: unknown): error is AnyResticError => + error instanceof ResticError || error instanceof ResticLockError; + +export const createResticError = (code: number, details: string) => { + if (code === 11) { + return new ResticLockError(details); + } + + return new ResticError(code, details); +}; diff --git a/packages/core/src/restic/index.ts b/packages/core/src/restic/index.ts index e7928da9..0900cf5c 100644 --- a/packages/core/src/restic/index.ts +++ b/packages/core/src/restic/index.ts @@ -1,6 +1,6 @@ export * from "./schemas"; export * from "./restic-dto"; -export { ResticError } from "./error"; +export { isResticError, ResticError, ResticLockError } from "./error"; export type { RestoreProgress } from "./commands/restore"; export type { diff --git a/packages/core/src/restic/lock-diagnostics.ts b/packages/core/src/restic/lock-diagnostics.ts index 2880edd6..03ef1ad6 100644 --- a/packages/core/src/restic/lock-diagnostics.ts +++ b/packages/core/src/restic/lock-diagnostics.ts @@ -1,6 +1,6 @@ import { logger, safeExec } from "../node"; import { toMessage } from "../utils"; -import { ResticError } from "./error"; +import { ResticLockError } from "./error"; import { addCommonArgs } from "./helpers/add-common-args"; import { buildEnv } from "./helpers/build-env"; import { buildRepoUrl } from "./helpers/build-repo-url"; @@ -26,7 +26,7 @@ const LOCK_ERROR_PATTERNS = [ ]; export const isResticLockFailure = (error: unknown) => { - if (error instanceof ResticError && error.code === 11) { + if (error instanceof ResticLockError) { return true; } diff --git a/packages/core/src/restic/server.ts b/packages/core/src/restic/server.ts index 1f307e52..3d6020d1 100644 --- a/packages/core/src/restic/server.ts +++ b/packages/core/src/restic/server.ts @@ -1,4 +1,5 @@ import { Effect } from "effect"; +import { logger } from "../node"; import { backup } from "./commands/backup"; import { check } from "./commands/check"; import { copy } from "./commands/copy"; @@ -14,6 +15,7 @@ import { snapshots } from "./commands/snapshots"; import { stats } from "./commands/stats"; import { tagSnapshots } from "./commands/tag-snapshots"; import { unlock } from "./commands/unlock"; +import { ResticLockError } from "./error"; import { logResticLockFailureDiagnostics } from "./lock-diagnostics"; import type { RepositoryConfig } from "./schemas"; import type { ResticDeps } from "./types"; @@ -24,7 +26,7 @@ export { buildRepoUrl } from "./helpers/build-repo-url"; export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys"; export { validateCustomResticParams } from "./helpers/validate-custom-params"; export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics"; -export { ResticError } from "./error"; +export { isResticError, ResticError, ResticLockError } from "./error"; type LockDiagnosticCommandContext = { repositoryConfig: RepositoryConfig; @@ -34,6 +36,12 @@ type LockDiagnosticCommandContext = { type ResticCommandOptions = { organizationId: string }; type ResticCommandResult = { error?: unknown }; +type ResticCommandFailure = Failure | ResticLockError; +type RunResticCommand = () => Effect.Effect< + Success, + ResticCommandFailure, + Requirements +>; const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => { const firstRepositoryConfig = args[0] as RepositoryConfig; @@ -68,15 +76,47 @@ const logLockFailure = async ( relatedRepositoryConfigs: context.relatedRepositoryConfigs, }); +const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect => + Effect.gen(function* () { + for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) { + yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps); + } + }); + +const recoverFromLockFailure = ( + error: ResticLockError, + operation: string, + context: LockDiagnosticCommandContext, + deps: ResticDeps, + runCommand: RunResticCommand, +): Effect.Effect | Error, Requirements> => + Effect.gen(function* () { + yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once."); + yield* Effect.promise(() => logLockFailure(error, operation, context, deps)); + yield* unlockStaleLocks(context, deps); + + const retryResult = yield* runCommand(); + yield* logger.effect.info("Restic lock failure recovered by removing stale locks and retrying once."); + + return retryResult; + }); + function withDeps( operation: string, - command: (...args: [...Args, ResticDeps]) => Effect.Effect, + command: (...args: [...Args, ResticDeps]) => Effect.Effect, Requirements>, deps: ResticDeps, -): (...args: Args) => Effect.Effect { +): (...args: Args) => Effect.Effect | Error, Requirements> { return (...args: Args) => { const context = getCommandContext(operation, args); - return command(...args, deps).pipe( - Effect.tapError((error) => Effect.promise(() => logLockFailure(error, operation, context, deps))), + const runCommand = () => command(...args, deps); + + const commandEffect = runCommand().pipe( + Effect.catchTag("ResticLockError", (error) => + recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand), + ), + ); + + return commandEffect.pipe( Effect.tap((result) => { const { error } = result as ResticCommandResult; return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void; diff --git a/packages/core/src/utils/errors.ts b/packages/core/src/utils/errors.ts index bd3b6c7e..ba5b6ed3 100644 --- a/packages/core/src/utils/errors.ts +++ b/packages/core/src/utils/errors.ts @@ -1,4 +1,4 @@ -import { ResticError } from "../restic/error.js"; +import { isResticError } from "../restic/error.js"; export const toMessage = (error: unknown) => { if (error instanceof Error) { @@ -9,7 +9,7 @@ export const toMessage = (error: unknown) => { }; export const toErrorDetails = (error: unknown) => { - if (error instanceof ResticError) { + if (isResticError(error)) { return error.details || error.summary; }