diff --git a/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts b/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts index 60d38847..d5c01fdf 100644 --- a/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts +++ b/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts @@ -213,9 +213,7 @@ describe("syncMirror", () => { }); await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id); - await expect( - backupsService.syncMirror(schedule.shortId, mirrorRepository.shortId as ShortId, ["snap1"]), - ).resolves.toEqual({ success: true }); + const firstSync = backupsService.syncMirror(schedule.shortId, mirrorRepository.shortId as ShortId, ["snap1"]); await copyStarted; @@ -230,6 +228,8 @@ describe("syncMirror", () => { releaseCopy?.(); + await expect(firstSync).resolves.toEqual({ success: true }); + await waitForExpect(async () => { const mirrors = await backupsService.getMirrors(schedule.shortId); expect(mirrors[0]?.lastCopyStatus).toBe("success"); diff --git a/app/server/modules/repositories/helpers/restic-lock-diagnostics.ts b/app/server/modules/repositories/helpers/restic-lock-diagnostics.ts deleted file mode 100644 index c3017fae..00000000 --- a/app/server/modules/repositories/helpers/restic-lock-diagnostics.ts +++ /dev/null @@ -1,251 +0,0 @@ -import { logger, safeExec } from "@zerobyte/core/node"; -import { ResticError, type RepositoryConfig } from "@zerobyte/core/restic"; -import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "@zerobyte/core/restic/server"; -import type { ResticDeps } from "@zerobyte/core/restic"; -import { db } from "~/server/db/db"; -import { type Repository } from "~/server/db/schema"; -import { toMessage } from "~/server/utils/errors"; - -type ResticLockDiagnosticContext = { - error: unknown; - operation: string; - repositoryConfig: RepositoryConfig; - organizationId: string; - resticDeps: ResticDeps; - relatedRepositoryConfigs?: RepositoryConfig[]; -}; - -const LOCK_ERROR_PATTERNS = [ - /unable to create lock in backend/i, - /repository is already locked/i, - /failed to lock repository/i, - /"code"\s*:\s*11/i, - /\bexit_error\b.*\b11\b/i, -]; - -export const isResticLockFailure = (error: unknown) => { - if (error instanceof ResticError && error.code === 11) { - return true; - } - - const message = toMessage(error); - return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message)); -}; - -const physicalRepositoryKey = (config: RepositoryConfig) => { - switch (config.backend) { - case "local": - return `local:${config.path}`; - case "s3": - case "r2": - return `${config.backend}:${config.endpoint.trim().replace(/\/$/, "")}/${config.bucket}`; - case "gcs": - return `gcs:${config.bucket}`; - case "azure": - return `azure:${config.accountName}/${config.container}`; - case "rclone": - return `rclone:${config.remote}:${config.path}`; - case "rest": - return `rest:${config.url.trim().replace(/\/$/, "")}/${config.path ?? ""}`; - case "sftp": - return `sftp:${config.user}@${config.host}:${config.port ?? 22}:${config.path}`; - } -}; - -const getMutexRows = (repositoryIds: string[]) => { - if (repositoryIds.length === 0) { - return { locks: [], waiters: [] }; - } - - const locks = db.query.repositoryLocksTable.findMany().sync(); - const waiters = db.query.repositoryLockWaitersTable.findMany().sync(); - const idSet = new Set(repositoryIds); - - return { - locks: locks.filter((lock) => idSet.has(lock.repositoryId)), - waiters: waiters.filter((waiter) => idSet.has(waiter.repositoryId)), - }; -}; - -const findRepositoriesByConfig = (config: RepositoryConfig, organizationId: string) => { - const target = physicalRepositoryKey(config); - const repositories = db.query.repositoriesTable - .findMany({ - where: { organizationId }, - }) - .sync(); - - return repositories.filter((candidate) => physicalRepositoryKey(candidate.config) === target); -}; - -const summarizeRepository = (repository: Repository) => ({ - id: repository.id, - shortId: repository.shortId, - name: repository.name, - type: repository.type, - status: repository.status, - physicalKey: physicalRepositoryKey(repository.config), - repoUrl: buildRepoUrl(repository.config), - lastChecked: repository.lastChecked, - lastError: repository.lastError, -}); - -const listPhysicalDuplicates = (config: RepositoryConfig, organizationId: string) => { - const repositories = findRepositoriesByConfig(config, organizationId); - - return repositories - .filter( - (candidate) => - repositories.length > 1 || physicalRepositoryKey(candidate.config) === physicalRepositoryKey(config), - ) - .map(summarizeRepository); -}; - -const parseLockIds = (stdout: string) => { - const ids = new Set(); - - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (!trimmed) continue; - - const jsonId = trimmed.match(/"id"\s*:\s*"([^"]+)"/)?.[1]; - if (jsonId) { - ids.add(jsonId); - continue; - } - - const hexId = trimmed.match(/[a-f0-9]{64}/i)?.[0]; - if (hexId) { - ids.add(hexId); - } - } - - return [...ids].slice(0, 20); -}; - -const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => { - const repoUrl = buildRepoUrl(config); - const env = await buildEnv(config, organizationId, deps); - const baseArgs = ["--repo", repoUrl]; - addCommonArgs(baseArgs, env, config); - - try { - const listResult = await safeExec({ - command: "restic", - args: [...baseArgs, "list", "locks"], - env, - timeout: 15_000, - }); - - const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : []; - const lockDetails = []; - - for (const lockId of lockIds) { - const catResult = await safeExec({ - command: "restic", - args: [...baseArgs, "cat", "lock", lockId], - env, - timeout: 15_000, - }); - - lockDetails.push({ - lockId, - exitCode: catResult.exitCode, - stdout: catResult.stdout, - stderr: catResult.stderr, - timedOut: catResult.timedOut, - }); - } - - return { - repoUrl, - list: { - exitCode: listResult.exitCode, - stdout: listResult.stdout, - stderr: listResult.stderr, - timedOut: listResult.timedOut, - }, - lockIds, - lockDetails, - }; - } finally { - await cleanupTemporaryKeys(env, deps); - } -}; - -export const logResticLockFailureDiagnostics = async ({ - error, - operation, - repositoryConfig, - organizationId, - resticDeps, - relatedRepositoryConfigs = [], -}: ResticLockDiagnosticContext) => { - if (!isResticLockFailure(error)) { - return false; - } - - try { - const repositoryRows = findRepositoriesByConfig(repositoryConfig, organizationId); - const relatedRows = relatedRepositoryConfigs.flatMap((config) => - findRepositoriesByConfig(config, organizationId), - ); - const uniqueRows = [...repositoryRows, ...relatedRows].filter( - (candidate, index, all) => all.findIndex((other) => other.id === candidate.id) === index, - ); - const duplicateRows = listPhysicalDuplicates(repositoryConfig, organizationId); - const repositoryIds = [ - ...new Set([...uniqueRows.map((candidate) => candidate.id), ...duplicateRows.map((row) => row.id)]), - ]; - const mutexRows = getMutexRows(repositoryIds); - - logger.error("[ResticLockFailure] Restic repository lock failure detected", { - operation, - error: toMessage(error), - process: { - pid: process.pid, - hostname: process.env.HOSTNAME, - nodeEnv: process.env.NODE_ENV, - }, - repository: { - physicalKey: physicalRepositoryKey(repositoryConfig), - repoUrl: buildRepoUrl(repositoryConfig), - matchingRows: repositoryRows.map(summarizeRepository), - }, - relatedRepositories: relatedRows.map(summarizeRepository), - duplicatePhysicalRepositoryRows: duplicateRows, - mutexState: mutexRows, - }); - - const configsToInspect = [repositoryConfig, ...relatedRepositoryConfigs].filter( - (config, index, all) => - all.findIndex((other) => physicalRepositoryKey(other) === physicalRepositoryKey(config)) === index, - ); - - for (const config of configsToInspect) { - try { - const resticLocks = await inspectResticLocks(config, organizationId, resticDeps); - logger.error("[ResticLockFailure] Restic backend lock dump", { - operation, - physicalKey: physicalRepositoryKey(config), - resticLocks, - }); - } catch (diagnosticError) { - logger.error("[ResticLockFailure] Failed to inspect restic backend locks", { - operation, - physicalKey: physicalRepositoryKey(config), - error: toMessage(diagnosticError), - }); - } - } - - return true; - } catch (diagnosticError) { - logger.error("[ResticLockFailure] Failed to collect lock diagnostics", { - operation, - physicalKey: physicalRepositoryKey(repositoryConfig), - error: toMessage(diagnosticError), - }); - return true; - } -}; diff --git a/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts b/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts index e3ffb668..1b35680a 100644 --- a/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts +++ b/packages/core/src/restic/commands/__tests__/delete-snapshots.test.ts @@ -50,4 +50,22 @@ describe("deleteSnapshots command", () => { expect(separatorIndex).toBeGreaterThan(-1); expect(getArgs().slice(separatorIndex + 1)).toEqual(snapshotIds); }); + + test("rejects empty snapshot IDs before building restic env", async () => { + setup(); + const deps = { + ...mockDeps, + resolveSecret: vi.fn(mockDeps.resolveSecret), + }; + + await expect( + Effect.runPromise(deleteSnapshots(config, [], { organizationId: "org-1" }, deps)), + ).rejects.toMatchObject({ + message: "No snapshot IDs provided for deletion.", + }); + + expect(deps.resolveSecret).not.toHaveBeenCalled(); + expect(nodeModule.safeExec).not.toHaveBeenCalled(); + expect(cleanupModule.cleanupTemporaryKeys).not.toHaveBeenCalled(); + }); }); diff --git a/packages/core/src/restic/commands/delete-snapshots.ts b/packages/core/src/restic/commands/delete-snapshots.ts index 98fa88d4..d6a521b7 100644 --- a/packages/core/src/restic/commands/delete-snapshots.ts +++ b/packages/core/src/restic/commands/delete-snapshots.ts @@ -22,13 +22,13 @@ export const deleteSnapshots = ( ) => { return Effect.tryPromise({ try: async () => { - const repoUrl = buildRepoUrl(config); - const env = await buildEnv(config, options.organizationId, deps); - if (snapshotIds.length === 0) { throw new Error("No snapshot IDs provided for deletion."); } + const repoUrl = buildRepoUrl(config); + const env = await buildEnv(config, options.organizationId, deps); + const args: string[] = ["--repo", repoUrl, "forget", "--prune"]; addCommonArgs(args, env, config); args.push("--", ...snapshotIds); diff --git a/packages/core/src/restic/commands/init.ts b/packages/core/src/restic/commands/init.ts index c106414a..58ffbb9b 100644 --- a/packages/core/src/restic/commands/init.ts +++ b/packages/core/src/restic/commands/init.ts @@ -65,11 +65,7 @@ export const init = ( logger.info(`Restic repository initialized: ${repoUrl}`); - await addDefaultKey( - config, - { organizationId: options.organizationId, timeoutMs: options?.timeoutMs }, - deps, - ); + void addDefaultKey(config, { organizationId: options.organizationId, timeoutMs: options?.timeoutMs }, deps); return { success: true, error: null }; }, diff --git a/packages/core/src/restic/server.ts b/packages/core/src/restic/server.ts index 3031fc49..1f307e52 100644 --- a/packages/core/src/restic/server.ts +++ b/packages/core/src/restic/server.ts @@ -23,6 +23,7 @@ export { buildEnv } from "./helpers/build-env"; export { buildRepoUrl } from "./helpers/build-repo-url"; export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys"; export { validateCustomResticParams } from "./helpers/validate-custom-params"; +export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics"; export { ResticError } from "./error"; type LockDiagnosticCommandContext = {