diff --git a/packages/core/src/restic/lock-diagnostics.ts b/packages/core/src/restic/lock-diagnostics.ts deleted file mode 100644 index 86e431ed..00000000 --- a/packages/core/src/restic/lock-diagnostics.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { logger, safeExec } from "../node"; -import { toMessage } from "../utils"; -import { safeJsonParse } from "../utils/json"; -import { ResticLockError } from "./error"; -import { addCommonArgs } from "./helpers/add-common-args"; -import { buildEnv } from "./helpers/build-env"; -import { buildRepoUrl } from "./helpers/build-repo-url"; -import { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys"; -import type { RepositoryConfig } from "./schemas"; -import type { ResticDeps } from "./types"; - -type ResticLockDiagnosticContext = { - error: unknown; - operation: string; - repositoryConfig: RepositoryConfig; - organizationId: string; - resticDeps: ResticDeps; - relatedRepositoryConfigs?: RepositoryConfig[]; -}; - -const LOCK_ERROR_PATTERNS = [ - /unable to create lock in backend/i, - /repository is already locked/i, - /failed to lock repository/i, - /"code"\s*:\s*11/i, - /\bexit_error\b.*\b11\b/i, -]; - -export const isResticLockFailure = (error: unknown) => { - if (error instanceof ResticLockError) { - return true; - } - - const message = toMessage(error); - return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message)); -}; - -const RESTIC_LOCK_ID_PATTERN = /^[a-f0-9]{64}$/i; - -const addLockId = (ids: Set, candidate: unknown) => { - if (typeof candidate === "string" && RESTIC_LOCK_ID_PATTERN.test(candidate)) { - ids.add(candidate); - } -}; - -const parseLockIds = (stdout: string) => { - const ids = new Set(); - - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (!trimmed) continue; - - const parsed = safeJsonParse<{ id?: unknown }>(trimmed); - if (parsed) { - addLockId(ids, parsed.id); - continue; - } - - addLockId(ids, trimmed); - } - - return [...ids].slice(0, 20); -}; - -const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => { - const repoUrl = buildRepoUrl(config); - const env = await buildEnv(config, organizationId, deps); - const baseArgs = ["--repo", repoUrl]; - addCommonArgs(baseArgs, env, config); - - try { - const listResult = await safeExec({ - command: "restic", - args: [...baseArgs, "list", "locks"], - env, - timeout: 15_000, - }); - - const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : []; - const lockDetails = []; - - for (const lockId of lockIds) { - const catResult = await safeExec({ - command: "restic", - args: [...baseArgs, "cat", "lock", "--", lockId], - env, - timeout: 15_000, - }); - - lockDetails.push({ - lockId, - exitCode: catResult.exitCode, - stdout: catResult.stdout, - stderr: catResult.stderr, - timedOut: catResult.timedOut, - }); - } - - return { - repoUrl, - list: { - exitCode: listResult.exitCode, - stdout: listResult.stdout, - stderr: listResult.stderr, - timedOut: listResult.timedOut, - }, - lockIds, - lockDetails, - }; - } finally { - await cleanupTemporaryKeys(env, deps); - } -}; - -export const logResticLockFailureDiagnostics = async ({ - error, - operation, - repositoryConfig, - organizationId, - resticDeps, - relatedRepositoryConfigs = [], -}: ResticLockDiagnosticContext) => { - if (!isResticLockFailure(error)) { - return false; - } - - try { - const configsByRepoUrl = new Map( - [repositoryConfig, ...relatedRepositoryConfigs].map((config) => [buildRepoUrl(config), config]), - ); - const configsToInspect = [...configsByRepoUrl.entries()]; - - logger.error("[ResticLockFailure] Restic repository lock failure detected", { - operation, - error: toMessage(error), - process: { - pid: process.pid, - hostname: process.env.HOSTNAME, - nodeEnv: process.env.NODE_ENV, - }, - repository: { - repoUrl: buildRepoUrl(repositoryConfig), - }, - relatedRepositories: relatedRepositoryConfigs.map((config) => ({ - repoUrl: buildRepoUrl(config), - })), - }); - - for (const [repoUrl, config] of configsToInspect) { - try { - const resticLocks = await inspectResticLocks(config, organizationId, resticDeps); - logger.error("[ResticLockFailure] Restic backend lock dump", { - operation, - repoUrl, - resticLocks, - }); - } catch (diagnosticError) { - logger.error("[ResticLockFailure] Failed to inspect restic backend locks", { - operation, - repoUrl, - error: toMessage(diagnosticError), - }); - } - } - - return true; - } catch (diagnosticError) { - logger.error("[ResticLockFailure] Failed to collect lock diagnostics", { - operation, - repoUrl: buildRepoUrl(repositoryConfig), - error: toMessage(diagnosticError), - }); - return true; - } -}; diff --git a/packages/core/src/restic/server.ts b/packages/core/src/restic/server.ts index 3d6020d1..08e50f47 100644 --- a/packages/core/src/restic/server.ts +++ b/packages/core/src/restic/server.ts @@ -16,7 +16,6 @@ import { stats } from "./commands/stats"; import { tagSnapshots } from "./commands/tag-snapshots"; import { unlock } from "./commands/unlock"; import { ResticLockError } from "./error"; -import { logResticLockFailureDiagnostics } from "./lock-diagnostics"; import type { RepositoryConfig } from "./schemas"; import type { ResticDeps } from "./types"; @@ -25,17 +24,14 @@ export { buildEnv } from "./helpers/build-env"; export { buildRepoUrl } from "./helpers/build-repo-url"; export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys"; export { validateCustomResticParams } from "./helpers/validate-custom-params"; -export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics"; export { isResticError, ResticError, ResticLockError } from "./error"; -type LockDiagnosticCommandContext = { - repositoryConfig: RepositoryConfig; +type LockRecoveryContext = { + repositoryConfigs: RepositoryConfig[]; organizationId: string; - relatedRepositoryConfigs?: RepositoryConfig[]; }; type ResticCommandOptions = { organizationId: string }; -type ResticCommandResult = { error?: unknown }; type ResticCommandFailure = Failure | ResticLockError; type RunResticCommand = () => Effect.Effect< Success, @@ -43,56 +39,37 @@ type RunResticCommand = () => Effect.Effect< Requirements >; -const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => { +const getLockRecoveryContext = (operation: string, args: unknown[]): LockRecoveryContext => { const firstRepositoryConfig = args[0] as RepositoryConfig; const options = args.at(-1) as ResticCommandOptions; if (operation === "restic.copy") { return { - repositoryConfig: args[1] as RepositoryConfig, + repositoryConfigs: [args[1] as RepositoryConfig, firstRepositoryConfig], organizationId: options.organizationId, - relatedRepositoryConfigs: [firstRepositoryConfig], }; } return { - repositoryConfig: firstRepositoryConfig, + repositoryConfigs: [firstRepositoryConfig], organizationId: options.organizationId, }; }; -const logLockFailure = async ( - error: unknown, - operation: string, - context: LockDiagnosticCommandContext, - deps: ResticDeps, -) => - logResticLockFailureDiagnostics({ - error, - operation, - repositoryConfig: context.repositoryConfig, - organizationId: context.organizationId, - resticDeps: deps, - relatedRepositoryConfigs: context.relatedRepositoryConfigs, - }); - -const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect => +const unlockStaleLocks = (context: LockRecoveryContext, deps: ResticDeps) => Effect.gen(function* () { - for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) { + for (const repositoryConfig of context.repositoryConfigs) { yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps); } - }); + }).pipe(Effect.catchAll(() => Effect.void)); const recoverFromLockFailure = ( - error: ResticLockError, - operation: string, - context: LockDiagnosticCommandContext, - deps: ResticDeps, + context: LockRecoveryContext, runCommand: RunResticCommand, + deps: ResticDeps, ): Effect.Effect | Error, Requirements> => Effect.gen(function* () { yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once."); - yield* Effect.promise(() => logLockFailure(error, operation, context, deps)); yield* unlockStaleLocks(context, deps); const retryResult = yield* runCommand(); @@ -107,20 +84,11 @@ function withDeps( deps: ResticDeps, ): (...args: Args) => Effect.Effect | Error, Requirements> { return (...args: Args) => { - const context = getCommandContext(operation, args); + const context = getLockRecoveryContext(operation, args); const runCommand = () => command(...args, deps); - const commandEffect = runCommand().pipe( - Effect.catchTag("ResticLockError", (error) => - recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand), - ), - ); - - return commandEffect.pipe( - Effect.tap((result) => { - const { error } = result as ResticCommandResult; - return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void; - }), + return runCommand().pipe( + Effect.catchTag("ResticLockError", () => recoverFromLockFailure(context, runCommand, deps)), ); }; }