refactor: remove repository lock diagnostics logs

This commit is contained in:
Nicolas Meienberger 2026-06-04 17:55:43 +02:00
parent 9d63f7cb2d
commit 4bba2c2493
No known key found for this signature in database
2 changed files with 13 additions and 220 deletions

View file

@ -1,175 +0,0 @@
import { logger, safeExec } from "../node";
import { toMessage } from "../utils";
import { safeJsonParse } from "../utils/json";
import { ResticLockError } from "./error";
import { addCommonArgs } from "./helpers/add-common-args";
import { buildEnv } from "./helpers/build-env";
import { buildRepoUrl } from "./helpers/build-repo-url";
import { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
import type { RepositoryConfig } from "./schemas";
import type { ResticDeps } from "./types";
type ResticLockDiagnosticContext = {
error: unknown;
operation: string;
repositoryConfig: RepositoryConfig;
organizationId: string;
resticDeps: ResticDeps;
relatedRepositoryConfigs?: RepositoryConfig[];
};
const LOCK_ERROR_PATTERNS = [
/unable to create lock in backend/i,
/repository is already locked/i,
/failed to lock repository/i,
/"code"\s*:\s*11/i,
/\bexit_error\b.*\b11\b/i,
];
export const isResticLockFailure = (error: unknown) => {
if (error instanceof ResticLockError) {
return true;
}
const message = toMessage(error);
return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message));
};
const RESTIC_LOCK_ID_PATTERN = /^[a-f0-9]{64}$/i;
const addLockId = (ids: Set<string>, candidate: unknown) => {
if (typeof candidate === "string" && RESTIC_LOCK_ID_PATTERN.test(candidate)) {
ids.add(candidate);
}
};
const parseLockIds = (stdout: string) => {
const ids = new Set<string>();
for (const line of stdout.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
const parsed = safeJsonParse<{ id?: unknown }>(trimmed);
if (parsed) {
addLockId(ids, parsed.id);
continue;
}
addLockId(ids, trimmed);
}
return [...ids].slice(0, 20);
};
const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, organizationId, deps);
const baseArgs = ["--repo", repoUrl];
addCommonArgs(baseArgs, env, config);
try {
const listResult = await safeExec({
command: "restic",
args: [...baseArgs, "list", "locks"],
env,
timeout: 15_000,
});
const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : [];
const lockDetails = [];
for (const lockId of lockIds) {
const catResult = await safeExec({
command: "restic",
args: [...baseArgs, "cat", "lock", "--", lockId],
env,
timeout: 15_000,
});
lockDetails.push({
lockId,
exitCode: catResult.exitCode,
stdout: catResult.stdout,
stderr: catResult.stderr,
timedOut: catResult.timedOut,
});
}
return {
repoUrl,
list: {
exitCode: listResult.exitCode,
stdout: listResult.stdout,
stderr: listResult.stderr,
timedOut: listResult.timedOut,
},
lockIds,
lockDetails,
};
} finally {
await cleanupTemporaryKeys(env, deps);
}
};
export const logResticLockFailureDiagnostics = async ({
error,
operation,
repositoryConfig,
organizationId,
resticDeps,
relatedRepositoryConfigs = [],
}: ResticLockDiagnosticContext) => {
if (!isResticLockFailure(error)) {
return false;
}
try {
const configsByRepoUrl = new Map(
[repositoryConfig, ...relatedRepositoryConfigs].map((config) => [buildRepoUrl(config), config]),
);
const configsToInspect = [...configsByRepoUrl.entries()];
logger.error("[ResticLockFailure] Restic repository lock failure detected", {
operation,
error: toMessage(error),
process: {
pid: process.pid,
hostname: process.env.HOSTNAME,
nodeEnv: process.env.NODE_ENV,
},
repository: {
repoUrl: buildRepoUrl(repositoryConfig),
},
relatedRepositories: relatedRepositoryConfigs.map((config) => ({
repoUrl: buildRepoUrl(config),
})),
});
for (const [repoUrl, config] of configsToInspect) {
try {
const resticLocks = await inspectResticLocks(config, organizationId, resticDeps);
logger.error("[ResticLockFailure] Restic backend lock dump", {
operation,
repoUrl,
resticLocks,
});
} catch (diagnosticError) {
logger.error("[ResticLockFailure] Failed to inspect restic backend locks", {
operation,
repoUrl,
error: toMessage(diagnosticError),
});
}
}
return true;
} catch (diagnosticError) {
logger.error("[ResticLockFailure] Failed to collect lock diagnostics", {
operation,
repoUrl: buildRepoUrl(repositoryConfig),
error: toMessage(diagnosticError),
});
return true;
}
};

View file

@ -16,7 +16,6 @@ import { stats } from "./commands/stats";
import { tagSnapshots } from "./commands/tag-snapshots"; import { tagSnapshots } from "./commands/tag-snapshots";
import { unlock } from "./commands/unlock"; import { unlock } from "./commands/unlock";
import { ResticLockError } from "./error"; import { ResticLockError } from "./error";
import { logResticLockFailureDiagnostics } from "./lock-diagnostics";
import type { RepositoryConfig } from "./schemas"; import type { RepositoryConfig } from "./schemas";
import type { ResticDeps } from "./types"; import type { ResticDeps } from "./types";
@ -25,17 +24,14 @@ export { buildEnv } from "./helpers/build-env";
export { buildRepoUrl } from "./helpers/build-repo-url"; export { buildRepoUrl } from "./helpers/build-repo-url";
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys"; export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
export { validateCustomResticParams } from "./helpers/validate-custom-params"; export { validateCustomResticParams } from "./helpers/validate-custom-params";
export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics";
export { isResticError, ResticError, ResticLockError } from "./error"; export { isResticError, ResticError, ResticLockError } from "./error";
type LockDiagnosticCommandContext = { type LockRecoveryContext = {
repositoryConfig: RepositoryConfig; repositoryConfigs: RepositoryConfig[];
organizationId: string; organizationId: string;
relatedRepositoryConfigs?: RepositoryConfig[];
}; };
type ResticCommandOptions = { organizationId: string }; type ResticCommandOptions = { organizationId: string };
type ResticCommandResult = { error?: unknown };
type ResticCommandFailure<Failure> = Failure | ResticLockError; type ResticCommandFailure<Failure> = Failure | ResticLockError;
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect< type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
Success, Success,
@ -43,56 +39,37 @@ type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
Requirements Requirements
>; >;
const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => { const getLockRecoveryContext = (operation: string, args: unknown[]): LockRecoveryContext => {
const firstRepositoryConfig = args[0] as RepositoryConfig; const firstRepositoryConfig = args[0] as RepositoryConfig;
const options = args.at(-1) as ResticCommandOptions; const options = args.at(-1) as ResticCommandOptions;
if (operation === "restic.copy") { if (operation === "restic.copy") {
return { return {
repositoryConfig: args[1] as RepositoryConfig, repositoryConfigs: [args[1] as RepositoryConfig, firstRepositoryConfig],
organizationId: options.organizationId, organizationId: options.organizationId,
relatedRepositoryConfigs: [firstRepositoryConfig],
}; };
} }
return { return {
repositoryConfig: firstRepositoryConfig, repositoryConfigs: [firstRepositoryConfig],
organizationId: options.organizationId, organizationId: options.organizationId,
}; };
}; };
const logLockFailure = async ( const unlockStaleLocks = (context: LockRecoveryContext, deps: ResticDeps) =>
error: unknown,
operation: string,
context: LockDiagnosticCommandContext,
deps: ResticDeps,
) =>
logResticLockFailureDiagnostics({
error,
operation,
repositoryConfig: context.repositoryConfig,
organizationId: context.organizationId,
resticDeps: deps,
relatedRepositoryConfigs: context.relatedRepositoryConfigs,
});
const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect<void, Error> =>
Effect.gen(function* () { Effect.gen(function* () {
for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) { for (const repositoryConfig of context.repositoryConfigs) {
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps); yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
} }
}); }).pipe(Effect.catchAll(() => Effect.void));
const recoverFromLockFailure = <Success, Failure, Requirements>( const recoverFromLockFailure = <Success, Failure, Requirements>(
error: ResticLockError, context: LockRecoveryContext,
operation: string,
context: LockDiagnosticCommandContext,
deps: ResticDeps,
runCommand: RunResticCommand<Success, Failure, Requirements>, runCommand: RunResticCommand<Success, Failure, Requirements>,
deps: ResticDeps,
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> => ): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
Effect.gen(function* () { Effect.gen(function* () {
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once."); yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
yield* Effect.promise(() => logLockFailure(error, operation, context, deps));
yield* unlockStaleLocks(context, deps); yield* unlockStaleLocks(context, deps);
const retryResult = yield* runCommand(); const retryResult = yield* runCommand();
@ -107,20 +84,11 @@ function withDeps<Args extends unknown[], Success, Failure, Requirements>(
deps: ResticDeps, deps: ResticDeps,
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> { ): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
return (...args: Args) => { return (...args: Args) => {
const context = getCommandContext(operation, args); const context = getLockRecoveryContext(operation, args);
const runCommand = () => command(...args, deps); const runCommand = () => command(...args, deps);
const commandEffect = runCommand().pipe( return runCommand().pipe(
Effect.catchTag("ResticLockError", (error) => Effect.catchTag("ResticLockError", () => recoverFromLockFailure(context, runCommand, deps)),
recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand),
),
);
return commandEffect.pipe(
Effect.tap((result) => {
const { error } = result as ResticCommandResult;
return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void;
}),
); );
}; };
} }