refactor: remove repository lock diagnostics logs
This commit is contained in:
Nicolas Meienberger
parent
9d63f7cb2d
commit
4bba2c2493
2 changed files with 13 additions and 220 deletions
|
|
@ -1,175 +0,0 @@
|
|||
import { logger, safeExec } from "../node";
|
||||
import { toMessage } from "../utils";
|
||||
import { safeJsonParse } from "../utils/json";
|
||||
import { ResticLockError } from "./error";
|
||||
import { addCommonArgs } from "./helpers/add-common-args";
|
||||
import { buildEnv } from "./helpers/build-env";
|
||||
import { buildRepoUrl } from "./helpers/build-repo-url";
|
||||
import { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
||||
import type { RepositoryConfig } from "./schemas";
|
||||
import type { ResticDeps } from "./types";
|
||||
|
||||
type ResticLockDiagnosticContext = {
|
||||
error: unknown;
|
||||
operation: string;
|
||||
repositoryConfig: RepositoryConfig;
|
||||
organizationId: string;
|
||||
resticDeps: ResticDeps;
|
||||
relatedRepositoryConfigs?: RepositoryConfig[];
|
||||
};
|
||||
|
||||
const LOCK_ERROR_PATTERNS = [
|
||||
/unable to create lock in backend/i,
|
||||
/repository is already locked/i,
|
||||
/failed to lock repository/i,
|
||||
/"code"\s*:\s*11/i,
|
||||
/\bexit_error\b.*\b11\b/i,
|
||||
];
|
||||
|
||||
export const isResticLockFailure = (error: unknown) => {
|
||||
if (error instanceof ResticLockError) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const message = toMessage(error);
|
||||
return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message));
|
||||
};
|
||||
|
||||
const RESTIC_LOCK_ID_PATTERN = /^[a-f0-9]{64}$/i;
|
||||
|
||||
const addLockId = (ids: Set<string>, candidate: unknown) => {
|
||||
if (typeof candidate === "string" && RESTIC_LOCK_ID_PATTERN.test(candidate)) {
|
||||
ids.add(candidate);
|
||||
}
|
||||
};
|
||||
|
||||
const parseLockIds = (stdout: string) => {
|
||||
const ids = new Set<string>();
|
||||
|
||||
for (const line of stdout.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
const parsed = safeJsonParse<{ id?: unknown }>(trimmed);
|
||||
if (parsed) {
|
||||
addLockId(ids, parsed.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
addLockId(ids, trimmed);
|
||||
}
|
||||
|
||||
return [...ids].slice(0, 20);
|
||||
};
|
||||
|
||||
const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config, organizationId, deps);
|
||||
const baseArgs = ["--repo", repoUrl];
|
||||
addCommonArgs(baseArgs, env, config);
|
||||
|
||||
try {
|
||||
const listResult = await safeExec({
|
||||
command: "restic",
|
||||
args: [...baseArgs, "list", "locks"],
|
||||
env,
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : [];
|
||||
const lockDetails = [];
|
||||
|
||||
for (const lockId of lockIds) {
|
||||
const catResult = await safeExec({
|
||||
command: "restic",
|
||||
args: [...baseArgs, "cat", "lock", "--", lockId],
|
||||
env,
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
lockDetails.push({
|
||||
lockId,
|
||||
exitCode: catResult.exitCode,
|
||||
stdout: catResult.stdout,
|
||||
stderr: catResult.stderr,
|
||||
timedOut: catResult.timedOut,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
repoUrl,
|
||||
list: {
|
||||
exitCode: listResult.exitCode,
|
||||
stdout: listResult.stdout,
|
||||
stderr: listResult.stderr,
|
||||
timedOut: listResult.timedOut,
|
||||
},
|
||||
lockIds,
|
||||
lockDetails,
|
||||
};
|
||||
} finally {
|
||||
await cleanupTemporaryKeys(env, deps);
|
||||
}
|
||||
};
|
||||
|
||||
export const logResticLockFailureDiagnostics = async ({
|
||||
error,
|
||||
operation,
|
||||
repositoryConfig,
|
||||
organizationId,
|
||||
resticDeps,
|
||||
relatedRepositoryConfigs = [],
|
||||
}: ResticLockDiagnosticContext) => {
|
||||
if (!isResticLockFailure(error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const configsByRepoUrl = new Map(
|
||||
[repositoryConfig, ...relatedRepositoryConfigs].map((config) => [buildRepoUrl(config), config]),
|
||||
);
|
||||
const configsToInspect = [...configsByRepoUrl.entries()];
|
||||
|
||||
logger.error("[ResticLockFailure] Restic repository lock failure detected", {
|
||||
operation,
|
||||
error: toMessage(error),
|
||||
process: {
|
||||
pid: process.pid,
|
||||
hostname: process.env.HOSTNAME,
|
||||
nodeEnv: process.env.NODE_ENV,
|
||||
},
|
||||
repository: {
|
||||
repoUrl: buildRepoUrl(repositoryConfig),
|
||||
},
|
||||
relatedRepositories: relatedRepositoryConfigs.map((config) => ({
|
||||
repoUrl: buildRepoUrl(config),
|
||||
})),
|
||||
});
|
||||
|
||||
for (const [repoUrl, config] of configsToInspect) {
|
||||
try {
|
||||
const resticLocks = await inspectResticLocks(config, organizationId, resticDeps);
|
||||
logger.error("[ResticLockFailure] Restic backend lock dump", {
|
||||
operation,
|
||||
repoUrl,
|
||||
resticLocks,
|
||||
});
|
||||
} catch (diagnosticError) {
|
||||
logger.error("[ResticLockFailure] Failed to inspect restic backend locks", {
|
||||
operation,
|
||||
repoUrl,
|
||||
error: toMessage(diagnosticError),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (diagnosticError) {
|
||||
logger.error("[ResticLockFailure] Failed to collect lock diagnostics", {
|
||||
operation,
|
||||
repoUrl: buildRepoUrl(repositoryConfig),
|
||||
error: toMessage(diagnosticError),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
|
@ -16,7 +16,6 @@ import { stats } from "./commands/stats";
|
|||
import { tagSnapshots } from "./commands/tag-snapshots";
|
||||
import { unlock } from "./commands/unlock";
|
||||
import { ResticLockError } from "./error";
|
||||
import { logResticLockFailureDiagnostics } from "./lock-diagnostics";
|
||||
import type { RepositoryConfig } from "./schemas";
|
||||
import type { ResticDeps } from "./types";
|
||||
|
||||
|
|
@ -25,17 +24,14 @@ export { buildEnv } from "./helpers/build-env";
|
|||
export { buildRepoUrl } from "./helpers/build-repo-url";
|
||||
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
||||
export { validateCustomResticParams } from "./helpers/validate-custom-params";
|
||||
export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics";
|
||||
export { isResticError, ResticError, ResticLockError } from "./error";
|
||||
|
||||
type LockDiagnosticCommandContext = {
|
||||
repositoryConfig: RepositoryConfig;
|
||||
type LockRecoveryContext = {
|
||||
repositoryConfigs: RepositoryConfig[];
|
||||
organizationId: string;
|
||||
relatedRepositoryConfigs?: RepositoryConfig[];
|
||||
};
|
||||
|
||||
type ResticCommandOptions = { organizationId: string };
|
||||
type ResticCommandResult = { error?: unknown };
|
||||
type ResticCommandFailure<Failure> = Failure | ResticLockError;
|
||||
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
||||
Success,
|
||||
|
|
@ -43,56 +39,37 @@ type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
|||
Requirements
|
||||
>;
|
||||
|
||||
const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => {
|
||||
const getLockRecoveryContext = (operation: string, args: unknown[]): LockRecoveryContext => {
|
||||
const firstRepositoryConfig = args[0] as RepositoryConfig;
|
||||
const options = args.at(-1) as ResticCommandOptions;
|
||||
|
||||
if (operation === "restic.copy") {
|
||||
return {
|
||||
repositoryConfig: args[1] as RepositoryConfig,
|
||||
repositoryConfigs: [args[1] as RepositoryConfig, firstRepositoryConfig],
|
||||
organizationId: options.organizationId,
|
||||
relatedRepositoryConfigs: [firstRepositoryConfig],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
repositoryConfig: firstRepositoryConfig,
|
||||
repositoryConfigs: [firstRepositoryConfig],
|
||||
organizationId: options.organizationId,
|
||||
};
|
||||
};
|
||||
|
||||
const logLockFailure = async (
|
||||
error: unknown,
|
||||
operation: string,
|
||||
context: LockDiagnosticCommandContext,
|
||||
deps: ResticDeps,
|
||||
) =>
|
||||
logResticLockFailureDiagnostics({
|
||||
error,
|
||||
operation,
|
||||
repositoryConfig: context.repositoryConfig,
|
||||
organizationId: context.organizationId,
|
||||
resticDeps: deps,
|
||||
relatedRepositoryConfigs: context.relatedRepositoryConfigs,
|
||||
});
|
||||
|
||||
const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect<void, Error> =>
|
||||
const unlockStaleLocks = (context: LockRecoveryContext, deps: ResticDeps) =>
|
||||
Effect.gen(function* () {
|
||||
for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) {
|
||||
for (const repositoryConfig of context.repositoryConfigs) {
|
||||
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
|
||||
}
|
||||
});
|
||||
}).pipe(Effect.catchAll(() => Effect.void));
|
||||
|
||||
const recoverFromLockFailure = <Success, Failure, Requirements>(
|
||||
error: ResticLockError,
|
||||
operation: string,
|
||||
context: LockDiagnosticCommandContext,
|
||||
deps: ResticDeps,
|
||||
context: LockRecoveryContext,
|
||||
runCommand: RunResticCommand<Success, Failure, Requirements>,
|
||||
deps: ResticDeps,
|
||||
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
|
||||
Effect.gen(function* () {
|
||||
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
|
||||
yield* Effect.promise(() => logLockFailure(error, operation, context, deps));
|
||||
yield* unlockStaleLocks(context, deps);
|
||||
|
||||
const retryResult = yield* runCommand();
|
||||
|
|
@ -107,20 +84,11 @@ function withDeps<Args extends unknown[], Success, Failure, Requirements>(
|
|||
deps: ResticDeps,
|
||||
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
|
||||
return (...args: Args) => {
|
||||
const context = getCommandContext(operation, args);
|
||||
const context = getLockRecoveryContext(operation, args);
|
||||
const runCommand = () => command(...args, deps);
|
||||
|
||||
const commandEffect = runCommand().pipe(
|
||||
Effect.catchTag("ResticLockError", (error) =>
|
||||
recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand),
|
||||
),
|
||||
);
|
||||
|
||||
return commandEffect.pipe(
|
||||
Effect.tap((result) => {
|
||||
const { error } = result as ResticCommandResult;
|
||||
return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void;
|
||||
}),
|
||||
return runCommand().pipe(
|
||||
Effect.catchTag("ResticLockError", () => recoverFromLockFailure(context, runCommand, deps)),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue