refactor: remove repository lock diagnostics logs
This commit is contained in:
parent
9d63f7cb2d
commit
4bba2c2493
2 changed files with 13 additions and 220 deletions
|
|
@ -1,175 +0,0 @@
|
||||||
import { logger, safeExec } from "../node";
|
|
||||||
import { toMessage } from "../utils";
|
|
||||||
import { safeJsonParse } from "../utils/json";
|
|
||||||
import { ResticLockError } from "./error";
|
|
||||||
import { addCommonArgs } from "./helpers/add-common-args";
|
|
||||||
import { buildEnv } from "./helpers/build-env";
|
|
||||||
import { buildRepoUrl } from "./helpers/build-repo-url";
|
|
||||||
import { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
|
||||||
import type { RepositoryConfig } from "./schemas";
|
|
||||||
import type { ResticDeps } from "./types";
|
|
||||||
|
|
||||||
type ResticLockDiagnosticContext = {
|
|
||||||
error: unknown;
|
|
||||||
operation: string;
|
|
||||||
repositoryConfig: RepositoryConfig;
|
|
||||||
organizationId: string;
|
|
||||||
resticDeps: ResticDeps;
|
|
||||||
relatedRepositoryConfigs?: RepositoryConfig[];
|
|
||||||
};
|
|
||||||
|
|
||||||
const LOCK_ERROR_PATTERNS = [
|
|
||||||
/unable to create lock in backend/i,
|
|
||||||
/repository is already locked/i,
|
|
||||||
/failed to lock repository/i,
|
|
||||||
/"code"\s*:\s*11/i,
|
|
||||||
/\bexit_error\b.*\b11\b/i,
|
|
||||||
];
|
|
||||||
|
|
||||||
export const isResticLockFailure = (error: unknown) => {
|
|
||||||
if (error instanceof ResticLockError) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const message = toMessage(error);
|
|
||||||
return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message));
|
|
||||||
};
|
|
||||||
|
|
||||||
const RESTIC_LOCK_ID_PATTERN = /^[a-f0-9]{64}$/i;
|
|
||||||
|
|
||||||
const addLockId = (ids: Set<string>, candidate: unknown) => {
|
|
||||||
if (typeof candidate === "string" && RESTIC_LOCK_ID_PATTERN.test(candidate)) {
|
|
||||||
ids.add(candidate);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const parseLockIds = (stdout: string) => {
|
|
||||||
const ids = new Set<string>();
|
|
||||||
|
|
||||||
for (const line of stdout.split("\n")) {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed) continue;
|
|
||||||
|
|
||||||
const parsed = safeJsonParse<{ id?: unknown }>(trimmed);
|
|
||||||
if (parsed) {
|
|
||||||
addLockId(ids, parsed.id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
addLockId(ids, trimmed);
|
|
||||||
}
|
|
||||||
|
|
||||||
return [...ids].slice(0, 20);
|
|
||||||
};
|
|
||||||
|
|
||||||
const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => {
|
|
||||||
const repoUrl = buildRepoUrl(config);
|
|
||||||
const env = await buildEnv(config, organizationId, deps);
|
|
||||||
const baseArgs = ["--repo", repoUrl];
|
|
||||||
addCommonArgs(baseArgs, env, config);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const listResult = await safeExec({
|
|
||||||
command: "restic",
|
|
||||||
args: [...baseArgs, "list", "locks"],
|
|
||||||
env,
|
|
||||||
timeout: 15_000,
|
|
||||||
});
|
|
||||||
|
|
||||||
const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : [];
|
|
||||||
const lockDetails = [];
|
|
||||||
|
|
||||||
for (const lockId of lockIds) {
|
|
||||||
const catResult = await safeExec({
|
|
||||||
command: "restic",
|
|
||||||
args: [...baseArgs, "cat", "lock", "--", lockId],
|
|
||||||
env,
|
|
||||||
timeout: 15_000,
|
|
||||||
});
|
|
||||||
|
|
||||||
lockDetails.push({
|
|
||||||
lockId,
|
|
||||||
exitCode: catResult.exitCode,
|
|
||||||
stdout: catResult.stdout,
|
|
||||||
stderr: catResult.stderr,
|
|
||||||
timedOut: catResult.timedOut,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
repoUrl,
|
|
||||||
list: {
|
|
||||||
exitCode: listResult.exitCode,
|
|
||||||
stdout: listResult.stdout,
|
|
||||||
stderr: listResult.stderr,
|
|
||||||
timedOut: listResult.timedOut,
|
|
||||||
},
|
|
||||||
lockIds,
|
|
||||||
lockDetails,
|
|
||||||
};
|
|
||||||
} finally {
|
|
||||||
await cleanupTemporaryKeys(env, deps);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const logResticLockFailureDiagnostics = async ({
|
|
||||||
error,
|
|
||||||
operation,
|
|
||||||
repositoryConfig,
|
|
||||||
organizationId,
|
|
||||||
resticDeps,
|
|
||||||
relatedRepositoryConfigs = [],
|
|
||||||
}: ResticLockDiagnosticContext) => {
|
|
||||||
if (!isResticLockFailure(error)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const configsByRepoUrl = new Map(
|
|
||||||
[repositoryConfig, ...relatedRepositoryConfigs].map((config) => [buildRepoUrl(config), config]),
|
|
||||||
);
|
|
||||||
const configsToInspect = [...configsByRepoUrl.entries()];
|
|
||||||
|
|
||||||
logger.error("[ResticLockFailure] Restic repository lock failure detected", {
|
|
||||||
operation,
|
|
||||||
error: toMessage(error),
|
|
||||||
process: {
|
|
||||||
pid: process.pid,
|
|
||||||
hostname: process.env.HOSTNAME,
|
|
||||||
nodeEnv: process.env.NODE_ENV,
|
|
||||||
},
|
|
||||||
repository: {
|
|
||||||
repoUrl: buildRepoUrl(repositoryConfig),
|
|
||||||
},
|
|
||||||
relatedRepositories: relatedRepositoryConfigs.map((config) => ({
|
|
||||||
repoUrl: buildRepoUrl(config),
|
|
||||||
})),
|
|
||||||
});
|
|
||||||
|
|
||||||
for (const [repoUrl, config] of configsToInspect) {
|
|
||||||
try {
|
|
||||||
const resticLocks = await inspectResticLocks(config, organizationId, resticDeps);
|
|
||||||
logger.error("[ResticLockFailure] Restic backend lock dump", {
|
|
||||||
operation,
|
|
||||||
repoUrl,
|
|
||||||
resticLocks,
|
|
||||||
});
|
|
||||||
} catch (diagnosticError) {
|
|
||||||
logger.error("[ResticLockFailure] Failed to inspect restic backend locks", {
|
|
||||||
operation,
|
|
||||||
repoUrl,
|
|
||||||
error: toMessage(diagnosticError),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (diagnosticError) {
|
|
||||||
logger.error("[ResticLockFailure] Failed to collect lock diagnostics", {
|
|
||||||
operation,
|
|
||||||
repoUrl: buildRepoUrl(repositoryConfig),
|
|
||||||
error: toMessage(diagnosticError),
|
|
||||||
});
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
@ -16,7 +16,6 @@ import { stats } from "./commands/stats";
|
||||||
import { tagSnapshots } from "./commands/tag-snapshots";
|
import { tagSnapshots } from "./commands/tag-snapshots";
|
||||||
import { unlock } from "./commands/unlock";
|
import { unlock } from "./commands/unlock";
|
||||||
import { ResticLockError } from "./error";
|
import { ResticLockError } from "./error";
|
||||||
import { logResticLockFailureDiagnostics } from "./lock-diagnostics";
|
|
||||||
import type { RepositoryConfig } from "./schemas";
|
import type { RepositoryConfig } from "./schemas";
|
||||||
import type { ResticDeps } from "./types";
|
import type { ResticDeps } from "./types";
|
||||||
|
|
||||||
|
|
@ -25,17 +24,14 @@ export { buildEnv } from "./helpers/build-env";
|
||||||
export { buildRepoUrl } from "./helpers/build-repo-url";
|
export { buildRepoUrl } from "./helpers/build-repo-url";
|
||||||
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
||||||
export { validateCustomResticParams } from "./helpers/validate-custom-params";
|
export { validateCustomResticParams } from "./helpers/validate-custom-params";
|
||||||
export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics";
|
|
||||||
export { isResticError, ResticError, ResticLockError } from "./error";
|
export { isResticError, ResticError, ResticLockError } from "./error";
|
||||||
|
|
||||||
type LockDiagnosticCommandContext = {
|
type LockRecoveryContext = {
|
||||||
repositoryConfig: RepositoryConfig;
|
repositoryConfigs: RepositoryConfig[];
|
||||||
organizationId: string;
|
organizationId: string;
|
||||||
relatedRepositoryConfigs?: RepositoryConfig[];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
type ResticCommandOptions = { organizationId: string };
|
type ResticCommandOptions = { organizationId: string };
|
||||||
type ResticCommandResult = { error?: unknown };
|
|
||||||
type ResticCommandFailure<Failure> = Failure | ResticLockError;
|
type ResticCommandFailure<Failure> = Failure | ResticLockError;
|
||||||
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
||||||
Success,
|
Success,
|
||||||
|
|
@ -43,56 +39,37 @@ type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
||||||
Requirements
|
Requirements
|
||||||
>;
|
>;
|
||||||
|
|
||||||
const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => {
|
const getLockRecoveryContext = (operation: string, args: unknown[]): LockRecoveryContext => {
|
||||||
const firstRepositoryConfig = args[0] as RepositoryConfig;
|
const firstRepositoryConfig = args[0] as RepositoryConfig;
|
||||||
const options = args.at(-1) as ResticCommandOptions;
|
const options = args.at(-1) as ResticCommandOptions;
|
||||||
|
|
||||||
if (operation === "restic.copy") {
|
if (operation === "restic.copy") {
|
||||||
return {
|
return {
|
||||||
repositoryConfig: args[1] as RepositoryConfig,
|
repositoryConfigs: [args[1] as RepositoryConfig, firstRepositoryConfig],
|
||||||
organizationId: options.organizationId,
|
organizationId: options.organizationId,
|
||||||
relatedRepositoryConfigs: [firstRepositoryConfig],
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
repositoryConfig: firstRepositoryConfig,
|
repositoryConfigs: [firstRepositoryConfig],
|
||||||
organizationId: options.organizationId,
|
organizationId: options.organizationId,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const logLockFailure = async (
|
const unlockStaleLocks = (context: LockRecoveryContext, deps: ResticDeps) =>
|
||||||
error: unknown,
|
|
||||||
operation: string,
|
|
||||||
context: LockDiagnosticCommandContext,
|
|
||||||
deps: ResticDeps,
|
|
||||||
) =>
|
|
||||||
logResticLockFailureDiagnostics({
|
|
||||||
error,
|
|
||||||
operation,
|
|
||||||
repositoryConfig: context.repositoryConfig,
|
|
||||||
organizationId: context.organizationId,
|
|
||||||
resticDeps: deps,
|
|
||||||
relatedRepositoryConfigs: context.relatedRepositoryConfigs,
|
|
||||||
});
|
|
||||||
|
|
||||||
const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect<void, Error> =>
|
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) {
|
for (const repositoryConfig of context.repositoryConfigs) {
|
||||||
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
|
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
|
||||||
}
|
}
|
||||||
});
|
}).pipe(Effect.catchAll(() => Effect.void));
|
||||||
|
|
||||||
const recoverFromLockFailure = <Success, Failure, Requirements>(
|
const recoverFromLockFailure = <Success, Failure, Requirements>(
|
||||||
error: ResticLockError,
|
context: LockRecoveryContext,
|
||||||
operation: string,
|
|
||||||
context: LockDiagnosticCommandContext,
|
|
||||||
deps: ResticDeps,
|
|
||||||
runCommand: RunResticCommand<Success, Failure, Requirements>,
|
runCommand: RunResticCommand<Success, Failure, Requirements>,
|
||||||
|
deps: ResticDeps,
|
||||||
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
|
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
|
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
|
||||||
yield* Effect.promise(() => logLockFailure(error, operation, context, deps));
|
|
||||||
yield* unlockStaleLocks(context, deps);
|
yield* unlockStaleLocks(context, deps);
|
||||||
|
|
||||||
const retryResult = yield* runCommand();
|
const retryResult = yield* runCommand();
|
||||||
|
|
@ -107,20 +84,11 @@ function withDeps<Args extends unknown[], Success, Failure, Requirements>(
|
||||||
deps: ResticDeps,
|
deps: ResticDeps,
|
||||||
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
|
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
|
||||||
return (...args: Args) => {
|
return (...args: Args) => {
|
||||||
const context = getCommandContext(operation, args);
|
const context = getLockRecoveryContext(operation, args);
|
||||||
const runCommand = () => command(...args, deps);
|
const runCommand = () => command(...args, deps);
|
||||||
|
|
||||||
const commandEffect = runCommand().pipe(
|
return runCommand().pipe(
|
||||||
Effect.catchTag("ResticLockError", (error) =>
|
Effect.catchTag("ResticLockError", () => recoverFromLockFailure(context, runCommand, deps)),
|
||||||
recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
return commandEffect.pipe(
|
|
||||||
Effect.tap((result) => {
|
|
||||||
const { error } = result as ResticCommandResult;
|
|
||||||
return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void;
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue