Compare commits

...

5 commits

Author SHA1 Message Date
Nico
7c1f0c52d5
refactor(repo-mutex): use effect.ts (#950)
Some checks failed
Checks / lint (push) Has been cancelled
Checks / typecheck (push) Has been cancelled
Checks / test-server (push) Has been cancelled
Checks / test-client (push) Has been cancelled
Checks / build (push) Has been cancelled
Scorecard analysis / Scorecard analysis (push) Has been cancelled
* refactor(repo-mutex): use effect.ts

* refactor(mutext): run with signal to preserve abort reason
2026-06-06 15:06:26 +02:00
Nico
2318b6bdd0
fix: limit concurrent ls to 2 in flight calls (#948)
* fix: limit concurrent ls to 2 in flight calls

* refactor: get shared lock after semaphore take
2026-06-05 18:07:59 +02:00
Copilot
34fc4f0cbb
docs: update Docker Compose Zerobyte image tags to v0.38 (#945)
* Initial plan

* docs: update Docker Compose Zerobyte image tags to v0.38

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-06-04 21:49:15 +02:00
Nico
68002b0308
fix: disable add passkey in insecure contexts (#943)
Some checks failed
Release Workflow / determine-release-type (push) Has been cancelled
Release Workflow / checks (push) Has been cancelled
Release Workflow / e2e-tests (push) Has been cancelled
Release Workflow / integration-tests (push) Has been cancelled
Release Workflow / request-docs-version-update (push) Has been cancelled
Release Workflow / build-images (push) Has been cancelled
Release Workflow / publish-release (push) Has been cancelled
2026-06-04 21:06:48 +02:00
Nicolas Meienberger
4bba2c2493
refactor: remove repository lock diagnostics logs 2026-06-04 19:58:02 +02:00
20 changed files with 615 additions and 505 deletions

View file

@ -50,7 +50,7 @@ In order to run Zerobyte, you need to have Docker and Docker Compose installed o
```yaml ```yaml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
cap_add: cap_add:
@ -156,7 +156,7 @@ If you only need to back up locally-mounted folders and don't require remote sha
```yaml ```yaml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
ports: ports:
@ -195,7 +195,7 @@ If you want to backup a local directory on the same host where Zerobyte is runni
```diff ```diff
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
cap_add: cap_add:
@ -270,7 +270,7 @@ Zerobyte can use [rclone](https://rclone.org/) to support 40+ cloud storage prov
```diff ```diff
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
cap_add: cap_add:

View file

@ -224,29 +224,6 @@ describe("SnapshotTreeBrowser", () => {
}); });
}); });
test("prefetches using the query path when display and query roots differ", async () => {
const requests = mockListSnapshotFiles();
renderSnapshotTreeBrowser();
const row = await screen.findByRole("button", { name: "project" });
const initialRequestCount = requests.length;
await userEvent.hover(row);
await waitFor(() => {
expect(requests.length).toBe(initialRequestCount + 1);
});
expect(requests.at(-1)).toEqual({
shortId: "repo-1",
snapshotId: "snap-1",
path: "/mnt/project",
offset: "0",
limit: "500",
});
});
test("shows the query root contents when display and query roots differ", async () => { test("shows the query root contents when display and query roots differ", async () => {
mockListSnapshotFiles(); mockListSnapshotFiles();

View file

@ -5,7 +5,6 @@ import { FileBrowser, type FileBrowserUiProps } from "~/client/components/file-b
import { useFileBrowser } from "~/client/hooks/use-file-browser"; import { useFileBrowser } from "~/client/hooks/use-file-browser";
import { parseError } from "~/client/lib/errors"; import { parseError } from "~/client/lib/errors";
import { isPathWithin, normalizeAbsolutePath } from "@zerobyte/core/utils"; import { isPathWithin, normalizeAbsolutePath } from "@zerobyte/core/utils";
import { logger } from "~/client/lib/logger";
function createPathPrefixFns(basePath: string) { function createPathPrefixFns(basePath: string) {
return { return {
@ -84,16 +83,6 @@ export const SnapshotTreeBrowser = (props: SnapshotTreeBrowserProps) => {
}), }),
); );
}, },
prefetchFolder: (displayPath) => {
void queryClient
.prefetchQuery(
listSnapshotFilesOptions({
path: { shortId: repositoryId, snapshotId },
query: { path: displayPath, offset: 0, limit: pageSize },
}),
)
.catch((e) => logger.error(e));
},
pathTransform: displayPathFns, pathTransform: displayPathFns,
}); });

View file

@ -0,0 +1,36 @@
import { describe, expect, test } from "vitest";
import { getIsSecureContextForRequest, isSecureContextOrigin } from "../is-secure-context";
describe("isSecureContextOrigin", () => {
test("treats HTTPS and loopback HTTP origins as secure contexts", () => {
expect(isSecureContextOrigin("https://example.com")).toBe(true);
expect(isSecureContextOrigin("http://localhost:3000")).toBe(true);
expect(isSecureContextOrigin("http://app.localhost")).toBe(true);
expect(isSecureContextOrigin("http://127.0.0.1:3000")).toBe(true);
expect(isSecureContextOrigin("http://[::1]:3000")).toBe(true);
});
test("treats non-local HTTP origins as insecure contexts", () => {
expect(isSecureContextOrigin("http://example.com")).toBe(false);
expect(isSecureContextOrigin("http://127.example.com")).toBe(false);
});
});
describe("getIsSecureContextForRequest", () => {
test("uses forwarded protocol and host when present", () => {
const request = new Request("http://internal.local/settings", {
headers: {
host: "internal.local",
"x-forwarded-host": "zerobyte.example.com",
"x-forwarded-proto": "https",
},
});
expect(getIsSecureContextForRequest(request)).toBe(true);
});
test("uses the request URL when forwarded headers are absent", () => {
expect(getIsSecureContextForRequest(new Request("http://zerobyte.example.com/settings"))).toBe(false);
expect(getIsSecureContextForRequest(new Request("https://zerobyte.example.com/settings"))).toBe(true);
});
});

View file

@ -0,0 +1,51 @@
import { createIsomorphicFn } from "@tanstack/react-start";
import { getRequest } from "@tanstack/react-start/server";
import ipaddr from "ipaddr.js";
const getFirstHeaderValue = (value: string | null) => value?.split(",")[0]?.trim();
const isLoopbackIp = (hostname: string) => {
try {
return ipaddr.parse(hostname).range() === "loopback";
} catch {
return false;
}
};
const getRequestOrigin = (request: Request) => {
const requestUrl = new URL(request.url);
const forwardedProto = getFirstHeaderValue(request.headers.get("x-forwarded-proto"));
const forwardedHost = getFirstHeaderValue(request.headers.get("x-forwarded-host"));
const host = forwardedHost || getFirstHeaderValue(request.headers.get("host"));
if (!host) {
return requestUrl.origin;
}
const protocol = (forwardedProto || requestUrl.protocol).replace(/:$/, "");
return `${protocol}://${host}`;
};
export const isSecureContextOrigin = (origin: string) => {
const url = new URL(origin);
if (url.protocol === "https:") {
return true;
}
const hostname = url.hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase();
return hostname === "localhost" || hostname.endsWith(".localhost") || isLoopbackIp(hostname);
};
export const getIsSecureContextForRequest = (request: Request) => isSecureContextOrigin(getRequestOrigin(request));
export const getIsSecureContext = createIsomorphicFn()
.server(() => {
try {
return getIsSecureContextForRequest(getRequest());
} catch {
return true;
}
})
.client(() => window.isSecureContext ?? true);

View file

@ -63,9 +63,7 @@ export function ScheduleDetailsPage(props: Props) {
const queryClient = useQueryClient(); const queryClient = useQueryClient();
const navigate = useNavigate(); const navigate = useNavigate();
const searchParams = useSearch({ from: "/(dashboard)/backups/$backupId/" }); const searchParams = useSearch({ from: "/(dashboard)/backups/$backupId/" });
const [selectedSnapshotId, setSelectedSnapshotId] = useState<string | undefined>( const [selectedSnapshotId, setSelectedSnapshotId] = useState<string | undefined>(initialSnapshotId);
initialSnapshotId ?? loaderData.snapshots?.at(-1)?.short_id,
);
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false); const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
const [snapshotToDelete, setSnapshotToDelete] = useState<string | null>(null); const [snapshotToDelete, setSnapshotToDelete] = useState<string | null>(null);

View file

@ -24,6 +24,8 @@ import {
} from "~/client/components/ui/dialog"; } from "~/client/components/ui/dialog";
import { Input } from "~/client/components/ui/input"; import { Input } from "~/client/components/ui/input";
import { Label } from "~/client/components/ui/label"; import { Label } from "~/client/components/ui/label";
import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip";
import { getIsSecureContext } from "~/client/functions/is-secure-context";
import { authClient } from "~/client/lib/auth-client"; import { authClient } from "~/client/lib/auth-client";
import { logger } from "~/client/lib/logger"; import { logger } from "~/client/lib/logger";
import { useTimeFormat } from "~/client/lib/datetime"; import { useTimeFormat } from "~/client/lib/datetime";
@ -38,6 +40,7 @@ type PasskeyEntry = {
export function PasskeysSection() { export function PasskeysSection() {
const { formatDateTime } = useTimeFormat(); const { formatDateTime } = useTimeFormat();
const isSecureContext = getIsSecureContext();
const { data: passkeys, isPending } = useQuery({ const { data: passkeys, isPending } = useQuery({
queryKey: ["passkeys"], queryKey: ["passkeys"],
queryFn: async () => { queryFn: async () => {
@ -111,6 +114,8 @@ export function PasskeysSection() {
const handleAddPasskey = (e: React.ChangeEvent) => { const handleAddPasskey = (e: React.ChangeEvent) => {
e.preventDefault(); e.preventDefault();
if (!isSecureContext) return;
const name = newPasskeyName.trim() || undefined; const name = newPasskeyName.trim() || undefined;
addPasskeyMutation.mutate(name); addPasskeyMutation.mutate(name);
}; };
@ -149,10 +154,19 @@ export function PasskeysSection() {
Passkeys use your device's biometrics or screen lock instead of a password. They are Passkeys use your device's biometrics or screen lock instead of a password. They are
phishing-resistant and cannot be reused across sites. phishing-resistant and cannot be reused across sites.
</p> </p>
<Button onClick={() => setAddDialogOpen(true)}> <Tooltip>
<Plus className="h-4 w-4 mr-2" /> <TooltipTrigger asChild>
Add passkey <span className="inline-flex">
</Button> <Button onClick={() => setAddDialogOpen(true)} disabled={!isSecureContext}>
<Plus className="h-4 w-4 mr-2" />
Add passkey
</Button>
</span>
</TooltipTrigger>
<TooltipContent className={cn({ hidden: isSecureContext })}>
<p>Passkeys can only be added over HTTPS or from localhost.</p>
</TooltipContent>
</Tooltip>
</div> </div>
<p className={cn("text-sm text-muted-foreground", { hidden: !isPending })}>Loading passkeys...</p> <p className={cn("text-sm text-muted-foreground", { hidden: !isPending })}>Loading passkeys...</p>
@ -233,7 +247,7 @@ export function PasskeysSection() {
<Button type="button" variant="outline" onClick={() => setAddDialogOpen(false)}> <Button type="button" variant="outline" onClick={() => setAddDialogOpen(false)}>
Cancel Cancel
</Button> </Button>
<Button type="submit" loading={addPasskeyMutation.isPending}> <Button type="submit" loading={addPasskeyMutation.isPending} disabled={!isSecureContext}>
<Fingerprint className="h-4 w-4 mr-2" /> <Fingerprint className="h-4 w-4 mr-2" />
Add passkey Add passkey
</Button> </Button>

View file

@ -134,6 +134,37 @@ describe("RepositoryMutex", () => {
expect(repoMutex.isLocked(repoId)).toBe(false); expect(repoMutex.isLocked(repoId)).toBe(false);
}); });
test("should reject aborted queued acquisitions only after waiter cleanup finishes", async () => {
const repoId = "abort-waits-for-cleanup";
const releaseHolder = await repoMutex.acquireExclusive(repoId, "holder");
const controller = new AbortController();
const waitingAcquisition = repoMutex.acquireShared(repoId, "waiter", controller.signal);
try {
await new Promise((resolve) => setTimeout(resolve, 50));
const waitersBeforeAbort = await db.query.repositoryLockWaitersTable.findMany({
where: { repositoryId: { eq: repoId } },
});
expect(waitersBeforeAbort.map((waiter) => waiter.operation)).toEqual(["waiter"]);
controller.abort(new Error("stop"));
await expect(waitingAcquisition).rejects.toThrow("stop");
const waitersAfterAbort = await db.query.repositoryLockWaitersTable.findMany({
where: { repositoryId: { eq: repoId } },
});
expect(waitersAfterAbort).toEqual([]);
expect(repoMutex.isLocked(repoId)).toBe(true);
} finally {
releaseHolder();
await db.delete(repositoryLockWaitersTable).where(eq(repositoryLockWaitersTable.repositoryId, repoId));
await db.delete(repositoryLocksTable).where(eq(repositoryLocksTable.repositoryId, repoId));
}
});
test("should allow concurrent shared locks", async () => { test("should allow concurrent shared locks", async () => {
const repoId = "concurrent-shared"; const repoId = "concurrent-shared";
const release1 = await repoMutex.acquireShared(repoId, "op1"); const release1 = await repoMutex.acquireShared(repoId, "op1");

View file

@ -7,6 +7,7 @@ import {
type RepositoryLock, type RepositoryLock,
type RepositoryLockWaiter, type RepositoryLockWaiter,
} from "../db/schema"; } from "../db/schema";
import { Effect, Exit, Fiber, Schedule, Scope } from "effect";
type LockType = "shared" | "exclusive"; type LockType = "shared" | "exclusive";
@ -33,59 +34,30 @@ const LOCK_HEARTBEAT_MS = 5_000;
const LOCK_POLL_MS = 250; const LOCK_POLL_MS = 250;
const LOCK_POLL_CLEANUP_MS = 5_000; const LOCK_POLL_CLEANUP_MS = 5_000;
const REPOSITORY_MUTEX_INSTANCE = Symbol.for("zerobyte.repositoryMutex.instance");
function getRepositoryMutex() {
const globalObject = globalThis as typeof globalThis & Record<symbol, RepositoryMutex | undefined>;
const mutex = globalObject[REPOSITORY_MUTEX_INSTANCE];
if (mutex) return mutex;
const newMutex = new RepositoryMutex();
globalObject[REPOSITORY_MUTEX_INSTANCE] = newMutex;
return newMutex;
}
class RepositoryMutex { class RepositoryMutex {
private ownerId = `owner_${Bun.randomUUIDv7()}`; private ownerId = `owner_${Bun.randomUUIDv7()}`;
private heartbeatTimers = new Map<string, ReturnType<typeof setInterval>>();
private nextPollCleanupAt = 0; private nextPollCleanupAt = 0;
private generateLockId(): string { private generateLockId(): string {
return `lock_${Bun.randomUUIDv7()}`; return `lock_${Bun.randomUUIDv7()}`;
} }
private abortReason(signal: AbortSignal) { private abortReason(signal: AbortSignal): Error {
return signal.reason || new Error("Operation aborted"); return signal.reason || new Error("Operation aborted");
} }
private throwIfAborted(signal?: AbortSignal) {
if (signal?.aborted) {
throw this.abortReason(signal);
}
}
private releaseIfAborted(releaseLock: () => void, signal?: AbortSignal) {
if (!signal?.aborted) return;
releaseLock();
throw this.abortReason(signal);
}
private waitForPoll(signal?: AbortSignal) {
this.throwIfAborted(signal);
return new Promise<void>((resolve, reject) => {
let settled = false;
const timeout = setTimeout(() => settle(resolve), LOCK_POLL_MS);
const onAbort = () => {
settle(() => reject(this.abortReason(signal!)));
};
const cleanup = () => {
clearTimeout(timeout);
signal?.removeEventListener("abort", onAbort);
};
const settle = (callback: () => void) => {
if (settled) return;
settled = true;
cleanup();
callback();
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
private cleanupExpired(tx: RepositoryMutexTransaction, now: number) { private cleanupExpired(tx: RepositoryMutexTransaction, now: number) {
tx.delete(repositoryLocksTable).where(lte(repositoryLocksTable.expiresAt, now)).run(); tx.delete(repositoryLocksTable).where(lte(repositoryLocksTable.expiresAt, now)).run();
tx.delete(repositoryLockWaitersTable).where(lte(repositoryLockWaitersTable.expiresAt, now)).run(); tx.delete(repositoryLockWaitersTable).where(lte(repositoryLockWaitersTable.expiresAt, now)).run();
@ -181,43 +153,49 @@ class RepositoryMutex {
}); });
} }
private tryAcquireImmediately(request: LockRequest, signal?: AbortSignal) { private tryAcquireImmediately(request: LockRequest) {
const locks = this.tryAcquireManyRows([request]); return Effect.gen(this, function* () {
if (!locks || locks.length === 0) return null; const locks = this.tryAcquireManyRows([request]);
if (!locks || locks.length === 0) return null;
const [lock] = locks; const [lock] = locks;
const releaseLock = this.createRelease(lock); return yield* this.createRelease(lock);
this.releaseIfAborted(releaseLock, signal); });
return releaseLock;
} }
private createWaiter(request: LockRequest, waiterId: string) { private createWaiter(request: LockRequest, waiterId: string) {
const now = Date.now(); return Effect.sync(() => {
const now = Date.now();
db.transaction((tx) => { db.transaction((tx) => {
this.cleanupExpired(tx, now); this.cleanupExpired(tx, now);
tx.insert(repositoryLockWaitersTable) tx.insert(repositoryLockWaitersTable)
.values({ .values({
id: waiterId, id: waiterId,
repositoryId: request.repositoryId, repositoryId: request.repositoryId,
type: request.type, type: request.type,
operation: request.operation, operation: request.operation,
ownerId: this.ownerId, ownerId: this.ownerId,
requestedAt: now, requestedAt: now,
expiresAt: now + LOCK_LEASE_MS, expiresAt: now + LOCK_LEASE_MS,
heartbeatAt: now, heartbeatAt: now,
}) })
.run(); .run();
});
}); });
} }
private deleteWaiter(waiterId: string) { private deleteWaiter(waiterId: string) {
db.delete(repositoryLockWaitersTable) return Effect.sync(() =>
.where( db
and(eq(repositoryLockWaitersTable.id, waiterId), eq(repositoryLockWaitersTable.ownerId, this.ownerId)), .delete(repositoryLockWaitersTable)
) .where(
.run(); and(
eq(repositoryLockWaitersTable.id, waiterId),
eq(repositoryLockWaitersTable.ownerId, this.ownerId),
),
)
.run(),
);
} }
private deleteWaiterRow(tx: RepositoryMutexTransaction, waiterId: string): void { private deleteWaiterRow(tx: RepositoryMutexTransaction, waiterId: string): void {
@ -293,97 +271,49 @@ class RepositoryMutex {
}); });
} }
private async waitForQueuedLock(request: LockRequest, signal?: AbortSignal) { private waitForQueuedLock(request: LockRequest) {
this.throwIfAborted(signal);
const waiterId = this.generateLockId(); const waiterId = this.generateLockId();
this.createWaiter(request, waiterId);
this.startHeartbeat("waiter", waiterId);
try { const attempt = Effect.sync(() => this.tryPromoteWaiter(waiterId)).pipe(
while (true) { Effect.flatMap((attempt) => {
this.throwIfAborted(signal);
const attempt = this.tryPromoteWaiter(waiterId);
if (attempt.status === "acquired") { if (attempt.status === "acquired") {
this.stopHeartbeat(waiterId); return Effect.succeed(attempt.lock);
const releaseLock = this.createRelease(attempt.lock);
this.releaseIfAborted(releaseLock, signal);
return releaseLock;
} }
if (attempt.status === "missing") { if (attempt.status === "missing") {
this.createWaiter(request, waiterId); return Effect.gen(this, function* () {
this.startHeartbeat("waiter", waiterId); yield* this.createWaiter(request, waiterId);
yield* this.startHeartbeat("waiter", waiterId);
return yield* Effect.fail("retry");
});
} }
await this.waitForPoll(signal); return Effect.fail("retry");
} }),
} catch (error) { );
this.stopHeartbeat(waiterId);
this.deleteWaiter(waiterId);
this.release({ id: waiterId });
throw error;
}
}
async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal) { const cleanupAbandonedWaiter = Effect.gen(this, function* () {
this.throwIfAborted(signal); yield* this.deleteWaiter(waiterId);
yield* this.release({ id: waiterId });
});
const request: LockRequest = { repositoryId, type: "shared", operation }; return Effect.scoped(
const releaseLock = this.tryAcquireImmediately(request, signal); Effect.gen(this, function* () {
if (releaseLock) { const lock = yield* attempt.pipe(
return releaseLock; Effect.retry(Schedule.spaced(LOCK_POLL_MS)),
} Effect.onExit((exit) => {
if (Exit.isSuccess(exit)) {
return Effect.void;
}
logger.debug(`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation}`); return cleanupAbandonedWaiter;
return await this.waitForQueuedLock(request, signal); }),
} );
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal) { return yield* this.createRelease(lock);
this.throwIfAborted(signal); }),
);
const request: LockRequest = { repositoryId, type: "exclusive", operation };
const releaseLock = this.tryAcquireImmediately(request, signal);
if (releaseLock) {
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
return releaseLock;
}
logger.debug(`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation}`);
const queuedReleaseLock = await this.waitForQueuedLock(request, signal);
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
return queuedReleaseLock;
}
async acquireMany(requests: LockRequest[], signal?: AbortSignal) {
this.throwIfAborted(signal);
if (requests.length === 0) {
return () => {};
}
const seenRepositoryIds = new Set<string>();
for (const request of requests) {
if (seenRepositoryIds.has(request.repositoryId)) {
throw new Error(`Duplicate repository lock request: ${request.repositoryId}`);
}
seenRepositoryIds.add(request.repositoryId);
}
const sortedRequests = [...requests].sort((a, b) => a.repositoryId.localeCompare(b.repositoryId));
while (true) {
const locks = this.tryAcquireManyRows(sortedRequests);
if (locks) {
const releaseLocks = this.createReleaseMany(locks);
this.releaseIfAborted(releaseLocks, signal);
return releaseLocks;
}
await this.waitForPoll(signal);
}
} }
isLocked(repositoryId: string) { isLocked(repositoryId: string) {
@ -396,69 +326,91 @@ class RepositoryMutex {
} }
private createReleaseMany(locks: AcquiredLock[]) { private createReleaseMany(locks: AcquiredLock[]) {
const releases = locks.map((lock) => this.createRelease(lock)); return Effect.gen(this, function* () {
let released = false; const releases = yield* Effect.all(locks.map((lock) => this.createRelease(lock)));
let released = false;
return () => { return () => {
if (released) return; if (released) return;
released = true; released = true;
for (const release of releases.toReversed()) { for (const release of releases.toReversed()) {
release(); release();
} }
}; };
});
} }
private createRelease(lock: AcquiredLock) { private createRelease(lock: AcquiredLock) {
this.startHeartbeat("lock", lock.id); return Effect.gen(this, function* () {
let released = false; const heartbeatFiber = yield* this.startHeartbeat("lock", lock.id);
let released = false;
return () => { return () => {
if (released) return; if (released) return;
released = true; released = true;
this.stopHeartbeat(lock.id); Effect.runFork(Fiber.interrupt(heartbeatFiber));
this.release(lock); Effect.runSync(this.release(lock));
}; };
});
} }
private release(lock: Pick<AcquiredLock, "id">) { private release(lock: Pick<AcquiredLock, "id">) {
const releasedLock = db.transaction((tx) => { return Effect.gen(this, function* () {
const row = tx.query.repositoryLocksTable const releasedLock = yield* Effect.sync(() =>
.findFirst({ where: { AND: [{ id: { eq: lock.id } }, { ownerId: { eq: this.ownerId } }] } }) db.transaction((tx) => {
.sync(); const row = tx.query.repositoryLocksTable
.findFirst({ where: { AND: [{ id: { eq: lock.id } }, { ownerId: { eq: this.ownerId } }] } })
.sync();
if (!row) return null; if (!row) return null;
tx.delete(repositoryLocksTable) tx.delete(repositoryLocksTable)
.where(and(eq(repositoryLocksTable.id, lock.id), eq(repositoryLocksTable.ownerId, this.ownerId))) .where(
.run(); and(eq(repositoryLocksTable.id, lock.id), eq(repositoryLocksTable.ownerId, this.ownerId)),
)
.run();
return row; return row;
}),
);
if (!releasedLock) return;
const duration = Date.now() - releasedLock.acquiredAt;
yield* logger.effect.debug(
`[Mutex] Released ${releasedLock.type} lock for repo ${releasedLock.repositoryId}: ${releasedLock.operation} (held for ${duration}ms)`,
);
}); });
if (!releasedLock) return;
const duration = Date.now() - releasedLock.acquiredAt;
logger.debug(
`[Mutex] Released ${releasedLock.type} lock for repo ${releasedLock.repositoryId}: ${releasedLock.operation} (held for ${duration}ms)`,
);
} }
private startHeartbeat(target: HeartbeatTarget, lockId: string) { private startHeartbeat(
this.stopHeartbeat(lockId); target: "waiter",
lockId: string,
const heartbeat = () => { ): Effect.Effect<Fiber.RuntimeFiber<void, never>, never, Scope.Scope>;
private startHeartbeat(
target: "lock",
lockId: string,
): Effect.Effect<Fiber.RuntimeFiber<void, never>, never, never>;
private startHeartbeat(
target: HeartbeatTarget,
lockId: string,
): Effect.Effect<Fiber.RuntimeFiber<unknown, never>, never, Scope.Scope> {
const heartbeat = Effect.gen(this, function* () {
const now = Date.now(); const now = Date.now();
const values = { heartbeatAt: now, expiresAt: now + LOCK_LEASE_MS }; const values = { heartbeatAt: now, expiresAt: now + LOCK_LEASE_MS };
try { if (target === "lock") {
if (target === "lock") { yield* Effect.try(() => {
db.update(repositoryLocksTable) db.update(repositoryLocksTable)
.set(values) .set(values)
.where(and(eq(repositoryLocksTable.id, lockId), eq(repositoryLocksTable.ownerId, this.ownerId))) .where(and(eq(repositoryLocksTable.id, lockId), eq(repositoryLocksTable.ownerId, this.ownerId)))
.run(); .run();
} else { });
} else {
yield* Effect.try(() => {
db.update(repositoryLockWaitersTable) db.update(repositoryLockWaitersTable)
.set(values) .set(values)
.where( .where(
@ -468,29 +420,142 @@ class RepositoryMutex {
), ),
) )
.run(); .run();
} });
} catch (error) {
logger.warn(`[Mutex] Failed to heartbeat ${target} ${lockId}: ${String(error)}`);
} }
}; }).pipe(
Effect.catchAll((error) =>
logger.effect.warn(`[Mutex] Failed to heartbeat ${target} ${lockId}: ${String(error)}`),
),
);
const timer = setInterval(heartbeat, LOCK_HEARTBEAT_MS); const repeat = heartbeat.pipe(Effect.repeat(Schedule.spaced(LOCK_HEARTBEAT_MS)));
if (timer && "unref" in timer) {
timer.unref(); if (target === "waiter") {
// For waiters, we can stop heartbeating when the releaser is dropped, so we use a scoped fiber
return repeat.pipe(Effect.forkScoped);
} }
this.heartbeatTimers.set(lockId, timer); // For locks, the heartbeat must outlive the acquire scope.
// It is interrupted manually by the returned release function.
// TODO: max lifetime for lock heartbeats to prevent leaks if the releaser is never called?
return repeat.pipe(Effect.forkDaemon);
} }
private stopHeartbeat(lockId: string) { private acquireSharedEffect(repositoryId: string, operation: string) {
const timer = this.heartbeatTimers.get(lockId); return Effect.gen(this, function* () {
if (!timer) { const request: LockRequest = { repositoryId, type: "shared", operation };
return; const releaseLock = yield* this.tryAcquireImmediately(request);
if (releaseLock) return releaseLock;
yield* logger.effect.debug(`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation}`);
return yield* this.waitForQueuedLock(request);
});
}
private acquireExclusiveEffect(repositoryId: string, operation: string) {
return Effect.gen(this, function* () {
const request: LockRequest = { repositoryId, type: "exclusive", operation };
const releaseLock = yield* this.tryAcquireImmediately(request);
if (releaseLock) {
yield* logger.effect.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
return releaseLock;
}
yield* logger.effect.debug(`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation}`);
const queuedReleaseLock = yield* this.waitForQueuedLock(request);
yield* logger.effect.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
return queuedReleaseLock;
});
}
private acquireManyEffect(requests: LockRequest[]) {
return Effect.gen(this, function* () {
if (requests.length === 0) {
return () => {};
}
const seenRepositoryIds = new Set<string>();
for (const request of requests) {
if (seenRepositoryIds.has(request.repositoryId)) {
throw new Error(`Duplicate repository lock request: ${request.repositoryId}`);
}
seenRepositoryIds.add(request.repositoryId);
}
const sortedRequests = [...requests].sort((a, b) => a.repositoryId.localeCompare(b.repositoryId));
const locks = yield* Effect.sync(() => this.tryAcquireManyRows(sortedRequests)).pipe(
Effect.flatMap((locks) => {
if (locks) return Effect.succeed(locks);
return Effect.fail("retry");
}),
Effect.retry(Schedule.spaced(LOCK_POLL_MS)),
);
return yield* this.createReleaseMany(locks);
});
}
private runWithSignal<A, E>(effect: Effect.Effect<A, E>, signal?: AbortSignal) {
if (!signal) return Effect.runPromise(effect);
if (signal.aborted) {
return Promise.reject(this.abortReason(signal));
} }
clearInterval(timer); return new Promise<A>((resolve, reject) => {
this.heartbeatTimers.delete(lockId); const fiber = Effect.runFork(effect);
let settled = false;
let aborting = false;
const complete = (callback: () => void) => {
if (settled) return;
settled = true;
signal.removeEventListener("abort", onAbort);
callback();
};
const onAbort = () => {
aborting = true;
Effect.runPromise(Fiber.interrupt(fiber)).then(
(exit) =>
complete(() => {
if (Exit.isSuccess(exit)) {
resolve(exit.value);
return;
}
reject(this.abortReason(signal));
}),
(error) => complete(() => reject(error)),
);
};
signal.addEventListener("abort", onAbort, { once: true });
Effect.runPromise(Fiber.join(fiber)).then(
(value) => complete(() => resolve(value)),
(error) => {
if (!aborting) {
complete(() => reject(error));
}
},
);
});
}
async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal) {
return await this.runWithSignal(this.acquireSharedEffect(repositoryId, operation), signal);
}
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal) {
return await this.runWithSignal(this.acquireExclusiveEffect(repositoryId, operation), signal);
}
async acquireMany(requests: LockRequest[], signal?: AbortSignal) {
return await this.runWithSignal(this.acquireManyEffect(requests), signal);
} }
} }
export const repoMutex = new RepositoryMutex(); export const repoMutex = getRepositoryMutex();

View file

@ -208,6 +208,141 @@ describe("repositoriesService repository stats", () => {
}); });
}); });
describe("repositoriesService.listSnapshotFiles", () => {
afterEach(() => {
vi.restoreAllMocks();
});
test("limits concurrent restic ls commands per repository", async () => {
const repository = await createTestRepository(session.organizationId);
let active = 0;
let maxActive = 0;
let releaseAll = false;
let exclusiveAcquired = false;
let releaseExclusive: (() => void) | undefined;
let exclusivePromise: Promise<() => void> | undefined;
const releaseWaiters: Array<() => void> = [];
const exclusiveController = new AbortController();
const releaseWaitingCommands = () => {
const waiters = releaseWaiters.splice(0);
for (const release of waiters) {
release();
}
};
const resolveWithin = async <T>(promise: Promise<T>, timeoutMs: number) => {
return await new Promise<T>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error(`Expected promise to resolve within ${timeoutMs}ms`));
}, timeoutMs);
promise.then(
(value) => {
clearTimeout(timeout);
resolve(value);
},
(error) => {
clearTimeout(timeout);
reject(error);
},
);
});
};
const lsSpy = vi.spyOn(restic, "ls").mockImplementation((_config, snapshotId, _path, options) =>
Effect.promise(async () => {
active++;
maxActive = Math.max(maxActive, active);
try {
if (!releaseAll) {
await new Promise<void>((resolve) => releaseWaiters.push(resolve));
}
return {
snapshot: {
id: snapshotId,
short_id: snapshotId,
time: new Date().toISOString(),
tree: "tree",
paths: ["/"],
hostname: "host",
struct_type: "snapshot" as const,
message_type: "snapshot" as const,
},
nodes: [],
pagination: {
offset: options.offset ?? 0,
limit: options.limit ?? 500,
total: 0,
hasMore: false,
},
};
} finally {
active--;
}
}),
);
const calls = Array.from({ length: 4 }, (_, index) =>
withContext({ organizationId: session.organizationId, userId: session.user.id }, () =>
repositoriesService.listSnapshotFiles(repository.shortId, `snapshot-${index}`, "/", {
offset: 0,
limit: 100,
}),
),
);
try {
await waitForExpect(() => {
expect(releaseWaiters).toHaveLength(2);
});
expect(maxActive).toBe(2);
exclusivePromise = repoMutex
.acquireExclusive(repository.id, "delete", exclusiveController.signal)
.then((release) => {
exclusiveAcquired = true;
releaseExclusive = release;
return release;
});
releaseWaitingCommands();
releaseExclusive = await resolveWithin(exclusivePromise, 2000);
expect(exclusiveAcquired).toBe(true);
expect(active).toBe(0);
releaseExclusive();
releaseExclusive = undefined;
await waitForExpect(() => {
expect(releaseWaiters).toHaveLength(2);
});
expect(maxActive).toBe(2);
releaseWaitingCommands();
await Promise.all(calls);
} finally {
if (releaseExclusive) {
releaseExclusive();
} else {
exclusiveController.abort();
}
releaseAll = true;
releaseWaitingCommands();
await Promise.allSettled(calls);
if (exclusivePromise) {
await Promise.allSettled([exclusivePromise]);
}
}
expect(lsSpy).toHaveBeenCalledTimes(4);
expect(maxActive).toBeLessThanOrEqual(2);
});
});
describe("repositoriesService.dumpSnapshot", () => { describe("repositoriesService.dumpSnapshot", () => {
afterEach(() => { afterEach(() => {
vi.restoreAllMocks(); vi.restoreAllMocks();

View file

@ -42,6 +42,7 @@ import type { ParsedTask, TaskInput } from "../tasks/tasks.schemas";
import { Effect } from "effect"; import { Effect } from "effect";
const runningDoctors = new Map<string, AbortController>(); const runningDoctors = new Map<string, AbortController>();
const lsLimiters = new Map<string, Effect.Semaphore>();
const RESTORE_TASK_RESOURCE_TYPE = "repository"; const RESTORE_TASK_RESOURCE_TYPE = "repository";
type RestoreTaskInput = Extract<TaskInput, { kind: "restore" }>; type RestoreTaskInput = Extract<TaskInput, { kind: "restore" }>;
@ -99,6 +100,15 @@ const updateActiveRestoreTask = (restoreId: string, eventName: string, update: (
} }
}; };
const getLsLimiter = (repositoryId: string) => {
let limiter = lsLimiters.get(repositoryId);
if (!limiter) {
limiter = Effect.runSync(Effect.makeSemaphore(2));
lsLimiters.set(repositoryId, limiter);
}
return limiter;
};
const findActiveRestoreTask = ( const findActiveRestoreTask = (
organizationId: string, organizationId: string,
repositoryShortId: string, repositoryShortId: string,
@ -469,36 +479,43 @@ const listSnapshotFiles = async (
}; };
} }
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`); const limiter = getLsLimiter(repository.id);
await runEffectPromise(limiter.take(1));
try { try {
const result = await runEffectPromise( const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
restic.ls(repository.config, snapshotId, path, { organizationId, offset, limit }), try {
); const result = await runEffectPromise(
restic.ls(repository.config, snapshotId, path, { organizationId, offset, limit }),
);
if (!result.snapshot) { if (!result.snapshot) {
throw new NotFoundError("Snapshot not found or empty"); throw new NotFoundError("Snapshot not found or empty");
}
const response = {
snapshot: {
id: result.snapshot.id,
short_id: result.snapshot.short_id,
time: result.snapshot.time,
hostname: result.snapshot.hostname,
paths: result.snapshot.paths,
},
files: result.nodes,
offset: result.pagination.offset,
limit: result.pagination.limit,
total: result.pagination.total,
hasMore: result.pagination.hasMore,
};
cache.set(cacheKey, result);
return response;
} finally {
releaseLock();
} }
const response = {
snapshot: {
id: result.snapshot.id,
short_id: result.snapshot.short_id,
time: result.snapshot.time,
hostname: result.snapshot.hostname,
paths: result.snapshot.paths,
},
files: result.nodes,
offset: result.pagination.offset,
limit: result.pagination.limit,
total: result.pagination.total,
hasMore: result.pagination.hasMore,
};
cache.set(cacheKey, result);
return response;
} finally { } finally {
releaseLock(); await runEffectPromise(limiter.release(1));
} }
}; };

View file

@ -99,7 +99,7 @@ Mount the provisioning file and set `PROVISIONING_PATH`:
```yaml docker-compose.yml ```yaml docker-compose.yml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
environment: environment:
- PROVISIONING_PATH=/config/provisioning.json - PROVISIONING_PATH=/config/provisioning.json
volumes: volumes:

View file

@ -100,7 +100,7 @@ If you run Traefik as your reverse proxy, add labels to the Zerobyte service in
```yaml docker-compose.yml ```yaml docker-compose.yml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
environment: environment:

View file

@ -52,7 +52,7 @@ services:
- "4096:4096" - "4096:4096"
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
# Uncomment if you need remote mounts (NFS/SMB/WebDAV): # Uncomment if you need remote mounts (NFS/SMB/WebDAV):

View file

@ -52,7 +52,7 @@ Create a `docker-compose.yml` file with the following configuration:
```yaml docker-compose.yml ```yaml docker-compose.yml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
cap_add: cap_add:
@ -182,7 +182,7 @@ If you only need to back up locally mounted directories and don't require remote
```yaml docker-compose.yml ```yaml docker-compose.yml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
ports: ports:
@ -259,7 +259,7 @@ If you use provisioning, Zerobyte can resolve secrets from environment variables
```yaml docker-compose.yml ```yaml docker-compose.yml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
cap_add: cap_add:

View file

@ -111,7 +111,7 @@ Update your `docker-compose.yml` to mount the directory you want to backup:
```yaml ```yaml
services: services:
zerobyte: zerobyte:
image: ghcr.io/nicotsx/zerobyte:v0.37 image: ghcr.io/nicotsx/zerobyte:v0.38
container_name: zerobyte container_name: zerobyte
restart: unless-stopped restart: unless-stopped
# ... other configuration ... # ... other configuration ...

View file

@ -55,6 +55,7 @@
"hono-rate-limiter": "^0.5.3", "hono-rate-limiter": "^0.5.3",
"http-errors-enhanced": "^4.0.2", "http-errors-enhanced": "^4.0.2",
"input-otp": "^1.4.2", "input-otp": "^1.4.2",
"ipaddr.js": "^2.4.0",
"lucide-react": "^1.17.0", "lucide-react": "^1.17.0",
"qrcode.react": "^4.2.0", "qrcode.react": "^4.2.0",
"react": "^19.2.6", "react": "^19.2.6",
@ -1683,7 +1684,7 @@
"ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="], "ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="],
"ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], "ipaddr.js": ["ipaddr.js@2.4.0", "", {}, "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ=="],
"is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="], "is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="],
@ -2665,6 +2666,8 @@
"prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], "prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="],
"proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
"recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], "recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
"recharts/es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="], "recharts/es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="],

View file

@ -87,6 +87,7 @@
"hono-rate-limiter": "^0.5.3", "hono-rate-limiter": "^0.5.3",
"http-errors-enhanced": "^4.0.2", "http-errors-enhanced": "^4.0.2",
"input-otp": "^1.4.2", "input-otp": "^1.4.2",
"ipaddr.js": "^2.4.0",
"lucide-react": "^1.17.0", "lucide-react": "^1.17.0",
"qrcode.react": "^4.2.0", "qrcode.react": "^4.2.0",
"react": "^19.2.6", "react": "^19.2.6",

View file

@ -1,175 +0,0 @@
import { logger, safeExec } from "../node";
import { toMessage } from "../utils";
import { safeJsonParse } from "../utils/json";
import { ResticLockError } from "./error";
import { addCommonArgs } from "./helpers/add-common-args";
import { buildEnv } from "./helpers/build-env";
import { buildRepoUrl } from "./helpers/build-repo-url";
import { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
import type { RepositoryConfig } from "./schemas";
import type { ResticDeps } from "./types";
type ResticLockDiagnosticContext = {
error: unknown;
operation: string;
repositoryConfig: RepositoryConfig;
organizationId: string;
resticDeps: ResticDeps;
relatedRepositoryConfigs?: RepositoryConfig[];
};
const LOCK_ERROR_PATTERNS = [
/unable to create lock in backend/i,
/repository is already locked/i,
/failed to lock repository/i,
/"code"\s*:\s*11/i,
/\bexit_error\b.*\b11\b/i,
];
export const isResticLockFailure = (error: unknown) => {
if (error instanceof ResticLockError) {
return true;
}
const message = toMessage(error);
return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message));
};
const RESTIC_LOCK_ID_PATTERN = /^[a-f0-9]{64}$/i;
const addLockId = (ids: Set<string>, candidate: unknown) => {
if (typeof candidate === "string" && RESTIC_LOCK_ID_PATTERN.test(candidate)) {
ids.add(candidate);
}
};
const parseLockIds = (stdout: string) => {
const ids = new Set<string>();
for (const line of stdout.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
const parsed = safeJsonParse<{ id?: unknown }>(trimmed);
if (parsed) {
addLockId(ids, parsed.id);
continue;
}
addLockId(ids, trimmed);
}
return [...ids].slice(0, 20);
};
const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, organizationId, deps);
const baseArgs = ["--repo", repoUrl];
addCommonArgs(baseArgs, env, config);
try {
const listResult = await safeExec({
command: "restic",
args: [...baseArgs, "list", "locks"],
env,
timeout: 15_000,
});
const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : [];
const lockDetails = [];
for (const lockId of lockIds) {
const catResult = await safeExec({
command: "restic",
args: [...baseArgs, "cat", "lock", "--", lockId],
env,
timeout: 15_000,
});
lockDetails.push({
lockId,
exitCode: catResult.exitCode,
stdout: catResult.stdout,
stderr: catResult.stderr,
timedOut: catResult.timedOut,
});
}
return {
repoUrl,
list: {
exitCode: listResult.exitCode,
stdout: listResult.stdout,
stderr: listResult.stderr,
timedOut: listResult.timedOut,
},
lockIds,
lockDetails,
};
} finally {
await cleanupTemporaryKeys(env, deps);
}
};
export const logResticLockFailureDiagnostics = async ({
error,
operation,
repositoryConfig,
organizationId,
resticDeps,
relatedRepositoryConfigs = [],
}: ResticLockDiagnosticContext) => {
if (!isResticLockFailure(error)) {
return false;
}
try {
const configsByRepoUrl = new Map(
[repositoryConfig, ...relatedRepositoryConfigs].map((config) => [buildRepoUrl(config), config]),
);
const configsToInspect = [...configsByRepoUrl.entries()];
logger.error("[ResticLockFailure] Restic repository lock failure detected", {
operation,
error: toMessage(error),
process: {
pid: process.pid,
hostname: process.env.HOSTNAME,
nodeEnv: process.env.NODE_ENV,
},
repository: {
repoUrl: buildRepoUrl(repositoryConfig),
},
relatedRepositories: relatedRepositoryConfigs.map((config) => ({
repoUrl: buildRepoUrl(config),
})),
});
for (const [repoUrl, config] of configsToInspect) {
try {
const resticLocks = await inspectResticLocks(config, organizationId, resticDeps);
logger.error("[ResticLockFailure] Restic backend lock dump", {
operation,
repoUrl,
resticLocks,
});
} catch (diagnosticError) {
logger.error("[ResticLockFailure] Failed to inspect restic backend locks", {
operation,
repoUrl,
error: toMessage(diagnosticError),
});
}
}
return true;
} catch (diagnosticError) {
logger.error("[ResticLockFailure] Failed to collect lock diagnostics", {
operation,
repoUrl: buildRepoUrl(repositoryConfig),
error: toMessage(diagnosticError),
});
return true;
}
};

View file

@ -16,7 +16,6 @@ import { stats } from "./commands/stats";
import { tagSnapshots } from "./commands/tag-snapshots"; import { tagSnapshots } from "./commands/tag-snapshots";
import { unlock } from "./commands/unlock"; import { unlock } from "./commands/unlock";
import { ResticLockError } from "./error"; import { ResticLockError } from "./error";
import { logResticLockFailureDiagnostics } from "./lock-diagnostics";
import type { RepositoryConfig } from "./schemas"; import type { RepositoryConfig } from "./schemas";
import type { ResticDeps } from "./types"; import type { ResticDeps } from "./types";
@ -25,17 +24,14 @@ export { buildEnv } from "./helpers/build-env";
export { buildRepoUrl } from "./helpers/build-repo-url"; export { buildRepoUrl } from "./helpers/build-repo-url";
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys"; export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
export { validateCustomResticParams } from "./helpers/validate-custom-params"; export { validateCustomResticParams } from "./helpers/validate-custom-params";
export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics";
export { isResticError, ResticError, ResticLockError } from "./error"; export { isResticError, ResticError, ResticLockError } from "./error";
type LockDiagnosticCommandContext = { type LockRecoveryContext = {
repositoryConfig: RepositoryConfig; repositoryConfigs: RepositoryConfig[];
organizationId: string; organizationId: string;
relatedRepositoryConfigs?: RepositoryConfig[];
}; };
type ResticCommandOptions = { organizationId: string }; type ResticCommandOptions = { organizationId: string };
type ResticCommandResult = { error?: unknown };
type ResticCommandFailure<Failure> = Failure | ResticLockError; type ResticCommandFailure<Failure> = Failure | ResticLockError;
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect< type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
Success, Success,
@ -43,56 +39,37 @@ type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
Requirements Requirements
>; >;
const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => { const getLockRecoveryContext = (operation: string, args: unknown[]): LockRecoveryContext => {
const firstRepositoryConfig = args[0] as RepositoryConfig; const firstRepositoryConfig = args[0] as RepositoryConfig;
const options = args.at(-1) as ResticCommandOptions; const options = args.at(-1) as ResticCommandOptions;
if (operation === "restic.copy") { if (operation === "restic.copy") {
return { return {
repositoryConfig: args[1] as RepositoryConfig, repositoryConfigs: [args[1] as RepositoryConfig, firstRepositoryConfig],
organizationId: options.organizationId, organizationId: options.organizationId,
relatedRepositoryConfigs: [firstRepositoryConfig],
}; };
} }
return { return {
repositoryConfig: firstRepositoryConfig, repositoryConfigs: [firstRepositoryConfig],
organizationId: options.organizationId, organizationId: options.organizationId,
}; };
}; };
const logLockFailure = async ( const unlockStaleLocks = (context: LockRecoveryContext, deps: ResticDeps) =>
error: unknown,
operation: string,
context: LockDiagnosticCommandContext,
deps: ResticDeps,
) =>
logResticLockFailureDiagnostics({
error,
operation,
repositoryConfig: context.repositoryConfig,
organizationId: context.organizationId,
resticDeps: deps,
relatedRepositoryConfigs: context.relatedRepositoryConfigs,
});
const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect<void, Error> =>
Effect.gen(function* () { Effect.gen(function* () {
for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) { for (const repositoryConfig of context.repositoryConfigs) {
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps); yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
} }
}); }).pipe(Effect.catchAll(() => Effect.void));
const recoverFromLockFailure = <Success, Failure, Requirements>( const recoverFromLockFailure = <Success, Failure, Requirements>(
error: ResticLockError, context: LockRecoveryContext,
operation: string,
context: LockDiagnosticCommandContext,
deps: ResticDeps,
runCommand: RunResticCommand<Success, Failure, Requirements>, runCommand: RunResticCommand<Success, Failure, Requirements>,
deps: ResticDeps,
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> => ): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
Effect.gen(function* () { Effect.gen(function* () {
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once."); yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
yield* Effect.promise(() => logLockFailure(error, operation, context, deps));
yield* unlockStaleLocks(context, deps); yield* unlockStaleLocks(context, deps);
const retryResult = yield* runCommand(); const retryResult = yield* runCommand();
@ -107,20 +84,11 @@ function withDeps<Args extends unknown[], Success, Failure, Requirements>(
deps: ResticDeps, deps: ResticDeps,
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> { ): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
return (...args: Args) => { return (...args: Args) => {
const context = getCommandContext(operation, args); const context = getLockRecoveryContext(operation, args);
const runCommand = () => command(...args, deps); const runCommand = () => command(...args, deps);
const commandEffect = runCommand().pipe( return runCommand().pipe(
Effect.catchTag("ResticLockError", (error) => Effect.catchTag("ResticLockError", () => recoverFromLockFailure(context, runCommand, deps)),
recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand),
),
);
return commandEffect.pipe(
Effect.tap((result) => {
const { error } = result as ResticCommandResult;
return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void;
}),
); );
}; };
} }