Compare commits
5 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c1f0c52d5 | ||
|
|
2318b6bdd0 | ||
|
|
34fc4f0cbb | ||
|
|
68002b0308 | ||
|
|
4bba2c2493 |
20 changed files with 615 additions and 505 deletions
|
|
@ -50,7 +50,7 @@ In order to run Zerobyte, you need to have Docker and Docker Compose installed o
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|
@ -156,7 +156,7 @@ If you only need to back up locally-mounted folders and don't require remote sha
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
|
|
@ -195,7 +195,7 @@ If you want to backup a local directory on the same host where Zerobyte is runni
|
||||||
```diff
|
```diff
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|
@ -270,7 +270,7 @@ Zerobyte can use [rclone](https://rclone.org/) to support 40+ cloud storage prov
|
||||||
```diff
|
```diff
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|
|
||||||
|
|
@ -224,29 +224,6 @@ describe("SnapshotTreeBrowser", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("prefetches using the query path when display and query roots differ", async () => {
|
|
||||||
const requests = mockListSnapshotFiles();
|
|
||||||
|
|
||||||
renderSnapshotTreeBrowser();
|
|
||||||
|
|
||||||
const row = await screen.findByRole("button", { name: "project" });
|
|
||||||
const initialRequestCount = requests.length;
|
|
||||||
|
|
||||||
await userEvent.hover(row);
|
|
||||||
|
|
||||||
await waitFor(() => {
|
|
||||||
expect(requests.length).toBe(initialRequestCount + 1);
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(requests.at(-1)).toEqual({
|
|
||||||
shortId: "repo-1",
|
|
||||||
snapshotId: "snap-1",
|
|
||||||
path: "/mnt/project",
|
|
||||||
offset: "0",
|
|
||||||
limit: "500",
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
test("shows the query root contents when display and query roots differ", async () => {
|
test("shows the query root contents when display and query roots differ", async () => {
|
||||||
mockListSnapshotFiles();
|
mockListSnapshotFiles();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ import { FileBrowser, type FileBrowserUiProps } from "~/client/components/file-b
|
||||||
import { useFileBrowser } from "~/client/hooks/use-file-browser";
|
import { useFileBrowser } from "~/client/hooks/use-file-browser";
|
||||||
import { parseError } from "~/client/lib/errors";
|
import { parseError } from "~/client/lib/errors";
|
||||||
import { isPathWithin, normalizeAbsolutePath } from "@zerobyte/core/utils";
|
import { isPathWithin, normalizeAbsolutePath } from "@zerobyte/core/utils";
|
||||||
import { logger } from "~/client/lib/logger";
|
|
||||||
|
|
||||||
function createPathPrefixFns(basePath: string) {
|
function createPathPrefixFns(basePath: string) {
|
||||||
return {
|
return {
|
||||||
|
|
@ -84,16 +83,6 @@ export const SnapshotTreeBrowser = (props: SnapshotTreeBrowserProps) => {
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
prefetchFolder: (displayPath) => {
|
|
||||||
void queryClient
|
|
||||||
.prefetchQuery(
|
|
||||||
listSnapshotFilesOptions({
|
|
||||||
path: { shortId: repositoryId, snapshotId },
|
|
||||||
query: { path: displayPath, offset: 0, limit: pageSize },
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.catch((e) => logger.error(e));
|
|
||||||
},
|
|
||||||
pathTransform: displayPathFns,
|
pathTransform: displayPathFns,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
36
app/client/functions/__tests__/is-secure-context.test.ts
Normal file
36
app/client/functions/__tests__/is-secure-context.test.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
import { getIsSecureContextForRequest, isSecureContextOrigin } from "../is-secure-context";
|
||||||
|
|
||||||
|
describe("isSecureContextOrigin", () => {
|
||||||
|
test("treats HTTPS and loopback HTTP origins as secure contexts", () => {
|
||||||
|
expect(isSecureContextOrigin("https://example.com")).toBe(true);
|
||||||
|
expect(isSecureContextOrigin("http://localhost:3000")).toBe(true);
|
||||||
|
expect(isSecureContextOrigin("http://app.localhost")).toBe(true);
|
||||||
|
expect(isSecureContextOrigin("http://127.0.0.1:3000")).toBe(true);
|
||||||
|
expect(isSecureContextOrigin("http://[::1]:3000")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("treats non-local HTTP origins as insecure contexts", () => {
|
||||||
|
expect(isSecureContextOrigin("http://example.com")).toBe(false);
|
||||||
|
expect(isSecureContextOrigin("http://127.example.com")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("getIsSecureContextForRequest", () => {
|
||||||
|
test("uses forwarded protocol and host when present", () => {
|
||||||
|
const request = new Request("http://internal.local/settings", {
|
||||||
|
headers: {
|
||||||
|
host: "internal.local",
|
||||||
|
"x-forwarded-host": "zerobyte.example.com",
|
||||||
|
"x-forwarded-proto": "https",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getIsSecureContextForRequest(request)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses the request URL when forwarded headers are absent", () => {
|
||||||
|
expect(getIsSecureContextForRequest(new Request("http://zerobyte.example.com/settings"))).toBe(false);
|
||||||
|
expect(getIsSecureContextForRequest(new Request("https://zerobyte.example.com/settings"))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
51
app/client/functions/is-secure-context.ts
Normal file
51
app/client/functions/is-secure-context.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
import { createIsomorphicFn } from "@tanstack/react-start";
|
||||||
|
import { getRequest } from "@tanstack/react-start/server";
|
||||||
|
import ipaddr from "ipaddr.js";
|
||||||
|
|
||||||
|
const getFirstHeaderValue = (value: string | null) => value?.split(",")[0]?.trim();
|
||||||
|
|
||||||
|
const isLoopbackIp = (hostname: string) => {
|
||||||
|
try {
|
||||||
|
return ipaddr.parse(hostname).range() === "loopback";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const getRequestOrigin = (request: Request) => {
|
||||||
|
const requestUrl = new URL(request.url);
|
||||||
|
const forwardedProto = getFirstHeaderValue(request.headers.get("x-forwarded-proto"));
|
||||||
|
const forwardedHost = getFirstHeaderValue(request.headers.get("x-forwarded-host"));
|
||||||
|
const host = forwardedHost || getFirstHeaderValue(request.headers.get("host"));
|
||||||
|
|
||||||
|
if (!host) {
|
||||||
|
return requestUrl.origin;
|
||||||
|
}
|
||||||
|
|
||||||
|
const protocol = (forwardedProto || requestUrl.protocol).replace(/:$/, "");
|
||||||
|
return `${protocol}://${host}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isSecureContextOrigin = (origin: string) => {
|
||||||
|
const url = new URL(origin);
|
||||||
|
|
||||||
|
if (url.protocol === "https:") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hostname = url.hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase();
|
||||||
|
|
||||||
|
return hostname === "localhost" || hostname.endsWith(".localhost") || isLoopbackIp(hostname);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getIsSecureContextForRequest = (request: Request) => isSecureContextOrigin(getRequestOrigin(request));
|
||||||
|
|
||||||
|
export const getIsSecureContext = createIsomorphicFn()
|
||||||
|
.server(() => {
|
||||||
|
try {
|
||||||
|
return getIsSecureContextForRequest(getRequest());
|
||||||
|
} catch {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.client(() => window.isSecureContext ?? true);
|
||||||
|
|
@ -63,9 +63,7 @@ export function ScheduleDetailsPage(props: Props) {
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
const searchParams = useSearch({ from: "/(dashboard)/backups/$backupId/" });
|
const searchParams = useSearch({ from: "/(dashboard)/backups/$backupId/" });
|
||||||
const [selectedSnapshotId, setSelectedSnapshotId] = useState<string | undefined>(
|
const [selectedSnapshotId, setSelectedSnapshotId] = useState<string | undefined>(initialSnapshotId);
|
||||||
initialSnapshotId ?? loaderData.snapshots?.at(-1)?.short_id,
|
|
||||||
);
|
|
||||||
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
|
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
|
||||||
const [snapshotToDelete, setSnapshotToDelete] = useState<string | null>(null);
|
const [snapshotToDelete, setSnapshotToDelete] = useState<string | null>(null);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,8 @@ import {
|
||||||
} from "~/client/components/ui/dialog";
|
} from "~/client/components/ui/dialog";
|
||||||
import { Input } from "~/client/components/ui/input";
|
import { Input } from "~/client/components/ui/input";
|
||||||
import { Label } from "~/client/components/ui/label";
|
import { Label } from "~/client/components/ui/label";
|
||||||
|
import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip";
|
||||||
|
import { getIsSecureContext } from "~/client/functions/is-secure-context";
|
||||||
import { authClient } from "~/client/lib/auth-client";
|
import { authClient } from "~/client/lib/auth-client";
|
||||||
import { logger } from "~/client/lib/logger";
|
import { logger } from "~/client/lib/logger";
|
||||||
import { useTimeFormat } from "~/client/lib/datetime";
|
import { useTimeFormat } from "~/client/lib/datetime";
|
||||||
|
|
@ -38,6 +40,7 @@ type PasskeyEntry = {
|
||||||
|
|
||||||
export function PasskeysSection() {
|
export function PasskeysSection() {
|
||||||
const { formatDateTime } = useTimeFormat();
|
const { formatDateTime } = useTimeFormat();
|
||||||
|
const isSecureContext = getIsSecureContext();
|
||||||
const { data: passkeys, isPending } = useQuery({
|
const { data: passkeys, isPending } = useQuery({
|
||||||
queryKey: ["passkeys"],
|
queryKey: ["passkeys"],
|
||||||
queryFn: async () => {
|
queryFn: async () => {
|
||||||
|
|
@ -111,6 +114,8 @@ export function PasskeysSection() {
|
||||||
|
|
||||||
const handleAddPasskey = (e: React.ChangeEvent) => {
|
const handleAddPasskey = (e: React.ChangeEvent) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
|
if (!isSecureContext) return;
|
||||||
|
|
||||||
const name = newPasskeyName.trim() || undefined;
|
const name = newPasskeyName.trim() || undefined;
|
||||||
addPasskeyMutation.mutate(name);
|
addPasskeyMutation.mutate(name);
|
||||||
};
|
};
|
||||||
|
|
@ -149,10 +154,19 @@ export function PasskeysSection() {
|
||||||
Passkeys use your device's biometrics or screen lock instead of a password. They are
|
Passkeys use your device's biometrics or screen lock instead of a password. They are
|
||||||
phishing-resistant and cannot be reused across sites.
|
phishing-resistant and cannot be reused across sites.
|
||||||
</p>
|
</p>
|
||||||
<Button onClick={() => setAddDialogOpen(true)}>
|
<Tooltip>
|
||||||
<Plus className="h-4 w-4 mr-2" />
|
<TooltipTrigger asChild>
|
||||||
Add passkey
|
<span className="inline-flex">
|
||||||
</Button>
|
<Button onClick={() => setAddDialogOpen(true)} disabled={!isSecureContext}>
|
||||||
|
<Plus className="h-4 w-4 mr-2" />
|
||||||
|
Add passkey
|
||||||
|
</Button>
|
||||||
|
</span>
|
||||||
|
</TooltipTrigger>
|
||||||
|
<TooltipContent className={cn({ hidden: isSecureContext })}>
|
||||||
|
<p>Passkeys can only be added over HTTPS or from localhost.</p>
|
||||||
|
</TooltipContent>
|
||||||
|
</Tooltip>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<p className={cn("text-sm text-muted-foreground", { hidden: !isPending })}>Loading passkeys...</p>
|
<p className={cn("text-sm text-muted-foreground", { hidden: !isPending })}>Loading passkeys...</p>
|
||||||
|
|
@ -233,7 +247,7 @@ export function PasskeysSection() {
|
||||||
<Button type="button" variant="outline" onClick={() => setAddDialogOpen(false)}>
|
<Button type="button" variant="outline" onClick={() => setAddDialogOpen(false)}>
|
||||||
Cancel
|
Cancel
|
||||||
</Button>
|
</Button>
|
||||||
<Button type="submit" loading={addPasskeyMutation.isPending}>
|
<Button type="submit" loading={addPasskeyMutation.isPending} disabled={!isSecureContext}>
|
||||||
<Fingerprint className="h-4 w-4 mr-2" />
|
<Fingerprint className="h-4 w-4 mr-2" />
|
||||||
Add passkey
|
Add passkey
|
||||||
</Button>
|
</Button>
|
||||||
|
|
|
||||||
|
|
@ -134,6 +134,37 @@ describe("RepositoryMutex", () => {
|
||||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("should reject aborted queued acquisitions only after waiter cleanup finishes", async () => {
|
||||||
|
const repoId = "abort-waits-for-cleanup";
|
||||||
|
const releaseHolder = await repoMutex.acquireExclusive(repoId, "holder");
|
||||||
|
const controller = new AbortController();
|
||||||
|
const waitingAcquisition = repoMutex.acquireShared(repoId, "waiter", controller.signal);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||||
|
|
||||||
|
const waitersBeforeAbort = await db.query.repositoryLockWaitersTable.findMany({
|
||||||
|
where: { repositoryId: { eq: repoId } },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(waitersBeforeAbort.map((waiter) => waiter.operation)).toEqual(["waiter"]);
|
||||||
|
|
||||||
|
controller.abort(new Error("stop"));
|
||||||
|
await expect(waitingAcquisition).rejects.toThrow("stop");
|
||||||
|
|
||||||
|
const waitersAfterAbort = await db.query.repositoryLockWaitersTable.findMany({
|
||||||
|
where: { repositoryId: { eq: repoId } },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(waitersAfterAbort).toEqual([]);
|
||||||
|
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||||
|
} finally {
|
||||||
|
releaseHolder();
|
||||||
|
await db.delete(repositoryLockWaitersTable).where(eq(repositoryLockWaitersTable.repositoryId, repoId));
|
||||||
|
await db.delete(repositoryLocksTable).where(eq(repositoryLocksTable.repositoryId, repoId));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("should allow concurrent shared locks", async () => {
|
test("should allow concurrent shared locks", async () => {
|
||||||
const repoId = "concurrent-shared";
|
const repoId = "concurrent-shared";
|
||||||
const release1 = await repoMutex.acquireShared(repoId, "op1");
|
const release1 = await repoMutex.acquireShared(repoId, "op1");
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import {
|
||||||
type RepositoryLock,
|
type RepositoryLock,
|
||||||
type RepositoryLockWaiter,
|
type RepositoryLockWaiter,
|
||||||
} from "../db/schema";
|
} from "../db/schema";
|
||||||
|
import { Effect, Exit, Fiber, Schedule, Scope } from "effect";
|
||||||
|
|
||||||
type LockType = "shared" | "exclusive";
|
type LockType = "shared" | "exclusive";
|
||||||
|
|
||||||
|
|
@ -33,59 +34,30 @@ const LOCK_HEARTBEAT_MS = 5_000;
|
||||||
const LOCK_POLL_MS = 250;
|
const LOCK_POLL_MS = 250;
|
||||||
const LOCK_POLL_CLEANUP_MS = 5_000;
|
const LOCK_POLL_CLEANUP_MS = 5_000;
|
||||||
|
|
||||||
|
const REPOSITORY_MUTEX_INSTANCE = Symbol.for("zerobyte.repositoryMutex.instance");
|
||||||
|
function getRepositoryMutex() {
|
||||||
|
const globalObject = globalThis as typeof globalThis & Record<symbol, RepositoryMutex | undefined>;
|
||||||
|
const mutex = globalObject[REPOSITORY_MUTEX_INSTANCE];
|
||||||
|
|
||||||
|
if (mutex) return mutex;
|
||||||
|
|
||||||
|
const newMutex = new RepositoryMutex();
|
||||||
|
globalObject[REPOSITORY_MUTEX_INSTANCE] = newMutex;
|
||||||
|
return newMutex;
|
||||||
|
}
|
||||||
|
|
||||||
class RepositoryMutex {
|
class RepositoryMutex {
|
||||||
private ownerId = `owner_${Bun.randomUUIDv7()}`;
|
private ownerId = `owner_${Bun.randomUUIDv7()}`;
|
||||||
private heartbeatTimers = new Map<string, ReturnType<typeof setInterval>>();
|
|
||||||
private nextPollCleanupAt = 0;
|
private nextPollCleanupAt = 0;
|
||||||
|
|
||||||
private generateLockId(): string {
|
private generateLockId(): string {
|
||||||
return `lock_${Bun.randomUUIDv7()}`;
|
return `lock_${Bun.randomUUIDv7()}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
private abortReason(signal: AbortSignal) {
|
private abortReason(signal: AbortSignal): Error {
|
||||||
return signal.reason || new Error("Operation aborted");
|
return signal.reason || new Error("Operation aborted");
|
||||||
}
|
}
|
||||||
|
|
||||||
private throwIfAborted(signal?: AbortSignal) {
|
|
||||||
if (signal?.aborted) {
|
|
||||||
throw this.abortReason(signal);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private releaseIfAborted(releaseLock: () => void, signal?: AbortSignal) {
|
|
||||||
if (!signal?.aborted) return;
|
|
||||||
releaseLock();
|
|
||||||
throw this.abortReason(signal);
|
|
||||||
}
|
|
||||||
|
|
||||||
private waitForPoll(signal?: AbortSignal) {
|
|
||||||
this.throwIfAborted(signal);
|
|
||||||
|
|
||||||
return new Promise<void>((resolve, reject) => {
|
|
||||||
let settled = false;
|
|
||||||
const timeout = setTimeout(() => settle(resolve), LOCK_POLL_MS);
|
|
||||||
|
|
||||||
const onAbort = () => {
|
|
||||||
settle(() => reject(this.abortReason(signal!)));
|
|
||||||
};
|
|
||||||
|
|
||||||
const cleanup = () => {
|
|
||||||
clearTimeout(timeout);
|
|
||||||
signal?.removeEventListener("abort", onAbort);
|
|
||||||
};
|
|
||||||
|
|
||||||
const settle = (callback: () => void) => {
|
|
||||||
if (settled) return;
|
|
||||||
|
|
||||||
settled = true;
|
|
||||||
cleanup();
|
|
||||||
callback();
|
|
||||||
};
|
|
||||||
|
|
||||||
signal?.addEventListener("abort", onAbort, { once: true });
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private cleanupExpired(tx: RepositoryMutexTransaction, now: number) {
|
private cleanupExpired(tx: RepositoryMutexTransaction, now: number) {
|
||||||
tx.delete(repositoryLocksTable).where(lte(repositoryLocksTable.expiresAt, now)).run();
|
tx.delete(repositoryLocksTable).where(lte(repositoryLocksTable.expiresAt, now)).run();
|
||||||
tx.delete(repositoryLockWaitersTable).where(lte(repositoryLockWaitersTable.expiresAt, now)).run();
|
tx.delete(repositoryLockWaitersTable).where(lte(repositoryLockWaitersTable.expiresAt, now)).run();
|
||||||
|
|
@ -181,43 +153,49 @@ class RepositoryMutex {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private tryAcquireImmediately(request: LockRequest, signal?: AbortSignal) {
|
private tryAcquireImmediately(request: LockRequest) {
|
||||||
const locks = this.tryAcquireManyRows([request]);
|
return Effect.gen(this, function* () {
|
||||||
if (!locks || locks.length === 0) return null;
|
const locks = this.tryAcquireManyRows([request]);
|
||||||
|
if (!locks || locks.length === 0) return null;
|
||||||
|
|
||||||
const [lock] = locks;
|
const [lock] = locks;
|
||||||
const releaseLock = this.createRelease(lock);
|
return yield* this.createRelease(lock);
|
||||||
this.releaseIfAborted(releaseLock, signal);
|
});
|
||||||
|
|
||||||
return releaseLock;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private createWaiter(request: LockRequest, waiterId: string) {
|
private createWaiter(request: LockRequest, waiterId: string) {
|
||||||
const now = Date.now();
|
return Effect.sync(() => {
|
||||||
|
const now = Date.now();
|
||||||
db.transaction((tx) => {
|
db.transaction((tx) => {
|
||||||
this.cleanupExpired(tx, now);
|
this.cleanupExpired(tx, now);
|
||||||
tx.insert(repositoryLockWaitersTable)
|
tx.insert(repositoryLockWaitersTable)
|
||||||
.values({
|
.values({
|
||||||
id: waiterId,
|
id: waiterId,
|
||||||
repositoryId: request.repositoryId,
|
repositoryId: request.repositoryId,
|
||||||
type: request.type,
|
type: request.type,
|
||||||
operation: request.operation,
|
operation: request.operation,
|
||||||
ownerId: this.ownerId,
|
ownerId: this.ownerId,
|
||||||
requestedAt: now,
|
requestedAt: now,
|
||||||
expiresAt: now + LOCK_LEASE_MS,
|
expiresAt: now + LOCK_LEASE_MS,
|
||||||
heartbeatAt: now,
|
heartbeatAt: now,
|
||||||
})
|
})
|
||||||
.run();
|
.run();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private deleteWaiter(waiterId: string) {
|
private deleteWaiter(waiterId: string) {
|
||||||
db.delete(repositoryLockWaitersTable)
|
return Effect.sync(() =>
|
||||||
.where(
|
db
|
||||||
and(eq(repositoryLockWaitersTable.id, waiterId), eq(repositoryLockWaitersTable.ownerId, this.ownerId)),
|
.delete(repositoryLockWaitersTable)
|
||||||
)
|
.where(
|
||||||
.run();
|
and(
|
||||||
|
eq(repositoryLockWaitersTable.id, waiterId),
|
||||||
|
eq(repositoryLockWaitersTable.ownerId, this.ownerId),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.run(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private deleteWaiterRow(tx: RepositoryMutexTransaction, waiterId: string): void {
|
private deleteWaiterRow(tx: RepositoryMutexTransaction, waiterId: string): void {
|
||||||
|
|
@ -293,97 +271,49 @@ class RepositoryMutex {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async waitForQueuedLock(request: LockRequest, signal?: AbortSignal) {
|
private waitForQueuedLock(request: LockRequest) {
|
||||||
this.throwIfAborted(signal);
|
|
||||||
|
|
||||||
const waiterId = this.generateLockId();
|
const waiterId = this.generateLockId();
|
||||||
this.createWaiter(request, waiterId);
|
|
||||||
this.startHeartbeat("waiter", waiterId);
|
|
||||||
|
|
||||||
try {
|
const attempt = Effect.sync(() => this.tryPromoteWaiter(waiterId)).pipe(
|
||||||
while (true) {
|
Effect.flatMap((attempt) => {
|
||||||
this.throwIfAborted(signal);
|
|
||||||
|
|
||||||
const attempt = this.tryPromoteWaiter(waiterId);
|
|
||||||
if (attempt.status === "acquired") {
|
if (attempt.status === "acquired") {
|
||||||
this.stopHeartbeat(waiterId);
|
return Effect.succeed(attempt.lock);
|
||||||
const releaseLock = this.createRelease(attempt.lock);
|
|
||||||
this.releaseIfAborted(releaseLock, signal);
|
|
||||||
|
|
||||||
return releaseLock;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (attempt.status === "missing") {
|
if (attempt.status === "missing") {
|
||||||
this.createWaiter(request, waiterId);
|
return Effect.gen(this, function* () {
|
||||||
this.startHeartbeat("waiter", waiterId);
|
yield* this.createWaiter(request, waiterId);
|
||||||
|
yield* this.startHeartbeat("waiter", waiterId);
|
||||||
|
|
||||||
|
return yield* Effect.fail("retry");
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.waitForPoll(signal);
|
return Effect.fail("retry");
|
||||||
}
|
}),
|
||||||
} catch (error) {
|
);
|
||||||
this.stopHeartbeat(waiterId);
|
|
||||||
this.deleteWaiter(waiterId);
|
|
||||||
this.release({ id: waiterId });
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal) {
|
const cleanupAbandonedWaiter = Effect.gen(this, function* () {
|
||||||
this.throwIfAborted(signal);
|
yield* this.deleteWaiter(waiterId);
|
||||||
|
yield* this.release({ id: waiterId });
|
||||||
|
});
|
||||||
|
|
||||||
const request: LockRequest = { repositoryId, type: "shared", operation };
|
return Effect.scoped(
|
||||||
const releaseLock = this.tryAcquireImmediately(request, signal);
|
Effect.gen(this, function* () {
|
||||||
if (releaseLock) {
|
const lock = yield* attempt.pipe(
|
||||||
return releaseLock;
|
Effect.retry(Schedule.spaced(LOCK_POLL_MS)),
|
||||||
}
|
Effect.onExit((exit) => {
|
||||||
|
if (Exit.isSuccess(exit)) {
|
||||||
|
return Effect.void;
|
||||||
|
}
|
||||||
|
|
||||||
logger.debug(`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation}`);
|
return cleanupAbandonedWaiter;
|
||||||
return await this.waitForQueuedLock(request, signal);
|
}),
|
||||||
}
|
);
|
||||||
|
|
||||||
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal) {
|
return yield* this.createRelease(lock);
|
||||||
this.throwIfAborted(signal);
|
}),
|
||||||
|
);
|
||||||
const request: LockRequest = { repositoryId, type: "exclusive", operation };
|
|
||||||
const releaseLock = this.tryAcquireImmediately(request, signal);
|
|
||||||
if (releaseLock) {
|
|
||||||
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
|
|
||||||
return releaseLock;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation}`);
|
|
||||||
const queuedReleaseLock = await this.waitForQueuedLock(request, signal);
|
|
||||||
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
|
|
||||||
return queuedReleaseLock;
|
|
||||||
}
|
|
||||||
|
|
||||||
async acquireMany(requests: LockRequest[], signal?: AbortSignal) {
|
|
||||||
this.throwIfAborted(signal);
|
|
||||||
|
|
||||||
if (requests.length === 0) {
|
|
||||||
return () => {};
|
|
||||||
}
|
|
||||||
|
|
||||||
const seenRepositoryIds = new Set<string>();
|
|
||||||
for (const request of requests) {
|
|
||||||
if (seenRepositoryIds.has(request.repositoryId)) {
|
|
||||||
throw new Error(`Duplicate repository lock request: ${request.repositoryId}`);
|
|
||||||
}
|
|
||||||
seenRepositoryIds.add(request.repositoryId);
|
|
||||||
}
|
|
||||||
|
|
||||||
const sortedRequests = [...requests].sort((a, b) => a.repositoryId.localeCompare(b.repositoryId));
|
|
||||||
while (true) {
|
|
||||||
const locks = this.tryAcquireManyRows(sortedRequests);
|
|
||||||
if (locks) {
|
|
||||||
const releaseLocks = this.createReleaseMany(locks);
|
|
||||||
this.releaseIfAborted(releaseLocks, signal);
|
|
||||||
|
|
||||||
return releaseLocks;
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.waitForPoll(signal);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
isLocked(repositoryId: string) {
|
isLocked(repositoryId: string) {
|
||||||
|
|
@ -396,69 +326,91 @@ class RepositoryMutex {
|
||||||
}
|
}
|
||||||
|
|
||||||
private createReleaseMany(locks: AcquiredLock[]) {
|
private createReleaseMany(locks: AcquiredLock[]) {
|
||||||
const releases = locks.map((lock) => this.createRelease(lock));
|
return Effect.gen(this, function* () {
|
||||||
let released = false;
|
const releases = yield* Effect.all(locks.map((lock) => this.createRelease(lock)));
|
||||||
|
let released = false;
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
if (released) return;
|
if (released) return;
|
||||||
|
|
||||||
released = true;
|
released = true;
|
||||||
for (const release of releases.toReversed()) {
|
for (const release of releases.toReversed()) {
|
||||||
release();
|
release();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private createRelease(lock: AcquiredLock) {
|
private createRelease(lock: AcquiredLock) {
|
||||||
this.startHeartbeat("lock", lock.id);
|
return Effect.gen(this, function* () {
|
||||||
let released = false;
|
const heartbeatFiber = yield* this.startHeartbeat("lock", lock.id);
|
||||||
|
let released = false;
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
if (released) return;
|
if (released) return;
|
||||||
|
|
||||||
released = true;
|
released = true;
|
||||||
this.stopHeartbeat(lock.id);
|
Effect.runFork(Fiber.interrupt(heartbeatFiber));
|
||||||
this.release(lock);
|
Effect.runSync(this.release(lock));
|
||||||
};
|
};
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private release(lock: Pick<AcquiredLock, "id">) {
|
private release(lock: Pick<AcquiredLock, "id">) {
|
||||||
const releasedLock = db.transaction((tx) => {
|
return Effect.gen(this, function* () {
|
||||||
const row = tx.query.repositoryLocksTable
|
const releasedLock = yield* Effect.sync(() =>
|
||||||
.findFirst({ where: { AND: [{ id: { eq: lock.id } }, { ownerId: { eq: this.ownerId } }] } })
|
db.transaction((tx) => {
|
||||||
.sync();
|
const row = tx.query.repositoryLocksTable
|
||||||
|
.findFirst({ where: { AND: [{ id: { eq: lock.id } }, { ownerId: { eq: this.ownerId } }] } })
|
||||||
|
.sync();
|
||||||
|
|
||||||
if (!row) return null;
|
if (!row) return null;
|
||||||
|
|
||||||
tx.delete(repositoryLocksTable)
|
tx.delete(repositoryLocksTable)
|
||||||
.where(and(eq(repositoryLocksTable.id, lock.id), eq(repositoryLocksTable.ownerId, this.ownerId)))
|
.where(
|
||||||
.run();
|
and(eq(repositoryLocksTable.id, lock.id), eq(repositoryLocksTable.ownerId, this.ownerId)),
|
||||||
|
)
|
||||||
|
.run();
|
||||||
|
|
||||||
return row;
|
return row;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!releasedLock) return;
|
||||||
|
|
||||||
|
const duration = Date.now() - releasedLock.acquiredAt;
|
||||||
|
|
||||||
|
yield* logger.effect.debug(
|
||||||
|
`[Mutex] Released ${releasedLock.type} lock for repo ${releasedLock.repositoryId}: ${releasedLock.operation} (held for ${duration}ms)`,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!releasedLock) return;
|
|
||||||
|
|
||||||
const duration = Date.now() - releasedLock.acquiredAt;
|
|
||||||
logger.debug(
|
|
||||||
`[Mutex] Released ${releasedLock.type} lock for repo ${releasedLock.repositoryId}: ${releasedLock.operation} (held for ${duration}ms)`,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private startHeartbeat(target: HeartbeatTarget, lockId: string) {
|
private startHeartbeat(
|
||||||
this.stopHeartbeat(lockId);
|
target: "waiter",
|
||||||
|
lockId: string,
|
||||||
const heartbeat = () => {
|
): Effect.Effect<Fiber.RuntimeFiber<void, never>, never, Scope.Scope>;
|
||||||
|
private startHeartbeat(
|
||||||
|
target: "lock",
|
||||||
|
lockId: string,
|
||||||
|
): Effect.Effect<Fiber.RuntimeFiber<void, never>, never, never>;
|
||||||
|
private startHeartbeat(
|
||||||
|
target: HeartbeatTarget,
|
||||||
|
lockId: string,
|
||||||
|
): Effect.Effect<Fiber.RuntimeFiber<unknown, never>, never, Scope.Scope> {
|
||||||
|
const heartbeat = Effect.gen(this, function* () {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const values = { heartbeatAt: now, expiresAt: now + LOCK_LEASE_MS };
|
const values = { heartbeatAt: now, expiresAt: now + LOCK_LEASE_MS };
|
||||||
|
|
||||||
try {
|
if (target === "lock") {
|
||||||
if (target === "lock") {
|
yield* Effect.try(() => {
|
||||||
db.update(repositoryLocksTable)
|
db.update(repositoryLocksTable)
|
||||||
.set(values)
|
.set(values)
|
||||||
.where(and(eq(repositoryLocksTable.id, lockId), eq(repositoryLocksTable.ownerId, this.ownerId)))
|
.where(and(eq(repositoryLocksTable.id, lockId), eq(repositoryLocksTable.ownerId, this.ownerId)))
|
||||||
.run();
|
.run();
|
||||||
} else {
|
});
|
||||||
|
} else {
|
||||||
|
yield* Effect.try(() => {
|
||||||
db.update(repositoryLockWaitersTable)
|
db.update(repositoryLockWaitersTable)
|
||||||
.set(values)
|
.set(values)
|
||||||
.where(
|
.where(
|
||||||
|
|
@ -468,29 +420,142 @@ class RepositoryMutex {
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.run();
|
.run();
|
||||||
}
|
});
|
||||||
} catch (error) {
|
|
||||||
logger.warn(`[Mutex] Failed to heartbeat ${target} ${lockId}: ${String(error)}`);
|
|
||||||
}
|
}
|
||||||
};
|
}).pipe(
|
||||||
|
Effect.catchAll((error) =>
|
||||||
|
logger.effect.warn(`[Mutex] Failed to heartbeat ${target} ${lockId}: ${String(error)}`),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
const timer = setInterval(heartbeat, LOCK_HEARTBEAT_MS);
|
const repeat = heartbeat.pipe(Effect.repeat(Schedule.spaced(LOCK_HEARTBEAT_MS)));
|
||||||
if (timer && "unref" in timer) {
|
|
||||||
timer.unref();
|
if (target === "waiter") {
|
||||||
|
// For waiters, we can stop heartbeating when the releaser is dropped, so we use a scoped fiber
|
||||||
|
return repeat.pipe(Effect.forkScoped);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.heartbeatTimers.set(lockId, timer);
|
// For locks, the heartbeat must outlive the acquire scope.
|
||||||
|
// It is interrupted manually by the returned release function.
|
||||||
|
// TODO: max lifetime for lock heartbeats to prevent leaks if the releaser is never called?
|
||||||
|
return repeat.pipe(Effect.forkDaemon);
|
||||||
}
|
}
|
||||||
|
|
||||||
private stopHeartbeat(lockId: string) {
|
private acquireSharedEffect(repositoryId: string, operation: string) {
|
||||||
const timer = this.heartbeatTimers.get(lockId);
|
return Effect.gen(this, function* () {
|
||||||
if (!timer) {
|
const request: LockRequest = { repositoryId, type: "shared", operation };
|
||||||
return;
|
const releaseLock = yield* this.tryAcquireImmediately(request);
|
||||||
|
if (releaseLock) return releaseLock;
|
||||||
|
|
||||||
|
yield* logger.effect.debug(`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation}`);
|
||||||
|
return yield* this.waitForQueuedLock(request);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private acquireExclusiveEffect(repositoryId: string, operation: string) {
|
||||||
|
return Effect.gen(this, function* () {
|
||||||
|
const request: LockRequest = { repositoryId, type: "exclusive", operation };
|
||||||
|
const releaseLock = yield* this.tryAcquireImmediately(request);
|
||||||
|
if (releaseLock) {
|
||||||
|
yield* logger.effect.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
|
||||||
|
return releaseLock;
|
||||||
|
}
|
||||||
|
|
||||||
|
yield* logger.effect.debug(`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation}`);
|
||||||
|
const queuedReleaseLock = yield* this.waitForQueuedLock(request);
|
||||||
|
yield* logger.effect.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
|
||||||
|
return queuedReleaseLock;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private acquireManyEffect(requests: LockRequest[]) {
|
||||||
|
return Effect.gen(this, function* () {
|
||||||
|
if (requests.length === 0) {
|
||||||
|
return () => {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const seenRepositoryIds = new Set<string>();
|
||||||
|
for (const request of requests) {
|
||||||
|
if (seenRepositoryIds.has(request.repositoryId)) {
|
||||||
|
throw new Error(`Duplicate repository lock request: ${request.repositoryId}`);
|
||||||
|
}
|
||||||
|
seenRepositoryIds.add(request.repositoryId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sortedRequests = [...requests].sort((a, b) => a.repositoryId.localeCompare(b.repositoryId));
|
||||||
|
|
||||||
|
const locks = yield* Effect.sync(() => this.tryAcquireManyRows(sortedRequests)).pipe(
|
||||||
|
Effect.flatMap((locks) => {
|
||||||
|
if (locks) return Effect.succeed(locks);
|
||||||
|
return Effect.fail("retry");
|
||||||
|
}),
|
||||||
|
Effect.retry(Schedule.spaced(LOCK_POLL_MS)),
|
||||||
|
);
|
||||||
|
|
||||||
|
return yield* this.createReleaseMany(locks);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private runWithSignal<A, E>(effect: Effect.Effect<A, E>, signal?: AbortSignal) {
|
||||||
|
if (!signal) return Effect.runPromise(effect);
|
||||||
|
|
||||||
|
if (signal.aborted) {
|
||||||
|
return Promise.reject(this.abortReason(signal));
|
||||||
}
|
}
|
||||||
|
|
||||||
clearInterval(timer);
|
return new Promise<A>((resolve, reject) => {
|
||||||
this.heartbeatTimers.delete(lockId);
|
const fiber = Effect.runFork(effect);
|
||||||
|
let settled = false;
|
||||||
|
let aborting = false;
|
||||||
|
|
||||||
|
const complete = (callback: () => void) => {
|
||||||
|
if (settled) return;
|
||||||
|
|
||||||
|
settled = true;
|
||||||
|
signal.removeEventListener("abort", onAbort);
|
||||||
|
callback();
|
||||||
|
};
|
||||||
|
|
||||||
|
const onAbort = () => {
|
||||||
|
aborting = true;
|
||||||
|
Effect.runPromise(Fiber.interrupt(fiber)).then(
|
||||||
|
(exit) =>
|
||||||
|
complete(() => {
|
||||||
|
if (Exit.isSuccess(exit)) {
|
||||||
|
resolve(exit.value);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
reject(this.abortReason(signal));
|
||||||
|
}),
|
||||||
|
(error) => complete(() => reject(error)),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
|
||||||
|
Effect.runPromise(Fiber.join(fiber)).then(
|
||||||
|
(value) => complete(() => resolve(value)),
|
||||||
|
(error) => {
|
||||||
|
if (!aborting) {
|
||||||
|
complete(() => reject(error));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal) {
|
||||||
|
return await this.runWithSignal(this.acquireSharedEffect(repositoryId, operation), signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal) {
|
||||||
|
return await this.runWithSignal(this.acquireExclusiveEffect(repositoryId, operation), signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
async acquireMany(requests: LockRequest[], signal?: AbortSignal) {
|
||||||
|
return await this.runWithSignal(this.acquireManyEffect(requests), signal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const repoMutex = new RepositoryMutex();
|
export const repoMutex = getRepositoryMutex();
|
||||||
|
|
|
||||||
|
|
@ -208,6 +208,141 @@ describe("repositoriesService repository stats", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("repositoriesService.listSnapshotFiles", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("limits concurrent restic ls commands per repository", async () => {
|
||||||
|
const repository = await createTestRepository(session.organizationId);
|
||||||
|
let active = 0;
|
||||||
|
let maxActive = 0;
|
||||||
|
let releaseAll = false;
|
||||||
|
let exclusiveAcquired = false;
|
||||||
|
let releaseExclusive: (() => void) | undefined;
|
||||||
|
let exclusivePromise: Promise<() => void> | undefined;
|
||||||
|
const releaseWaiters: Array<() => void> = [];
|
||||||
|
const exclusiveController = new AbortController();
|
||||||
|
|
||||||
|
const releaseWaitingCommands = () => {
|
||||||
|
const waiters = releaseWaiters.splice(0);
|
||||||
|
for (const release of waiters) {
|
||||||
|
release();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const resolveWithin = async <T>(promise: Promise<T>, timeoutMs: number) => {
|
||||||
|
return await new Promise<T>((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
reject(new Error(`Expected promise to resolve within ${timeoutMs}ms`));
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
promise.then(
|
||||||
|
(value) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
resolve(value);
|
||||||
|
},
|
||||||
|
(error) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
reject(error);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const lsSpy = vi.spyOn(restic, "ls").mockImplementation((_config, snapshotId, _path, options) =>
|
||||||
|
Effect.promise(async () => {
|
||||||
|
active++;
|
||||||
|
maxActive = Math.max(maxActive, active);
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!releaseAll) {
|
||||||
|
await new Promise<void>((resolve) => releaseWaiters.push(resolve));
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
snapshot: {
|
||||||
|
id: snapshotId,
|
||||||
|
short_id: snapshotId,
|
||||||
|
time: new Date().toISOString(),
|
||||||
|
tree: "tree",
|
||||||
|
paths: ["/"],
|
||||||
|
hostname: "host",
|
||||||
|
struct_type: "snapshot" as const,
|
||||||
|
message_type: "snapshot" as const,
|
||||||
|
},
|
||||||
|
nodes: [],
|
||||||
|
pagination: {
|
||||||
|
offset: options.offset ?? 0,
|
||||||
|
limit: options.limit ?? 500,
|
||||||
|
total: 0,
|
||||||
|
hasMore: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
active--;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const calls = Array.from({ length: 4 }, (_, index) =>
|
||||||
|
withContext({ organizationId: session.organizationId, userId: session.user.id }, () =>
|
||||||
|
repositoriesService.listSnapshotFiles(repository.shortId, `snapshot-${index}`, "/", {
|
||||||
|
offset: 0,
|
||||||
|
limit: 100,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await waitForExpect(() => {
|
||||||
|
expect(releaseWaiters).toHaveLength(2);
|
||||||
|
});
|
||||||
|
expect(maxActive).toBe(2);
|
||||||
|
|
||||||
|
exclusivePromise = repoMutex
|
||||||
|
.acquireExclusive(repository.id, "delete", exclusiveController.signal)
|
||||||
|
.then((release) => {
|
||||||
|
exclusiveAcquired = true;
|
||||||
|
releaseExclusive = release;
|
||||||
|
return release;
|
||||||
|
});
|
||||||
|
|
||||||
|
releaseWaitingCommands();
|
||||||
|
|
||||||
|
releaseExclusive = await resolveWithin(exclusivePromise, 2000);
|
||||||
|
expect(exclusiveAcquired).toBe(true);
|
||||||
|
expect(active).toBe(0);
|
||||||
|
|
||||||
|
releaseExclusive();
|
||||||
|
releaseExclusive = undefined;
|
||||||
|
|
||||||
|
await waitForExpect(() => {
|
||||||
|
expect(releaseWaiters).toHaveLength(2);
|
||||||
|
});
|
||||||
|
expect(maxActive).toBe(2);
|
||||||
|
|
||||||
|
releaseWaitingCommands();
|
||||||
|
await Promise.all(calls);
|
||||||
|
} finally {
|
||||||
|
if (releaseExclusive) {
|
||||||
|
releaseExclusive();
|
||||||
|
} else {
|
||||||
|
exclusiveController.abort();
|
||||||
|
}
|
||||||
|
releaseAll = true;
|
||||||
|
releaseWaitingCommands();
|
||||||
|
await Promise.allSettled(calls);
|
||||||
|
if (exclusivePromise) {
|
||||||
|
await Promise.allSettled([exclusivePromise]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(lsSpy).toHaveBeenCalledTimes(4);
|
||||||
|
expect(maxActive).toBeLessThanOrEqual(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("repositoriesService.dumpSnapshot", () => {
|
describe("repositoriesService.dumpSnapshot", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vi.restoreAllMocks();
|
vi.restoreAllMocks();
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ import type { ParsedTask, TaskInput } from "../tasks/tasks.schemas";
|
||||||
import { Effect } from "effect";
|
import { Effect } from "effect";
|
||||||
|
|
||||||
const runningDoctors = new Map<string, AbortController>();
|
const runningDoctors = new Map<string, AbortController>();
|
||||||
|
const lsLimiters = new Map<string, Effect.Semaphore>();
|
||||||
const RESTORE_TASK_RESOURCE_TYPE = "repository";
|
const RESTORE_TASK_RESOURCE_TYPE = "repository";
|
||||||
|
|
||||||
type RestoreTaskInput = Extract<TaskInput, { kind: "restore" }>;
|
type RestoreTaskInput = Extract<TaskInput, { kind: "restore" }>;
|
||||||
|
|
@ -99,6 +100,15 @@ const updateActiveRestoreTask = (restoreId: string, eventName: string, update: (
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const getLsLimiter = (repositoryId: string) => {
|
||||||
|
let limiter = lsLimiters.get(repositoryId);
|
||||||
|
if (!limiter) {
|
||||||
|
limiter = Effect.runSync(Effect.makeSemaphore(2));
|
||||||
|
lsLimiters.set(repositoryId, limiter);
|
||||||
|
}
|
||||||
|
return limiter;
|
||||||
|
};
|
||||||
|
|
||||||
const findActiveRestoreTask = (
|
const findActiveRestoreTask = (
|
||||||
organizationId: string,
|
organizationId: string,
|
||||||
repositoryShortId: string,
|
repositoryShortId: string,
|
||||||
|
|
@ -469,36 +479,43 @@ const listSnapshotFiles = async (
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
|
const limiter = getLsLimiter(repository.id);
|
||||||
|
await runEffectPromise(limiter.take(1));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await runEffectPromise(
|
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
|
||||||
restic.ls(repository.config, snapshotId, path, { organizationId, offset, limit }),
|
try {
|
||||||
);
|
const result = await runEffectPromise(
|
||||||
|
restic.ls(repository.config, snapshotId, path, { organizationId, offset, limit }),
|
||||||
|
);
|
||||||
|
|
||||||
if (!result.snapshot) {
|
if (!result.snapshot) {
|
||||||
throw new NotFoundError("Snapshot not found or empty");
|
throw new NotFoundError("Snapshot not found or empty");
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = {
|
||||||
|
snapshot: {
|
||||||
|
id: result.snapshot.id,
|
||||||
|
short_id: result.snapshot.short_id,
|
||||||
|
time: result.snapshot.time,
|
||||||
|
hostname: result.snapshot.hostname,
|
||||||
|
paths: result.snapshot.paths,
|
||||||
|
},
|
||||||
|
files: result.nodes,
|
||||||
|
offset: result.pagination.offset,
|
||||||
|
limit: result.pagination.limit,
|
||||||
|
total: result.pagination.total,
|
||||||
|
hasMore: result.pagination.hasMore,
|
||||||
|
};
|
||||||
|
|
||||||
|
cache.set(cacheKey, result);
|
||||||
|
|
||||||
|
return response;
|
||||||
|
} finally {
|
||||||
|
releaseLock();
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = {
|
|
||||||
snapshot: {
|
|
||||||
id: result.snapshot.id,
|
|
||||||
short_id: result.snapshot.short_id,
|
|
||||||
time: result.snapshot.time,
|
|
||||||
hostname: result.snapshot.hostname,
|
|
||||||
paths: result.snapshot.paths,
|
|
||||||
},
|
|
||||||
files: result.nodes,
|
|
||||||
offset: result.pagination.offset,
|
|
||||||
limit: result.pagination.limit,
|
|
||||||
total: result.pagination.total,
|
|
||||||
hasMore: result.pagination.hasMore,
|
|
||||||
};
|
|
||||||
|
|
||||||
cache.set(cacheKey, result);
|
|
||||||
|
|
||||||
return response;
|
|
||||||
} finally {
|
} finally {
|
||||||
releaseLock();
|
await runEffectPromise(limiter.release(1));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -99,7 +99,7 @@ Mount the provisioning file and set `PROVISIONING_PATH`:
|
||||||
```yaml docker-compose.yml
|
```yaml docker-compose.yml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
environment:
|
environment:
|
||||||
- PROVISIONING_PATH=/config/provisioning.json
|
- PROVISIONING_PATH=/config/provisioning.json
|
||||||
volumes:
|
volumes:
|
||||||
|
|
|
||||||
|
|
@ -100,7 +100,7 @@ If you run Traefik as your reverse proxy, add labels to the Zerobyte service in
|
||||||
```yaml docker-compose.yml
|
```yaml docker-compose.yml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,7 @@ services:
|
||||||
- "4096:4096"
|
- "4096:4096"
|
||||||
|
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
# Uncomment if you need remote mounts (NFS/SMB/WebDAV):
|
# Uncomment if you need remote mounts (NFS/SMB/WebDAV):
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,7 @@ Create a `docker-compose.yml` file with the following configuration:
|
||||||
```yaml docker-compose.yml
|
```yaml docker-compose.yml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|
@ -182,7 +182,7 @@ If you only need to back up locally mounted directories and don't require remote
|
||||||
```yaml docker-compose.yml
|
```yaml docker-compose.yml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
|
|
@ -259,7 +259,7 @@ If you use provisioning, Zerobyte can resolve secrets from environment variables
|
||||||
```yaml docker-compose.yml
|
```yaml docker-compose.yml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|
|
||||||
|
|
@ -111,7 +111,7 @@ Update your `docker-compose.yml` to mount the directory you want to backup:
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
zerobyte:
|
zerobyte:
|
||||||
image: ghcr.io/nicotsx/zerobyte:v0.37
|
image: ghcr.io/nicotsx/zerobyte:v0.38
|
||||||
container_name: zerobyte
|
container_name: zerobyte
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
# ... other configuration ...
|
# ... other configuration ...
|
||||||
|
|
|
||||||
5
bun.lock
5
bun.lock
|
|
@ -55,6 +55,7 @@
|
||||||
"hono-rate-limiter": "^0.5.3",
|
"hono-rate-limiter": "^0.5.3",
|
||||||
"http-errors-enhanced": "^4.0.2",
|
"http-errors-enhanced": "^4.0.2",
|
||||||
"input-otp": "^1.4.2",
|
"input-otp": "^1.4.2",
|
||||||
|
"ipaddr.js": "^2.4.0",
|
||||||
"lucide-react": "^1.17.0",
|
"lucide-react": "^1.17.0",
|
||||||
"qrcode.react": "^4.2.0",
|
"qrcode.react": "^4.2.0",
|
||||||
"react": "^19.2.6",
|
"react": "^19.2.6",
|
||||||
|
|
@ -1683,7 +1684,7 @@
|
||||||
|
|
||||||
"ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="],
|
"ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="],
|
||||||
|
|
||||||
"ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
|
"ipaddr.js": ["ipaddr.js@2.4.0", "", {}, "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ=="],
|
||||||
|
|
||||||
"is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="],
|
"is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="],
|
||||||
|
|
||||||
|
|
@ -2665,6 +2666,8 @@
|
||||||
|
|
||||||
"prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="],
|
"prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="],
|
||||||
|
|
||||||
|
"proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
|
||||||
|
|
||||||
"recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
|
"recast/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
|
||||||
|
|
||||||
"recharts/es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="],
|
"recharts/es-toolkit": ["es-toolkit@1.46.1", "", {}, "sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ=="],
|
||||||
|
|
|
||||||
|
|
@ -87,6 +87,7 @@
|
||||||
"hono-rate-limiter": "^0.5.3",
|
"hono-rate-limiter": "^0.5.3",
|
||||||
"http-errors-enhanced": "^4.0.2",
|
"http-errors-enhanced": "^4.0.2",
|
||||||
"input-otp": "^1.4.2",
|
"input-otp": "^1.4.2",
|
||||||
|
"ipaddr.js": "^2.4.0",
|
||||||
"lucide-react": "^1.17.0",
|
"lucide-react": "^1.17.0",
|
||||||
"qrcode.react": "^4.2.0",
|
"qrcode.react": "^4.2.0",
|
||||||
"react": "^19.2.6",
|
"react": "^19.2.6",
|
||||||
|
|
|
||||||
|
|
@ -1,175 +0,0 @@
|
||||||
import { logger, safeExec } from "../node";
|
|
||||||
import { toMessage } from "../utils";
|
|
||||||
import { safeJsonParse } from "../utils/json";
|
|
||||||
import { ResticLockError } from "./error";
|
|
||||||
import { addCommonArgs } from "./helpers/add-common-args";
|
|
||||||
import { buildEnv } from "./helpers/build-env";
|
|
||||||
import { buildRepoUrl } from "./helpers/build-repo-url";
|
|
||||||
import { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
|
||||||
import type { RepositoryConfig } from "./schemas";
|
|
||||||
import type { ResticDeps } from "./types";
|
|
||||||
|
|
||||||
type ResticLockDiagnosticContext = {
|
|
||||||
error: unknown;
|
|
||||||
operation: string;
|
|
||||||
repositoryConfig: RepositoryConfig;
|
|
||||||
organizationId: string;
|
|
||||||
resticDeps: ResticDeps;
|
|
||||||
relatedRepositoryConfigs?: RepositoryConfig[];
|
|
||||||
};
|
|
||||||
|
|
||||||
const LOCK_ERROR_PATTERNS = [
|
|
||||||
/unable to create lock in backend/i,
|
|
||||||
/repository is already locked/i,
|
|
||||||
/failed to lock repository/i,
|
|
||||||
/"code"\s*:\s*11/i,
|
|
||||||
/\bexit_error\b.*\b11\b/i,
|
|
||||||
];
|
|
||||||
|
|
||||||
export const isResticLockFailure = (error: unknown) => {
|
|
||||||
if (error instanceof ResticLockError) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const message = toMessage(error);
|
|
||||||
return LOCK_ERROR_PATTERNS.some((pattern) => pattern.test(message));
|
|
||||||
};
|
|
||||||
|
|
||||||
const RESTIC_LOCK_ID_PATTERN = /^[a-f0-9]{64}$/i;
|
|
||||||
|
|
||||||
const addLockId = (ids: Set<string>, candidate: unknown) => {
|
|
||||||
if (typeof candidate === "string" && RESTIC_LOCK_ID_PATTERN.test(candidate)) {
|
|
||||||
ids.add(candidate);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const parseLockIds = (stdout: string) => {
|
|
||||||
const ids = new Set<string>();
|
|
||||||
|
|
||||||
for (const line of stdout.split("\n")) {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed) continue;
|
|
||||||
|
|
||||||
const parsed = safeJsonParse<{ id?: unknown }>(trimmed);
|
|
||||||
if (parsed) {
|
|
||||||
addLockId(ids, parsed.id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
addLockId(ids, trimmed);
|
|
||||||
}
|
|
||||||
|
|
||||||
return [...ids].slice(0, 20);
|
|
||||||
};
|
|
||||||
|
|
||||||
const inspectResticLocks = async (config: RepositoryConfig, organizationId: string, deps: ResticDeps) => {
|
|
||||||
const repoUrl = buildRepoUrl(config);
|
|
||||||
const env = await buildEnv(config, organizationId, deps);
|
|
||||||
const baseArgs = ["--repo", repoUrl];
|
|
||||||
addCommonArgs(baseArgs, env, config);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const listResult = await safeExec({
|
|
||||||
command: "restic",
|
|
||||||
args: [...baseArgs, "list", "locks"],
|
|
||||||
env,
|
|
||||||
timeout: 15_000,
|
|
||||||
});
|
|
||||||
|
|
||||||
const lockIds = listResult.exitCode === 0 ? parseLockIds(listResult.stdout) : [];
|
|
||||||
const lockDetails = [];
|
|
||||||
|
|
||||||
for (const lockId of lockIds) {
|
|
||||||
const catResult = await safeExec({
|
|
||||||
command: "restic",
|
|
||||||
args: [...baseArgs, "cat", "lock", "--", lockId],
|
|
||||||
env,
|
|
||||||
timeout: 15_000,
|
|
||||||
});
|
|
||||||
|
|
||||||
lockDetails.push({
|
|
||||||
lockId,
|
|
||||||
exitCode: catResult.exitCode,
|
|
||||||
stdout: catResult.stdout,
|
|
||||||
stderr: catResult.stderr,
|
|
||||||
timedOut: catResult.timedOut,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
repoUrl,
|
|
||||||
list: {
|
|
||||||
exitCode: listResult.exitCode,
|
|
||||||
stdout: listResult.stdout,
|
|
||||||
stderr: listResult.stderr,
|
|
||||||
timedOut: listResult.timedOut,
|
|
||||||
},
|
|
||||||
lockIds,
|
|
||||||
lockDetails,
|
|
||||||
};
|
|
||||||
} finally {
|
|
||||||
await cleanupTemporaryKeys(env, deps);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const logResticLockFailureDiagnostics = async ({
|
|
||||||
error,
|
|
||||||
operation,
|
|
||||||
repositoryConfig,
|
|
||||||
organizationId,
|
|
||||||
resticDeps,
|
|
||||||
relatedRepositoryConfigs = [],
|
|
||||||
}: ResticLockDiagnosticContext) => {
|
|
||||||
if (!isResticLockFailure(error)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const configsByRepoUrl = new Map(
|
|
||||||
[repositoryConfig, ...relatedRepositoryConfigs].map((config) => [buildRepoUrl(config), config]),
|
|
||||||
);
|
|
||||||
const configsToInspect = [...configsByRepoUrl.entries()];
|
|
||||||
|
|
||||||
logger.error("[ResticLockFailure] Restic repository lock failure detected", {
|
|
||||||
operation,
|
|
||||||
error: toMessage(error),
|
|
||||||
process: {
|
|
||||||
pid: process.pid,
|
|
||||||
hostname: process.env.HOSTNAME,
|
|
||||||
nodeEnv: process.env.NODE_ENV,
|
|
||||||
},
|
|
||||||
repository: {
|
|
||||||
repoUrl: buildRepoUrl(repositoryConfig),
|
|
||||||
},
|
|
||||||
relatedRepositories: relatedRepositoryConfigs.map((config) => ({
|
|
||||||
repoUrl: buildRepoUrl(config),
|
|
||||||
})),
|
|
||||||
});
|
|
||||||
|
|
||||||
for (const [repoUrl, config] of configsToInspect) {
|
|
||||||
try {
|
|
||||||
const resticLocks = await inspectResticLocks(config, organizationId, resticDeps);
|
|
||||||
logger.error("[ResticLockFailure] Restic backend lock dump", {
|
|
||||||
operation,
|
|
||||||
repoUrl,
|
|
||||||
resticLocks,
|
|
||||||
});
|
|
||||||
} catch (diagnosticError) {
|
|
||||||
logger.error("[ResticLockFailure] Failed to inspect restic backend locks", {
|
|
||||||
operation,
|
|
||||||
repoUrl,
|
|
||||||
error: toMessage(diagnosticError),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (diagnosticError) {
|
|
||||||
logger.error("[ResticLockFailure] Failed to collect lock diagnostics", {
|
|
||||||
operation,
|
|
||||||
repoUrl: buildRepoUrl(repositoryConfig),
|
|
||||||
error: toMessage(diagnosticError),
|
|
||||||
});
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
@ -16,7 +16,6 @@ import { stats } from "./commands/stats";
|
||||||
import { tagSnapshots } from "./commands/tag-snapshots";
|
import { tagSnapshots } from "./commands/tag-snapshots";
|
||||||
import { unlock } from "./commands/unlock";
|
import { unlock } from "./commands/unlock";
|
||||||
import { ResticLockError } from "./error";
|
import { ResticLockError } from "./error";
|
||||||
import { logResticLockFailureDiagnostics } from "./lock-diagnostics";
|
|
||||||
import type { RepositoryConfig } from "./schemas";
|
import type { RepositoryConfig } from "./schemas";
|
||||||
import type { ResticDeps } from "./types";
|
import type { ResticDeps } from "./types";
|
||||||
|
|
||||||
|
|
@ -25,17 +24,14 @@ export { buildEnv } from "./helpers/build-env";
|
||||||
export { buildRepoUrl } from "./helpers/build-repo-url";
|
export { buildRepoUrl } from "./helpers/build-repo-url";
|
||||||
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
export { cleanupTemporaryKeys } from "./helpers/cleanup-temporary-keys";
|
||||||
export { validateCustomResticParams } from "./helpers/validate-custom-params";
|
export { validateCustomResticParams } from "./helpers/validate-custom-params";
|
||||||
export { isResticLockFailure, logResticLockFailureDiagnostics } from "./lock-diagnostics";
|
|
||||||
export { isResticError, ResticError, ResticLockError } from "./error";
|
export { isResticError, ResticError, ResticLockError } from "./error";
|
||||||
|
|
||||||
type LockDiagnosticCommandContext = {
|
type LockRecoveryContext = {
|
||||||
repositoryConfig: RepositoryConfig;
|
repositoryConfigs: RepositoryConfig[];
|
||||||
organizationId: string;
|
organizationId: string;
|
||||||
relatedRepositoryConfigs?: RepositoryConfig[];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
type ResticCommandOptions = { organizationId: string };
|
type ResticCommandOptions = { organizationId: string };
|
||||||
type ResticCommandResult = { error?: unknown };
|
|
||||||
type ResticCommandFailure<Failure> = Failure | ResticLockError;
|
type ResticCommandFailure<Failure> = Failure | ResticLockError;
|
||||||
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
||||||
Success,
|
Success,
|
||||||
|
|
@ -43,56 +39,37 @@ type RunResticCommand<Success, Failure, Requirements> = () => Effect.Effect<
|
||||||
Requirements
|
Requirements
|
||||||
>;
|
>;
|
||||||
|
|
||||||
const getCommandContext = (operation: string, args: unknown[]): LockDiagnosticCommandContext => {
|
const getLockRecoveryContext = (operation: string, args: unknown[]): LockRecoveryContext => {
|
||||||
const firstRepositoryConfig = args[0] as RepositoryConfig;
|
const firstRepositoryConfig = args[0] as RepositoryConfig;
|
||||||
const options = args.at(-1) as ResticCommandOptions;
|
const options = args.at(-1) as ResticCommandOptions;
|
||||||
|
|
||||||
if (operation === "restic.copy") {
|
if (operation === "restic.copy") {
|
||||||
return {
|
return {
|
||||||
repositoryConfig: args[1] as RepositoryConfig,
|
repositoryConfigs: [args[1] as RepositoryConfig, firstRepositoryConfig],
|
||||||
organizationId: options.organizationId,
|
organizationId: options.organizationId,
|
||||||
relatedRepositoryConfigs: [firstRepositoryConfig],
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
repositoryConfig: firstRepositoryConfig,
|
repositoryConfigs: [firstRepositoryConfig],
|
||||||
organizationId: options.organizationId,
|
organizationId: options.organizationId,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const logLockFailure = async (
|
const unlockStaleLocks = (context: LockRecoveryContext, deps: ResticDeps) =>
|
||||||
error: unknown,
|
|
||||||
operation: string,
|
|
||||||
context: LockDiagnosticCommandContext,
|
|
||||||
deps: ResticDeps,
|
|
||||||
) =>
|
|
||||||
logResticLockFailureDiagnostics({
|
|
||||||
error,
|
|
||||||
operation,
|
|
||||||
repositoryConfig: context.repositoryConfig,
|
|
||||||
organizationId: context.organizationId,
|
|
||||||
resticDeps: deps,
|
|
||||||
relatedRepositoryConfigs: context.relatedRepositoryConfigs,
|
|
||||||
});
|
|
||||||
|
|
||||||
const unlockStaleLocks = (context: LockDiagnosticCommandContext, deps: ResticDeps): Effect.Effect<void, Error> =>
|
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
for (const repositoryConfig of [context.repositoryConfig, ...(context.relatedRepositoryConfigs ?? [])]) {
|
for (const repositoryConfig of context.repositoryConfigs) {
|
||||||
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
|
yield* unlock(repositoryConfig, { organizationId: context.organizationId }, deps);
|
||||||
}
|
}
|
||||||
});
|
}).pipe(Effect.catchAll(() => Effect.void));
|
||||||
|
|
||||||
const recoverFromLockFailure = <Success, Failure, Requirements>(
|
const recoverFromLockFailure = <Success, Failure, Requirements>(
|
||||||
error: ResticLockError,
|
context: LockRecoveryContext,
|
||||||
operation: string,
|
|
||||||
context: LockDiagnosticCommandContext,
|
|
||||||
deps: ResticDeps,
|
|
||||||
runCommand: RunResticCommand<Success, Failure, Requirements>,
|
runCommand: RunResticCommand<Success, Failure, Requirements>,
|
||||||
|
deps: ResticDeps,
|
||||||
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
|
): Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
|
yield* logger.effect.warn("Restic lock failure detected. Removing stale locks and retrying once.");
|
||||||
yield* Effect.promise(() => logLockFailure(error, operation, context, deps));
|
|
||||||
yield* unlockStaleLocks(context, deps);
|
yield* unlockStaleLocks(context, deps);
|
||||||
|
|
||||||
const retryResult = yield* runCommand();
|
const retryResult = yield* runCommand();
|
||||||
|
|
@ -107,20 +84,11 @@ function withDeps<Args extends unknown[], Success, Failure, Requirements>(
|
||||||
deps: ResticDeps,
|
deps: ResticDeps,
|
||||||
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
|
): (...args: Args) => Effect.Effect<Success, ResticCommandFailure<Failure> | Error, Requirements> {
|
||||||
return (...args: Args) => {
|
return (...args: Args) => {
|
||||||
const context = getCommandContext(operation, args);
|
const context = getLockRecoveryContext(operation, args);
|
||||||
const runCommand = () => command(...args, deps);
|
const runCommand = () => command(...args, deps);
|
||||||
|
|
||||||
const commandEffect = runCommand().pipe(
|
return runCommand().pipe(
|
||||||
Effect.catchTag("ResticLockError", (error) =>
|
Effect.catchTag("ResticLockError", () => recoverFromLockFailure(context, runCommand, deps)),
|
||||||
recoverFromLockFailure(error as ResticLockError, operation, context, deps, runCommand),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
return commandEffect.pipe(
|
|
||||||
Effect.tap((result) => {
|
|
||||||
const { error } = result as ResticCommandResult;
|
|
||||||
return error ? Effect.promise(() => logLockFailure(error, operation, context, deps)) : Effect.void;
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue