refactor(repo-mutex): use effect.ts (#950)

* refactor(repo-mutex): use effect.ts

* refactor(mutext): run with signal to preserve abort reason

.dockerignore

@@ -3,6 +3,7 @@
 !bun.lock
 !package.json
 !.gitignore
+!bunfig.toml
 
 !**/package.json
 !**/bun.lock
@@ -23,3 +24,4 @@
 node_modules/**
 dist/**
 .output/**
+app/test/integration/artifacts/**

.github/workflows/docs-deploy.yml

@@ -13,6 +13,7 @@ on:
 jobs:
   deploy:
     name: Deploy docs
+    if: github.event.repository.default_branch == github.ref_name
     runs-on: ubuntu-latest
     timeout-minutes: 10
 

.github/workflows/integration.yml

@@ -0,0 +1,35 @@
+name: Integration Tests
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  integration:
+    name: Integration
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+
+      - name: Install dependencies
+        uses: "./.github/actions/install-dependencies"
+
+      - name: Run integration tests
+        env:
+          SKIP_VOLUME_MOUNT_INTEGRATION_TESTS: "true"
+        run: bun run test:integration
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: failure()
+        with:
+          name: integration-artifacts
+          path: |
+            app/test/integration/artifacts/compose.log
+            app/test/integration/artifacts/docker-output.log
+            app/test/integration/artifacts/runs/**
+          retention-days: 5

.github/workflows/nightly.yml

@@ -20,20 +20,20 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
         with:
           driver: cloud
           endpoint: "meienberger/runtipi-builder"
           install: true
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -41,14 +41,14 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6
+        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6
         with:
           images: ghcr.io/${{ github.repository_owner }}/zerobyte
           tags: |
             type=raw,value=nightly
 
       - name: Push images to GitHub Container Registry
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
         with:
           context: .
           target: production

.github/workflows/release.yml

@@ -43,10 +43,13 @@ jobs:
   e2e-tests:
     uses: ./.github/workflows/e2e.yml
 
+  integration-tests:
+    uses: ./.github/workflows/integration.yml
+
   build-images:
     environment: release
     timeout-minutes: 15
-    needs: [determine-release-type, checks, e2e-tests]
+    needs: [determine-release-type, checks, e2e-tests, integration-tests]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -60,20 +63,20 @@ jobs:
           ref: ${{ github.ref }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
         with:
           driver: cloud
           endpoint: "meienberger/runtipi-builder"
           install: true
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -81,7 +84,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6
+        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6
         with:
           images: ghcr.io/${{ github.repository_owner }}/zerobyte
           tags: |
@@ -93,7 +96,7 @@ jobs:
             latest=${{ needs.determine-release-type.outputs.release_type == 'release' }}
 
       - name: Push images to GitHub Container Registry
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
         with:
           context: .
           target: production

Dockerfile

@@ -44,6 +44,15 @@ RUN bzip2 -d restic.bz2 && chmod +x restic
 RUN mv rclone-v*-linux-*/rclone /deps/rclone && chmod +x /deps/rclone
 RUN tar -xzf shoutrrr.tar.gz && chmod +x shoutrrr
 
+# ------------------------------
+# RUNTIME TOOLS
+# ------------------------------
+FROM base AS runtime-tools
+
+COPY --from=deps /deps/restic /usr/local/bin/restic
+COPY --from=deps /deps/rclone /usr/local/bin/rclone
+COPY --from=deps /deps/shoutrrr /usr/local/bin/shoutrrr
+
 # ------------------------------
 # DEVELOPMENT
 # ------------------------------

README.md

@@ -50,7 +50,7 @@ In order to run Zerobyte, you need to have Docker and Docker Compose installed o
 ```yaml
 services:
   zerobyte:
-    image: ghcr.io/nicotsx/zerobyte:v0.37
+    image: ghcr.io/nicotsx/zerobyte:v0.38
     container_name: zerobyte
     restart: unless-stopped
     cap_add:
@@ -156,7 +156,7 @@ If you only need to back up locally-mounted folders and don't require remote sha
 ```yaml
 services:
   zerobyte:
-    image: ghcr.io/nicotsx/zerobyte:v0.37
+    image: ghcr.io/nicotsx/zerobyte:v0.38
     container_name: zerobyte
     restart: unless-stopped
     ports:
@@ -195,7 +195,7 @@ If you want to backup a local directory on the same host where Zerobyte is runni
 ```diff
 services:
   zerobyte:
-    image: ghcr.io/nicotsx/zerobyte:v0.37
+    image: ghcr.io/nicotsx/zerobyte:v0.38
     container_name: zerobyte
     restart: unless-stopped
     cap_add:
@@ -270,7 +270,7 @@ Zerobyte can use [rclone](https://rclone.org/) to support 40+ cloud storage prov
    ```diff
    services:
      zerobyte:
-       image: ghcr.io/nicotsx/zerobyte:v0.37
+       image: ghcr.io/nicotsx/zerobyte:v0.38
        container_name: zerobyte
        restart: unless-stopped
        cap_add:
@@ -336,6 +336,12 @@ Zerobyte allows you to easily restore your data from backups. To restore data, n
 
 Zerobyte uses [better-auth](https://github.com/better-auth/better-auth) for authentication and session management. The authentication system automatically adapts to your deployment scenario:
 
+### Users and roles
+
+Zerobyte does not currently provide fine-grained RBAC for backup operations. Authenticated organization members are trusted operators for repositories, volumes, backup schedules, restores, and notification destinations in their organization.
+
+Organization roles mainly restrict organization-management and instance-management actions. For example, normal members cannot manage members, SSO settings, invitations, registration, or global users, but they can still operate backup resources. Only add users to an organization if you are comfortable with them using the storage, volume, and notification capabilities configured for the instance.
+
 ### Cookie security
 
 - **IP Address/HTTP access**: Set `BASE_URL=http://192.168.1.50:4096` (or your IP). Cookies will use `Secure: false`, allowing immediate login without SSL.

app/client/api-client/client/client.gen.ts

@@ -49,10 +49,7 @@ export const createClient = (config: Config = {}): Client => {
     };
 
     if (opts.security) {
-      await setAuthParams({
-        ...opts,
-        security: opts.security,
-      });
+      await setAuthParams(opts);
     }
 
     if (opts.requestValidator) {

app/client/api-client/client/utils.gen.ts

@@ -119,14 +119,12 @@ const checkForExistence = (
   return false;
 };
 
-export const setAuthParams = async ({
-  security,
-  ...options
-}: Pick<Required<RequestOptions>, 'security'> &
-  Pick<RequestOptions, 'auth' | 'query'> & {
+export async function setAuthParams(
+  options: Pick<RequestOptions, 'auth' | 'query' | 'security'> & {
     headers: Headers;
-  }) => {
-  for (const auth of security) {
+  },
+): Promise<void> {
+  for (const auth of options.security ?? []) {
     if (checkForExistence(options, auth.name)) {
       continue;
     }
@@ -155,7 +153,7 @@ export const setAuthParams = async ({
         break;
     }
   }
-};
+}
 
 export const buildUrl: Client['buildUrl'] = (options) =>
   getUrl({

app/client/api-client/types.gen.ts

@@ -2267,6 +2267,7 @@ export type RestoreSnapshotData = {
         excludeXattr?: Array<string>;
         delete?: boolean;
         targetPath?: string;
+        targetAgentId?: string;
         overwrite?: 'always' | 'if-changed' | 'if-newer' | 'never';
     };
     path: {
@@ -2278,13 +2279,11 @@ export type RestoreSnapshotData = {
 
 export type RestoreSnapshotResponses = {
     /**
-     * Snapshot restored successfully
+     * Snapshot restore started
      */
-    200: {
-        success: boolean;
-        message: string;
-        filesRestored: number;
-        filesSkipped: number;
+    202: {
+        restoreId: string;
+        status: 'started';
     };
 };
 

app/client/components/file-browsers/__tests__/snapshot-tree-browser.test.tsx

@@ -224,29 +224,6 @@ describe("SnapshotTreeBrowser", () => {
 		});
 	});
 
-	test("prefetches using the query path when display and query roots differ", async () => {
-		const requests = mockListSnapshotFiles();
-
-		renderSnapshotTreeBrowser();
-
-		const row = await screen.findByRole("button", { name: "project" });
-		const initialRequestCount = requests.length;
-
-		await userEvent.hover(row);
-
-		await waitFor(() => {
-			expect(requests.length).toBe(initialRequestCount + 1);
-		});
-
-		expect(requests.at(-1)).toEqual({
-			shortId: "repo-1",
-			snapshotId: "snap-1",
-			path: "/mnt/project",
-			offset: "0",
-			limit: "500",
-		});
-	});
-
 	test("shows the query root contents when display and query roots differ", async () => {
 		mockListSnapshotFiles();
 

app/client/components/file-browsers/snapshot-tree-browser.tsx

@@ -5,7 +5,6 @@ import { FileBrowser, type FileBrowserUiProps } from "~/client/components/file-b
 import { useFileBrowser } from "~/client/hooks/use-file-browser";
 import { parseError } from "~/client/lib/errors";
 import { isPathWithin, normalizeAbsolutePath } from "@zerobyte/core/utils";
-import { logger } from "~/client/lib/logger";
 
 function createPathPrefixFns(basePath: string) {
 	return {
@@ -84,16 +83,6 @@ export const SnapshotTreeBrowser = (props: SnapshotTreeBrowserProps) => {
 				}),
 			);
 		},
-		prefetchFolder: (displayPath) => {
-			void queryClient
-				.prefetchQuery(
-					listSnapshotFilesOptions({
-						path: { shortId: repositoryId, snapshotId },
-						query: { path: displayPath, offset: 0, limit: pageSize },
-					}),
-				)
-				.catch((e) => logger.error(e));
-		},
 		pathTransform: displayPathFns,
 	});
 

app/client/components/ui/input-otp.tsx

@@ -1,6 +1,5 @@
 import * as React from "react";
 import { OTPInput, OTPInputContext } from "input-otp";
-import { MinusIcon } from "lucide-react";
 
 import { cn } from "~/client/lib/utils";
 
@@ -55,12 +54,8 @@ function InputOTPSlot({
 	);
 }
 
-function InputOTPSeparator({ ...props }: React.ComponentProps<"div">) {
-	return (
-		<div data-slot="input-otp-separator" role="separator" {...props}>
-			<MinusIcon />
-		</div>
-	);
+function InputOTPSeparator({ ...props }: React.ComponentProps<"hr">) {
+	return <hr data-slot="input-otp-separator" {...props} />;
 }
 
 export { InputOTP, InputOTPGroup, InputOTPSlot, InputOTPSeparator };

app/client/functions/__tests__/is-secure-context.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, test } from "vitest";
+import { getIsSecureContextForRequest, isSecureContextOrigin } from "../is-secure-context";
+
+describe("isSecureContextOrigin", () => {
+	test("treats HTTPS and loopback HTTP origins as secure contexts", () => {
+		expect(isSecureContextOrigin("https://example.com")).toBe(true);
+		expect(isSecureContextOrigin("http://localhost:3000")).toBe(true);
+		expect(isSecureContextOrigin("http://app.localhost")).toBe(true);
+		expect(isSecureContextOrigin("http://127.0.0.1:3000")).toBe(true);
+		expect(isSecureContextOrigin("http://[::1]:3000")).toBe(true);
+	});
+
+	test("treats non-local HTTP origins as insecure contexts", () => {
+		expect(isSecureContextOrigin("http://example.com")).toBe(false);
+		expect(isSecureContextOrigin("http://127.example.com")).toBe(false);
+	});
+});
+
+describe("getIsSecureContextForRequest", () => {
+	test("uses forwarded protocol and host when present", () => {
+		const request = new Request("http://internal.local/settings", {
+			headers: {
+				host: "internal.local",
+				"x-forwarded-host": "zerobyte.example.com",
+				"x-forwarded-proto": "https",
+			},
+		});
+
+		expect(getIsSecureContextForRequest(request)).toBe(true);
+	});
+
+	test("uses the request URL when forwarded headers are absent", () => {
+		expect(getIsSecureContextForRequest(new Request("http://zerobyte.example.com/settings"))).toBe(false);
+		expect(getIsSecureContextForRequest(new Request("https://zerobyte.example.com/settings"))).toBe(true);
+	});
+});

app/client/functions/is-secure-context.ts

@@ -0,0 +1,51 @@
+import { createIsomorphicFn } from "@tanstack/react-start";
+import { getRequest } from "@tanstack/react-start/server";
+import ipaddr from "ipaddr.js";
+
+const getFirstHeaderValue = (value: string | null) => value?.split(",")[0]?.trim();
+
+const isLoopbackIp = (hostname: string) => {
+	try {
+		return ipaddr.parse(hostname).range() === "loopback";
+	} catch {
+		return false;
+	}
+};
+
+const getRequestOrigin = (request: Request) => {
+	const requestUrl = new URL(request.url);
+	const forwardedProto = getFirstHeaderValue(request.headers.get("x-forwarded-proto"));
+	const forwardedHost = getFirstHeaderValue(request.headers.get("x-forwarded-host"));
+	const host = forwardedHost || getFirstHeaderValue(request.headers.get("host"));
+
+	if (!host) {
+		return requestUrl.origin;
+	}
+
+	const protocol = (forwardedProto || requestUrl.protocol).replace(/:$/, "");
+	return `${protocol}://${host}`;
+};
+
+export const isSecureContextOrigin = (origin: string) => {
+	const url = new URL(origin);
+
+	if (url.protocol === "https:") {
+		return true;
+	}
+
+	const hostname = url.hostname.replace(/^\[(.*)\]$/, "$1").toLowerCase();
+
+	return hostname === "localhost" || hostname.endsWith(".localhost") || isLoopbackIp(hostname);
+};
+
+export const getIsSecureContextForRequest = (request: Request) => isSecureContextOrigin(getRequestOrigin(request));
+
+export const getIsSecureContext = createIsomorphicFn()
+	.server(() => {
+		try {
+			return getIsSecureContextForRequest(getRequest());
+		} catch {
+			return true;
+		}
+	})
+	.client(() => window.isSecureContext ?? true);

app/client/modules/auth/components/alternative-sign-in-section.tsx

@@ -0,0 +1,35 @@
+import { useSuspenseQuery } from "@tanstack/react-query";
+import { useServerFn } from "@tanstack/react-start";
+import { getPublicSsoProvidersOptions } from "~/client/api-client/@tanstack/react-query.gen";
+import { SsoLoginButtons } from "~/client/modules/sso/components/sso-login-buttons";
+import { getLoginOptions } from "~/server/lib/functions/login-options";
+import { PasskeySignInButton } from "./passkey-sign-in-button";
+
+type AlternativeSignInSectionProps = {
+	onPasskeySignIn: () => Promise<void>;
+};
+
+export function AlternativeSignInSection({ onPasskeySignIn }: AlternativeSignInSectionProps) {
+	const getOptions = useServerFn(getLoginOptions);
+	const { data: ssoProviders } = useSuspenseQuery({
+		...getPublicSsoProvidersOptions(),
+	});
+	const { data: loginOptions } = useSuspenseQuery({
+		queryKey: ["login-options"],
+		queryFn: getOptions,
+	});
+
+	if (ssoProviders.providers.length === 0 && !loginOptions.hasPasskeySignIn) {
+		return null;
+	}
+
+	return (
+		<div className="pt-4 border-t border-border/60 space-y-3">
+			<p className="text-sm font-medium">Alternative Sign-in</p>
+			<div className="flex flex-col gap-2">
+				{loginOptions.hasPasskeySignIn && <PasskeySignInButton onSignIn={onPasskeySignIn} />}
+				<SsoLoginButtons providers={ssoProviders.providers} />
+			</div>
+		</div>
+	);
+}

app/client/modules/auth/components/passkey-sign-in-button.tsx

@@ -0,0 +1,64 @@
+import { useMutation } from "@tanstack/react-query";
+import { useNavigate } from "@tanstack/react-router";
+import { Fingerprint } from "lucide-react";
+import { Button } from "~/client/components/ui/button";
+import { authClient } from "~/client/lib/auth-client";
+import { logger } from "~/client/lib/logger";
+import { LOGIN_ERROR_CODES, PASSKEY_LOGIN_FAILED_ERROR, type LoginErrorCode } from "~/lib/sso-errors";
+
+type PasskeySignInButtonProps = {
+	onSignIn: () => Promise<void>;
+};
+
+type PasskeySignInError = {
+	code?: string;
+	message?: string;
+	status?: number;
+	statusText?: string;
+};
+
+const LOGIN_ERROR_CODE_SET = new Set<string>(LOGIN_ERROR_CODES);
+
+function getPasskeyLoginErrorCode(code: string | undefined): LoginErrorCode {
+	if (code && LOGIN_ERROR_CODE_SET.has(code)) {
+		return code as LoginErrorCode;
+	}
+
+	return PASSKEY_LOGIN_FAILED_ERROR;
+}
+
+export function PasskeySignInButton({ onSignIn }: PasskeySignInButtonProps) {
+	const navigate = useNavigate();
+	const passkeyLoginMutation = useMutation<void, PasskeySignInError>({
+		mutationFn: async () => {
+			const { error } = await authClient.signIn.passkey();
+			if (error) throw error;
+
+			await onSignIn();
+		},
+		onError: (error) => {
+			logger.error(error);
+
+			void navigate({
+				to: "/login",
+				search: {
+					error: getPasskeyLoginErrorCode(error.code),
+				},
+			});
+		},
+	});
+
+	return (
+		<Button
+			type="button"
+			variant="outline"
+			className="w-full"
+			loading={passkeyLoginMutation.isPending}
+			disabled={passkeyLoginMutation.isPending}
+			onClick={() => passkeyLoginMutation.mutate()}
+		>
+			<Fingerprint className="h-4 w-4 mr-2" />
+			Sign in with passkey
+		</Button>
+	);
+}

app/client/modules/auth/routes/__tests__/login.test.tsx

@@ -1,16 +1,49 @@
 import { afterEach, describe, expect, test, vi } from "vitest";
 import { HttpResponse, http, server } from "~/test/msw/server";
-import { cleanup, render, screen } from "~/test/test-utils";
+import { cleanup, render, screen, userEvent, waitFor } from "~/test/test-utils";
+import { getLoginErrorDescription, PASSKEY_LOGIN_FAILED_ERROR } from "~/lib/sso-errors";
+
+const { mockGetLoginOptions, mockNavigate, mockPasskeySignIn } = vi.hoisted(() => ({
+	mockGetLoginOptions: vi.fn(async () => ({ hasPasskeySignIn: false })),
+	mockNavigate: vi.fn(async () => {}),
+	mockPasskeySignIn: vi.fn(
+		async (): Promise<{
+			data: unknown;
+			error: { code: string; message: string } | null;
+		}> => ({ data: null, error: null }),
+	),
+}));
 
 vi.mock("@tanstack/react-router", async (importOriginal) => {
 	const actual = await importOriginal<typeof import("@tanstack/react-router")>();
 
 	return {
 		...actual,
-		useNavigate: (() => vi.fn(async () => {})) as typeof actual.useNavigate,
+		useNavigate: (() => mockNavigate) as typeof actual.useNavigate,
 	};
 });
 
+vi.mock("@tanstack/react-start", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("@tanstack/react-start")>();
+
+	return {
+		...actual,
+		useServerFn: (fn: unknown) => fn,
+	};
+});
+
+vi.mock("~/server/lib/functions/login-options", () => ({
+	getLoginOptions: mockGetLoginOptions,
+}));
+
+vi.mock("~/client/lib/auth-client", () => ({
+	authClient: {
+		signIn: {
+			passkey: mockPasskeySignIn,
+		},
+	},
+}));
+
 import { LoginPage } from "../login";
 const inviteOnlyMessage =
 	"Access is invite-only. Ask an organization admin to send you an invitation before signing in with SSO.";
@@ -29,6 +62,12 @@ const mockSsoProvidersRequest = (
 };
 
 afterEach(() => {
+	mockGetLoginOptions.mockClear();
+	mockGetLoginOptions.mockResolvedValue({ hasPasskeySignIn: false });
+	mockNavigate.mockClear();
+	mockPasskeySignIn.mockClear();
+	mockPasskeySignIn.mockResolvedValue({ data: null, error: null });
+	vi.unstubAllGlobals();
 	cleanup();
 });
 
@@ -81,6 +120,14 @@ describe("LoginPage", () => {
 		expect(await screen.findByText("SSO authentication failed. Please try again.")).toBeTruthy();
 	});
 
+	test("shows passkey login failure message when passkey returns the login error code", async () => {
+		mockSsoProvidersRequest();
+
+		render(<LoginPage error={"PASSKEY_LOGIN_FAILED"} />, { withSuspense: true });
+
+		expect(await screen.findByText(getLoginErrorDescription(PASSKEY_LOGIN_FAILED_ERROR))).toBeTruthy();
+	});
+
 	test("does not show error message for invalid error codes", async () => {
 		mockSsoProvidersRequest();
 
@@ -90,11 +137,141 @@ describe("LoginPage", () => {
 		expect(screen.queryByText(inviteOnlyMessage)).toBeNull();
 	});
 
-	test("renders available SSO providers from the real SSO section", async () => {
+	test("renders available SSO providers from the alternative sign-in section", async () => {
 		mockSsoProvidersRequest([{ providerId: "acme", organizationSlug: "acme-org" }]);
 
 		render(<LoginPage />, { withSuspense: true });
 
 		expect(await screen.findByRole("button", { name: "Log in with acme" })).toBeTruthy();
 	});
+
+	test("renders passkey sign-in when an active user has a passkey", async () => {
+		mockSsoProvidersRequest();
+		mockGetLoginOptions.mockResolvedValue({ hasPasskeySignIn: true });
+
+		render(<LoginPage />, { withSuspense: true });
+
+		expect(await screen.findByRole("button", { name: "Sign in with passkey" })).toBeTruthy();
+	});
+
+	test("redirects passkey verification failures to the login error box", async () => {
+		mockSsoProvidersRequest();
+		mockGetLoginOptions.mockResolvedValue({ hasPasskeySignIn: true });
+		mockPasskeySignIn.mockResolvedValue({
+			data: null,
+			error: { code: "AUTHENTICATION_FAILED", message: "Authentication failed" },
+		});
+
+		render(<LoginPage />, { withSuspense: true });
+
+		await userEvent.click(await screen.findByRole("button", { name: "Sign in with passkey" }));
+
+		await waitFor(() => {
+			expect(mockNavigate).toHaveBeenCalledWith({
+				to: "/login",
+				search: {
+					error: PASSKEY_LOGIN_FAILED_ERROR,
+				},
+			});
+		});
+	});
+
+	test("redirects unauthorized passkey failures to the login error box", async () => {
+		mockSsoProvidersRequest();
+		mockGetLoginOptions.mockResolvedValue({ hasPasskeySignIn: true });
+		mockPasskeySignIn.mockResolvedValue({
+			data: null,
+			error: { code: "UNAUTHORIZED", message: "Unauthorized" },
+		});
+
+		render(<LoginPage />, { withSuspense: true });
+
+		await userEvent.click(await screen.findByRole("button", { name: "Sign in with passkey" }));
+
+		await waitFor(() => {
+			expect(mockNavigate).toHaveBeenCalledWith({
+				to: "/login",
+				search: {
+					error: PASSKEY_LOGIN_FAILED_ERROR,
+				},
+			});
+		});
+	});
+
+	test("preserves specific passkey login error codes", async () => {
+		mockSsoProvidersRequest();
+		mockGetLoginOptions.mockResolvedValue({ hasPasskeySignIn: true });
+		mockPasskeySignIn.mockResolvedValue({
+			data: null,
+			error: { code: "ERROR_INVALID_RP_ID", message: "Auth cancelled" },
+		});
+
+		render(<LoginPage />, { withSuspense: true });
+
+		await userEvent.click(await screen.findByRole("button", { name: "Sign in with passkey" }));
+
+		await waitFor(() => {
+			expect(mockNavigate).toHaveBeenCalledWith({
+				to: "/login",
+				search: {
+					error: "ERROR_INVALID_RP_ID",
+				},
+			});
+		});
+	});
+
+	test("redirects conditional passkey autofill failures to the login error box", async () => {
+		mockSsoProvidersRequest();
+		mockPasskeySignIn.mockResolvedValue({
+			data: null,
+			error: { code: "AUTHENTICATION_FAILED", message: "Authentication failed" },
+		});
+		vi.stubGlobal("PublicKeyCredential", {
+			isConditionalMediationAvailable: vi.fn(async () => true),
+		});
+
+		render(<LoginPage />, { withSuspense: true });
+
+		await waitFor(() => {
+			expect(mockPasskeySignIn).toHaveBeenCalledWith({
+				autoFill: true,
+			});
+			expect(mockNavigate).toHaveBeenCalledWith({
+				to: "/login",
+				search: {
+					error: "PASSKEY_LOGIN_FAILED",
+				},
+			});
+		});
+	});
+
+	test("ignores conditional passkey autofill cancellation", async () => {
+		mockSsoProvidersRequest();
+		mockPasskeySignIn.mockResolvedValue({
+			data: null,
+			error: { code: "AUTH_CANCELLED", message: "Authentication cancelled" },
+		});
+		vi.stubGlobal("PublicKeyCredential", {
+			isConditionalMediationAvailable: vi.fn(async () => true),
+		});
+
+		render(<LoginPage />, { withSuspense: true });
+
+		await waitFor(() => {
+			expect(mockPasskeySignIn).toHaveBeenCalledWith({
+				autoFill: true,
+			});
+		});
+		expect(mockNavigate).not.toHaveBeenCalled();
+	});
+
+	test("hides alternative sign-in when no SSO providers or passkeys are available", async () => {
+		mockSsoProvidersRequest();
+
+		render(<LoginPage />, { withSuspense: true });
+
+		await screen.findByText("Login to your account");
+		expect(screen.queryByText("Alternative Sign-in")).toBeNull();
+		expect(screen.queryByRole("button", { name: "Sign in with passkey" })).toBeNull();
+	});
 });

app/client/modules/auth/routes/login.tsx

@@ -1,5 +1,5 @@
 import { zodResolver } from "@hookform/resolvers/zod";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
 import { useForm } from "react-hook-form";
 import { toast } from "sonner";
 import { AuthLayout } from "~/client/components/auth-layout";
@@ -12,11 +12,12 @@ import { authClient } from "~/client/lib/auth-client";
 import { logger } from "~/client/lib/logger";
 import { RECOVERY_KEY_DOWNLOAD_SKIPPED_COOKIE_NAME } from "~/lib/recovery-key-skip";
 import { decodeLoginError, getLoginErrorDescription } from "~/client/lib/sso-errors";
+import { PASSKEY_LOGIN_FAILED_ERROR } from "~/lib/sso-errors";
 import { ResetPasswordDialog } from "../components/reset-password-dialog";
 import { useNavigate } from "@tanstack/react-router";
 import { normalizeUsername } from "~/lib/username";
 import { cn } from "~/client/lib/utils";
-import { SsoLoginSection } from "~/client/modules/sso/components/sso-login-section";
+import { AlternativeSignInSection } from "../components/alternative-sign-in-section";
 import { z } from "zod";
 
 const loginSchema = z.object({
@@ -33,6 +34,17 @@ type LoginPageProps = {
 	error?: string;
 };
 
+type PasskeySignInError = {
+	code?: string;
+	message?: string;
+	status?: number;
+	statusText?: string;
+};
+
+function isPasskeyVerificationFailure(error: PasskeySignInError | null) {
+	return error?.code === "AUTHENTICATION_FAILED" || error?.code === "UNAUTHORIZED";
+}
+
 function hasSkippedRecoveryKeyDownload(userId: string) {
 	return document.cookie
 		.split(";")
@@ -50,6 +62,20 @@ export function LoginPage({ error }: LoginPageProps = {}) {
 	const errorCode = decodeLoginError(error);
 	const errorDescription = errorCode ? getLoginErrorDescription(errorCode) : null;
 
+	const navigateAfterLogin = useCallback(async () => {
+		const session = await authClient.getSession();
+
+		if (
+			session.data?.user &&
+			!session.data.user.hasDownloadedResticPassword &&
+			!hasSkippedRecoveryKeyDownload(session.data.user.id)
+		) {
+			void navigate({ to: "/download-recovery-key" });
+		} else {
+			void navigate({ to: "/volumes" });
+		}
+	}, [navigate]);
+
 	useEffect(() => {
 		const autoSignIn = async () => {
 			if (
@@ -60,28 +86,27 @@ export function LoginPage({ error }: LoginPageProps = {}) {
 				return;
 			}
 
-			await authClient.signIn.passkey({
+			const { data, error } = await authClient.signIn.passkey({
 				autoFill: true,
-				fetchOptions: {
-					onResponse: async () => {
-						const session = await authClient.getSession();
-
-						if (
-							session.data?.user &&
-							!session.data.user.hasDownloadedResticPassword &&
-							!hasSkippedRecoveryKeyDownload(session.data.user.id)
-						) {
-							void navigate({ to: "/download-recovery-key" });
-						} else {
-							void navigate({ to: "/volumes" });
-						}
-					},
-				},
 			});
+
+			if (isPasskeyVerificationFailure(error)) {
+				void navigate({
+					to: "/login",
+					search: {
+						error: PASSKEY_LOGIN_FAILED_ERROR,
+					},
+				});
+				return;
+			}
+
+			if (data) {
+				await navigateAfterLogin();
+			}
 		};
 
 		void autoSignIn();
-	}, [navigate]);
+	}, [navigate, navigateAfterLogin]);
 
 	const form = useForm<LoginFormValues>({
 		resolver: zodResolver(loginSchema),
@@ -116,12 +141,7 @@ export function LoginPage({ error }: LoginPageProps = {}) {
 			return;
 		}
 
-		const d = await authClient.getSession();
-		if (data.user && !d.data?.user.hasDownloadedResticPassword && !hasSkippedRecoveryKeyDownload(data.user.id)) {
-			void navigate({ to: "/download-recovery-key" });
-		} else {
-			void navigate({ to: "/volumes" });
-		}
+		await navigateAfterLogin();
 	};
 
 	const handleVerify2FA = async () => {
@@ -208,6 +228,7 @@ export function LoginPage({ error }: LoginPageProps = {}) {
 						<input
 							type="checkbox"
 							id="trust-device"
+							aria-label="Trust this device for 30 days"
 							checked={trustDevice}
 							onChange={(e) => setTrustDevice(e.target.checked)}
 							className="h-4 w-4"
@@ -305,7 +326,7 @@ export function LoginPage({ error }: LoginPageProps = {}) {
 				</form>
 			</Form>
 
-			<SsoLoginSection />
+			<AlternativeSignInSection onPasskeySignIn={navigateAfterLogin} />
 
 			<ResetPasswordDialog open={showResetDialog} onOpenChange={setShowResetDialog} />
 		</AuthLayout>

app/client/modules/backups/routes/backup-details.tsx

@@ -63,9 +63,7 @@ export function ScheduleDetailsPage(props: Props) {
 	const queryClient = useQueryClient();
 	const navigate = useNavigate();
 	const searchParams = useSearch({ from: "/(dashboard)/backups/$backupId/" });
-	const [selectedSnapshotId, setSelectedSnapshotId] = useState<string | undefined>(
-		initialSnapshotId ?? loaderData.snapshots?.at(-1)?.short_id,
-	);
+	const [selectedSnapshotId, setSelectedSnapshotId] = useState<string | undefined>(initialSnapshotId);
 	const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
 	const [snapshotToDelete, setSnapshotToDelete] = useState<string | null>(null);
 

app/client/modules/settings/components/passkeys-section.tsx

@@ -24,6 +24,8 @@ import {
 } from "~/client/components/ui/dialog";
 import { Input } from "~/client/components/ui/input";
 import { Label } from "~/client/components/ui/label";
+import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip";
+import { getIsSecureContext } from "~/client/functions/is-secure-context";
 import { authClient } from "~/client/lib/auth-client";
 import { logger } from "~/client/lib/logger";
 import { useTimeFormat } from "~/client/lib/datetime";
@@ -38,6 +40,7 @@ type PasskeyEntry = {
 
 export function PasskeysSection() {
 	const { formatDateTime } = useTimeFormat();
+	const isSecureContext = getIsSecureContext();
 	const { data: passkeys, isPending } = useQuery({
 		queryKey: ["passkeys"],
 		queryFn: async () => {
@@ -111,6 +114,8 @@ export function PasskeysSection() {
 
 	const handleAddPasskey = (e: React.ChangeEvent) => {
 		e.preventDefault();
+		if (!isSecureContext) return;
+
 		const name = newPasskeyName.trim() || undefined;
 		addPasskeyMutation.mutate(name);
 	};
@@ -149,10 +154,19 @@ export function PasskeysSection() {
 						Passkeys use your device's biometrics or screen lock instead of a password. They are
 						phishing-resistant and cannot be reused across sites.
 					</p>
-					<Button onClick={() => setAddDialogOpen(true)}>
-						<Plus className="h-4 w-4 mr-2" />
-						Add passkey
-					</Button>
+					<Tooltip>
+						<TooltipTrigger asChild>
+							<span className="inline-flex">
+								<Button onClick={() => setAddDialogOpen(true)} disabled={!isSecureContext}>
+									<Plus className="h-4 w-4 mr-2" />
+									Add passkey
+								</Button>
+							</span>
+						</TooltipTrigger>
+						<TooltipContent className={cn({ hidden: isSecureContext })}>
+							<p>Passkeys can only be added over HTTPS or from localhost.</p>
+						</TooltipContent>
+					</Tooltip>
 				</div>
 
 				<p className={cn("text-sm text-muted-foreground", { hidden: !isPending })}>Loading passkeys...</p>
@@ -233,7 +247,7 @@ export function PasskeysSection() {
 							<Button type="button" variant="outline" onClick={() => setAddDialogOpen(false)}>
 								Cancel
 							</Button>
-							<Button type="submit" loading={addPasskeyMutation.isPending}>
+							<Button type="submit" loading={addPasskeyMutation.isPending} disabled={!isSecureContext}>
 								<Fingerprint className="h-4 w-4 mr-2" />
 								Add passkey
 							</Button>

app/client/modules/sso/components/sso-login-buttons.tsx

@@ -0,0 +1,54 @@
+import { useMutation } from "@tanstack/react-query";
+import { toast } from "sonner";
+import { Button } from "~/client/components/ui/button";
+import { authClient } from "~/client/lib/auth-client";
+import { logger } from "~/client/lib/logger";
+
+type SsoProvider = {
+	providerId: string;
+};
+
+type SsoLoginButtonsProps = {
+	providers: SsoProvider[];
+};
+
+export function SsoLoginButtons({ providers }: SsoLoginButtonsProps) {
+	const ssoLoginMutation = useMutation({
+		mutationFn: async (providerId: string) => {
+			const callbackPath = "/login";
+			const { data, error } = await authClient.signIn.sso({
+				providerId: providerId,
+				callbackURL: callbackPath,
+				errorCallbackURL: "/api/v1/auth/login-error",
+			});
+			if (error) throw error;
+
+			return data;
+		},
+		onSuccess: (data) => {
+			window.location.href = data.url;
+		},
+		onError: (error) => {
+			logger.error(error);
+			toast.error("SSO Login failed", { description: error.message });
+		},
+	});
+
+	return (
+		<>
+			{providers.map((provider) => (
+				<Button
+					key={provider.providerId}
+					type="button"
+					variant="outline"
+					className="w-full"
+					loading={ssoLoginMutation.isPending}
+					disabled={ssoLoginMutation.isPending}
+					onClick={() => ssoLoginMutation.mutate(provider.providerId)}
+				>
+					Log in with {provider.providerId}
+				</Button>
+			))}
+		</>
+	);
+}

app/client/modules/sso/components/sso-login-section.tsx

@@ -1,58 +0,0 @@
-import { useMutation, useSuspenseQuery } from "@tanstack/react-query";
-import { toast } from "sonner";
-import { Button } from "~/client/components/ui/button";
-import { authClient } from "~/client/lib/auth-client";
-import { logger } from "~/client/lib/logger";
-import { getPublicSsoProvidersOptions } from "~/client/api-client/@tanstack/react-query.gen";
-
-export function SsoLoginSection() {
-	const { data: ssoProviders } = useSuspenseQuery({
-		...getPublicSsoProvidersOptions(),
-	});
-
-	const ssoLoginMutation = useMutation({
-		mutationFn: async (providerId: string) => {
-			const callbackPath = "/login";
-			const { data, error } = await authClient.signIn.sso({
-				providerId: providerId,
-				callbackURL: callbackPath,
-				errorCallbackURL: "/api/v1/auth/login-error",
-			});
-			if (error) throw error;
-
-			return data;
-		},
-		onSuccess: (data) => {
-			window.location.href = data.url;
-		},
-		onError: (error) => {
-			logger.error(error);
-			toast.error("SSO Login failed", { description: error.message });
-		},
-	});
-
-	if (ssoProviders.providers.length === 0) {
-		return null;
-	}
-
-	return (
-		<div className="pt-4 border-t border-border/60 space-y-3">
-			<p className="text-sm font-medium">Alternative Sign-in</p>
-			<div className="flex flex-col gap-2">
-				{ssoProviders.providers.map((provider) => (
-					<Button
-						key={provider.providerId}
-						type="button"
-						variant="outline"
-						className="w-full"
-						loading={ssoLoginMutation.isPending}
-						disabled={ssoLoginMutation.isPending}
-						onClick={() => ssoLoginMutation.mutate(provider.providerId)}
-					>
-						Log in with {provider.providerId}
-					</Button>
-				))}
-			</div>
-		</div>
-	);
-}

app/client/modules/volumes/components/volume-forms/nfs-form.tsx

@@ -101,6 +101,7 @@ export const NFSForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Mount volume as read-only"
 									checked={field.value ?? false}
 									onChange={(e) => field.onChange(e.target.checked)}
 									className="rounded border-gray-300"

app/client/modules/volumes/components/volume-forms/rclone-form.tsx

@@ -108,6 +108,7 @@ export const RcloneForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Mount volume as read-only"
 									checked={field.value ?? false}
 									onChange={(e) => field.onChange(e.target.checked)}
 									className="rounded border-gray-300"

app/client/modules/volumes/components/volume-forms/smb-form.tsx

@@ -60,6 +60,7 @@ export const SMBForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Connect as guest"
 									checked={field.value ?? false}
 									onChange={(e) => {
 										field.onChange(e.target.checked);
@@ -179,6 +180,7 @@ export const SMBForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Map all files to container user/group"
 									checked={field.value ?? false}
 									onChange={(e) => field.onChange(e.target.checked)}
 									className="rounded border-gray-300"
@@ -187,8 +189,8 @@ export const SMBForm = ({ form }: Props) => {
 							</div>
 						</FormControl>
 						<FormDescription>
-							Keep the old behavior by forcing the SMB mount to present every file and directory as owned by the
-							container user and group instead of using server reported ownership.
+							Keep the old behavior by forcing the SMB mount to present every file and directory as owned
+							by the container user and group instead of using server reported ownership.
 						</FormDescription>
 						<FormMessage />
 					</FormItem>
@@ -205,6 +207,7 @@ export const SMBForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Mount volume as read-only"
 									checked={field.value ?? false}
 									onChange={(e) => field.onChange(e.target.checked)}
 									className="rounded border-gray-300"

app/client/modules/volumes/components/volume-forms/webdav-form.tsx

@@ -105,6 +105,7 @@ export const WebDAVForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Enable HTTPS for secure connections"
 									checked={field.value ?? false}
 									onChange={(e) => field.onChange(e.target.checked)}
 									className="rounded border-gray-300"
@@ -128,6 +129,7 @@ export const WebDAVForm = ({ form }: Props) => {
 							<div className="flex items-center space-x-2">
 								<input
 									type="checkbox"
+									aria-label="Mount volume as read-only"
 									checked={field.value ?? false}
 									onChange={(e) => field.onChange(e.target.checked)}
 									className="rounded border-gray-300"

app/drizzle/20260530131042_wakeful_santa_claus/migration.sql

@@ -0,0 +1,22 @@
+CREATE TABLE `tasks` (
+	`id` text PRIMARY KEY,
+	`organization_id` text NOT NULL,
+	`kind` text NOT NULL,
+	`status` text NOT NULL,
+	`resource_type` text NOT NULL,
+	`resource_id` text NOT NULL,
+	`target_agent_id` text,
+	`input` text NOT NULL,
+	`progress` text,
+	`result` text,
+	`error` text,
+	`cancellation_requested` integer DEFAULT false NOT NULL,
+	`created_at` integer DEFAULT (unixepoch() * 1000) NOT NULL,
+	`started_at` integer,
+	`updated_at` integer DEFAULT (unixepoch() * 1000) NOT NULL,
+	`finished_at` integer,
+	CONSTRAINT `fk_tasks_organization_id_organization_id_fk` FOREIGN KEY (`organization_id`) REFERENCES `organization`(`id`) ON DELETE CASCADE
+);
+--> statement-breakpoint
+CREATE INDEX `tasks_org_kind_resource_status_idx` ON `tasks` (`organization_id`,`kind`,`resource_type`,`resource_id`,`status`);--> statement-breakpoint
+CREATE INDEX `tasks_org_status_updated_at_idx` ON `tasks` (`organization_id`,`status`,`updated_at`);

app/lib/sso-errors.ts

@@ -1,9 +1,13 @@
+export const PASSKEY_LOGIN_FAILED_ERROR = "PASSKEY_LOGIN_FAILED";
+
 export const LOGIN_ERROR_CODES = [
 	"ACCOUNT_LINK_REQUIRED",
 	"EMAIL_NOT_VERIFIED",
 	"INVITE_REQUIRED",
 	"BANNED_USER",
 	"SSO_LOGIN_FAILED",
+	PASSKEY_LOGIN_FAILED_ERROR,
+	"ERROR_INVALID_RP_ID",
 ] as const;
 
 export type LoginErrorCode = (typeof LOGIN_ERROR_CODES)[number];
@@ -20,5 +24,9 @@ export function getLoginErrorDescription(errorCode: LoginErrorCode): string {
 			return "You have been banned from this application. Please contact support if you believe this is an error.";
 		case "SSO_LOGIN_FAILED":
 			return "SSO authentication failed. Please try again.";
+		case PASSKEY_LOGIN_FAILED_ERROR:
+			return "Passkey sign-in failed. The passkey didn't verify your identity with a PIN, biometrics, or screen lock. Please use a verified passkey or sign in with your password.";
+		case "ERROR_INVALID_RP_ID":
+			return "You can only sign in with a passkey on the domain set by the BASE_URL environment variable";
 	}
 }

app/schemas/events-dto.ts

@@ -2,7 +2,7 @@ import { z } from "zod";
 import { resticBackupProgressMetricsSchema, resticBackupRunSummarySchema } from "@zerobyte/core/restic";
 
 const backupEventStatusSchema = z.enum(["success", "error", "stopped", "warning"]);
-const restoreEventStatusSchema = z.enum(["success", "error"]);
+const restoreEventStatusSchema = z.enum(["success", "error", "cancelled"]);
 
 const backupEventBaseSchema = z.object({
 	scheduleId: z.string(),
@@ -15,6 +15,7 @@ const organizationScopedSchema = z.object({
 });
 
 const restoreEventBaseSchema = z.object({
+	restoreId: z.string(),
 	repositoryId: z.string(),
 	snapshotId: z.string(),
 });
@@ -51,6 +52,8 @@ const restoreProgressEventSchema = restoreEventBaseSchema.extend(restoreProgress
 const restoreCompletedEventSchema = restoreEventBaseSchema.extend({
 	status: restoreEventStatusSchema,
 	error: z.string().optional(),
+	filesRestored: z.number().optional(),
+	filesSkipped: z.number().optional(),
 });
 
 const serverBackupStartedEventSchema = organizationScopedSchema.extend(backupStartedEventSchema.shape);

app/server/core/__tests__/repository-mutex.test.ts

@@ -134,6 +134,37 @@ describe("RepositoryMutex", () => {
 		expect(repoMutex.isLocked(repoId)).toBe(false);
 	});
 
+	test("should reject aborted queued acquisitions only after waiter cleanup finishes", async () => {
+		const repoId = "abort-waits-for-cleanup";
+		const releaseHolder = await repoMutex.acquireExclusive(repoId, "holder");
+		const controller = new AbortController();
+		const waitingAcquisition = repoMutex.acquireShared(repoId, "waiter", controller.signal);
+
+		try {
+			await new Promise((resolve) => setTimeout(resolve, 50));
+
+			const waitersBeforeAbort = await db.query.repositoryLockWaitersTable.findMany({
+				where: { repositoryId: { eq: repoId } },
+			});
+
+			expect(waitersBeforeAbort.map((waiter) => waiter.operation)).toEqual(["waiter"]);
+
+			controller.abort(new Error("stop"));
+			await expect(waitingAcquisition).rejects.toThrow("stop");
+
+			const waitersAfterAbort = await db.query.repositoryLockWaitersTable.findMany({
+				where: { repositoryId: { eq: repoId } },
+			});
+
+			expect(waitersAfterAbort).toEqual([]);
+			expect(repoMutex.isLocked(repoId)).toBe(true);
+		} finally {
+			releaseHolder();
+			await db.delete(repositoryLockWaitersTable).where(eq(repositoryLockWaitersTable.repositoryId, repoId));
+			await db.delete(repositoryLocksTable).where(eq(repositoryLocksTable.repositoryId, repoId));
+		}
+	});
+
 	test("should allow concurrent shared locks", async () => {
 		const repoId = "concurrent-shared";
 		const release1 = await repoMutex.acquireShared(repoId, "op1");

app/server/core/config.ts

@@ -18,6 +18,8 @@ const envSchema = z
 		MIGRATIONS_PATH: z.string().optional(),
 		APP_VERSION: z.string().default("dev"),
 		TRUSTED_ORIGINS: z.string().optional(),
+		PORTLESS_URL: z.string().optional(),
+		PORTLESS_TAILSCALE_URL: z.string().optional(),
 		TRUST_PROXY: z.string().default("false"),
 		DISABLE_RATE_LIMITING: z.string().default("false"),
 		APP_SECRET: z.preprocess((value) => (value === "" ? undefined : value), z.string().min(32).max(256).optional()),
@@ -29,10 +31,24 @@ const envSchema = z
 		PROVISIONING_PATH: z.string().optional(),
 	})
 	.transform((s, ctx) => {
-		const baseUrl = unquote(s.BASE_URL);
-		const trustedOrigins = s.TRUSTED_ORIGINS?.split(",").map(unquote).filter(Boolean).concat(baseUrl) ?? [baseUrl];
+		let baseUrl = unquote(s.BASE_URL);
+		const trustedOrigins = s.TRUSTED_ORIGINS?.split(",").map(unquote).filter(Boolean) ?? [];
+
+		if (s.NODE_ENV === "development") {
+			if (s.PORTLESS_URL) {
+				trustedOrigins.push(unquote(s.PORTLESS_URL));
+			}
+
+			if (s.PORTLESS_TAILSCALE_URL) {
+				baseUrl = unquote(s.PORTLESS_TAILSCALE_URL);
+				trustedOrigins.push(baseUrl);
+			}
+		}
+
+		trustedOrigins.push(baseUrl);
+		const uniqueTrustedOrigins = Array.from(new Set(trustedOrigins));
 		const webhookAllowedOrigins = s.WEBHOOK_ALLOWED_ORIGINS?.split(",").map(unquote).filter(Boolean) ?? [];
-		const authOrigins = [baseUrl, ...trustedOrigins];
+		const authOrigins = [baseUrl, ...uniqueTrustedOrigins];
 		const { allowedHosts, invalidOrigins } = buildAllowedHosts(authOrigins);
 		let appSecret = s.APP_SECRET;
 
@@ -108,7 +124,7 @@ const envSchema = z
 			port: s.PORT,
 			migrationsPath: s.MIGRATIONS_PATH,
 			appVersion: s.APP_VERSION,
-			trustedOrigins: trustedOrigins,
+			trustedOrigins: uniqueTrustedOrigins,
 			trustProxy: s.TRUST_PROXY === "true",
 			appSecret: appSecret ?? "",
 			baseUrl,

app/server/core/repository-mutex.ts

@@ -7,6 +7,7 @@ import {
 	type RepositoryLock,
 	type RepositoryLockWaiter,
 } from "../db/schema";
+import { Effect, Exit, Fiber, Schedule, Scope } from "effect";
 
 type LockType = "shared" | "exclusive";
 
@@ -33,59 +34,30 @@ const LOCK_HEARTBEAT_MS = 5_000;
 const LOCK_POLL_MS = 250;
 const LOCK_POLL_CLEANUP_MS = 5_000;
 
+const REPOSITORY_MUTEX_INSTANCE = Symbol.for("zerobyte.repositoryMutex.instance");
+function getRepositoryMutex() {
+	const globalObject = globalThis as typeof globalThis & Record<symbol, RepositoryMutex | undefined>;
+	const mutex = globalObject[REPOSITORY_MUTEX_INSTANCE];
+
+	if (mutex) return mutex;
+
+	const newMutex = new RepositoryMutex();
+	globalObject[REPOSITORY_MUTEX_INSTANCE] = newMutex;
+	return newMutex;
+}
+
 class RepositoryMutex {
 	private ownerId = `owner_${Bun.randomUUIDv7()}`;
-	private heartbeatTimers = new Map<string, ReturnType<typeof setInterval>>();
 	private nextPollCleanupAt = 0;
 
 	private generateLockId(): string {
 		return `lock_${Bun.randomUUIDv7()}`;
 	}
 
-	private abortReason(signal: AbortSignal) {
+	private abortReason(signal: AbortSignal): Error {
 		return signal.reason || new Error("Operation aborted");
 	}
 
-	private throwIfAborted(signal?: AbortSignal) {
-		if (signal?.aborted) {
-			throw this.abortReason(signal);
-		}
-	}
-
-	private releaseIfAborted(releaseLock: () => void, signal?: AbortSignal) {
-		if (!signal?.aborted) return;
-		releaseLock();
-		throw this.abortReason(signal);
-	}
-
-	private waitForPoll(signal?: AbortSignal) {
-		this.throwIfAborted(signal);
-
-		return new Promise<void>((resolve, reject) => {
-			let settled = false;
-			const timeout = setTimeout(() => settle(resolve), LOCK_POLL_MS);
-
-			const onAbort = () => {
-				settle(() => reject(this.abortReason(signal!)));
-			};
-
-			const cleanup = () => {
-				clearTimeout(timeout);
-				signal?.removeEventListener("abort", onAbort);
-			};
-
-			const settle = (callback: () => void) => {
-				if (settled) return;
-
-				settled = true;
-				cleanup();
-				callback();
-			};
-
-			signal?.addEventListener("abort", onAbort, { once: true });
-		});
-	}
-
 	private cleanupExpired(tx: RepositoryMutexTransaction, now: number) {
 		tx.delete(repositoryLocksTable).where(lte(repositoryLocksTable.expiresAt, now)).run();
 		tx.delete(repositoryLockWaitersTable).where(lte(repositoryLockWaitersTable.expiresAt, now)).run();
@@ -181,43 +153,49 @@ class RepositoryMutex {
 		});
 	}
 
-	private tryAcquireImmediately(request: LockRequest, signal?: AbortSignal) {
-		const locks = this.tryAcquireManyRows([request]);
-		if (!locks || locks.length === 0) return null;
+	private tryAcquireImmediately(request: LockRequest) {
+		return Effect.gen(this, function* () {
+			const locks = this.tryAcquireManyRows([request]);
+			if (!locks || locks.length === 0) return null;
 
-		const [lock] = locks;
-		const releaseLock = this.createRelease(lock);
-		this.releaseIfAborted(releaseLock, signal);
-
-		return releaseLock;
+			const [lock] = locks;
+			return yield* this.createRelease(lock);
+		});
 	}
 
 	private createWaiter(request: LockRequest, waiterId: string) {
-		const now = Date.now();
-
-		db.transaction((tx) => {
-			this.cleanupExpired(tx, now);
-			tx.insert(repositoryLockWaitersTable)
-				.values({
-					id: waiterId,
-					repositoryId: request.repositoryId,
-					type: request.type,
-					operation: request.operation,
-					ownerId: this.ownerId,
-					requestedAt: now,
-					expiresAt: now + LOCK_LEASE_MS,
-					heartbeatAt: now,
-				})
-				.run();
+		return Effect.sync(() => {
+			const now = Date.now();
+			db.transaction((tx) => {
+				this.cleanupExpired(tx, now);
+				tx.insert(repositoryLockWaitersTable)
+					.values({
+						id: waiterId,
+						repositoryId: request.repositoryId,
+						type: request.type,
+						operation: request.operation,
+						ownerId: this.ownerId,
+						requestedAt: now,
+						expiresAt: now + LOCK_LEASE_MS,
+						heartbeatAt: now,
+					})
+					.run();
+			});
 		});
 	}
 
 	private deleteWaiter(waiterId: string) {
-		db.delete(repositoryLockWaitersTable)
-			.where(
-				and(eq(repositoryLockWaitersTable.id, waiterId), eq(repositoryLockWaitersTable.ownerId, this.ownerId)),
-			)
-			.run();
+		return Effect.sync(() =>
+			db
+				.delete(repositoryLockWaitersTable)
+				.where(
+					and(
+						eq(repositoryLockWaitersTable.id, waiterId),
+						eq(repositoryLockWaitersTable.ownerId, this.ownerId),
+					),
+				)
+				.run(),
+		);
 	}
 
 	private deleteWaiterRow(tx: RepositoryMutexTransaction, waiterId: string): void {
@@ -293,97 +271,49 @@ class RepositoryMutex {
 		});
 	}
 
-	private async waitForQueuedLock(request: LockRequest, signal?: AbortSignal) {
-		this.throwIfAborted(signal);
-
+	private waitForQueuedLock(request: LockRequest) {
 		const waiterId = this.generateLockId();
-		this.createWaiter(request, waiterId);
-		this.startHeartbeat("waiter", waiterId);
 
-		try {
-			while (true) {
-				this.throwIfAborted(signal);
-
-				const attempt = this.tryPromoteWaiter(waiterId);
+		const attempt = Effect.sync(() => this.tryPromoteWaiter(waiterId)).pipe(
+			Effect.flatMap((attempt) => {
 				if (attempt.status === "acquired") {
-					this.stopHeartbeat(waiterId);
-					const releaseLock = this.createRelease(attempt.lock);
-					this.releaseIfAborted(releaseLock, signal);
-
-					return releaseLock;
+					return Effect.succeed(attempt.lock);
 				}
 
 				if (attempt.status === "missing") {
-					this.createWaiter(request, waiterId);
-					this.startHeartbeat("waiter", waiterId);
+					return Effect.gen(this, function* () {
+						yield* this.createWaiter(request, waiterId);
+						yield* this.startHeartbeat("waiter", waiterId);
+
+						return yield* Effect.fail("retry");
+					});
 				}
 
-				await this.waitForPoll(signal);
-			}
-		} catch (error) {
-			this.stopHeartbeat(waiterId);
-			this.deleteWaiter(waiterId);
-			this.release({ id: waiterId });
-			throw error;
-		}
-	}
+				return Effect.fail("retry");
+			}),
+		);
 
-	async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal) {
-		this.throwIfAborted(signal);
+		const cleanupAbandonedWaiter = Effect.gen(this, function* () {
+			yield* this.deleteWaiter(waiterId);
+			yield* this.release({ id: waiterId });
+		});
 
-		const request: LockRequest = { repositoryId, type: "shared", operation };
-		const releaseLock = this.tryAcquireImmediately(request, signal);
-		if (releaseLock) {
-			return releaseLock;
-		}
+		return Effect.scoped(
+			Effect.gen(this, function* () {
+				const lock = yield* attempt.pipe(
+					Effect.retry(Schedule.spaced(LOCK_POLL_MS)),
+					Effect.onExit((exit) => {
+						if (Exit.isSuccess(exit)) {
+							return Effect.void;
+						}
 
-		logger.debug(`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation}`);
-		return await this.waitForQueuedLock(request, signal);
-	}
+						return cleanupAbandonedWaiter;
+					}),
+				);
 
-	async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal) {
-		this.throwIfAborted(signal);
-
-		const request: LockRequest = { repositoryId, type: "exclusive", operation };
-		const releaseLock = this.tryAcquireImmediately(request, signal);
-		if (releaseLock) {
-			logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
-			return releaseLock;
-		}
-
-		logger.debug(`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation}`);
-		const queuedReleaseLock = await this.waitForQueuedLock(request, signal);
-		logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
-		return queuedReleaseLock;
-	}
-
-	async acquireMany(requests: LockRequest[], signal?: AbortSignal) {
-		this.throwIfAborted(signal);
-
-		if (requests.length === 0) {
-			return () => {};
-		}
-
-		const seenRepositoryIds = new Set<string>();
-		for (const request of requests) {
-			if (seenRepositoryIds.has(request.repositoryId)) {
-				throw new Error(`Duplicate repository lock request: ${request.repositoryId}`);
-			}
-			seenRepositoryIds.add(request.repositoryId);
-		}
-
-		const sortedRequests = [...requests].sort((a, b) => a.repositoryId.localeCompare(b.repositoryId));
-		while (true) {
-			const locks = this.tryAcquireManyRows(sortedRequests);
-			if (locks) {
-				const releaseLocks = this.createReleaseMany(locks);
-				this.releaseIfAborted(releaseLocks, signal);
-
-				return releaseLocks;
-			}
-
-			await this.waitForPoll(signal);
-		}
+				return yield* this.createRelease(lock);
+			}),
+		);
 	}
 
 	isLocked(repositoryId: string) {
@@ -396,69 +326,91 @@ class RepositoryMutex {
 	}
 
 	private createReleaseMany(locks: AcquiredLock[]) {
-		const releases = locks.map((lock) => this.createRelease(lock));
-		let released = false;
+		return Effect.gen(this, function* () {
+			const releases = yield* Effect.all(locks.map((lock) => this.createRelease(lock)));
+			let released = false;
 
-		return () => {
-			if (released) return;
+			return () => {
+				if (released) return;
 
-			released = true;
-			for (const release of releases.toReversed()) {
-				release();
-			}
-		};
+				released = true;
+				for (const release of releases.toReversed()) {
+					release();
+				}
+			};
+		});
 	}
 
 	private createRelease(lock: AcquiredLock) {
-		this.startHeartbeat("lock", lock.id);
-		let released = false;
+		return Effect.gen(this, function* () {
+			const heartbeatFiber = yield* this.startHeartbeat("lock", lock.id);
+			let released = false;
 
-		return () => {
-			if (released) return;
+			return () => {
+				if (released) return;
 
-			released = true;
-			this.stopHeartbeat(lock.id);
-			this.release(lock);
-		};
+				released = true;
+				Effect.runFork(Fiber.interrupt(heartbeatFiber));
+				Effect.runSync(this.release(lock));
+			};
+		});
 	}
 
 	private release(lock: Pick<AcquiredLock, "id">) {
-		const releasedLock = db.transaction((tx) => {
-			const row = tx.query.repositoryLocksTable
-				.findFirst({ where: { AND: [{ id: { eq: lock.id } }, { ownerId: { eq: this.ownerId } }] } })
-				.sync();
+		return Effect.gen(this, function* () {
+			const releasedLock = yield* Effect.sync(() =>
+				db.transaction((tx) => {
+					const row = tx.query.repositoryLocksTable
+						.findFirst({ where: { AND: [{ id: { eq: lock.id } }, { ownerId: { eq: this.ownerId } }] } })
+						.sync();
 
-			if (!row) return null;
+					if (!row) return null;
 
-			tx.delete(repositoryLocksTable)
-				.where(and(eq(repositoryLocksTable.id, lock.id), eq(repositoryLocksTable.ownerId, this.ownerId)))
-				.run();
+					tx.delete(repositoryLocksTable)
+						.where(
+							and(eq(repositoryLocksTable.id, lock.id), eq(repositoryLocksTable.ownerId, this.ownerId)),
+						)
+						.run();
 
-			return row;
+					return row;
+				}),
+			);
+
+			if (!releasedLock) return;
+
+			const duration = Date.now() - releasedLock.acquiredAt;
+
+			yield* logger.effect.debug(
+				`[Mutex] Released ${releasedLock.type} lock for repo ${releasedLock.repositoryId}: ${releasedLock.operation} (held for ${duration}ms)`,
+			);
 		});
-
-		if (!releasedLock) return;
-
-		const duration = Date.now() - releasedLock.acquiredAt;
-		logger.debug(
-			`[Mutex] Released ${releasedLock.type} lock for repo ${releasedLock.repositoryId}: ${releasedLock.operation} (held for ${duration}ms)`,
-		);
 	}
 
-	private startHeartbeat(target: HeartbeatTarget, lockId: string) {
-		this.stopHeartbeat(lockId);
-
-		const heartbeat = () => {
+	private startHeartbeat(
+		target: "waiter",
+		lockId: string,
+	): Effect.Effect<Fiber.RuntimeFiber<void, never>, never, Scope.Scope>;
+	private startHeartbeat(
+		target: "lock",
+		lockId: string,
+	): Effect.Effect<Fiber.RuntimeFiber<void, never>, never, never>;
+	private startHeartbeat(
+		target: HeartbeatTarget,
+		lockId: string,
+	): Effect.Effect<Fiber.RuntimeFiber<unknown, never>, never, Scope.Scope> {
+		const heartbeat = Effect.gen(this, function* () {
 			const now = Date.now();
 			const values = { heartbeatAt: now, expiresAt: now + LOCK_LEASE_MS };
 
-			try {
-				if (target === "lock") {
+			if (target === "lock") {
+				yield* Effect.try(() => {
 					db.update(repositoryLocksTable)
 						.set(values)
 						.where(and(eq(repositoryLocksTable.id, lockId), eq(repositoryLocksTable.ownerId, this.ownerId)))
 						.run();
-				} else {
+				});
+			} else {
+				yield* Effect.try(() => {
 					db.update(repositoryLockWaitersTable)
 						.set(values)
 						.where(
@@ -468,29 +420,142 @@ class RepositoryMutex {
 							),
 						)
 						.run();
-				}
-			} catch (error) {
-				logger.warn(`[Mutex] Failed to heartbeat ${target} ${lockId}: ${String(error)}`);
+				});
 			}
-		};
+		}).pipe(
+			Effect.catchAll((error) =>
+				logger.effect.warn(`[Mutex] Failed to heartbeat ${target} ${lockId}: ${String(error)}`),
+			),
+		);
 
-		const timer = setInterval(heartbeat, LOCK_HEARTBEAT_MS);
-		if (timer && "unref" in timer) {
-			timer.unref();
+		const repeat = heartbeat.pipe(Effect.repeat(Schedule.spaced(LOCK_HEARTBEAT_MS)));
+
+		if (target === "waiter") {
+			// For waiters, we can stop heartbeating when the releaser is dropped, so we use a scoped fiber
+			return repeat.pipe(Effect.forkScoped);
 		}
 
-		this.heartbeatTimers.set(lockId, timer);
+		// For locks, the heartbeat must outlive the acquire scope.
+		// It is interrupted manually by the returned release function.
+		// TODO: max lifetime for lock heartbeats to prevent leaks if the releaser is never called?
+		return repeat.pipe(Effect.forkDaemon);
 	}
 
-	private stopHeartbeat(lockId: string) {
-		const timer = this.heartbeatTimers.get(lockId);
-		if (!timer) {
-			return;
+	private acquireSharedEffect(repositoryId: string, operation: string) {
+		return Effect.gen(this, function* () {
+			const request: LockRequest = { repositoryId, type: "shared", operation };
+			const releaseLock = yield* this.tryAcquireImmediately(request);
+			if (releaseLock) return releaseLock;
+
+			yield* logger.effect.debug(`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation}`);
+			return yield* this.waitForQueuedLock(request);
+		});
+	}
+
+	private acquireExclusiveEffect(repositoryId: string, operation: string) {
+		return Effect.gen(this, function* () {
+			const request: LockRequest = { repositoryId, type: "exclusive", operation };
+			const releaseLock = yield* this.tryAcquireImmediately(request);
+			if (releaseLock) {
+				yield* logger.effect.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
+				return releaseLock;
+			}
+
+			yield* logger.effect.debug(`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation}`);
+			const queuedReleaseLock = yield* this.waitForQueuedLock(request);
+			yield* logger.effect.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation}`);
+			return queuedReleaseLock;
+		});
+	}
+
+	private acquireManyEffect(requests: LockRequest[]) {
+		return Effect.gen(this, function* () {
+			if (requests.length === 0) {
+				return () => {};
+			}
+
+			const seenRepositoryIds = new Set<string>();
+			for (const request of requests) {
+				if (seenRepositoryIds.has(request.repositoryId)) {
+					throw new Error(`Duplicate repository lock request: ${request.repositoryId}`);
+				}
+				seenRepositoryIds.add(request.repositoryId);
+			}
+
+			const sortedRequests = [...requests].sort((a, b) => a.repositoryId.localeCompare(b.repositoryId));
+
+			const locks = yield* Effect.sync(() => this.tryAcquireManyRows(sortedRequests)).pipe(
+				Effect.flatMap((locks) => {
+					if (locks) return Effect.succeed(locks);
+					return Effect.fail("retry");
+				}),
+				Effect.retry(Schedule.spaced(LOCK_POLL_MS)),
+			);
+
+			return yield* this.createReleaseMany(locks);
+		});
+	}
+
+	private runWithSignal<A, E>(effect: Effect.Effect<A, E>, signal?: AbortSignal) {
+		if (!signal) return Effect.runPromise(effect);
+
+		if (signal.aborted) {
+			return Promise.reject(this.abortReason(signal));
 		}
 
-		clearInterval(timer);
-		this.heartbeatTimers.delete(lockId);
+		return new Promise<A>((resolve, reject) => {
+			const fiber = Effect.runFork(effect);
+			let settled = false;
+			let aborting = false;
+
+			const complete = (callback: () => void) => {
+				if (settled) return;
+
+				settled = true;
+				signal.removeEventListener("abort", onAbort);
+				callback();
+			};
+
+			const onAbort = () => {
+				aborting = true;
+				Effect.runPromise(Fiber.interrupt(fiber)).then(
+					(exit) =>
+						complete(() => {
+							if (Exit.isSuccess(exit)) {
+								resolve(exit.value);
+								return;
+							}
+
+							reject(this.abortReason(signal));
+						}),
+					(error) => complete(() => reject(error)),
+				);
+			};
+
+			signal.addEventListener("abort", onAbort, { once: true });
+
+			Effect.runPromise(Fiber.join(fiber)).then(
+				(value) => complete(() => resolve(value)),
+				(error) => {
+					if (!aborting) {
+						complete(() => reject(error));
+					}
+				},
+			);
+		});
+	}
+
+	async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal) {
+		return await this.runWithSignal(this.acquireSharedEffect(repositoryId, operation), signal);
+	}
+
+	async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal) {
+		return await this.runWithSignal(this.acquireExclusiveEffect(repositoryId, operation), signal);
+	}
+
+	async acquireMany(requests: LockRequest[], signal?: AbortSignal) {
+		return await this.runWithSignal(this.acquireManyEffect(requests), signal);
 	}
 }
 
-export const repoMutex = new RepositoryMutex();
+export const repoMutex = getRepositoryMutex();

app/server/db/schema.ts

@@ -352,6 +352,48 @@ export const repositoryLockWaitersTable = sqliteTable(
 );
 export type RepositoryLockWaiter = typeof repositoryLockWaitersTable.$inferSelect;
 
+type TaskJson = Record<string, unknown>;
+
+export const tasksTable = sqliteTable(
+	"tasks",
+	{
+		id: text("id").primaryKey(),
+		organizationId: text("organization_id")
+			.notNull()
+			.references(() => organization.id, { onDelete: "cascade" }),
+		kind: text("kind").notNull(),
+		status: text("status").notNull(),
+		resourceType: text("resource_type").notNull(),
+		resourceId: text("resource_id").notNull(),
+		targetAgentId: text("target_agent_id"),
+		input: text("input", { mode: "json" }).$type<TaskJson>().notNull(),
+		progress: text("progress", { mode: "json" }).$type<TaskJson | null>(),
+		result: text("result", { mode: "json" }).$type<TaskJson | null>(),
+		error: text("error"),
+		cancellationRequested: int("cancellation_requested", { mode: "boolean" }).notNull().default(false),
+		createdAt: int("created_at", { mode: "number" })
+			.notNull()
+			.default(sql`(unixepoch() * 1000)`),
+		startedAt: int("started_at", { mode: "number" }),
+		updatedAt: int("updated_at", { mode: "number" })
+			.notNull()
+			.default(sql`(unixepoch() * 1000)`),
+		finishedAt: int("finished_at", { mode: "number" }),
+	},
+	(table) => [
+		index("tasks_org_kind_resource_status_idx").on(
+			table.organizationId,
+			table.kind,
+			table.resourceType,
+			table.resourceId,
+			table.status,
+		),
+		index("tasks_org_status_updated_at_idx").on(table.organizationId, table.status, table.updatedAt),
+	],
+);
+export type Task = typeof tasksTable.$inferSelect;
+export type TaskInsert = typeof tasksTable.$inferInsert;
+
 /**
  * Backup Schedules Table
  */

app/server/lib/auth.ts

@@ -174,6 +174,22 @@ export const auth = betterAuth({
 		passkey({
 			rpID: new URL(config.baseUrl).hostname,
 			rpName: "Zerobyte",
+			authenticatorSelection: {
+				userVerification: "required",
+				residentKey: "required",
+			},
+			authentication: {
+				afterVerification: async ({ verification }) => {
+					if (verification.authenticationInfo.userVerified) {
+						return;
+					}
+
+					throw new APIError("UNAUTHORIZED", {
+						message:
+							"Your passkey was accepted, but it did not confirm your identity with a PIN, biometrics, or screen lock. Please use a verified passkey or sign in with your password.",
+					});
+				},
+			},
 		}),
 		tanstackStartCookies(),
 		...(process.env.NODE_ENV === "test" ? [testUtils()] : []),

app/server/lib/functions/login-options.ts

@@ -0,0 +1,6 @@
+import { createServerFn } from "@tanstack/react-start";
+import { hasActivePasskeyUser } from "~/server/modules/auth/helpers";
+
+export const getLoginOptions = createServerFn({ method: "GET" }).handler(async () => ({
+	hasPasskeySignIn: await hasActivePasskeyUser(),
+}));

app/server/modules/agents/__tests__/agents-manager.events.test.ts

@@ -1,7 +1,7 @@
 import { afterEach, expect, test, vi } from "vitest";
 import { Effect } from "effect";
 import { fromAny, fromPartial } from "@total-typescript/shoehorn";
-import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
+import type { BackupRunPayload, RestoreRunPayload } from "@zerobyte/contracts/agent-protocol";
 import type { AgentManagerEvent } from "../controller/server";
 import type { ProcessWithAgentRuntime } from "../agents-manager";
 
@@ -9,6 +9,8 @@ const controllerMock = vi.hoisted(() => ({
 	onEvent: null as null | ((event: AgentManagerEvent) => void),
 	sendBackup: vi.fn(),
 	cancelBackup: vi.fn(),
+	sendRestore: vi.fn(),
+	cancelRestore: vi.fn(),
 	stop: vi.fn(),
 }));
 
@@ -22,6 +24,8 @@ vi.mock("../controller/server", async () => {
 				stop: Effect.sync(controllerMock.stop),
 				sendBackup: controllerMock.sendBackup,
 				cancelBackup: controllerMock.cancelBackup,
+				sendRestore: controllerMock.sendRestore,
+				cancelRestore: controllerMock.cancelRestore,
 			};
 		}),
 	};
@@ -37,6 +41,7 @@ const resetAgentRuntime = () => {
 		localAgentRestartTimeout: null,
 		activeBackupsByScheduleId: new Map(),
 		activeBackupScheduleIdsByJobId: new Map(),
+		activeRestoresByRestoreId: new Map(),
 	};
 };
 
@@ -44,12 +49,17 @@ const backupPayload = fromPartial<BackupRunPayload>({
 	jobId: "job-1",
 	scheduleId: "schedule-1",
 });
+const restorePayload = fromPartial<RestoreRunPayload>({
+	restoreId: "restore-1",
+});
 
 afterEach(() => {
 	delete processWithAgentRuntime.__zerobyteAgentRuntime;
 	controllerMock.onEvent = null;
 	controllerMock.sendBackup.mockReset();
 	controllerMock.cancelBackup.mockReset();
+	controllerMock.sendRestore.mockReset();
+	controllerMock.cancelRestore.mockReset();
 	controllerMock.stop.mockReset();
 	vi.resetModules();
 	vi.restoreAllMocks();
@@ -73,7 +83,11 @@ test("backup progress is delivered to the running backup callback", async () =>
 		type: "backup.progress",
 		agentId: "local",
 		agentName: "Local Agent",
-		payload: fromAny({ jobId: "job-1", scheduleId: "schedule-1", progress: { percentDone: 0.5 } }),
+		payload: fromAny({
+			jobId: "job-1",
+			scheduleId: "schedule-1",
+			progress: { percentDone: 0.5 },
+		}),
 	});
 	controllerMock.onEvent?.({
 		type: "backup.completed",
@@ -92,6 +106,41 @@ test("backup progress is delivered to the running backup callback", async () =>
 	await stopAgentController();
 });
 
+test("backup events from agents that do not own the active run are ignored", async () => {
+	resetAgentRuntime();
+	controllerMock.sendBackup.mockImplementation(() => Effect.succeed(true));
+	const { agentManager, startAgentController, stopAgentController } = await import("../agents-manager");
+
+	await startAgentController();
+	const resultPromise = agentManager.runBackup("local", {
+		scheduleId: 42,
+		payload: backupPayload,
+		signal: new AbortController().signal,
+		onProgress: vi.fn(),
+	});
+
+	controllerMock.onEvent?.({
+		type: "backup.completed",
+		agentId: "remote",
+		agentName: "Remote Agent",
+		payload: { jobId: "job-1", scheduleId: "schedule-1", exitCode: 0, result: null },
+	});
+	controllerMock.onEvent?.({
+		type: "backup.completed",
+		agentId: "local",
+		agentName: "Local Agent",
+		payload: { jobId: "job-1", scheduleId: "schedule-1", exitCode: 0, result: null },
+	});
+
+	await expect(resultPromise).resolves.toEqual({
+		status: "completed",
+		exitCode: 0,
+		result: null,
+		warningDetails: null,
+	});
+	await stopAgentController();
+});
+
 test("backup failed and cancelled events resolve the matching running backup", async () => {
 	resetAgentRuntime();
 	controllerMock.sendBackup.mockImplementation(() => Effect.succeed(true));
@@ -108,7 +157,12 @@ test("backup failed and cancelled events resolve the matching running backup", a
 		type: "backup.failed",
 		agentId: "local",
 		agentName: "Local Agent",
-		payload: { jobId: "job-1", scheduleId: "schedule-1", error: "failed", errorDetails: "restic failed" },
+		payload: {
+			jobId: "job-1",
+			scheduleId: "schedule-1",
+			error: "failed",
+			errorDetails: "restic failed",
+		},
 	});
 	await expect(failedPromise).resolves.toEqual({ status: "failed", error: "restic failed" });
 
@@ -124,7 +178,10 @@ test("backup failed and cancelled events resolve the matching running backup", a
 		agentName: "Local Agent",
 		payload: { jobId: "job-2", scheduleId: "schedule-2", message: "cancelled remotely" },
 	});
-	await expect(cancelledPromise).resolves.toEqual({ status: "cancelled", message: "cancelled remotely" });
+	await expect(cancelledPromise).resolves.toEqual({
+		status: "cancelled",
+		message: "cancelled remotely",
+	});
 	await stopAgentController();
 });
 
@@ -147,7 +204,11 @@ test("agent disconnect cancels only backups owned by that agent", async () => {
 		onProgress: vi.fn(),
 	});
 
-	controllerMock.onEvent?.({ type: "agent.disconnected", agentId: "local", agentName: "Local Agent" });
+	controllerMock.onEvent?.({
+		type: "agent.disconnected",
+		agentId: "local",
+		agentName: "Local Agent",
+	});
 	controllerMock.onEvent?.({
 		type: "backup.completed",
 		agentId: "remote",
@@ -230,6 +291,117 @@ test("runBackup requests cancellation when the abort signal fires while sending"
 	});
 
 	expect(result).toEqual({ status: "cancelled" });
-	expect(controllerMock.cancelBackup).toHaveBeenCalledWith("local", { jobId: "job-1", scheduleId: "schedule-1" });
+	expect(controllerMock.cancelBackup).toHaveBeenCalledWith("local", {
+		jobId: "job-1",
+		scheduleId: "schedule-1",
+	});
+	await stopAgentController();
+});
+
+test("restore events are delivered to the running restore callbacks", async () => {
+	resetAgentRuntime();
+	controllerMock.sendRestore.mockImplementation(() => Effect.succeed(true));
+	const { agentManager, startAgentController, stopAgentController } = await import("../agents-manager");
+	const onStarted = vi.fn();
+	const onProgress = vi.fn();
+
+	await startAgentController();
+	const started = await agentManager.startRestore("local", {
+		payload: restorePayload,
+		signal: new AbortController().signal,
+		onStarted,
+		onProgress,
+	});
+	if (started.status !== "started") {
+		throw new Error("Expected restore to start");
+	}
+
+	controllerMock.onEvent?.({
+		type: "restore.started",
+		agentId: "local",
+		agentName: "Local Agent",
+		payload: {
+			restoreId: "restore-1",
+			organizationId: "org-1",
+			repositoryId: "repo-1",
+			snapshotId: "snapshot-1",
+		},
+	});
+	controllerMock.onEvent?.({
+		type: "restore.progress",
+		agentId: "local",
+		agentName: "Local Agent",
+		payload: fromAny({ restoreId: "restore-1", progress: { percent_done: 0.5 } }),
+	});
+	controllerMock.onEvent?.({
+		type: "restore.completed",
+		agentId: "local",
+		agentName: "Local Agent",
+		payload: {
+			restoreId: "restore-1",
+			organizationId: "org-1",
+			repositoryId: "repo-1",
+			snapshotId: "snapshot-1",
+			result: { message_type: "summary", files_restored: 2, files_skipped: 1 },
+		},
+	});
+
+	await expect(started.result).resolves.toEqual({
+		status: "completed",
+		result: { message_type: "summary", files_restored: 2, files_skipped: 1 },
+	});
+	expect(onStarted).toHaveBeenCalledOnce();
+	expect(onProgress).toHaveBeenCalledWith({ percent_done: 0.5 });
+	await stopAgentController();
+});
+
+test("agent disconnect cancels only restores owned by that agent", async () => {
+	resetAgentRuntime();
+	controllerMock.sendRestore.mockImplementation(() => Effect.succeed(true));
+	const { agentManager, startAgentController, stopAgentController } = await import("../agents-manager");
+
+	await startAgentController();
+	const localStarted = await agentManager.startRestore("local", {
+		payload: restorePayload,
+		signal: new AbortController().signal,
+		onStarted: vi.fn(),
+		onProgress: vi.fn(),
+	});
+	const remoteStarted = await agentManager.startRestore("remote", {
+		payload: fromPartial<RestoreRunPayload>({ restoreId: "restore-2" }),
+		signal: new AbortController().signal,
+		onStarted: vi.fn(),
+		onProgress: vi.fn(),
+	});
+	if (localStarted.status !== "started" || remoteStarted.status !== "started") {
+		throw new Error("Expected restores to start");
+	}
+
+	controllerMock.onEvent?.({
+		type: "agent.disconnected",
+		agentId: "local",
+		agentName: "Local Agent",
+	});
+	controllerMock.onEvent?.({
+		type: "restore.completed",
+		agentId: "remote",
+		agentName: "Remote Agent",
+		payload: {
+			restoreId: "restore-2",
+			organizationId: "org-1",
+			repositoryId: "repo-1",
+			snapshotId: "snapshot-1",
+			result: { message_type: "summary", files_restored: 1, files_skipped: 0 },
+		},
+	});
+
+	await expect(localStarted.result).resolves.toEqual({
+		status: "cancelled",
+		message: "The connection to the restore agent was lost. Restart the restore to ensure it completes.",
+	});
+	await expect(remoteStarted.result).resolves.toEqual({
+		status: "completed",
+		result: { message_type: "summary", files_restored: 1, files_skipped: 0 },
+	});
 	await stopAgentController();
 });

app/server/modules/agents/__tests__/controller-runtime.test.ts

@@ -207,6 +207,7 @@ test("websocket lifecycle updates agent connection status", async () => {
 	expect(agentsServiceMocks.markAgentOnline).toHaveBeenCalledWith(LOCAL_AGENT_ID, expect.any(Number), {
 		backup: true,
 		protocolVersion: 1,
+		protocolCompatible: true,
 		hostname: "host",
 		platform: "linux",
 	});
@@ -215,6 +216,73 @@ test("websocket lifecycle updates agent connection status", async () => {
 	expect(stop).toHaveBeenCalledWith(true);
 });
 
+test("websocket protocol rejection forwards the event and closes the connection", async () => {
+	const serve = vi
+		.spyOn(Bun, "serve")
+		.mockReturnValue(fromPartial({ port: 3001, stop: vi.fn(() => Promise.resolve()) }));
+	const { runtime, onEvent } = await startRuntime(vi.fn());
+	const websocket = serve.mock.calls[0]?.[0].websocket;
+	const socket = createSocket("connection-1");
+
+	await websocket?.open?.(fromPartial(socket));
+	await websocket?.message?.(
+		fromPartial(socket),
+		JSON.stringify({
+			type: "agent.ready",
+			payload: {
+				protocolVersion: 2,
+				hostname: "host",
+				platform: "linux",
+			},
+		}),
+	);
+	await Effect.runPromise(runtime.stop);
+
+	expect(onEvent).toHaveBeenCalledWith(
+		expect.objectContaining({
+			type: "agent.protocolRejected",
+			agentId: LOCAL_AGENT_ID,
+			agentName: LOCAL_AGENT_NAME,
+			payload: expect.objectContaining({ reason: "agent_too_new" }),
+		}),
+	);
+	expect(agentsServiceMocks.markAgentOffline).toHaveBeenCalledWith(LOCAL_AGENT_ID);
+	expect(socket.close).toHaveBeenCalledWith(1002, "agent_too_new");
+});
+
+test("websocket restore events are forwarded with agent metadata", async () => {
+	const serve = vi
+		.spyOn(Bun, "serve")
+		.mockReturnValue(fromPartial({ port: 3001, stop: vi.fn(() => Promise.resolve()) }));
+	const { runtime, onEvent } = await startRuntime(vi.fn());
+	const websocket = serve.mock.calls[0]?.[0].websocket;
+	const socket = createSocket("connection-1");
+
+	await websocket?.open?.(fromPartial(socket));
+	await websocket?.message?.(fromPartial(socket), createAgentMessage("agent.ready", readyPayload));
+	onEvent.mockClear();
+	await websocket?.message?.(
+		fromPartial(socket),
+		createAgentMessage("restore.completed", {
+			restoreId: "restore-1",
+			organizationId: "org-1",
+			repositoryId: "repo-1",
+			snapshotId: "snapshot-1",
+			result: { message_type: "summary", files_restored: 2, files_skipped: 0 },
+		}),
+	);
+	await Effect.runPromise(runtime.stop);
+
+	expect(onEvent).toHaveBeenCalledWith(
+		expect.objectContaining({
+			type: "restore.completed",
+			agentId: LOCAL_AGENT_ID,
+			agentName: LOCAL_AGENT_NAME,
+			payload: expect.objectContaining({ restoreId: "restore-1" }),
+		}),
+	);
+});
+
 test("websocket open failure closes the upgraded socket", async () => {
 	agentsServiceMocks.markAgentConnecting.mockRejectedValueOnce(new Error("db unavailable"));
 	const serve = vi

app/server/modules/agents/__tests__/session.test.ts

@@ -2,7 +2,11 @@ import { Effect, Exit, Fiber, Scope } from "effect";
 import { expect, test, vi } from "vitest";
 import waitForExpect from "wait-for-expect";
 import { fromPartial } from "@total-typescript/shoehorn";
-import { createAgentMessage, type AgentMessage } from "@zerobyte/contracts/agent-protocol";
+import {
+	createAgentMessage,
+	SUPPORTED_AGENT_PROTOCOL_MAX_VERSION,
+	type AgentMessage,
+} from "@zerobyte/contracts/agent-protocol";
 import type { Volume } from "@zerobyte/contracts/volumes";
 import { LOCAL_AGENT_ID, LOCAL_AGENT_KIND, LOCAL_AGENT_NAME } from "../constants";
 import { createControllerAgentSession } from "../controller/session";
@@ -133,6 +137,19 @@ test("invalid inbound messages are ignored", () => {
 	const onEvent = vi.fn(() => Effect.void);
 	const { session, close } = createSession(onEvent);
 
+	Effect.runSync(
+		session.handleMessage(
+			createAgentMessage("agent.ready", {
+				agentId: LOCAL_AGENT_ID,
+				protocolVersion: 1,
+				hostname: "host",
+				platform: "linux",
+				capabilities: { backup: true },
+			}),
+		),
+	);
+	onEvent.mockClear();
+
 	Effect.runSync(session.handleMessage("not json"));
 	Effect.runSync(session.handleMessage(JSON.stringify({ type: "backup.progress", payload: {} })));
 
@@ -193,6 +210,19 @@ test("backup agent messages are forwarded unchanged", () => {
 		},
 	} satisfies Extract<AgentMessage, { type: "backup.progress" }>;
 
+	Effect.runSync(
+		session.handleMessage(
+			createAgentMessage("agent.ready", {
+				agentId: LOCAL_AGENT_ID,
+				protocolVersion: 1,
+				hostname: "host",
+				platform: "linux",
+				capabilities: { backup: true },
+			}),
+		),
+	);
+	onEvent.mockClear();
+
 	Effect.runSync(session.handleMessage(createAgentMessage(message.type, message.payload)));
 
 	expect(onEvent).toHaveBeenCalledWith(
@@ -208,6 +238,53 @@ test("backup agent messages are forwarded unchanged", () => {
 	close();
 });
 
+test("unsupported agent protocol rejects startup and closes the session", () => {
+	const onEvent = vi.fn(() => Effect.void);
+	const { session, socket } = createSession(onEvent);
+
+	Effect.runSync(
+		session.handleMessage(
+			JSON.stringify({
+				type: "agent.ready",
+				payload: {
+					protocolVersion: SUPPORTED_AGENT_PROTOCOL_MAX_VERSION + 1,
+					hostname: "host",
+					platform: "linux",
+				},
+			}),
+		),
+	);
+
+	expect(Effect.runSync(session.isReady())).toBe(false);
+	expect(onEvent).toHaveBeenCalledWith({
+		type: "agent.protocolRejected",
+		payload: expect.objectContaining({
+			reason: "agent_too_new",
+			protocolVersion: SUPPORTED_AGENT_PROTOCOL_MAX_VERSION + 1,
+			hostname: "host",
+			platform: "linux",
+		}),
+	});
+	expect(socket.close).toHaveBeenCalledWith(1002, "agent_too_new");
+});
+
+test("pre-ready non-ready messages reject startup and close the session", () => {
+	const onEvent = vi.fn(() => Effect.void);
+	const { session, socket } = createSession(onEvent);
+
+	Effect.runSync(session.handleMessage(createAgentMessage("heartbeat.pong", { sentAt: 123 })));
+
+	expect(Effect.runSync(session.isReady())).toBe(false);
+	expect(onEvent).toHaveBeenCalledWith({
+		type: "agent.protocolRejected",
+		payload: expect.objectContaining({
+			reason: "unexpected_startup_message",
+			messageType: "heartbeat.pong",
+		}),
+	});
+	expect(socket.close).toHaveBeenCalledWith(1002, "unexpected_startup_message");
+});
+
 test("a dropped backup.cancel closes the session and reports a transport disconnect", async () => {
 	const send = vi.fn(() => 0);
 	const socket = createSocket({ send, close: vi.fn() });

app/server/modules/agents/agents-manager.ts

@@ -1,13 +1,12 @@
 import { logger } from "@zerobyte/core/node";
 import type {
 	BackupRunPayload,
-	RestoreCommandPayload,
+	RestoreRunPayload,
 	VolumeCommand,
 	VolumeCommandResult,
 } from "@zerobyte/contracts/agent-protocol";
 import { Effect } from "effect";
 import { config } from "../../core/config";
-import { serverEvents } from "../../core/events";
 import { createAgentManagerRuntime, type AgentManagerEvent } from "./controller/server";
 import { LOCAL_AGENT_ID } from "./constants";
 import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process";
@@ -16,9 +15,16 @@ import {
 	type AgentRuntimeState,
 	type BackupExecutionProgress,
 	type BackupExecutionResult,
+	type RestoreExecutionProgress,
+	type RestoreExecutionResult,
 } from "./helpers/runtime-state";
 import { getDevAgentRuntimeState } from "./helpers/runtime-state.dev";
-export type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
+export type {
+	BackupExecutionProgress,
+	BackupExecutionResult,
+	RestoreExecutionProgress,
+	RestoreExecutionResult,
+} from "./helpers/runtime-state";
 export type { ProcessWithAgentRuntime } from "./helpers/runtime-state.dev";
 
 type ProcessWithProductionAgentRuntime = NodeJS.Process & {
@@ -32,6 +38,17 @@ type AgentRunBackupRequest = {
 	onProgress: (progress: BackupExecutionProgress) => void;
 };
 
+type AgentStartRestoreRequest = {
+	payload: RestoreRunPayload;
+	signal: AbortSignal;
+	onStarted: () => void;
+	onProgress: (progress: RestoreExecutionProgress) => void;
+};
+
+type AgentRestoreStartResult =
+	| { status: "started"; result: Promise<RestoreExecutionResult> }
+	| { status: "unavailable"; error: Error };
+
 const getProductionAgentRuntimeState = () => {
 	// Nitro production builds can bundle startup plugins and API handlers into separate chunks.
 	// Keep the live controller on process so both chunks see the same agent sessions.
@@ -47,6 +64,7 @@ const getAgentRuntimeState = () => (config.__prod__ ? getProductionAgentRuntimeS
 const getAgentManagerRuntime = () => getAgentRuntimeState().agentManager;
 const getActiveBackupsByScheduleId = () => getAgentRuntimeState().activeBackupsByScheduleId;
 const getActiveBackupScheduleIdsByJobId = () => getAgentRuntimeState().activeBackupScheduleIdsByJobId;
+const getActiveRestoresByRestoreId = () => getAgentRuntimeState().activeRestoresByRestoreId;
 
 const clearActiveBackupRun = (scheduleId: number) => {
 	const activeBackupsByScheduleId = getActiveBackupsByScheduleId();
@@ -71,6 +89,27 @@ const resolveActiveBackupRun = (scheduleId: number, result: BackupExecutionResul
 	return true;
 };
 
+const clearActiveRestoreRun = (restoreId: string) => {
+	const activeRestoresByRestoreId = getActiveRestoresByRestoreId();
+	const activeRestoreRun = activeRestoresByRestoreId.get(restoreId);
+	if (!activeRestoreRun) {
+		return null;
+	}
+
+	activeRestoresByRestoreId.delete(restoreId);
+	return activeRestoreRun;
+};
+
+const resolveActiveRestoreRun = (restoreId: string, result: RestoreExecutionResult) => {
+	const activeRestoreRun = clearActiveRestoreRun(restoreId);
+	if (!activeRestoreRun) {
+		return false;
+	}
+
+	activeRestoreRun.resolve(result);
+	return true;
+};
+
 const cancelActiveBackupRunsForAgent = (agentId: string, message: string) => {
 	const activeBackupsByScheduleId = getActiveBackupsByScheduleId();
 	const matchingScheduleIds = [...activeBackupsByScheduleId.values()]
@@ -82,6 +121,17 @@ const cancelActiveBackupRunsForAgent = (agentId: string, message: string) => {
 	}
 };
 
+const cancelActiveRestoreRunsForAgent = (agentId: string, message: string) => {
+	const activeRestoresByRestoreId = getActiveRestoresByRestoreId();
+	const matchingRestoreIds = [...activeRestoresByRestoreId.values()]
+		.filter((activeRestoreRun) => activeRestoreRun.agentId === agentId)
+		.map((activeRestoreRun) => activeRestoreRun.restoreId);
+
+	for (const restoreId of matchingRestoreIds) {
+		resolveActiveRestoreRun(restoreId, { status: "cancelled", message });
+	}
+};
+
 const getActiveBackupRun = (jobId: string, scheduleId: string, eventName: string, agentId: string) => {
 	const trackedScheduleId = getActiveBackupScheduleIdsByJobId().get(jobId);
 	if (trackedScheduleId === undefined) {
@@ -102,9 +152,29 @@ const getActiveBackupRun = (jobId: string, scheduleId: string, eventName: string
 		return null;
 	}
 
+	if (activeBackupRun.agentId !== agentId) {
+		logger.warn(`Ignoring ${eventName} for job ${jobId} from unexpected agent ${agentId}`);
+		return null;
+	}
+
 	return activeBackupRun;
 };
 
+const getActiveRestoreRun = (restoreId: string, eventName: string, agentId: string) => {
+	const activeRestoreRun = getActiveRestoresByRestoreId().get(restoreId);
+	if (!activeRestoreRun) {
+		logger.warn(`Received ${eventName} for unknown restore ${restoreId} from agent ${agentId}`);
+		return null;
+	}
+
+	if (activeRestoreRun.agentId !== agentId) {
+		logger.warn(`Ignoring ${eventName} for restore ${restoreId} from unexpected agent ${agentId}`);
+		return null;
+	}
+
+	return activeRestoreRun;
+};
+
 const requestBackupCancellation = async (agentId: string, scheduleId: number) => {
 	const activeBackupRun = getActiveBackupsByScheduleId().get(scheduleId);
 	if (!activeBackupRun) {
@@ -138,6 +208,32 @@ const requestBackupCancellation = async (agentId: string, scheduleId: number) =>
 	return true;
 };
 
+const requestRestoreCancellation = async (agentId: string, restoreId: string) => {
+	const activeRestoreRun = getActiveRestoresByRestoreId().get(restoreId);
+	if (!activeRestoreRun) {
+		return false;
+	}
+
+	if (activeRestoreRun.cancellationRequested) {
+		return true;
+	}
+
+	activeRestoreRun.cancellationRequested = true;
+
+	const runtime = getAgentManagerRuntime();
+	if (!runtime) {
+		resolveActiveRestoreRun(restoreId, { status: "cancelled" });
+		return true;
+	}
+
+	if (await Effect.runPromise(runtime.cancelRestore(agentId, { restoreId }))) {
+		return true;
+	}
+
+	resolveActiveRestoreRun(restoreId, { status: "cancelled" });
+	return true;
+};
+
 const handleAgentManagerEvent = (event: AgentManagerEvent) => {
 	switch (event.type) {
 		case "agent.disconnected": {
@@ -145,6 +241,14 @@ const handleAgentManagerEvent = (event: AgentManagerEvent) => {
 				event.agentId,
 				"The connection to the backup agent was lost. Restart the backup to ensure it completes.",
 			);
+			cancelActiveRestoreRunsForAgent(
+				event.agentId,
+				"The connection to the restore agent was lost. Restart the restore to ensure it completes.",
+			);
+			break;
+		}
+		case "agent.protocolRejected": {
+			logger.warn(`Rejected agent protocol for ${event.agentName} (${event.agentId}): ${event.payload.reason}`);
 			break;
 		}
 		case "backup.started": {
@@ -218,12 +322,57 @@ const handleAgentManagerEvent = (event: AgentManagerEvent) => {
 			});
 			break;
 		}
+		case "restore.started": {
+			const activeRestoreRun = getActiveRestoreRun(event.payload.restoreId, event.type, event.agentId);
+			if (!activeRestoreRun) {
+				break;
+			}
+
+			activeRestoreRun.onStarted();
+			break;
+		}
 		case "restore.progress": {
-			serverEvents.emit("restore:progress", {
-				organizationId: event.payload.organizationId,
-				repositoryId: event.payload.repositoryId,
-				snapshotId: event.payload.snapshotId,
-				...event.payload.progress,
+			const activeRestoreRun = getActiveRestoreRun(event.payload.restoreId, event.type, event.agentId);
+			if (!activeRestoreRun) {
+				break;
+			}
+
+			activeRestoreRun.onProgress(event.payload.progress);
+			break;
+		}
+		case "restore.completed": {
+			const activeRestoreRun = getActiveRestoreRun(event.payload.restoreId, event.type, event.agentId);
+			if (!activeRestoreRun) {
+				break;
+			}
+
+			resolveActiveRestoreRun(activeRestoreRun.restoreId, {
+				status: "completed",
+				result: event.payload.result,
+			});
+			break;
+		}
+		case "restore.failed": {
+			const activeRestoreRun = getActiveRestoreRun(event.payload.restoreId, event.type, event.agentId);
+			if (!activeRestoreRun) {
+				break;
+			}
+
+			resolveActiveRestoreRun(activeRestoreRun.restoreId, {
+				status: "failed",
+				error: event.payload.errorDetails ?? event.payload.error,
+			});
+			break;
+		}
+		case "restore.cancelled": {
+			const activeRestoreRun = getActiveRestoreRun(event.payload.restoreId, event.type, event.agentId);
+			if (!activeRestoreRun) {
+				break;
+			}
+
+			resolveActiveRestoreRun(activeRestoreRun.restoreId, {
+				status: "cancelled",
+				message: activeRestoreRun.cancellationRequested ? undefined : event.payload.message,
 			});
 			break;
 		}
@@ -322,22 +471,51 @@ export const agentManager = {
 
 		return response.command;
 	},
-	runRestoreCommand: async (agentId: string, payload: Omit<RestoreCommandPayload, "commandId">) => {
+	startRestore: async (agentId: string, request: AgentStartRestoreRequest): Promise<AgentRestoreStartResult> => {
 		const runtime = getAgentManagerRuntime();
 		if (!runtime) {
-			throw new Error(`Restore agent ${agentId} is not connected`);
+			return {
+				status: "unavailable",
+				error: new Error(`Restore agent ${agentId} is not connected`),
+			};
 		}
 
-		const response = await Effect.runPromise(runtime.runRestoreCommand(agentId, payload));
-		if (!response) {
-			throw new Error(`Failed to send restore command ${payload.snapshotId} to agent ${agentId}`);
+		if (request.signal.aborted) {
+			throw request.signal.reason || new Error("Operation aborted");
 		}
 
-		if (response.status === "error") {
-			throw new Error(response.error);
-		}
+		const completion = new Promise<RestoreExecutionResult>((resolve) => {
+			getActiveRestoresByRestoreId().set(request.payload.restoreId, {
+				agentId,
+				restoreId: request.payload.restoreId,
+				onStarted: request.onStarted,
+				onProgress: request.onProgress,
+				resolve,
+				cancellationRequested: false,
+			});
+		});
 
-		return response.result;
+		try {
+			if (!(await Effect.runPromise(runtime.sendRestore(agentId, request.payload)))) {
+				clearActiveRestoreRun(request.payload.restoreId);
+				return {
+					status: "unavailable",
+					error: new Error(`Failed to send restore command to agent ${agentId}`),
+				};
+			}
+
+			if (request.signal.aborted) {
+				await requestRestoreCancellation(agentId, request.payload.restoreId);
+			}
+
+			return { status: "started", result: completion };
+		} catch (error) {
+			clearActiveRestoreRun(request.payload.restoreId);
+			throw error;
+		}
+	},
+	cancelRestore: async (agentId: string, restoreId: string) => {
+		return requestRestoreCancellation(agentId, restoreId);
 	},
 };
 

app/server/modules/agents/controller/server.ts

@@ -3,10 +3,11 @@ import { logger } from "@zerobyte/core/node";
 import { toMessage } from "@zerobyte/core/utils";
 import type {
 	AgentMessage,
+	AgentProtocolRejection,
 	BackupCancelPayload,
 	BackupRunPayload,
-	RestoreCommandPayload,
-	RestoreCommandResponsePayload,
+	RestoreCancelPayload,
+	RestoreRunPayload,
 	VolumeCommand,
 	VolumeCommandResponsePayload,
 } from "@zerobyte/contracts/agent-protocol";
@@ -26,6 +27,7 @@ type AgentEventContext = {
 
 export type AgentManagerEvent =
 	| (AgentEventContext & { type: "agent.disconnected" })
+	| (AgentEventContext & { type: "agent.protocolRejected"; payload: AgentProtocolRejection })
 	| (AgentEventContext & AgentMessage);
 
 type ControllerAgentSessionHandle = {
@@ -81,7 +83,7 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
 		return !!session && Effect.runSync(session.isReady());
 	};
 
-	const handleSessionEvent = (params: { agentId: string; agentName: string; sessionId: string }) => {
+	const handleSessionEvent = (params: { agentId: string; agentName: string }) => {
 		const { agentId, agentName } = params;
 
 		return (event: ControllerAgentSessionEvent) => {
@@ -92,11 +94,17 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
 						await agentsService.markAgentOnline(agentId, at, {
 							...event.payload.capabilities,
 							protocolVersion: event.payload.protocolVersion,
+							protocolCompatible: true,
 							hostname: event.payload.hostname,
 							platform: event.payload.platform,
 						});
 					});
 				}
+				case "agent.protocolRejected": {
+					return Effect.sync(() =>
+						onEvent({ type: "agent.protocolRejected", agentId, agentName, payload: event.payload }),
+					);
+				}
 				case "heartbeat.pong": {
 					const at = Date.now();
 					return Effect.promise(() => agentsService.markAgentSeen(agentId, at));
@@ -121,7 +129,6 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
 					handleSessionEvent({
 						agentId: ws.data.agentId,
 						agentName: ws.data.agentName,
-						sessionId: ws.data.id,
 					}),
 				),
 				scope,
@@ -354,6 +361,47 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
 				logger.info(`Sent backup cancel for command ${payload.jobId} to agent ${agentId}`);
 				return true;
 			}),
+		sendRestore: (agentId: string, payload: RestoreRunPayload) =>
+			Effect.gen(function* () {
+				const session = getSession(agentId);
+
+				if (!session) {
+					logger.warn(`Cannot send restore command. Agent ${agentId} is not connected.`);
+					return false;
+				}
+
+				if (!(yield* session.isReady())) {
+					logger.warn(`Cannot send restore command. Agent ${agentId} is not ready.`);
+					return false;
+				}
+
+				if (!(yield* session.sendRestore(payload))) {
+					logger.warn(`Cannot send restore command. Agent ${agentId} is no longer accepting commands.`);
+					return false;
+				}
+
+				logger.info(
+					`Sent restore command ${payload.restoreId} to agent ${agentId} for snapshot ${payload.snapshotId}`,
+				);
+				return true;
+			}),
+		cancelRestore: (agentId: string, payload: RestoreCancelPayload) =>
+			Effect.gen(function* () {
+				const session = getSession(agentId);
+
+				if (!session) {
+					logger.warn(`Cannot cancel restore command. Agent ${agentId} is not connected.`);
+					return false;
+				}
+
+				if (!(yield* session.sendRestoreCancel(payload))) {
+					logger.warn(`Cannot cancel restore command. Agent ${agentId} is no longer accepting commands.`);
+					return false;
+				}
+
+				logger.info(`Sent restore cancel for command ${payload.restoreId} to agent ${agentId}`);
+				return true;
+			}),
 		runVolumeCommand: (
 			agentId: string,
 			command: VolumeCommand,
@@ -379,31 +427,6 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
 				yield* logger.effect.info(`Completed volume command ${command.name} on agent ${agentId}`);
 				return result;
 			}),
-		runRestoreCommand: (
-			agentId: string,
-			payload: Omit<RestoreCommandPayload, "commandId">,
-		): Effect.Effect<RestoreCommandResponsePayload | null, Error> =>
-			Effect.gen(function* () {
-				const session = getSession(agentId);
-
-				if (!session) {
-					yield* logger.effect.warn(
-						`Cannot send restore command ${payload.snapshotId}. Agent ${agentId} is not connected.`,
-					);
-					return null;
-				}
-
-				if (!(yield* session.isReady())) {
-					yield* logger.effect.warn(
-						`Cannot send restore command ${payload.snapshotId}. Agent ${agentId} is not ready.`,
-					);
-					return null;
-				}
-
-				const result = yield* session.runRestoreCommand(payload);
-				yield* logger.effect.info(`Completed restore command ${payload.snapshotId} on agent ${agentId}`);
-				return result;
-			}),
 		stop,
 	};
 }

app/server/modules/agents/controller/session.ts

@@ -3,12 +3,16 @@ import type { AgentKind } from "../../../db/schema";
 import {
 	createControllerMessage,
 	parseAgentMessage,
+	parseAgentStartupMessage,
+	SUPPORTED_AGENT_PROTOCOL_MAX_VERSION,
+	SUPPORTED_AGENT_PROTOCOL_MIN_VERSION,
 	type AgentMessage,
+	type AgentProtocolRejection,
 	type BackupCancelPayload,
 	type BackupRunPayload,
 	type ControllerWireMessage,
-	type RestoreCommandPayload,
-	type RestoreCommandResponsePayload,
+	type RestoreCancelPayload,
+	type RestoreRunPayload,
 	type VolumeCommand,
 	type VolumeCommandResponsePayload,
 } from "@zerobyte/contracts/agent-protocol";
@@ -27,14 +31,16 @@ type AgentSocket = Bun.ServerWebSocket<AgentConnectionData>;
 
 type SessionState = {
 	isReady: boolean;
+	protocolVersion: number | null;
 	lastSeenAt: number | null;
 	lastPongAt: number | null;
 };
 
-type PendingCommand = { deferred: Deferred.Deferred<any, Error>; description: string };
+type PendingCommand = { deferred: Deferred.Deferred<VolumeCommandResponsePayload, Error>; description: string };
 
 export type ControllerAgentSessionEvent =
-	| Exclude<AgentMessage, { type: "volume.commandResult" | "restore.commandResult" }>
+	| Exclude<AgentMessage, { type: "volume.commandResult" }>
+	| { type: "agent.protocolRejected"; payload: AgentProtocolRejection }
 	| { type: "agent.disconnected" };
 
 export type ControllerAgentSession = {
@@ -42,10 +48,9 @@ export type ControllerAgentSession = {
 	handleMessage: (data: string) => Effect.Effect<void>;
 	sendBackup: (payload: BackupRunPayload) => Effect.Effect<boolean>;
 	sendBackupCancel: (payload: BackupCancelPayload) => Effect.Effect<boolean>;
+	sendRestore: (payload: RestoreRunPayload) => Effect.Effect<boolean>;
+	sendRestoreCancel: (payload: RestoreCancelPayload) => Effect.Effect<boolean>;
 	runVolumeCommand: (command: VolumeCommand) => Effect.Effect<VolumeCommandResponsePayload, Error>;
-	runRestoreCommand: (
-		payload: Omit<RestoreCommandPayload, "commandId">,
-	) => Effect.Effect<RestoreCommandResponsePayload, Error>;
 	isReady: () => Effect.Effect<boolean>;
 	run: Effect.Effect<void, never, Scope.Scope>;
 };
@@ -60,6 +65,7 @@ export const createControllerAgentSession = (
 		const pendingCommands = yield* Ref.make(new Map<string, PendingCommand>());
 		const state = yield* Ref.make<SessionState>({
 			isReady: false,
+			protocolVersion: null,
 			lastSeenAt: null,
 			lastPongAt: null,
 		});
@@ -179,23 +185,17 @@ export const createControllerAgentSession = (
 				yield* Deferred.succeed(pending.deferred, payload);
 			});
 
-		const handleRestoreCommandResult = (payload: RestoreCommandResponsePayload) =>
-			Effect.gen(function* () {
-				const pending = yield* removePendingCommand(payload.commandId);
-				if (!pending) {
-					yield* logger.effect.warn(`Received response for unknown restore command ${payload.commandId}`);
-					return;
-				}
-
-				yield* Deferred.succeed(pending.deferred, payload);
-			});
-
 		const handleAgentMessage = (message: AgentMessage) =>
 			Effect.gen(function* () {
 				switch (message.type) {
 					case "agent.ready": {
 						const readyAt = Date.now();
-						yield* updateState((current) => ({ ...current, isReady: true, lastSeenAt: readyAt }));
+						yield* updateState((current) => ({
+							...current,
+							isReady: true,
+							protocolVersion: message.payload.protocolVersion,
+							lastSeenAt: readyAt,
+						}));
 						yield* logger.effect.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
 						yield* onEvent(message);
 						break;
@@ -214,10 +214,6 @@ export const createControllerAgentSession = (
 						yield* handleVolumeCommandResult(message.payload);
 						break;
 					}
-					case "restore.commandResult": {
-						yield* handleRestoreCommandResult(message.payload);
-						break;
-					}
 					default: {
 						yield* onEvent(message);
 						break;
@@ -225,10 +221,30 @@ export const createControllerAgentSession = (
 				}
 			});
 
+		const rejectStartupMessage = (rejection: AgentProtocolRejection) =>
+			Effect.gen(function* () {
+				yield* logger.effect.warn(
+					`Rejecting startup message from agent ${socket.data.agentId}: ${rejection.reason}`,
+				);
+				yield* onEvent({ type: "agent.protocolRejected", payload: rejection });
+				yield* Effect.sync(() => socket.close(1002, rejection.reason));
+				yield* closeSession();
+			});
+
 		return {
 			connectionId: socket.data.id,
 			handleMessage: (data: string) => {
 				return Effect.gen(function* () {
+					const currentState = yield* Ref.get(state);
+
+					if (!currentState.isReady) {
+						const startupMessage = parseAgentStartupMessage(data);
+						if (!("success" in startupMessage)) {
+							yield* rejectStartupMessage(startupMessage);
+							return;
+						}
+					}
+
 					const parsed = parseAgentMessage(data);
 
 					if (parsed === null) {
@@ -237,6 +253,15 @@ export const createControllerAgentSession = (
 					}
 
 					if (!parsed.success) {
+						if (!currentState.isReady) {
+							yield* rejectStartupMessage({
+								reason: "invalid_agent_ready",
+								supportedProtocolMinVersion: SUPPORTED_AGENT_PROTOCOL_MIN_VERSION,
+								supportedProtocolMaxVersion: SUPPORTED_AGENT_PROTOCOL_MAX_VERSION,
+							});
+							return;
+						}
+
 						yield* logger.effect.warn(
 							`Invalid agent message from ${socket.data.agentId}: ${parsed.error.message}`,
 						);
@@ -248,6 +273,8 @@ export const createControllerAgentSession = (
 			},
 			sendBackup: (payload) => offerOutbound(createControllerMessage("backup.run", payload)),
 			sendBackupCancel: (payload) => offerOutbound(createControllerMessage("backup.cancel", payload)),
+			sendRestore: (payload) => offerOutbound(createControllerMessage("restore.run", payload)),
+			sendRestoreCancel: (payload) => offerOutbound(createControllerMessage("restore.cancel", payload)),
 			runVolumeCommand: (command) =>
 				Effect.gen(function* () {
 					const commandId = Bun.randomUUIDv7();
@@ -263,36 +290,13 @@ export const createControllerAgentSession = (
 						return yield* Effect.fail(new Error(`Failed to queue volume command ${command.name}`));
 					}
 
-					return (yield* Deferred.await(deferred).pipe(
+					return yield* Deferred.await(deferred).pipe(
 						Effect.timeoutFail({
 							duration: "60 seconds",
 							onTimeout: () => new Error(`Volume command ${command.name} timed out`),
 						}),
 						Effect.ensuring(removePendingCommand(commandId)),
-					)) as VolumeCommandResponsePayload;
-				}),
-			runRestoreCommand: (payload) =>
-				Effect.gen(function* () {
-					const commandId = Bun.randomUUIDv7();
-					const description = `restore command ${payload.snapshotId}`;
-					const deferred = yield* Deferred.make<RestoreCommandResponsePayload, Error>();
-					yield* setPendingCommand(commandId, { deferred, description });
-
-					const queued = yield* offerOutbound(
-						createControllerMessage("restore.command", { commandId, ...payload }),
 					);
-					if (!queued) {
-						yield* removePendingCommand(commandId);
-						return yield* Effect.fail(new Error(`Failed to queue restore command ${payload.snapshotId}`));
-					}
-
-					return (yield* Deferred.await(deferred).pipe(
-						Effect.timeoutFail({
-							duration: "1 hour",
-							onTimeout: () => new Error(`Restore command ${payload.snapshotId} timed out`),
-						}),
-						Effect.ensuring(removePendingCommand(commandId)),
-					)) as RestoreCommandResponsePayload;
 				}),
 			isReady: () => Ref.get(state).pipe(Effect.map((current) => current.isReady)),
 			run,

app/server/modules/agents/helpers/runtime-state.dev.ts

@@ -1,20 +1,26 @@
 import { createAgentRuntimeState, type AgentRuntimeState } from "./runtime-state";
 
-type LegacyAgentRuntimeState = Omit<AgentRuntimeState, "activeBackupsByScheduleId" | "activeBackupScheduleIdsByJobId"> &
-	Partial<Pick<AgentRuntimeState, "activeBackupsByScheduleId" | "activeBackupScheduleIdsByJobId">>;
+type RuntimeMapKey = "activeBackupsByScheduleId" | "activeBackupScheduleIdsByJobId" | "activeRestoresByRestoreId";
+
+type LegacyAgentRuntimeState = Omit<AgentRuntimeState, RuntimeMapKey> & Partial<Pick<AgentRuntimeState, RuntimeMapKey>>;
 
 export type ProcessWithAgentRuntime = NodeJS.Process & {
 	__zerobyteAgentRuntime?: LegacyAgentRuntimeState;
 };
 
-const hasActiveBackupMaps = (runtime: LegacyAgentRuntimeState): runtime is AgentRuntimeState => {
-	return runtime.activeBackupsByScheduleId instanceof Map && runtime.activeBackupScheduleIdsByJobId instanceof Map;
+const hasActiveRuntimeMaps = (runtime: LegacyAgentRuntimeState): runtime is AgentRuntimeState => {
+	return (
+		runtime.activeBackupsByScheduleId instanceof Map &&
+		runtime.activeBackupScheduleIdsByJobId instanceof Map &&
+		runtime.activeRestoresByRestoreId instanceof Map
+	);
 };
 
 const hydrateAgentRuntimeState = (runtime: LegacyAgentRuntimeState): AgentRuntimeState => ({
 	...runtime,
 	activeBackupsByScheduleId: runtime.activeBackupsByScheduleId ?? new Map(),
 	activeBackupScheduleIdsByJobId: runtime.activeBackupScheduleIdsByJobId ?? new Map(),
+	activeRestoresByRestoreId: runtime.activeRestoresByRestoreId ?? new Map(),
 });
 
 export const getDevAgentRuntimeState = (): AgentRuntimeState => {
@@ -27,7 +33,7 @@ export const getDevAgentRuntimeState = (): AgentRuntimeState => {
 		return runtime;
 	}
 
-	if (hasActiveBackupMaps(existingRuntime)) {
+	if (hasActiveRuntimeMaps(existingRuntime)) {
 		return existingRuntime;
 	}
 

app/server/modules/agents/helpers/runtime-state.ts

@@ -1,12 +1,27 @@
 import type { ChildProcess } from "node:child_process";
 import type { ResticBackupOutputDto } from "@zerobyte/core/restic";
-import type { BackupProgressPayload } from "@zerobyte/contracts/agent-protocol";
+import type {
+	BackupProgressPayload,
+	RestoreCompletedPayload,
+	RestoreProgressPayload,
+} from "@zerobyte/contracts/agent-protocol";
 import type { AgentManagerRuntime } from "../controller/server";
 
 export type BackupExecutionProgress = BackupProgressPayload["progress"];
 export type BackupExecutionResult =
 	| { status: "unavailable"; error: Error }
-	| { status: "completed"; exitCode: number; result: ResticBackupOutputDto | null; warningDetails: string | null }
+	| {
+			status: "completed";
+			exitCode: number;
+			result: ResticBackupOutputDto | null;
+			warningDetails: string | null;
+	  }
+	| { status: "failed"; error: string }
+	| { status: "cancelled"; message?: string };
+export type RestoreExecutionProgress = RestoreProgressPayload["progress"];
+export type RestoreExecutionResult =
+	| { status: "unavailable"; error: Error }
+	| { status: "completed"; result: RestoreCompletedPayload["result"] }
 	| { status: "failed"; error: string }
 	| { status: "cancelled"; message?: string };
 
@@ -20,6 +35,15 @@ type ActiveBackupRun = {
 	cancellationRequested: boolean;
 };
 
+type ActiveRestoreRun = {
+	agentId: string;
+	restoreId: string;
+	onStarted: () => void;
+	onProgress: (progress: RestoreExecutionProgress) => void;
+	resolve: (result: RestoreExecutionResult) => void;
+	cancellationRequested: boolean;
+};
+
 export type AgentRuntimeState = {
 	agentManager: AgentManagerRuntime | null;
 	localAgent: ChildProcess | null;
@@ -27,6 +51,7 @@ export type AgentRuntimeState = {
 	localAgentRestartTimeout: ReturnType<typeof setTimeout> | null;
 	activeBackupsByScheduleId: Map<number, ActiveBackupRun>;
 	activeBackupScheduleIdsByJobId: Map<string, number>;
+	activeRestoresByRestoreId: Map<string, ActiveRestoreRun>;
 };
 
 export const createAgentRuntimeState = (): AgentRuntimeState => ({
@@ -36,4 +61,5 @@ export const createAgentRuntimeState = (): AgentRuntimeState => ({
 	localAgentRestartTimeout: null,
 	activeBackupsByScheduleId: new Map(),
 	activeBackupScheduleIdsByJobId: new Map(),
+	activeRestoresByRestoreId: new Map(),
 });

app/server/modules/auth/__tests__/helpers.test.ts

@@ -1,8 +1,9 @@
 import { beforeEach, describe, expect, test, vi } from "vitest";
+import { eq } from "drizzle-orm";
 import { db } from "~/server/db/db";
-import { account, usersTable } from "~/server/db/schema";
+import { account, passkey, usersTable } from "~/server/db/schema";
 import { createUser, randomId, randomSlug } from "~/test/helpers/user-org";
-import { userHasCredentialPassword, verifyUserPassword } from "../helpers";
+import { hasActivePasskeyUser, userHasCredentialPassword, verifyUserPassword } from "../helpers";
 
 const { verifyPassword } = vi.hoisted(() => ({
 	verifyPassword: vi.fn(async ({ hash }: { hash: string }) => hash === "credential-password-hash"),
@@ -30,9 +31,24 @@ async function createAccount({
 	});
 }
 
+async function createPasskey(userId: string) {
+	await db.insert(passkey).values({
+		id: randomId(),
+		name: "Test passkey",
+		publicKey: randomSlug("public-key"),
+		userId,
+		credentialID: randomSlug("credential"),
+		counter: 0,
+		deviceType: "singleDevice",
+		backedUp: false,
+		transports: "internal",
+	});
+}
+
 describe("verifyUserPassword", () => {
 	beforeEach(async () => {
 		verifyPassword.mockClear();
+		await db.delete(passkey);
 		await db.delete(account);
 		await db.delete(usersTable);
 	});
@@ -64,6 +80,7 @@ describe("verifyUserPassword", () => {
 
 describe("userHasCredentialPassword", () => {
 	beforeEach(async () => {
+		await db.delete(passkey);
 		await db.delete(account);
 		await db.delete(usersTable);
 	});
@@ -89,3 +106,32 @@ describe("userHasCredentialPassword", () => {
 		await expect(userHasCredentialPassword(userId)).resolves.toBe(false);
 	});
 });
+
+describe("hasActivePasskeyUser", () => {
+	beforeEach(async () => {
+		await db.delete(passkey);
+		await db.delete(account);
+		await db.delete(usersTable);
+	});
+
+	test("returns true when a non-banned user has a passkey", async () => {
+		const userId = await createUser(`${randomSlug("user")}@example.com`);
+		await createPasskey(userId);
+
+		await expect(hasActivePasskeyUser()).resolves.toBe(true);
+	});
+
+	test("returns false when only banned users have passkeys", async () => {
+		const userId = await createUser(`${randomSlug("user")}@example.com`);
+		await db.update(usersTable).set({ banned: true }).where(eq(usersTable.id, userId));
+		await createPasskey(userId);
+
+		await expect(hasActivePasskeyUser()).resolves.toBe(false);
+	});
+
+	test("returns false when no users have passkeys", async () => {
+		await createUser(`${randomSlug("user")}@example.com`);
+
+		await expect(hasActivePasskeyUser()).resolves.toBe(false);
+	});
+});

app/server/modules/auth/helpers.ts

@@ -1,5 +1,7 @@
+import { eq } from "drizzle-orm";
 import { verifyPassword } from "better-auth/crypto";
 import { db } from "~/server/db/db";
+import { passkey, usersTable } from "~/server/db/schema";
 
 type PasswordVerificationBody = {
 	userId: string;
@@ -31,3 +33,14 @@ export const userHasCredentialPassword = async (userId: string) => {
 
 	return Boolean(userAccount?.password);
 };
+
+export const hasActivePasskeyUser = async () => {
+	const [user] = await db
+		.select({ id: usersTable.id })
+		.from(passkey)
+		.innerJoin(usersTable, eq(passkey.userId, usersTable.id))
+		.where(eq(usersTable.banned, false))
+		.limit(1);
+
+	return Boolean(user);
+};

app/server/modules/backups/__tests__/backups.service.execution.test.ts

@@ -25,6 +25,7 @@ import { volumeService } from "~/server/modules/volumes/volume.service";
 import { db } from "~/server/db/db";
 import { config } from "~/server/core/config";
 import { Effect } from "effect";
+import { taskStore } from "~/server/modules/tasks/tasks.store";
 
 const setup = () => {
 	const resticBackupMock = vi.fn((_: SafeSpawnParams) =>
@@ -90,6 +91,18 @@ const setup = () => {
 	};
 };
 
+const getBackupTaskForSchedule = (scheduleId: number) =>
+	db.query.tasksTable.findFirst({
+		where: {
+			AND: [
+				{ organizationId: TEST_ORG_ID },
+				{ kind: "backup" },
+				{ resourceType: "backup_schedule" },
+				{ resourceId: String(scheduleId) },
+			],
+		},
+	});
+
 afterEach(() => {
 	vi.restoreAllMocks();
 	config.flags.enableLocalAgent = true;
@@ -233,6 +246,102 @@ describe("backup execution - validation failures", () => {
 		).toBe(false);
 	});
 
+	test("creates a backup task and uses the task id as the agent job id", async () => {
+		const { runBackupMock } = setup();
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+		});
+
+		await backupsService.executeBackup(schedule.id);
+
+		const task = await getBackupTaskForSchedule(schedule.id);
+		expect(task).toBeDefined();
+		expect(task).toMatchObject({
+			organizationId: TEST_ORG_ID,
+			kind: "backup",
+			status: "succeeded",
+			resourceType: "backup_schedule",
+			resourceId: String(schedule.id),
+			targetAgentId: "local",
+			input: {
+				kind: "backup",
+				scheduleId: schedule.id,
+				scheduleShortId: schedule.shortId,
+				manual: false,
+			},
+			result: expect.objectContaining({
+				kind: "backup",
+				exitCode: 0,
+				warningDetails: null,
+			}),
+		});
+		expect(runBackupMock.mock.calls[0]?.[1].payload.jobId).toBe(task!.id);
+	});
+
+	test("does not leave a queued task behind when backup startup state fails", async () => {
+		const { runBackupMock } = setup();
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+		});
+
+		vi.spyOn(scheduleQueries, "updateStatus").mockRejectedValueOnce(new Error("status update failed"));
+
+		await expect(backupsService.executeBackup(schedule.id)).rejects.toThrow("status update failed");
+
+		expect(await getBackupTaskForSchedule(schedule.id)).toBeUndefined();
+		expect(runBackupMock).not.toHaveBeenCalled();
+	});
+
+	test("persists latest backup progress while preserving execution", async () => {
+		const { runBackupMock } = setup();
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+		});
+
+		runBackupMock.mockImplementationOnce(async (_agentId, request) => {
+			request.onProgress({
+				message_type: "status",
+				seconds_elapsed: 1,
+				seconds_remaining: 9,
+				percent_done: 0.25,
+				total_files: 100,
+				files_done: 25,
+				total_bytes: 1000,
+				bytes_done: 250,
+				current_files: ["file.txt"],
+			});
+
+			return {
+				status: "completed",
+				exitCode: 0,
+				result: JSON.parse(generateBackupOutput()),
+				warningDetails: null,
+			};
+		});
+
+		await backupsService.executeBackup(schedule.id);
+
+		const task = await getBackupTaskForSchedule(schedule.id);
+		expect(task?.status).toBe("succeeded");
+		expect(task?.progress).toMatchObject({
+			kind: "backup",
+			progress: {
+				percent_done: 0.25,
+				bytes_done: 250,
+				current_files: ["file.txt"],
+			},
+		});
+	});
+
 	test("passes configured backup webhooks to the backup agent", async () => {
 		const { runBackupMock } = setup();
 		const volume = await createTestVolume();
@@ -374,8 +483,52 @@ describe("backup execution - validation failures", () => {
 		await backupsService.executeBackup(schedule.id);
 
 		const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
+		const task = await getBackupTaskForSchedule(schedule.id);
 		expect(updatedSchedule.lastBackupStatus).toBe("error");
 		expect(updatedSchedule.lastBackupError).toBe("Local backup agent is not connected");
+		expect(task?.status).toBe("failed");
+		expect(task?.error).toBe("Local backup agent is not connected");
+	});
+
+	test("removes stale locks and retries once when the local backup fallback hits a restic lock", async () => {
+		const { resticBackupMock, runBackupMock } = setup();
+		config.flags.enableLocalAgent = false;
+		const safeExecMock = vi.spyOn(spawnModule, "safeExec").mockResolvedValue({
+			exitCode: 0,
+			stdout: "",
+			stderr: "",
+			timedOut: false,
+		});
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+		});
+
+		runBackupMock.mockResolvedValueOnce({
+			status: "unavailable",
+			error: new Error("Local backup agent is not connected"),
+		});
+		resticBackupMock
+			.mockImplementationOnce((params: SafeSpawnParams) => {
+				params.onStderr?.("unable to create lock in backend: repository is already locked");
+				return Promise.resolve({
+					exitCode: 11,
+					summary: "",
+					error: "unable to create lock in backend: repository is already locked",
+				});
+			})
+			.mockImplementationOnce(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }));
+
+		await backupsService.executeBackup(schedule.id);
+
+		const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
+		expect(updatedSchedule.lastBackupStatus).toBe("success");
+		expect(resticBackupMock).toHaveBeenCalledTimes(2);
+		const unlockCalls = safeExecMock.mock.calls.filter(([params]) => params.args?.includes("unlock"));
+		expect(unlockCalls).toHaveLength(1);
+		expect(unlockCalls[0]?.[0].args).not.toContain("--remove-all");
 	});
 });
 
@@ -450,8 +603,15 @@ describe("stop backup", () => {
 		await backupsService.executeBackup(schedule.id);
 
 		const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
+		const task = await getBackupTaskForSchedule(schedule.id);
 		expect(updatedSchedule.lastBackupStatus).toBe("warning");
 		expect(updatedSchedule.lastBackupError).toBe("error: open /mnt/data/private.db: permission denied");
+		expect(task?.status).toBe("succeeded");
+		expect(task?.result).toMatchObject({
+			kind: "backup",
+			exitCode: 3,
+			warningDetails: "error: open /mnt/data/private.db: permission denied",
+		});
 		expect(notificationSpy).toHaveBeenLastCalledWith(
 			schedule.id,
 			"warning",
@@ -604,6 +764,48 @@ describe("stop backup", () => {
 		expect(updatedSchedule.lastBackupError).toBe("Backup was stopped by the user");
 	});
 
+	test("treats a task that finishes during stop as no longer running", async () => {
+		setup();
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+			lastBackupStatus: "in_progress",
+		});
+		const task = taskStore.create({
+			id: "task-stop-race",
+			organizationId: TEST_ORG_ID,
+			resourceType: "backup_schedule",
+			resourceId: String(schedule.id),
+			targetAgentId: "local",
+			input: {
+				kind: "backup",
+				scheduleId: schedule.id,
+				scheduleShortId: schedule.shortId,
+				manual: false,
+			},
+		});
+		const requestCancel = taskStore.requestCancel;
+		vi.spyOn(taskStore, "requestCancel").mockImplementation((taskId) => {
+			taskStore.complete(taskId, {
+				kind: "backup",
+				exitCode: 0,
+				result: JSON.parse(generateBackupOutput()),
+				warningDetails: null,
+			});
+
+			return requestCancel(taskId);
+		});
+
+		await expect(backupsService.stopBackup(schedule.id)).rejects.toThrow(
+			"No backup is currently running for this schedule",
+		);
+
+		const updatedTask = await getBackupTaskForSchedule(schedule.id);
+		expect(updatedTask).toMatchObject({ id: task.id, status: "succeeded" });
+	});
+
 	test("should stop a running backup when the cancel command cannot be delivered", async () => {
 		const { resticBackupMock, cancelBackupMock } = setup();
 		const volume = await createTestVolume();
@@ -647,6 +849,8 @@ describe("stop backup", () => {
 			await waitForExpect(async () => {
 				const queuedSchedule = await getScheduleByIdOrShortId(schedule.id);
 				expect(queuedSchedule.lastBackupStatus).toBe("in_progress");
+				const task = await getBackupTaskForSchedule(schedule.id);
+				expect(task?.status).toBe("queued");
 			});
 
 			expect(resticBackupMock).not.toHaveBeenCalled();
@@ -659,8 +863,11 @@ describe("stop backup", () => {
 		await executePromise;
 
 		const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
+		const task = await getBackupTaskForSchedule(schedule.id);
 		expect(updatedSchedule.lastBackupStatus).toBe("warning");
 		expect(updatedSchedule.lastBackupError).toBe("Backup was stopped by the user");
+		expect(task?.status).toBe("cancelled");
+		expect(task?.cancellationRequested).toBe(true);
 		expect(resticBackupMock).not.toHaveBeenCalled();
 	});
 
@@ -761,6 +968,41 @@ describe("stop backup", () => {
 		expect(updatedSchedule.lastBackupError).toBe("Backup was stopped by the user");
 	});
 
+	test("marks an active task stale when stopping a stuck schedule without a live executor", async () => {
+		setup();
+		const volume = await createTestVolume();
+		const repository = await createTestRepository();
+		const schedule = await createTestBackupSchedule({
+			volumeId: volume.id,
+			repositoryId: repository.id,
+			lastBackupStatus: "in_progress",
+		});
+		taskStore.create({
+			id: "task-stuck-backup",
+			organizationId: TEST_ORG_ID,
+			resourceType: "backup_schedule",
+			resourceId: String(schedule.id),
+			targetAgentId: "local",
+			input: {
+				kind: "backup",
+				scheduleId: schedule.id,
+				scheduleShortId: schedule.shortId,
+				manual: false,
+			},
+		});
+
+		await expect(backupsService.stopBackup(schedule.id)).rejects.toThrow(
+			"No backup is currently running for this schedule",
+		);
+
+		const task = await getBackupTaskForSchedule(schedule.id);
+		const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
+		expect(task?.status).toBe("stale");
+		expect(task?.error).toBe("No live backup execution was found for this schedule");
+		expect(updatedSchedule.lastBackupStatus).toBe("warning");
+		expect(updatedSchedule.lastBackupError).toBe("Backup was stopped by the user");
+	});
+
 	test("should throw NotFoundError when schedule does not exist", async () => {
 		setup();
 		// act & assert

app/server/modules/backups/backup-executor.ts

@@ -15,6 +15,7 @@ import { BadRequestError } from "http-errors-enhanced";
 const FUSE_VOLUME_BACKENDS = new Set<Volume["type"]>(["rclone", "sftp", "webdav"]);
 const IGNORE_INODE_FLAG = "--ignore-inode";
 type BackupExecutionRequest = {
+	jobId: string;
 	scheduleId: number;
 	schedule: BackupSchedule;
 	volume: Volume;
@@ -42,7 +43,7 @@ const createBackupRunPayload = async ({
 	volume,
 	repository,
 	organizationId,
-}: BackupExecutionRequest & { jobId: string }): Promise<BackupRunPayload> => {
+}: BackupExecutionRequest): Promise<BackupRunPayload> => {
 	const agentVolume = { ...volume, config: await decryptVolumeConfig(volume.config) };
 	const customResticParams = schedule.customResticParams ?? [];
 
@@ -119,7 +120,7 @@ export const backupExecutor = {
 			activeControllersByScheduleId.delete(scheduleId);
 		}
 	},
-	execute: async (request: Omit<BackupExecutionRequest, "jobId">) => {
+	execute: async (request: BackupExecutionRequest) => {
 		const trackedExecution = activeControllersByScheduleId.get(request.scheduleId);
 		if (!trackedExecution || trackedExecution.abortController.signal !== request.signal) {
 			throw new Error(`Backup execution for schedule ${request.scheduleId} was not tracked`);
@@ -129,9 +130,7 @@ export const backupExecutor = {
 			throw request.signal.reason || new Error("Operation aborted");
 		}
 
-		const jobId = Bun.randomUUIDv7();
-
-		const payload = await createBackupRunPayload({ ...request, jobId });
+		const payload = await createBackupRunPayload(request);
 
 		if (request.signal.aborted) {
 			throw request.signal.reason || new Error("Operation aborted");

app/server/modules/backups/backups.service.ts

@@ -30,6 +30,27 @@ import { restic } from "../../core/restic";
 import { mirrorQueries } from "./backups.queries";
 import { runEffectPromise, toMessage } from "../../utils/errors";
 import { Effect } from "effect";
+import { taskStore } from "../tasks/tasks.store";
+
+const BACKUP_TASK_RESOURCE_TYPE = "backup_schedule";
+
+const tryCancelTask = (
+	taskId: string,
+	activeTaskResource: { organizationId: string; kind: "backup"; resourceType: string; resourceId: string },
+) => {
+	try {
+		taskStore.requestCancel(taskId);
+		return true;
+	} catch (error) {
+		const currentTask = taskStore.findActiveByResource(activeTaskResource);
+		if (!currentTask || currentTask.id !== taskId) {
+			return false;
+		}
+
+		throw error;
+	}
+};
+
 const listSchedules = async () => {
 	const organizationId = getOrganizationId();
 	const schedules = await db.query.backupSchedulesTable.findMany({
@@ -418,7 +439,16 @@ const executeBackup = async (scheduleId: number, manual = false) => {
 		...(ctx.schedule.cronExpression ? { nextBackupAt: calculateNextRun(ctx.schedule.cronExpression) } : {}),
 	});
 
+	const task = taskStore.create({
+		organizationId: ctx.organizationId,
+		resourceType: BACKUP_TASK_RESOURCE_TYPE,
+		resourceId: String(scheduleId),
+		targetAgentId: ctx.volume.agentId,
+		input: { kind: "backup", scheduleId, scheduleShortId: ctx.schedule.shortId, manual },
+	});
+
 	const abortController = backupExecutor.track(scheduleId);
+	let domainHandlerCompleted = false;
 
 	try {
 		const releaseLock = await repoMutex.acquireShared(
@@ -428,7 +458,10 @@ const executeBackup = async (scheduleId: number, manual = false) => {
 		);
 
 		try {
+			taskStore.markRunning(task.id);
+
 			const executionResult = await backupExecutor.execute({
+				jobId: task.id,
 				scheduleId,
 				schedule: ctx.schedule,
 				volume: ctx.volume,
@@ -437,33 +470,63 @@ const executeBackup = async (scheduleId: number, manual = false) => {
 				signal: abortController.signal,
 				onProgress: (progress) => {
 					updateBackupProgress(ctx, progress);
+					try {
+						taskStore.updateProgress(task.id, { kind: "backup", progress });
+					} catch (error) {
+						logger.error(`Failed to persist backup task progress for ${task.id}: ${toMessage(error)}`);
+					}
 				},
 			});
 
 			switch (executionResult.status) {
-				case "unavailable":
-					return handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, manual, ctx);
+				case "unavailable": {
+					await handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, manual, ctx);
+					domainHandlerCompleted = true;
+					taskStore.fail(task.id, toMessage(executionResult.error));
+					return;
+				}
 				case "completed":
-					return finalizeSuccessfulBackup(
+					await finalizeSuccessfulBackup(
 						ctx,
 						executionResult.exitCode,
 						executionResult.result,
 						executionResult.warningDetails,
 					);
-				case "failed":
-					return handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, manual, ctx);
+					domainHandlerCompleted = true;
+					taskStore.complete(task.id, {
+						kind: "backup",
+						exitCode: executionResult.exitCode,
+						result: executionResult.result,
+						warningDetails: executionResult.warningDetails,
+					});
+					return;
+				case "failed": {
+					await handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, manual, ctx);
+					domainHandlerCompleted = true;
+					taskStore.fail(task.id, toMessage(executionResult.error));
+					return;
+				}
 				case "cancelled":
-					return handleBackupCancellation(scheduleId, ctx.organizationId, executionResult.message);
+					await handleBackupCancellation(scheduleId, ctx.organizationId, executionResult.message);
+					domainHandlerCompleted = true;
+					taskStore.cancel(task.id, executionResult.message ?? "Backup was stopped by the user");
+					return;
 			}
 		} finally {
 			releaseLock();
 		}
 	} catch (error) {
 		if (abortController.signal.aborted) {
+			taskStore.cancel(task.id, "Backup was stopped by the user");
 			return;
 		}
 
-		return handleBackupFailure(scheduleId, ctx.organizationId, error, manual, ctx);
+		if (domainHandlerCompleted) {
+			throw error;
+		}
+
+		await handleBackupFailure(scheduleId, ctx.organizationId, error, manual, ctx);
+		taskStore.fail(task.id, toMessage(error));
 	} finally {
 		backupExecutor.untrack(scheduleId, abortController);
 		cache.del(cacheKeys.backup.progress(scheduleId));
@@ -483,8 +546,26 @@ const stopBackup = async (scheduleId: number) => {
 		throw new NotFoundError("Backup schedule not found");
 	}
 
+	const activeTaskResource = {
+		organizationId,
+		kind: "backup",
+		resourceType: BACKUP_TASK_RESOURCE_TYPE,
+		resourceId: String(scheduleId),
+	} as const;
+	const activeTask = taskStore.findActiveByResource(activeTaskResource);
+	let shouldMarkActiveTaskStale = false;
+	if (activeTask) {
+		shouldMarkActiveTaskStale = tryCancelTask(activeTask.id, activeTaskResource);
+	}
+
 	try {
 		if (!(await backupExecutor.cancel(scheduleId))) {
+			if (shouldMarkActiveTaskStale) {
+				taskStore.markActiveStale({
+					...activeTaskResource,
+					error: "No live backup execution was found for this schedule",
+				});
+			}
 			throw new ConflictError("No backup is currently running for this schedule");
 		}
 

app/server/modules/lifecycle/__tests__/startup.test.ts

@@ -0,0 +1,82 @@
+import { afterEach, beforeEach, expect, test, vi } from "vitest";
+import { Scheduler } from "~/server/core/scheduler";
+import { serverEvents } from "~/server/core/events";
+import { config } from "~/server/core/config";
+import { db } from "~/server/db/db";
+import { backupsService } from "~/server/modules/backups/backups.service";
+import { repositoriesService } from "~/server/modules/repositories/repositories.service";
+import { notificationsService } from "~/server/modules/notifications/notifications.service";
+import { volumeService } from "~/server/modules/volumes/volume.service";
+import * as provisioningModule from "~/server/modules/provisioning/provisioning";
+import { taskStore } from "~/server/modules/tasks/tasks.store";
+import { createTestBackupSchedule } from "~/test/helpers/backup";
+import { createTestRepository } from "~/test/helpers/repository";
+import { createTestVolume } from "~/test/helpers/volume";
+import { TEST_ORG_ID } from "~/test/helpers/organization";
+
+const loadStartupModule = async () => {
+	const moduleUrl = new URL("../startup.ts", import.meta.url);
+	moduleUrl.searchParams.set("test", crypto.randomUUID());
+	return import(moduleUrl.href);
+};
+
+let originalEnableLocalAgent: boolean;
+
+beforeEach(() => {
+	originalEnableLocalAgent = config.flags.enableLocalAgent;
+	config.flags.enableLocalAgent = true;
+
+	vi.spyOn(Scheduler, "start").mockResolvedValue();
+	vi.spyOn(Scheduler, "clear").mockResolvedValue();
+	vi.spyOn(Scheduler, "build").mockImplementation(() => ({ schedule: vi.fn() }));
+	vi.spyOn(provisioningModule, "syncProvisionedResources").mockResolvedValue();
+	vi.spyOn(backupsService, "cleanupOrphanedSchedules").mockResolvedValue({ deletedSchedules: 0 });
+	vi.spyOn(volumeService, "updateVolume").mockResolvedValue(undefined as never);
+	vi.spyOn(repositoriesService, "updateRepository").mockResolvedValue(undefined as never);
+	vi.spyOn(notificationsService, "updateDestination").mockResolvedValue(undefined as never);
+});
+
+afterEach(() => {
+	config.flags.enableLocalAgent = originalEnableLocalAgent;
+	vi.restoreAllMocks();
+});
+
+test("marks active tasks stale and keeps stuck backup schedule recovery silent", async () => {
+	const emitSpy = vi.spyOn(serverEvents, "emit");
+	const notificationSpy = vi.spyOn(notificationsService, "sendBackupNotification").mockResolvedValue();
+	const volume = await createTestVolume();
+	const repository = await createTestRepository();
+	const schedule = await createTestBackupSchedule({
+		volumeId: volume.id,
+		repositoryId: repository.id,
+		lastBackupStatus: "in_progress",
+	});
+	const task = taskStore.create({
+		id: "task-startup-active",
+		organizationId: TEST_ORG_ID,
+		resourceType: "backup_schedule",
+		resourceId: String(schedule.id),
+		targetAgentId: "local",
+		input: {
+			kind: "backup",
+			scheduleId: schedule.id,
+			scheduleShortId: schedule.shortId,
+			manual: false,
+		},
+	});
+	taskStore.markRunning(task.id);
+
+	const { startup } = await loadStartupModule();
+
+	await startup();
+
+	const updatedTask = await db.query.tasksTable.findFirst({ where: { id: task.id } });
+	const updatedSchedule = await db.query.backupSchedulesTable.findFirst({ where: { id: schedule.id } });
+	expect(updatedTask?.status).toBe("stale");
+	expect(updatedTask?.error).toBe("Zerobyte was restarted before this task completed");
+	expect(updatedTask?.finishedAt).toEqual(expect.any(Number));
+	expect(updatedSchedule?.lastBackupStatus).toBe("warning");
+	expect(updatedSchedule?.lastBackupError).toBe("Zerobyte was restarted during the last scheduled backup");
+	expect(emitSpy).not.toHaveBeenCalledWith("backup:completed", expect.anything());
+	expect(notificationSpy).not.toHaveBeenCalled();
+});

app/server/modules/lifecycle/startup.ts

@@ -18,6 +18,7 @@ import { config } from "~/server/core/config";
 import { syncProvisionedResources } from "../provisioning/provisioning";
 import { toMessage } from "~/server/utils/errors";
 import { LOCAL_AGENT_ID } from "../agents/constants";
+import { taskStore } from "../tasks/tasks.store";
 
 const ensureLatestConfigurationSchema = async () => {
 	const volumes = await db.query.volumesTable.findMany({});
@@ -98,6 +99,16 @@ export const startup = async () => {
 		}
 	}
 
+	try {
+		const staleTasks = taskStore.markActiveStale({ error: "Zerobyte was restarted before this task completed" });
+
+		if (staleTasks.length > 0) {
+			logger.warn(`Marked ${staleTasks.length} active task(s) stale during startup`);
+		}
+	} catch (err) {
+		logger.error(`Failed to mark stale tasks on startup: ${toMessage(err)}`);
+	}
+
 	await db
 		.update(backupSchedulesTable)
 		.set({

app/server/modules/repositories/__tests__/repositories.service.test.ts

@@ -11,14 +11,16 @@ import { REPOSITORY_BASE } from "~/server/core/constants";
 import { config } from "~/server/core/config";
 import { withContext } from "~/server/core/request-context";
 import { db } from "~/server/db/db";
-import { repositoriesTable, type RepositoryInsert } from "~/server/db/schema";
+import { agentsTable, repositoriesTable, type RepositoryInsert } from "~/server/db/schema";
 import { generateShortId } from "~/server/utils/id";
 import { restic } from "~/server/core/restic";
-import { agentManager } from "~/server/modules/agents/agents-manager";
+import { agentManager, type RestoreExecutionResult } from "~/server/modules/agents/agents-manager";
 import { createTestSession } from "~/test/helpers/auth";
 import { createTestBackupSchedule } from "~/test/helpers/backup";
 import { cache, cacheKeys } from "~/server/utils/cache";
 import { ResticError } from "@zerobyte/core/restic/server";
+import { repoMutex } from "~/server/core/repository-mutex";
+import { taskStore } from "~/server/modules/tasks/tasks.store";
 import { repositoriesService } from "../repositories.service";
 
 const createTestRepository = async (organizationId: string, overrides: Partial<RepositoryInsert> = {}) => {
@@ -118,7 +120,11 @@ describe("repositoriesService.createRepository", () => {
 	test("keeps an explicit local repository path unchanged when importing existing repository", async () => {
 		// arrange
 		const explicitPath = `${REPOSITORY_BASE}/custom-${randomUUID()}`;
-		const config: RepositoryConfig = { backend: "local", path: explicitPath, isExistingRepository: true };
+		const config: RepositoryConfig = {
+			backend: "local",
+			path: explicitPath,
+			isExistingRepository: true,
+		};
 
 		vi.spyOn(restic, "snapshots").mockImplementation(() => Effect.succeed([]));
 
@@ -187,7 +193,9 @@ describe("repositoriesService repository stats", () => {
 		expect(refreshed).toEqual(expectedStats);
 		expect(statsSpy).toHaveBeenCalledTimes(1);
 
-		const persistedRepository = await db.query.repositoriesTable.findFirst({ where: { id: repository.id } });
+		const persistedRepository = await db.query.repositoriesTable.findFirst({
+			where: { id: repository.id },
+		});
 		expect(persistedRepository?.stats).toEqual(expectedStats);
 		expect(typeof persistedRepository?.statsUpdatedAt).toBe("number");
 
@@ -200,6 +208,141 @@ describe("repositoriesService repository stats", () => {
 	});
 });
 
+describe("repositoriesService.listSnapshotFiles", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	test("limits concurrent restic ls commands per repository", async () => {
+		const repository = await createTestRepository(session.organizationId);
+		let active = 0;
+		let maxActive = 0;
+		let releaseAll = false;
+		let exclusiveAcquired = false;
+		let releaseExclusive: (() => void) | undefined;
+		let exclusivePromise: Promise<() => void> | undefined;
+		const releaseWaiters: Array<() => void> = [];
+		const exclusiveController = new AbortController();
+
+		const releaseWaitingCommands = () => {
+			const waiters = releaseWaiters.splice(0);
+			for (const release of waiters) {
+				release();
+			}
+		};
+
+		const resolveWithin = async <T>(promise: Promise<T>, timeoutMs: number) => {
+			return await new Promise<T>((resolve, reject) => {
+				const timeout = setTimeout(() => {
+					reject(new Error(`Expected promise to resolve within ${timeoutMs}ms`));
+				}, timeoutMs);
+
+				promise.then(
+					(value) => {
+						clearTimeout(timeout);
+						resolve(value);
+					},
+					(error) => {
+						clearTimeout(timeout);
+						reject(error);
+					},
+				);
+			});
+		};
+
+		const lsSpy = vi.spyOn(restic, "ls").mockImplementation((_config, snapshotId, _path, options) =>
+			Effect.promise(async () => {
+				active++;
+				maxActive = Math.max(maxActive, active);
+
+				try {
+					if (!releaseAll) {
+						await new Promise<void>((resolve) => releaseWaiters.push(resolve));
+					}
+
+					return {
+						snapshot: {
+							id: snapshotId,
+							short_id: snapshotId,
+							time: new Date().toISOString(),
+							tree: "tree",
+							paths: ["/"],
+							hostname: "host",
+							struct_type: "snapshot" as const,
+							message_type: "snapshot" as const,
+						},
+						nodes: [],
+						pagination: {
+							offset: options.offset ?? 0,
+							limit: options.limit ?? 500,
+							total: 0,
+							hasMore: false,
+						},
+					};
+				} finally {
+					active--;
+				}
+			}),
+		);
+
+		const calls = Array.from({ length: 4 }, (_, index) =>
+			withContext({ organizationId: session.organizationId, userId: session.user.id }, () =>
+				repositoriesService.listSnapshotFiles(repository.shortId, `snapshot-${index}`, "/", {
+					offset: 0,
+					limit: 100,
+				}),
+			),
+		);
+
+		try {
+			await waitForExpect(() => {
+				expect(releaseWaiters).toHaveLength(2);
+			});
+			expect(maxActive).toBe(2);
+
+			exclusivePromise = repoMutex
+				.acquireExclusive(repository.id, "delete", exclusiveController.signal)
+				.then((release) => {
+					exclusiveAcquired = true;
+					releaseExclusive = release;
+					return release;
+				});
+
+			releaseWaitingCommands();
+
+			releaseExclusive = await resolveWithin(exclusivePromise, 2000);
+			expect(exclusiveAcquired).toBe(true);
+			expect(active).toBe(0);
+
+			releaseExclusive();
+			releaseExclusive = undefined;
+
+			await waitForExpect(() => {
+				expect(releaseWaiters).toHaveLength(2);
+			});
+			expect(maxActive).toBe(2);
+
+			releaseWaitingCommands();
+			await Promise.all(calls);
+		} finally {
+			if (releaseExclusive) {
+				releaseExclusive();
+			} else {
+				exclusiveController.abort();
+			}
+			releaseAll = true;
+			releaseWaitingCommands();
+			await Promise.allSettled(calls);
+			if (exclusivePromise) {
+				await Promise.allSettled([exclusivePromise]);
+			}
+		}
+
+		expect(lsSpy).toHaveBeenCalledTimes(4);
+		expect(maxActive).toBeLessThanOrEqual(2);
+	});
+});
+
 describe("repositoriesService.dumpSnapshot", () => {
 	afterEach(() => {
 		vi.restoreAllMocks();
@@ -408,6 +551,28 @@ describe("repositoriesService.dumpSnapshot", () => {
 
 describe("repositoriesService.restoreSnapshot", () => {
 	let originalEnableLocalAgent: boolean;
+	const createPendingRestoreStart = () => ({
+		status: "started" as const,
+		result: new Promise<RestoreExecutionResult>(() => {}),
+	});
+	const resolveWithin = async <T>(promise: Promise<T>, timeoutMs: number) => {
+		return await new Promise<T>((resolve, reject) => {
+			const timeout = setTimeout(() => {
+				reject(new Error(`Expected promise to resolve within ${timeoutMs}ms`));
+			}, timeoutMs);
+
+			promise.then(
+				(value) => {
+					clearTimeout(timeout);
+					resolve(value);
+				},
+				(error) => {
+					clearTimeout(timeout);
+					reject(error);
+				},
+			);
+		});
+	};
 
 	beforeEach(() => {
 		originalEnableLocalAgent = config.flags.enableLocalAgent;
@@ -435,44 +600,31 @@ describe("repositoriesService.restoreSnapshot", () => {
 			]),
 		);
 
-		const restoreMock = vi.fn(() =>
-			Promise.resolve({
-				message_type: "summary" as const,
-				seconds_elapsed: 1,
-				percent_done: 100,
-				files_skipped: 0,
-				total_files: 1,
-				files_restored: 1,
-				total_bytes: 1,
-				bytes_restored: 1,
-			}),
-		);
-		vi.spyOn(agentManager, "runRestoreCommand").mockImplementation(restoreMock);
+		const restoreMock = vi.fn(() => Promise.resolve(createPendingRestoreStart()));
+		vi.spyOn(agentManager, "startRestore").mockImplementation(restoreMock);
 
 		return {
 			organizationId,
 			userId: session.user.id,
+			repositoryId: repository.id,
 			repositoryShortId: repository.shortId,
 			restoreMock,
 		};
 	};
 
-	test("delegates restore target protection to the selected agent", async () => {
+	test("rejects protected targets even when the local agent is enabled", async () => {
 		const { organizationId, userId, repositoryShortId, restoreMock } = await setupRestoreSnapshotScenario();
 		const targetPath = nodePath.join(os.tmpdir(), "zerobyte-restore-target");
-		restoreMock.mockRejectedValueOnce(
-			new Error(
-				"Restore target path is not allowed. Restoring to this path could overwrite critical system files or application data.",
-			),
-		);
 
 		await expect(
 			withContext({ organizationId, userId }, () =>
-				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", { targetPath }),
+				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+					targetPath,
+				}),
 			),
 		).rejects.toThrow("Restore target path is not allowed");
 
-		expect(restoreMock).toHaveBeenCalledOnce();
+		expect(restoreMock).not.toHaveBeenCalled();
 	});
 
 	test("restores to a custom target outside protected roots", async () => {
@@ -481,28 +633,125 @@ describe("repositoriesService.restoreSnapshot", () => {
 
 		try {
 			await withContext({ organizationId, userId }, () =>
-				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", { targetPath }),
+				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+					targetPath,
+				}),
 			);
 		} finally {
 			await fs.rm(targetPath, { recursive: true, force: true });
 		}
 
-		expect(restoreMock).toHaveBeenCalledWith(
-			"local",
-			expect.objectContaining({
-				snapshotId: "snapshot-restore",
-				target: targetPath,
-				repositoryConfig: expect.objectContaining({ backend: "local" }),
-				options: expect.objectContaining({
-					organizationId,
-					basePath: "/var/lib/zerobyte/volumes/vol123/_data",
+		await waitForExpect(() => {
+			expect(restoreMock).toHaveBeenCalledWith(
+				"local",
+				expect.objectContaining({
+					payload: expect.objectContaining({
+						snapshotId: "snapshot-restore",
+						target: targetPath,
+						repositoryConfig: expect.objectContaining({ backend: "local" }),
+						options: expect.objectContaining({
+							organizationId,
+							basePath: "/var/lib/zerobyte/volumes/vol123/_data",
+						}),
+					}),
 				}),
-			}),
+			);
+		});
+	});
+
+	test("rejects starting a second active restore for the same snapshot", async () => {
+		const { organizationId, userId, repositoryShortId } = await setupRestoreSnapshotScenario();
+		const targetPath = await fs.mkdtemp(nodePath.join(process.cwd(), "restore-target-"));
+
+		try {
+			await withContext({ organizationId, userId }, () =>
+				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+					targetPath,
+				}),
+			);
+
+			await expect(
+				withContext({ organizationId, userId }, () =>
+					repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+						targetPath,
+					}),
+				),
+			).rejects.toThrow("A restore is already running for this snapshot");
+		} finally {
+			await fs.rm(targetPath, { recursive: true, force: true });
+		}
+	});
+
+	test("returns a restore id while waiting for the repository mutex", async () => {
+		const { organizationId, userId, repositoryId, repositoryShortId, restoreMock } =
+			await setupRestoreSnapshotScenario();
+		const targetPath = await fs.mkdtemp(nodePath.join(process.cwd(), "restore-target-"));
+		await withContext({ organizationId, userId }, () =>
+			repositoriesService.getSnapshotDetails(repositoryShortId, "snapshot-restore"),
 		);
+
+		let finishRestore: (result: RestoreExecutionResult) => void = () => {};
+		const restoreResult = new Promise<RestoreExecutionResult>((resolve) => {
+			finishRestore = resolve;
+		});
+		restoreMock.mockResolvedValueOnce({ status: "started", result: restoreResult });
+
+		const releaseExclusive = await repoMutex.acquireExclusive(repositoryId, "check");
+		let restoreId = "";
+		try {
+			const restoreStart = withContext({ organizationId, userId }, () =>
+				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+					targetPath,
+				}),
+			);
+
+			const result = await resolveWithin(restoreStart, 1000);
+			restoreId = result.restoreId;
+
+			expect(result.status).toBe("started");
+			expect(restoreMock).not.toHaveBeenCalled();
+
+			const task = taskStore.findActiveByResource({
+				organizationId,
+				kind: "restore",
+				resourceType: "repository",
+				resourceId: repositoryShortId,
+			});
+			expect(task?.id).toBe(restoreId);
+			expect(task?.status).toBe("queued");
+		} finally {
+			releaseExclusive();
+			await fs.rm(targetPath, { recursive: true, force: true });
+		}
+
+		await waitForExpect(() => {
+			expect(restoreMock).toHaveBeenCalledTimes(1);
+		});
+
+		finishRestore({
+			status: "completed",
+			result: {
+				message_type: "summary",
+				files_skipped: 0,
+				files_restored: 1,
+			},
+		});
+
+		await waitForExpect(() => {
+			expect(
+				taskStore.findActiveByResource({
+					organizationId,
+					kind: "restore",
+					resourceType: "repository",
+					resourceId: repositoryShortId,
+				}),
+			).toBeNull();
+		});
 	});
 
 	test("routes restore to the requested target agent", async () => {
 		const organizationId = session.organizationId;
+		const agentId = `agent-${randomUUID()}`;
 		const repository = await createTestRepository(organizationId, {
 			type: "s3",
 			config: {
@@ -513,6 +762,15 @@ describe("repositoriesService.restoreSnapshot", () => {
 				secretAccessKey: "secret-key",
 			},
 		});
+		await db.insert(agentsTable).values({
+			id: agentId,
+			organizationId,
+			name: "Remote Agent",
+			kind: "remote",
+			status: "online",
+			capabilities: {},
+			updatedAt: Date.now(),
+		});
 		vi.spyOn(restic, "snapshots").mockReturnValue(
 			Effect.succeed([
 				{
@@ -524,33 +782,80 @@ describe("repositoriesService.restoreSnapshot", () => {
 				},
 			]),
 		);
-		const restoreMock = vi.fn(() =>
-			Promise.resolve({
-				message_type: "summary" as const,
-				files_skipped: 0,
-				files_restored: 1,
-			}),
-		);
-		vi.spyOn(agentManager, "runRestoreCommand").mockImplementation(restoreMock);
+		const restoreMock = vi.fn(() => Promise.resolve(createPendingRestoreStart()));
+		vi.spyOn(agentManager, "startRestore").mockImplementation(restoreMock);
 		const targetPath = await fs.mkdtemp(nodePath.join(process.cwd(), "restore-target-"));
 
 		try {
 			await withContext({ organizationId, userId: session.user.id }, () =>
 				repositoriesService.restoreSnapshot(repository.shortId, "snapshot-restore", {
 					targetPath,
-					targetAgentId: "agent-remote",
+					targetAgentId: agentId,
 				}),
 			);
 		} finally {
 			await fs.rm(targetPath, { recursive: true, force: true });
 		}
 
-		expect(restoreMock).toHaveBeenCalledWith(
-			"agent-remote",
-			expect.objectContaining({
-				target: targetPath,
-			}),
+		await waitForExpect(() => {
+			expect(restoreMock).toHaveBeenCalledWith(
+				agentId,
+				expect.objectContaining({
+					payload: expect.objectContaining({
+						target: targetPath,
+					}),
+				}),
+			);
+		});
+	});
+
+	test("rejects a target agent outside the current organization", async () => {
+		const organizationId = session.organizationId;
+		const otherSession = await createTestSession();
+		const otherAgentId = `agent-${randomUUID()}`;
+		const repository = await createTestRepository(organizationId, {
+			type: "s3",
+			config: {
+				backend: "s3",
+				endpoint: "https://s3.example.com",
+				bucket: "bucket",
+				accessKeyId: "access-key",
+				secretAccessKey: "secret-key",
+			},
+		});
+
+		await db.insert(agentsTable).values({
+			id: otherAgentId,
+			organizationId: otherSession.organizationId,
+			name: "Other Org Agent",
+			kind: "remote",
+			status: "online",
+			capabilities: {},
+			updatedAt: Date.now(),
+		});
+		vi.spyOn(restic, "snapshots").mockReturnValue(
+			Effect.succeed([
+				{
+					id: "snapshot-restore",
+					short_id: "snapshot-restore",
+					time: new Date().toISOString(),
+					paths: ["/var/lib/zerobyte/volumes/vol123/_data"],
+					hostname: "host",
+				},
+			]),
 		);
+		const restoreMock = vi.fn(() => Promise.resolve(createPendingRestoreStart()));
+		vi.spyOn(agentManager, "startRestore").mockImplementation(restoreMock);
+
+		await expect(
+			withContext({ organizationId, userId: session.user.id }, () =>
+				repositoriesService.restoreSnapshot(repository.shortId, "snapshot-restore", {
+					targetAgentId: otherAgentId,
+				}),
+			),
+		).rejects.toThrow("Restore target agent not found");
+
+		expect(restoreMock).not.toHaveBeenCalled();
 	});
 
 	test("uses controller-local restore fallback when local agent supervision is disabled", async () => {
@@ -567,19 +872,27 @@ describe("repositoriesService.restoreSnapshot", () => {
 
 		try {
 			await withContext({ organizationId, userId }, () =>
-				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", { targetPath }),
+				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+					targetPath,
+				}),
 			);
 		} finally {
 			await fs.rm(targetPath, { recursive: true, force: true });
 		}
 
 		expect(restoreMock).not.toHaveBeenCalled();
-		expect(resticRestoreMock).toHaveBeenCalledWith(
-			expect.objectContaining({ backend: "local" }),
-			"snapshot-restore",
-			targetPath,
-			expect.objectContaining({ organizationId, basePath: "/var/lib/zerobyte/volumes/vol123/_data" }),
-		);
+		await waitForExpect(() => {
+			expect(resticRestoreMock).toHaveBeenCalledWith(
+				expect.objectContaining({ backend: "local" }),
+				"snapshot-restore",
+				targetPath,
+				expect.objectContaining({
+					organizationId,
+					basePath: "/var/lib/zerobyte/volumes/vol123/_data",
+					signal: expect.any(AbortSignal),
+				}),
+			);
+		});
 	});
 
 	test("rejects original-location restore for snapshots with non-posix source paths", async () => {
@@ -607,24 +920,30 @@ describe("repositoriesService.restoreSnapshot", () => {
 
 		try {
 			await withContext({ organizationId, userId }, () =>
-				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", { targetPath }),
+				repositoriesService.restoreSnapshot(repositoryShortId, "snapshot-restore", {
+					targetPath,
+				}),
 			);
 		} finally {
 			await fs.rm(targetPath, { recursive: true, force: true });
 		}
 
-		expect(restoreMock).toHaveBeenCalledWith(
-			"local",
-			expect.objectContaining({
-				snapshotId: "snapshot-restore",
-				target: targetPath,
-				repositoryConfig: expect.objectContaining({ backend: "local" }),
-				options: expect.objectContaining({
-					organizationId,
-					basePath: "/",
+		await waitForExpect(() => {
+			expect(restoreMock).toHaveBeenCalledWith(
+				"local",
+				expect.objectContaining({
+					payload: expect.objectContaining({
+						snapshotId: "snapshot-restore",
+						target: targetPath,
+						repositoryConfig: expect.objectContaining({ backend: "local" }),
+						options: expect.objectContaining({
+							organizationId,
+							basePath: "/",
+						}),
+					}),
 				}),
-			}),
-		);
+			);
+		});
 	});
 });
 
@@ -635,9 +954,14 @@ describe("repositoriesService.getRetentionCategories", () => {
 
 	test("recomputes retention categories after repository cache invalidation", async () => {
 		const organizationId = session.organizationId;
-		const schedule = await createTestBackupSchedule({ organizationId, retentionPolicy: { keepLast: 1 } });
+		const schedule = await createTestBackupSchedule({
+			organizationId,
+			retentionPolicy: { keepLast: 1 },
+		});
 
-		const repository = await db.query.repositoriesTable.findFirst({ where: { id: schedule.repositoryId } });
+		const repository = await db.query.repositoriesTable.findFirst({
+			where: { id: schedule.repositoryId },
+		});
 
 		expect(repository).toBeTruthy();
 		if (!repository) {
@@ -721,7 +1045,9 @@ describe("repositoriesService.deleteSnapshot", () => {
 			expect(statsSpy).toHaveBeenCalledTimes(1);
 		});
 
-		const updatedRepository = await db.query.repositoriesTable.findFirst({ where: { id: repository.id } });
+		const updatedRepository = await db.query.repositoriesTable.findFirst({
+			where: { id: repository.id },
+		});
 		expect(updatedRepository?.stats).toEqual(expectedStats);
 		expect(typeof updatedRepository?.statsUpdatedAt).toBe("number");
 	});

app/server/modules/repositories/repositories.controller.ts

@@ -182,7 +182,10 @@ export const repositoriesController = new Hono()
 			const offset = Math.max(0, query.offset ?? 0);
 			const limit = Math.min(1000, Math.max(1, query.limit ?? 500));
 
-			const result = await repositoriesService.listSnapshotFiles(shortId, snapshotId, decodedPath, { offset, limit });
+			const result = await repositoriesService.listSnapshotFiles(shortId, snapshotId, decodedPath, {
+				offset,
+				limit,
+			});
 
 			c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");
 
@@ -235,7 +238,7 @@ export const repositoriesController = new Hono()
 		const { snapshotId, ...options } = c.req.valid("json");
 		const result = await repositoriesService.restoreSnapshot(shortId, snapshotId, options);
 
-		return c.json<RestoreSnapshotDto>(result, 200);
+		return c.json<RestoreSnapshotDto>(result, 202);
 	})
 	.post("/:shortId/doctor", startDoctorDto, async (c) => {
 		const shortId = asShortId(c.req.param("shortId"));

app/server/modules/repositories/repositories.dto.ts

@@ -348,10 +348,8 @@ export const restoreSnapshotBody = z.object({
 });
 
 const restoreSnapshotResponse = z.object({
-	success: z.boolean(),
-	message: z.string(),
-	filesRestored: z.number(),
-	filesSkipped: z.number(),
+	restoreId: z.string(),
+	status: z.literal("started"),
 });
 
 export type RestoreSnapshotDto = z.infer<typeof restoreSnapshotResponse>;
@@ -361,8 +359,8 @@ export const restoreSnapshotDto = describeRoute({
 	tags: ["Repositories"],
 	operationId: "restoreSnapshot",
 	responses: {
-		200: {
-			description: "Snapshot restored successfully",
+		202: {
+			description: "Snapshot restore started",
 			content: {
 				"application/json": {
 					schema: resolver(restoreSnapshotResponse),

app/server/modules/repositories/repositories.service.ts

@@ -30,14 +30,23 @@ import type { DumpPathKind, UpdateRepositoryBody } from "./repositories.dto";
 import { findCommonAncestor } from "@zerobyte/core/utils";
 import { prepareSnapshotDump } from "./helpers/dump";
 import { executeDoctor } from "./helpers/doctor";
+import { restoreExecutor } from "./restore-executor";
 import type { ShortId } from "~/server/utils/branded";
 import { decryptRepositoryConfig, encryptRepositoryConfig } from "./repository-config-secrets";
 import { getScheduleByIdOrShortId } from "../backups/helpers/backup-schedule-lookups";
-import { agentManager } from "../agents/agents-manager";
+import type { RestoreExecutionProgress, RestoreExecutionResult } from "../agents/agents-manager";
+import { agentsService } from "../agents/agents.service";
 import { LOCAL_AGENT_ID } from "../agents/constants";
+import { taskStore } from "../tasks/tasks.store";
+import type { ParsedTask, TaskInput } from "../tasks/tasks.schemas";
 import { Effect } from "effect";
 
 const runningDoctors = new Map<string, AbortController>();
+const lsLimiters = new Map<string, Effect.Semaphore>();
+const RESTORE_TASK_RESOURCE_TYPE = "repository";
+
+type RestoreTaskInput = Extract<TaskInput, { kind: "restore" }>;
+type RestoreTask = ParsedTask & { kind: "restore"; input: RestoreTaskInput };
 
 const emptyRepositoryStats: ResticStatsDto = {
 	total_size: 0,
@@ -70,8 +79,171 @@ const assertAllowedControllerLocalRestoreTarget = (target: string) => {
 	}
 };
 
-const shouldUseControllerLocalRestoreFallback = (agentId: string) =>
-	agentId === LOCAL_AGENT_ID && !appConfig.flags.enableLocalAgent;
+const isRestoreTask = (task: ParsedTask): task is RestoreTask =>
+	task.kind === "restore" && task.input.kind === "restore";
+
+const asRestoreTask = (task: ParsedTask, restoreId: string, eventName: string) => {
+	if (!isRestoreTask(task)) {
+		logger.warn(`Received ${eventName} for non-restore task ${restoreId}`);
+		return null;
+	}
+
+	return task;
+};
+
+const updateActiveRestoreTask = (restoreId: string, eventName: string, update: () => ParsedTask) => {
+	try {
+		return asRestoreTask(update(), restoreId, eventName);
+	} catch (error) {
+		logger.warn(`Received ${eventName} for inactive restore ${restoreId}: ${toMessage(error)}`);
+		return null;
+	}
+};
+
+const getLsLimiter = (repositoryId: string) => {
+	let limiter = lsLimiters.get(repositoryId);
+	if (!limiter) {
+		limiter = Effect.runSync(Effect.makeSemaphore(2));
+		lsLimiters.set(repositoryId, limiter);
+	}
+	return limiter;
+};
+
+const findActiveRestoreTask = (
+	organizationId: string,
+	repositoryShortId: string,
+	snapshotId: string,
+): RestoreTask | null => {
+	return (
+		taskStore
+			.listActiveByResource({
+				organizationId,
+				kind: "restore",
+				resourceType: RESTORE_TASK_RESOURCE_TYPE,
+				resourceId: repositoryShortId,
+			})
+			.find((task): task is RestoreTask => isRestoreTask(task) && task.input.snapshotId === snapshotId) ?? null
+	);
+};
+
+const emitRestoreStarted = (task: RestoreTask) => {
+	serverEvents.emit("restore:started", {
+		restoreId: task.id,
+		organizationId: task.organizationId,
+		repositoryId: task.input.repositoryId,
+		snapshotId: task.input.snapshotId,
+	});
+};
+
+const emitRestoreProgress = (task: RestoreTask, progress: RestoreExecutionProgress) => {
+	serverEvents.emit("restore:progress", {
+		restoreId: task.id,
+		organizationId: task.organizationId,
+		repositoryId: task.input.repositoryId,
+		snapshotId: task.input.snapshotId,
+		...progress,
+	});
+};
+
+const emitRestoreCompleted = (
+	task: RestoreTask,
+	payload: {
+		status: "success" | "error" | "cancelled";
+		error?: string;
+		filesRestored?: number;
+		filesSkipped?: number;
+	},
+) => {
+	serverEvents.emit("restore:completed", {
+		restoreId: task.id,
+		organizationId: task.organizationId,
+		repositoryId: task.input.repositoryId,
+		snapshotId: task.input.snapshotId,
+		...payload,
+	});
+};
+
+const markRestoreStarted = (restoreId: string) => {
+	const task = updateActiveRestoreTask(restoreId, "restore.started", () => taskStore.markRunning(restoreId));
+	if (!task) return;
+
+	emitRestoreStarted(task);
+};
+
+const updateRestoreProgress = (restoreId: string, progress: RestoreExecutionProgress) => {
+	const task = updateActiveRestoreTask(restoreId, "restore.progress", () =>
+		taskStore.updateProgress(restoreId, { kind: "restore", progress }),
+	);
+	if (!task) return;
+
+	emitRestoreProgress(task, progress);
+};
+
+const completeRestoreTask = (
+	restoreId: string,
+	result: Extract<RestoreExecutionResult, { status: "completed" }>["result"],
+) => {
+	const task = updateActiveRestoreTask(restoreId, "restore.completed", () =>
+		taskStore.complete(restoreId, { kind: "restore", result }),
+	);
+	if (!task) return;
+
+	emitRestoreCompleted(task, {
+		status: "success",
+		filesRestored: result.files_restored,
+		filesSkipped: result.files_skipped,
+	});
+};
+
+const failRestoreTask = (restoreId: string, error: string) => {
+	const task = updateActiveRestoreTask(restoreId, "restore.failed", () => taskStore.fail(restoreId, error));
+	if (!task) return;
+
+	emitRestoreCompleted(task, { status: "error", error });
+};
+
+const cancelRestoreTask = (restoreId: string, message?: string) => {
+	const task = updateActiveRestoreTask(restoreId, "restore.cancelled", () =>
+		taskStore.cancel(restoreId, message ?? null),
+	);
+	if (!task) return;
+
+	emitRestoreCompleted(task, { status: "cancelled", error: task.cancellationRequested ? undefined : message });
+};
+
+const finishRestoreExecution = async (restoreId: string, resultPromise: Promise<RestoreExecutionResult>) => {
+	try {
+		const result = await resultPromise;
+
+		switch (result.status) {
+			case "completed":
+				completeRestoreTask(restoreId, result.result);
+				return;
+			case "failed":
+				failRestoreTask(restoreId, result.error);
+				return;
+			case "cancelled":
+				cancelRestoreTask(restoreId, result.message);
+				return;
+			case "unavailable":
+				failRestoreTask(restoreId, result.error.message);
+				return;
+		}
+	} catch (error) {
+		failRestoreTask(restoreId, toMessage(error));
+	}
+};
+
+const assertAllowedRestoreAgent = async (agentId: string, organizationId: string) => {
+	if (agentId === LOCAL_AGENT_ID) {
+		return;
+	}
+
+	const agent = await agentsService.getAgent(agentId);
+	if (!agent || agent.organizationId !== organizationId) {
+		throw new NotFoundError("Restore target agent not found");
+	}
+};
 
 const findRepository = async (shortId: ShortId) => {
 	const organizationId = getOrganizationId();
@@ -285,7 +457,13 @@ const listSnapshotFiles = async (
 
 	const cacheKey = cacheKeys.repository.ls(repository.id, snapshotId, path, offset, limit);
 	type LsResult = {
-		snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null;
+		snapshot: {
+			id: string;
+			short_id: string;
+			time: string;
+			hostname: string;
+			paths: string[];
+		} | null;
 		nodes: { name: string; type: string; path: string; size?: number; mode?: number }[];
 		pagination: { offset: number; limit: number; total: number; hasMore: boolean };
 	};
@@ -301,36 +479,43 @@ const listSnapshotFiles = async (
 		};
 	}
 
-	const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
+	const limiter = getLsLimiter(repository.id);
+	await runEffectPromise(limiter.take(1));
+
 	try {
-		const result = await runEffectPromise(
-			restic.ls(repository.config, snapshotId, path, { organizationId, offset, limit }),
-		);
+		const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
+		try {
+			const result = await runEffectPromise(
+				restic.ls(repository.config, snapshotId, path, { organizationId, offset, limit }),
+			);
 
-		if (!result.snapshot) {
-			throw new NotFoundError("Snapshot not found or empty");
+			if (!result.snapshot) {
+				throw new NotFoundError("Snapshot not found or empty");
+			}
+
+			const response = {
+				snapshot: {
+					id: result.snapshot.id,
+					short_id: result.snapshot.short_id,
+					time: result.snapshot.time,
+					hostname: result.snapshot.hostname,
+					paths: result.snapshot.paths,
+				},
+				files: result.nodes,
+				offset: result.pagination.offset,
+				limit: result.pagination.limit,
+				total: result.pagination.total,
+				hasMore: result.pagination.hasMore,
+			};
+
+			cache.set(cacheKey, result);
+
+			return response;
+		} finally {
+			releaseLock();
 		}
-
-		const response = {
-			snapshot: {
-				id: result.snapshot.id,
-				short_id: result.snapshot.short_id,
-				time: result.snapshot.time,
-				hostname: result.snapshot.hostname,
-				paths: result.snapshot.paths,
-			},
-			files: result.nodes,
-			offset: result.pagination.offset,
-			limit: result.pagination.limit,
-			total: result.pagination.total,
-			hasMore: result.pagination.hasMore,
-		};
-
-		cache.set(cacheKey, result);
-
-		return response;
 	} finally {
-		releaseLock();
+		await runEffectPromise(limiter.release(1));
 	}
 };
 
@@ -355,20 +540,22 @@ const restoreSnapshot = async (
 		throw new NotFoundError("Repository not found");
 	}
 
-	const target = options?.targetPath || "/";
+	const { targetAgentId, targetPath, ...restoreExecutionOptions } = options ?? {};
+	const target = targetPath || "/";
 
 	const snapshot = await getSnapshotDetails(repository.shortId, snapshotId);
 	const hasNonPosixSnapshotPaths = snapshot.paths.some((path) => !path.startsWith("/"));
 
-	if (hasNonPosixSnapshotPaths && !options?.targetPath) {
+	if (hasNonPosixSnapshotPaths && !targetPath) {
 		throw new BadRequestError(
 			"Original location restore is unavailable for this snapshot. Restore it to a custom location instead.",
 		);
 	}
 
 	const basePath = hasNonPosixSnapshotPaths ? "/" : findCommonAncestor(snapshot.paths);
-	const executionAgentId = options?.targetAgentId ?? LOCAL_AGENT_ID;
-	const useControllerLocalRestoreFallback = shouldUseControllerLocalRestoreFallback(executionAgentId);
+	const executionAgentId = targetAgentId ?? LOCAL_AGENT_ID;
+	const useControllerLocalRestoreFallback = executionAgentId === LOCAL_AGENT_ID && !appConfig.flags.enableLocalAgent;
+	await assertAllowedRestoreAgent(executionAgentId, organizationId);
 
 	if (!useControllerLocalRestoreFallback && repository.type === "local" && executionAgentId !== LOCAL_AGENT_ID) {
 		throw new BadRequestError(
@@ -376,76 +563,48 @@ const restoreSnapshot = async (
 		);
 	}
 
-	const releaseLock = await repoMutex.acquireShared(repository.id, `restore:${snapshotId}`);
+	if (executionAgentId === LOCAL_AGENT_ID) {
+		assertAllowedControllerLocalRestoreTarget(target);
+	}
+
+	const activeRestore = findActiveRestoreTask(organizationId, repository.shortId, snapshotId);
+	if (activeRestore) {
+		throw new ConflictError("A restore is already running for this snapshot");
+	}
+
+	const task = taskStore.create({
+		organizationId,
+		resourceType: RESTORE_TASK_RESOURCE_TYPE,
+		resourceId: repository.shortId,
+		targetAgentId: useControllerLocalRestoreFallback ? null : executionAgentId,
+		input: { kind: "restore", repositoryId: repository.shortId, snapshotId, target },
+	});
+	const restoreId = task.id;
 	try {
-		serverEvents.emit("restore:started", {
-			organizationId,
-			repositoryId: repository.shortId,
-			snapshotId,
-		});
-
 		const repositoryConfig = await decryptRepositoryConfig(repository.config);
-		let result;
-
-		if (useControllerLocalRestoreFallback) {
-			assertAllowedControllerLocalRestoreTarget(target);
-			result = await runEffectPromise(
-				restic.restore(repositoryConfig, snapshotId, target, {
-					basePath,
-					...options,
-					organizationId,
-					onProgress: (progress) => {
-						serverEvents.emit("restore:progress", {
-							organizationId,
-							repositoryId: repository.shortId,
-							snapshotId,
-							...progress,
-						});
-					},
-				}),
-			);
-		} else {
-			const encryptedResticPassword = await resticDeps.getOrganizationResticPassword(organizationId);
-			const resticPassword = await resticDeps.resolveSecret(encryptedResticPassword);
-			result = await agentManager.runRestoreCommand(executionAgentId, {
-				organizationId,
-				repositoryId: repository.shortId,
-				snapshotId,
-				target,
-				repositoryConfig,
-				runtime: { password: resticPassword },
-				options: {
-					basePath,
-					...options,
-					organizationId,
-				},
-			});
-		}
-
-		serverEvents.emit("restore:completed", {
+		const execution = restoreExecutor.start({
+			restoreId,
 			organizationId,
-			repositoryId: repository.shortId,
+			repositoryId: repository.id,
+			repositoryShortId: repository.shortId,
+			repositoryConfig,
 			snapshotId,
-			status: "success",
+			target,
+			executionAgentId,
+			options: {
+				basePath,
+				...restoreExecutionOptions,
+			},
+			onStarted: () => markRestoreStarted(restoreId),
+			onProgress: (progress) => updateRestoreProgress(restoreId, progress),
 		});
 
-		return {
-			success: true,
-			message: "Snapshot restored successfully",
-			filesRestored: result.files_restored,
-			filesSkipped: result.files_skipped,
-		};
+		void finishRestoreExecution(restoreId, execution.result);
+
+		return { restoreId, status: "started" as const };
 	} catch (error) {
-		serverEvents.emit("restore:completed", {
-			organizationId,
-			repositoryId: repository.shortId,
-			snapshotId,
-			status: "error",
-			error: toMessage(error),
-		});
+		failRestoreTask(restoreId, toMessage(error));
 		throw error;
-	} finally {
-		releaseLock();
 	}
 };
 
@@ -462,7 +621,11 @@ const dumpSnapshot = async (shortId: ShortId, snapshotId: string, path?: string,
 
 	try {
 		const snapshot = await getSnapshotDetails(repository.shortId, snapshotId);
-		const preparedDump = prepareSnapshotDump({ snapshotId, snapshotPaths: snapshot.paths, requestedPath: path });
+		const preparedDump = prepareSnapshotDump({
+			snapshotId,
+			snapshotPaths: snapshot.paths,
+			requestedPath: path,
+		});
 		const dumpOptions: Parameters<typeof restic.dump>[2] = {
 			organizationId,
 			path: preparedDump.path,
@@ -853,7 +1016,14 @@ const execResticCommand = async (
 	addCommonArgs(resticArgs, env, repository.config);
 
 	try {
-		const result = await safeSpawn({ command: "restic", args: resticArgs, env, signal, onStdout, onStderr });
+		const result = await safeSpawn({
+			command: "restic",
+			args: resticArgs,
+			env,
+			signal,
+			onStdout,
+			onStderr,
+		});
 		return { exitCode: result.exitCode };
 	} finally {
 		await cleanupTemporaryKeys(env, resticDeps);

app/server/modules/repositories/restore-executor.ts

@@ -0,0 +1,145 @@
+import type { RepositoryConfig } from "@zerobyte/core/restic";
+import type { RestoreRunPayload } from "@zerobyte/contracts/agent-protocol";
+import { config as appConfig } from "~/server/core/config";
+import { repoMutex } from "../../core/repository-mutex";
+import { restic, resticDeps } from "../../core/restic";
+import { runEffectPromise, toMessage } from "../../utils/errors";
+import { agentManager, type RestoreExecutionProgress, type RestoreExecutionResult } from "../agents/agents-manager";
+import { LOCAL_AGENT_ID } from "../agents/constants";
+
+type RestoreExecutionOptions = Omit<Parameters<typeof restic.restore>[3], "organizationId" | "signal" | "onProgress">;
+
+type RestoreExecutionRequest = {
+	restoreId: string;
+	organizationId: string;
+	repositoryId: string;
+	repositoryShortId: string;
+	repositoryConfig: RepositoryConfig;
+	snapshotId: string;
+	target: string;
+	executionAgentId: string;
+	options: RestoreExecutionOptions;
+	onStarted: () => void;
+	onProgress: (progress: RestoreExecutionProgress) => void;
+};
+
+type RestoreExecutionHandle = {
+	result: Promise<RestoreExecutionResult>;
+};
+
+const shouldRunInController = (agentId: string) => agentId === LOCAL_AGENT_ID && !appConfig.flags.enableLocalAgent;
+
+const createRestoreRunPayload = async (request: RestoreExecutionRequest): Promise<RestoreRunPayload> => {
+	const encryptedResticPassword = await resticDeps.getOrganizationResticPassword(request.organizationId);
+	const resticPassword = await resticDeps.resolveSecret(encryptedResticPassword);
+
+	return {
+		restoreId: request.restoreId,
+		organizationId: request.organizationId,
+		repositoryId: request.repositoryShortId,
+		snapshotId: request.snapshotId,
+		target: request.target,
+		repositoryConfig: request.repositoryConfig,
+		runtime: { password: resticPassword },
+		options: {
+			...request.options,
+			organizationId: request.organizationId,
+		},
+	};
+};
+
+const executeControllerRestore = async (
+	request: RestoreExecutionRequest,
+	signal: AbortSignal,
+): Promise<RestoreExecutionResult> => {
+	if (signal.aborted) {
+		return { status: "cancelled", message: "Restore was cancelled" };
+	}
+
+	request.onStarted();
+
+	try {
+		const result = await runEffectPromise(
+			restic.restore(request.repositoryConfig, request.snapshotId, request.target, {
+				...request.options,
+				organizationId: request.organizationId,
+				signal,
+				onProgress: request.onProgress,
+			}),
+		);
+
+		return { status: "completed", result };
+	} catch (error) {
+		if (signal.aborted) {
+			return { status: "cancelled", message: "Restore was cancelled" };
+		}
+
+		return { status: "failed", error: toMessage(error) };
+	}
+};
+
+const executeAgentRestore = async (
+	request: RestoreExecutionRequest,
+	signal: AbortSignal,
+): Promise<RestoreExecutionResult> => {
+	if (signal.aborted) {
+		return { status: "cancelled", message: "Restore was cancelled" };
+	}
+
+	try {
+		const payload = await createRestoreRunPayload(request);
+		const started = await agentManager.startRestore(request.executionAgentId, {
+			payload,
+			signal,
+			onStarted: request.onStarted,
+			onProgress: request.onProgress,
+		});
+
+		if (started.status === "unavailable") {
+			return started;
+		}
+
+		return await started.result;
+	} catch (error) {
+		if (signal.aborted) {
+			return { status: "cancelled", message: "Restore was cancelled" };
+		}
+
+		return { status: "failed", error: toMessage(error) };
+	}
+};
+
+const executeRestoreWithRepositoryLock = async (
+	request: RestoreExecutionRequest,
+	signal: AbortSignal,
+): Promise<RestoreExecutionResult> => {
+	let releaseLock: (() => void) | null = null;
+
+	try {
+		releaseLock = await repoMutex.acquireShared(request.repositoryId, `restore:${request.restoreId}`, signal);
+
+		if (shouldRunInController(request.executionAgentId)) {
+			return await executeControllerRestore(request, signal);
+		}
+
+		return await executeAgentRestore(request, signal);
+	} catch (error) {
+		if (signal.aborted) {
+			return { status: "cancelled", message: "Restore was cancelled" };
+		}
+
+		return { status: "failed", error: toMessage(error) };
+	} finally {
+		releaseLock?.();
+	}
+};
+
+export const restoreExecutor = {
+	start: (request: RestoreExecutionRequest): RestoreExecutionHandle => {
+		const abortController = new AbortController();
+
+		return {
+			result: executeRestoreWithRepositoryLock(request, abortController.signal),
+		};
+	},
+};

app/server/modules/tasks/__tests__/tasks.store.test.ts

@@ -0,0 +1,251 @@
+import { beforeEach, expect, test } from "vitest";
+import { db } from "~/server/db/db";
+import { tasksTable } from "~/server/db/schema";
+import { ensureTestOrganization, TEST_ORG_ID } from "~/test/helpers/organization";
+import { generateBackupOutput } from "~/test/helpers/restic";
+import { taskStore } from "../tasks.store";
+import type { TaskInput, TaskProgress, TaskResult } from "../tasks.schemas";
+
+type BackupTaskInput = Extract<TaskInput, { kind: "backup" }>;
+type BackupTaskProgress = Extract<TaskProgress, { kind: "backup" }>;
+type BackupTaskResult = Extract<TaskResult, { kind: "backup" }>;
+
+const backupInput = (scheduleId = 1): BackupTaskInput => ({
+	kind: "backup",
+	scheduleId,
+	scheduleShortId: `schedule-${scheduleId}`,
+	manual: false,
+});
+
+const backupProgress = (percentDone = 0.5): BackupTaskProgress => ({
+	kind: "backup",
+	progress: {
+		message_type: "status",
+		seconds_elapsed: 1,
+		seconds_remaining: 1,
+		percent_done: percentDone,
+		total_files: 10,
+		files_done: 5,
+		total_bytes: 100,
+		bytes_done: 50,
+		current_files: [],
+	},
+});
+
+const backupResult = (): BackupTaskResult => ({
+	kind: "backup",
+	exitCode: 0,
+	result: JSON.parse(generateBackupOutput()),
+	warningDetails: null,
+});
+
+const createBackupTask = (overrides: Partial<Parameters<typeof taskStore.create>[0]> = {}) =>
+	taskStore.create({
+		organizationId: TEST_ORG_ID,
+		resourceType: "backup_schedule",
+		resourceId: "1",
+		targetAgentId: "local",
+		input: backupInput(),
+		...overrides,
+	});
+
+const createRestoreTask = (overrides: Partial<Parameters<typeof taskStore.create>[0]> = {}) =>
+	taskStore.create({
+		organizationId: TEST_ORG_ID,
+		resourceType: "repository",
+		resourceId: "repo-short",
+		targetAgentId: "local",
+		input: { kind: "restore", repositoryId: "repo-short", snapshotId: "snapshot-1", target: "/tmp/restore" },
+		...overrides,
+	});
+
+beforeEach(async () => {
+	await ensureTestOrganization();
+	await db.delete(tasksTable);
+});
+
+test("creates queued backup tasks with parsed input and durable metadata only", () => {
+	const task = createBackupTask({
+		id: "task-create",
+		input: { ...backupInput(12), manual: true },
+		resourceId: "12",
+	});
+
+	expect(task).toMatchObject({
+		id: "task-create",
+		organizationId: TEST_ORG_ID,
+		kind: "backup",
+		status: "queued",
+		resourceType: "backup_schedule",
+		resourceId: "12",
+		targetAgentId: "local",
+		input: {
+			kind: "backup",
+			scheduleId: 12,
+			scheduleShortId: "schedule-12",
+			manual: true,
+		},
+		progress: null,
+		result: null,
+		error: null,
+		cancellationRequested: false,
+		startedAt: null,
+		finishedAt: null,
+	});
+	expect(Object.keys(task.input)).toEqual(["kind", "scheduleId", "scheduleShortId", "manual"]);
+});
+
+test("moves an active task through running, progress, cancellation request, and success", () => {
+	const task = createBackupTask({ id: "task-success" });
+
+	const running = taskStore.markRunning(task.id);
+	expect(running.status).toBe("running");
+	expect(running.startedAt).toEqual(expect.any(Number));
+
+	const progressed = taskStore.updateProgress(task.id, backupProgress(0.7));
+	expect(progressed.progress?.progress.percent_done).toBe(0.7);
+
+	const cancelling = taskStore.requestCancel(task.id);
+	expect(cancelling.status).toBe("cancelling");
+	expect(cancelling.cancellationRequested).toBe(true);
+
+	const completed = taskStore.complete(task.id, backupResult());
+	expect(completed.status).toBe("succeeded");
+	expect(completed.result?.kind).toBe("backup");
+	if (completed.result?.kind !== "backup") {
+		throw new Error("Expected backup result");
+	}
+	expect(completed.result?.result?.snapshot_id).toBe("abcd1234");
+	expect(completed.finishedAt).toEqual(expect.any(Number));
+	expect(completed.cancellationRequested).toBe(true);
+});
+
+test("records failed and cancelled terminal task states", () => {
+	const failedTask = createBackupTask({ id: "task-failed", resourceId: "failed" });
+	const failed = taskStore.fail(failedTask.id, "restic failed");
+	expect(failed.status).toBe("failed");
+	expect(failed.error).toBe("restic failed");
+	expect(failed.finishedAt).toEqual(expect.any(Number));
+
+	const cancelledTask = createBackupTask({ id: "task-cancelled", resourceId: "cancelled" });
+	taskStore.requestCancel(cancelledTask.id);
+	const cancelled = taskStore.cancel(cancelledTask.id, "Backup was stopped by the user");
+	expect(cancelled.status).toBe("cancelled");
+	expect(cancelled.error).toBe("Backup was stopped by the user");
+	expect(cancelled.cancellationRequested).toBe(true);
+});
+
+test("moves restore tasks through progress and success", () => {
+	const task = createRestoreTask({ id: "restore-task" });
+
+	const running = taskStore.markRunning(task.id);
+	expect(running.kind).toBe("restore");
+	expect(running.input).toMatchObject({ kind: "restore", snapshotId: "snapshot-1" });
+
+	const progressed = taskStore.updateProgress(task.id, {
+		kind: "restore",
+		progress: {
+			message_type: "status",
+			seconds_elapsed: 2,
+			percent_done: 0.25,
+			total_files: 4,
+			files_restored: 1,
+			total_bytes: 400,
+			bytes_restored: 100,
+		},
+	});
+	expect(progressed.progress?.progress.percent_done).toBe(0.25);
+
+	const completed = taskStore.complete(task.id, {
+		kind: "restore",
+		result: {
+			message_type: "summary",
+			total_files: 4,
+			files_restored: 4,
+			files_skipped: 0,
+		},
+	});
+	expect(completed.status).toBe("succeeded");
+	expect(completed.result?.kind).toBe("restore");
+	if (completed.result?.kind !== "restore") {
+		throw new Error("Expected restore result");
+	}
+	expect(completed.result?.result.files_restored).toBe(4);
+});
+
+test("finds the newest active task for a resource and marks only matching active tasks stale", async () => {
+	createBackupTask({ id: "task-a-old", resourceId: "shared" });
+	const newest = createBackupTask({ id: "task-z-new", resourceId: "shared" });
+	const otherResource = createBackupTask({ id: "task-other", resourceId: "other" });
+	const terminal = taskStore.complete(
+		createBackupTask({ id: "task-terminal", resourceId: "shared" }).id,
+		backupResult(),
+	);
+
+	const active = taskStore.findActiveByResource({
+		organizationId: TEST_ORG_ID,
+		kind: "backup",
+		resourceType: "backup_schedule",
+		resourceId: "shared",
+	});
+	expect(active?.id).toBe(newest.id);
+
+	const staleTasks = taskStore.markActiveStale({
+		organizationId: TEST_ORG_ID,
+		kind: "backup",
+		resourceType: "backup_schedule",
+		resourceId: "shared",
+		error: "process restarted",
+	});
+	expect(staleTasks.map((task) => task.id).sort()).toEqual(["task-a-old", "task-z-new"]);
+
+	const other = await db.query.tasksTable.findFirst({ where: { id: otherResource.id } });
+	const completed = await db.query.tasksTable.findFirst({ where: { id: terminal.id } });
+	expect(other?.status).toBe("queued");
+	expect(completed?.status).toBe("succeeded");
+});
+
+test("parses task JSON on reads and rejects invalid persisted shapes", () => {
+	db.insert(tasksTable)
+		.values({
+			id: "task-invalid-json",
+			organizationId: TEST_ORG_ID,
+			kind: "backup",
+			status: "queued",
+			resourceType: "backup_schedule",
+			resourceId: "invalid",
+			input: {
+				kind: "backup",
+				scheduleId: "not-a-number",
+				scheduleShortId: "schedule-invalid",
+				manual: false,
+			},
+			cancellationRequested: false,
+			createdAt: Date.now(),
+			updatedAt: Date.now(),
+		})
+		.run();
+
+	expect(() =>
+		taskStore.findActiveByResource({
+			organizationId: TEST_ORG_ID,
+			kind: "backup",
+			resourceType: "backup_schedule",
+			resourceId: "invalid",
+		}),
+	).toThrow();
+});
+
+test("terminal updates do not mutate unrelated task rows", async () => {
+	const task = createBackupTask({ id: "task-target", resourceId: "target" });
+	const unrelated = createBackupTask({ id: "task-unrelated", resourceId: "unrelated" });
+
+	taskStore.complete(task.id, backupResult());
+
+	const targetRow = await db.query.tasksTable.findFirst({ where: { id: task.id } });
+	const unrelatedRow = await db.query.tasksTable.findFirst({ where: { id: unrelated.id } });
+
+	expect(targetRow?.status).toBe("succeeded");
+	expect(unrelatedRow?.status).toBe("queued");
+	expect(unrelatedRow?.finishedAt).toBeNull();
+});

app/server/modules/tasks/tasks.schemas.ts

@@ -0,0 +1,106 @@
+import {
+	resticBackupOutputSchema,
+	resticBackupProgressSchema,
+	resticRestoreOutputSchema,
+	restoreProgressSchema,
+} from "@zerobyte/core/restic";
+import { z } from "zod";
+
+export const taskStatuses = ["queued", "running", "cancelling", "cancelled", "succeeded", "failed", "stale"] as const;
+export const activeTaskStatuses = ["queued", "running", "cancelling"] as const;
+
+export const taskStatusSchema = z.enum(taskStatuses);
+export const activeTaskStatusSchema = z.enum(activeTaskStatuses);
+export const taskKindSchema = z.enum(["backup", "restore"]);
+
+export const taskInputSchema = z.discriminatedUnion("kind", [
+	z.object({
+		kind: z.literal("backup"),
+		scheduleId: z.number(),
+		scheduleShortId: z.string(),
+		manual: z.boolean(),
+	}),
+	z.object({
+		kind: z.literal("restore"),
+		repositoryId: z.string(),
+		snapshotId: z.string(),
+		target: z.string(),
+	}),
+]);
+
+export const taskProgressSchema = z.discriminatedUnion("kind", [
+	z.object({
+		kind: z.literal("backup"),
+		progress: resticBackupProgressSchema,
+	}),
+	z.object({
+		kind: z.literal("restore"),
+		progress: restoreProgressSchema,
+	}),
+]);
+
+export const taskResultSchema = z.discriminatedUnion("kind", [
+	z.object({
+		kind: z.literal("backup"),
+		exitCode: z.number(),
+		result: resticBackupOutputSchema.nullable(),
+		warningDetails: z.string().nullable(),
+	}),
+	z.object({
+		kind: z.literal("restore"),
+		result: resticRestoreOutputSchema,
+	}),
+]);
+
+export const taskSchema = z
+	.object({
+		id: z.string(),
+		organizationId: z.string(),
+		kind: taskKindSchema,
+		status: taskStatusSchema,
+		resourceType: z.string(),
+		resourceId: z.string(),
+		targetAgentId: z.string().nullable(),
+		input: taskInputSchema,
+		progress: taskProgressSchema.nullable(),
+		result: taskResultSchema.nullable(),
+		error: z.string().nullable(),
+		cancellationRequested: z.boolean(),
+		createdAt: z.number(),
+		startedAt: z.number().nullable(),
+		updatedAt: z.number(),
+		finishedAt: z.number().nullable(),
+	})
+	.superRefine((task, ctx) => {
+		if (task.kind !== task.input.kind) {
+			ctx.addIssue({
+				code: "custom",
+				path: ["input", "kind"],
+				message: "Task input kind must match task kind",
+			});
+		}
+
+		if (task.progress && task.kind !== task.progress.kind) {
+			ctx.addIssue({
+				code: "custom",
+				path: ["progress", "kind"],
+				message: "Task progress kind must match task kind",
+			});
+		}
+
+		if (task.result && task.kind !== task.result.kind) {
+			ctx.addIssue({
+				code: "custom",
+				path: ["result", "kind"],
+				message: "Task result kind must match task kind",
+			});
+		}
+	});
+
+export type TaskStatus = z.infer<typeof taskStatusSchema>;
+export type ActiveTaskStatus = z.infer<typeof activeTaskStatusSchema>;
+export type TaskKind = z.infer<typeof taskKindSchema>;
+export type TaskInput = z.infer<typeof taskInputSchema>;
+export type TaskProgress = z.infer<typeof taskProgressSchema>;
+export type TaskResult = z.infer<typeof taskResultSchema>;
+export type ParsedTask = z.infer<typeof taskSchema>;

app/server/modules/tasks/tasks.store.ts

@@ -0,0 +1,216 @@
+import { and, desc, eq, inArray, type SQL } from "drizzle-orm";
+import { db } from "~/server/db/db";
+import { tasksTable } from "~/server/db/schema";
+import {
+	activeTaskStatuses,
+	taskInputSchema,
+	taskProgressSchema,
+	taskResultSchema,
+	taskSchema,
+	type ParsedTask,
+	type TaskInput,
+	type TaskKind,
+	type TaskProgress,
+	type TaskResult,
+} from "./tasks.schemas";
+
+type TaskResource = {
+	organizationId: string;
+	kind: TaskKind;
+	resourceType: string;
+	resourceId: string;
+};
+
+type CreateTaskParams = {
+	id?: string;
+	organizationId: string;
+	resourceType: string;
+	resourceId: string;
+	targetAgentId?: string | null;
+	input: TaskInput;
+};
+
+type MarkActiveStaleParams = Partial<TaskResource> & { error?: string };
+
+const parseTask = (row: unknown): ParsedTask => taskSchema.parse(row);
+
+const activeStatusCondition = () => inArray(tasksTable.status, activeTaskStatuses);
+
+const byIdCondition = (id: string) => eq(tasksTable.id, id);
+
+const buildActiveConditions = (params: Partial<TaskResource> = {}) => {
+	const conditions: SQL[] = [activeStatusCondition()];
+
+	if (params.organizationId) conditions.push(eq(tasksTable.organizationId, params.organizationId));
+	if (params.kind) conditions.push(eq(tasksTable.kind, params.kind));
+	if (params.resourceType) conditions.push(eq(tasksTable.resourceType, params.resourceType));
+	if (params.resourceId) conditions.push(eq(tasksTable.resourceId, params.resourceId));
+
+	return conditions;
+};
+
+const getUpdatedTask = (row: unknown, taskId: string, operation: string) => {
+	if (!row) {
+		throw new Error(`Task ${taskId} was not ${operation}`);
+	}
+
+	return parseTask(row);
+};
+
+export const taskStore = {
+	create: (params: CreateTaskParams): ParsedTask => {
+		const input = taskInputSchema.parse(params.input);
+		const now = Date.now();
+		const row = db
+			.insert(tasksTable)
+			.values({
+				id: params.id ?? Bun.randomUUIDv7(),
+				organizationId: params.organizationId,
+				kind: input.kind,
+				status: "queued",
+				resourceType: params.resourceType,
+				resourceId: params.resourceId,
+				targetAgentId: params.targetAgentId ?? null,
+				input,
+				progress: null,
+				result: null,
+				error: null,
+				cancellationRequested: false,
+				createdAt: now,
+				updatedAt: now,
+			})
+			.returning()
+			.get();
+
+		return parseTask(row);
+	},
+
+	markRunning: (taskId: string): ParsedTask => {
+		const now = Date.now();
+		const row = db
+			.update(tasksTable)
+			.set({ status: "running", startedAt: now, updatedAt: now })
+			.where(and(byIdCondition(taskId), activeStatusCondition()))
+			.returning()
+			.get();
+
+		return getUpdatedTask(row, taskId, "marked running");
+	},
+
+	updateProgress: (taskId: string, progress: TaskProgress): ParsedTask => {
+		const parsedProgress = taskProgressSchema.parse(progress);
+		const row = db
+			.update(tasksTable)
+			.set({ progress: parsedProgress, updatedAt: Date.now() })
+			.where(and(byIdCondition(taskId), activeStatusCondition()))
+			.returning()
+			.get();
+
+		return getUpdatedTask(row, taskId, "updated with progress");
+	},
+
+	requestCancel: (taskId: string): ParsedTask => {
+		const row = db
+			.update(tasksTable)
+			.set({ status: "cancelling", cancellationRequested: true, updatedAt: Date.now() })
+			.where(and(byIdCondition(taskId), activeStatusCondition()))
+			.returning()
+			.get();
+
+		return getUpdatedTask(row, taskId, "marked cancelling");
+	},
+
+	complete: (taskId: string, result: TaskResult): ParsedTask => {
+		const parsedResult = taskResultSchema.parse(result);
+		const now = Date.now();
+		const row = db
+			.update(tasksTable)
+			.set({
+				status: "succeeded",
+				result: parsedResult,
+				error: null,
+				updatedAt: now,
+				finishedAt: now,
+			})
+			.where(and(byIdCondition(taskId), activeStatusCondition()))
+			.returning()
+			.get();
+
+		return getUpdatedTask(row, taskId, "completed");
+	},
+
+	fail: (taskId: string, error: string): ParsedTask => {
+		const now = Date.now();
+		const row = db
+			.update(tasksTable)
+			.set({
+				status: "failed",
+				error,
+				updatedAt: now,
+				finishedAt: now,
+			})
+			.where(and(byIdCondition(taskId), activeStatusCondition()))
+			.returning()
+			.get();
+
+		return getUpdatedTask(row, taskId, "failed");
+	},
+
+	cancel: (taskId: string, error: string | null = null): ParsedTask => {
+		const now = Date.now();
+		const row = db
+			.update(tasksTable)
+			.set({
+				status: "cancelled",
+				error,
+				updatedAt: now,
+				finishedAt: now,
+			})
+			.where(and(byIdCondition(taskId), activeStatusCondition()))
+			.returning()
+			.get();
+
+		return getUpdatedTask(row, taskId, "cancelled");
+	},
+
+	findActiveByResource: (params: TaskResource): ParsedTask | null => {
+		const rows = db
+			.select()
+			.from(tasksTable)
+			.where(and(...buildActiveConditions(params)))
+			.orderBy(desc(tasksTable.createdAt), desc(tasksTable.id))
+			.limit(1)
+			.all();
+
+		const [row] = rows;
+		return row ? parseTask(row) : null;
+	},
+
+	listActiveByResource: (params: TaskResource): ParsedTask[] => {
+		const rows = db
+			.select()
+			.from(tasksTable)
+			.where(and(...buildActiveConditions(params)))
+			.orderBy(desc(tasksTable.createdAt), desc(tasksTable.id))
+			.all();
+
+		return rows.map(parseTask);
+	},
+
+	markActiveStale: (params: MarkActiveStaleParams = {}): ParsedTask[] => {
+		const now = Date.now();
+		const rows = db
+			.update(tasksTable)
+			.set({
+				status: "stale",
+				error: params.error ?? "Task was interrupted before it completed",
+				updatedAt: now,
+				finishedAt: now,
+			})
+			.where(and(...buildActiveConditions(params)))
+			.returning()
+			.all();
+
+		return rows.map(parseTask);
+	},
+};

app/server/utils/errors.ts

@@ -1,6 +1,6 @@
 import { HttpError } from "http-errors-enhanced";
 import { sanitizeSensitiveData } from "@zerobyte/core/node";
-import { ResticError } from "@zerobyte/core/restic";
+import { isResticError } from "@zerobyte/core/restic";
 import { toErrorDetails as getErrorDetails, toMessage as getMessage } from "@zerobyte/core/utils";
 import { Cause, Effect, Exit, Option } from "effect";
 
@@ -31,7 +31,7 @@ export const handleServiceError = (error: unknown) => {
 		return { message: sanitizeSensitiveData(error.message), status: error.statusCode };
 	}
 
-	if (error instanceof ResticError) {
+	if (isResticError(error)) {
 		return {
 			message: sanitizeSensitiveData(error.summary),
 			details: error.details ? sanitizeSensitiveData(error.details) : undefined,

app/test/backend-integration/setup-target.sh

@@ -4,18 +4,12 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 TARGET_HOST="192.168.2.41"
 TARGET="root@$TARGET_HOST"
-FIXTURE_UID="1000"
-FIXTURE_GID="1000"
 
 ARTIFACTS_DIR="$SCRIPT_DIR/artifacts/$TARGET_HOST"
-KEY_PATH="$ARTIFACTS_DIR/zerobyte-sftp-ed25519"
 KNOWN_HOSTS_PATH="$ARTIFACTS_DIR/known_hosts"
 CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json"
 
-SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt"
 SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt"
-WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt"
-RESTIC_PASSWORD_FILE="$ARTIFACTS_DIR/restic-password.txt"
 
 read_or_create_secret() {
 	local file_path="$1"
@@ -32,31 +26,12 @@ read_or_create_secret() {
 mkdir -p "$ARTIFACTS_DIR"
 chmod 700 "$ARTIFACTS_DIR"
 
-SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")"
 SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")"
-WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")"
-RESTIC_PASSWORD="$(read_or_create_secret "$RESTIC_PASSWORD_FILE")"
 
-if [[ ! -f "$KEY_PATH" || ! -f "$KEY_PATH.pub" ]]; then
-	ssh-keygen -q -t ed25519 -N "" -C "zerobyte-backend-integration@$TARGET_HOST" -f "$KEY_PATH"
-	chmod 600 "$KEY_PATH"
-fi
-
-PUBLIC_KEY_BASE64="$(base64 <"$KEY_PATH.pub" | tr -d '\n')"
-
-ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" "$RESTIC_PASSWORD" "$PUBLIC_KEY_BASE64" <<'REMOTE'
+ssh "$TARGET" bash -s -- "$SFTP_PASSWORD" <<'REMOTE'
 set -euo pipefail
 
-fixture_uid="$1"
-fixture_gid="$2"
-smb_password="$3"
-sftp_password="$4"
-webdav_password="$5"
-restic_password="$6"
-public_key="$(printf '%s' "$7" | base64 -d)"
-repo_path="/srv/zerobyte-backend-integration/restic-repo"
-repo_password_fingerprint_path="$repo_path/.zerobyte-password-sha256"
-repo_password_fingerprint="$(printf '%s' "$restic_password" | sha256sum | cut -d' ' -f1)"
+sftp_password="$1"
 legacy_sshd_dir="/etc/ssh/zerobyte-backend-integration-legacy"
 
 export DEBIAN_FRONTEND=noninteractive
@@ -66,98 +41,19 @@ write_file() {
 	cat >"$file_path"
 }
 
-initialize_restic_repo() {
-	local password_file
-
-	rm -rf "$repo_path"
-	install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 "$repo_path"
-
-	password_file="$(mktemp)"
-	printf '%s\n' "$restic_password" >"$password_file"
-	chown zerobyte-sftp:zerobyte-sftp "$password_file"
-	chmod 0600 "$password_file"
-	su -s /bin/sh -c "restic init --repo '$repo_path' --password-file '$password_file'" zerobyte-sftp
-	rm -f "$password_file"
-
-	printf '%s\n' "$repo_password_fingerprint" >"$repo_password_fingerprint_path"
-	chmod 0600 "$repo_password_fingerprint_path"
-}
-
 apt-get update
-apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server restic rpcbind samba
+apt-get install -y openssh-server
 
 id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp
-id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb
 
 install -d -m 0755 /srv/zerobyte-backend-integration/fixtures/case-a/docs
 printf 'hello from zerobyte integration\n' >/srv/zerobyte-backend-integration/fixtures/case-a/hello.txt
 printf 'fixture documentation\n' >/srv/zerobyte-backend-integration/fixtures/case-a/docs/readme.md
-chown -R "$fixture_uid:$fixture_gid" /srv/zerobyte-backend-integration/fixtures
 find /srv/zerobyte-backend-integration/fixtures -type d -exec chmod 0755 {} +
 find /srv/zerobyte-backend-integration/fixtures -type f -exec chmod 0644 {} +
 
-install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp
-install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp/.ssh
-printf '%s\n' "$public_key" >/home/zerobyte-sftp/.ssh/authorized_keys
-chown zerobyte-sftp:zerobyte-sftp /home/zerobyte-sftp/.ssh/authorized_keys
-chmod 0600 /home/zerobyte-sftp/.ssh/authorized_keys
-
-printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb >/dev/null
-smbpasswd -e zerobyte-smb >/dev/null
 printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd
 passwd -u zerobyte-sftp >/dev/null 2>&1 || true
-htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null
-
-if [[ ! -f "$repo_path/config" ]]; then
-	initialize_restic_repo
-elif [[ ! -f "$repo_password_fingerprint_path" ]] || [[ "$(cat "$repo_password_fingerprint_path")" != "$repo_password_fingerprint" ]]; then
-	initialize_restic_repo
-fi
-
-write_file /etc/exports <<'EOF'
-/srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure)
-EOF
-exportfs -ra
-systemctl unmask rpcbind rpcbind.socket >/dev/null 2>&1
-systemctl start rpcbind.socket
-systemctl start rpcbind
-systemctl start proc-fs-nfsd.mount
-systemctl restart nfs-kernel-server
-
-write_file /etc/samba/smb.conf <<'EOF'
-[zerobyte-backend-integration]
-	path = /srv/zerobyte-backend-integration/fixtures
-	browseable = yes
-	read only = yes
-	guest ok = no
-	valid users = zerobyte-smb
-EOF
-
-install -d -o www-data -g www-data -m 0755 /var/lib/dav
-a2enmod dav dav_fs auth_basic >/dev/null
-printf 'ServerName localhost\n' >/etc/apache2/conf-available/zerobyte-backend-integration-servername.conf
-a2enconf zerobyte-backend-integration-servername >/dev/null
-write_file /etc/apache2/sites-available/zerobyte-backend-integration-dav.conf <<'EOF'
-Alias /zerobyte-backend-integration /srv/zerobyte-backend-integration/fixtures
-
-DAVLockDB /var/lib/dav/lockdb
-
-<Location /zerobyte-backend-integration>
-	DAV On
-	AuthType Basic
-	AuthName "Zerobyte Backend Integration WebDAV"
-	AuthUserFile /etc/apache2/zerobyte-backend-integration.htpasswd
-	Require valid-user
-</Location>
-
-<Directory /srv/zerobyte-backend-integration/fixtures>
-	Options Indexes FollowSymLinks
-	AllowOverride None
-	Require all granted
-</Directory>
-EOF
-a2ensite zerobyte-backend-integration-dav >/dev/null
-apache2ctl configtest
 
 install -d -m 0700 "$legacy_sshd_dir"
 if [[ ! -f "$legacy_sshd_dir/ssh_host_rsa_key" ]]; then
@@ -213,8 +109,6 @@ EOF
 systemctl daemon-reload
 systemctl enable --now zerobyte-backend-integration-legacy-sshd.service
 
-systemctl restart apache2
-systemctl restart smbd
 systemctl restart ssh
 systemctl restart zerobyte-backend-integration-legacy-sshd.service
 systemctl is-active --quiet zerobyte-backend-integration-legacy-sshd.service
@@ -233,14 +127,8 @@ if ! ssh-keyscan -T 5 -p 2222 "$TARGET_HOST" >>"$KNOWN_HOSTS_PATH" 2>/dev/null;
 	exit 1
 fi
 
-INTEGRATION_HOST="$TARGET_HOST" \
-	FIXTURE_UID="$FIXTURE_UID" \
-	FIXTURE_GID="$FIXTURE_GID" \
-	SMB_PASSWORD="$SMB_PASSWORD" \
+	INTEGRATION_HOST="$TARGET_HOST" \
 	SFTP_PASSWORD="$SFTP_PASSWORD" \
-	WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \
-	RESTIC_PASSWORD="$RESTIC_PASSWORD" \
-	SFTP_KEY_PATH="$KEY_PATH" \
 	KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \
 	CONFIG_PATH="$CONFIG_PATH" \
 	bun run "$SCRIPT_DIR/write-generated-config.ts"

app/test/backend-integration/write-generated-config.ts

@@ -4,36 +4,14 @@ function getRequiredEnv(name: string) {
 	return process.env[name]!;
 }
 
-function getRequiredNumberEnv(name: string) {
-	return Number.parseInt(getRequiredEnv(name), 10);
-}
-
 const host = getRequiredEnv("INTEGRATION_HOST");
-const fixtureUid = getRequiredNumberEnv("FIXTURE_UID");
-const fixtureGid = getRequiredNumberEnv("FIXTURE_GID");
-const smbPassword = getRequiredEnv("SMB_PASSWORD");
 const sftpPassword = getRequiredEnv("SFTP_PASSWORD");
-const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD");
-const resticPassword = getRequiredEnv("RESTIC_PASSWORD");
-const privateKey = fs.readFileSync(getRequiredEnv("SFTP_KEY_PATH"), "utf8");
 const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8");
 const configPath = getRequiredEnv("CONFIG_PATH");
 
 const fileText = "hello from zerobyte integration\n";
 const readmeText = "fixture documentation\n";
 
-const nfsEntries = [
-	{ path: "hello.txt", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: fileText },
-	{ path: "docs", type: "directory", uid: fixtureUid, gid: fixtureGid, mode: "0755" },
-	{ path: "docs/readme.md", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: readmeText },
-];
-
-const smbEntries = [
-	{ path: "hello.txt", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: fileText },
-	{ path: "docs", type: "directory", uid: fixtureUid, gid: fixtureGid, mode: "1755" },
-	{ path: "docs/readme.md", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: readmeText },
-];
-
 const contentOnlyEntries = [
 	{ path: "hello.txt", type: "file", text: fileText },
 	{ path: "docs", type: "directory" },
@@ -43,71 +21,6 @@ const contentOnlyEntries = [
 const config = {
 	version: 1,
 	scenarios: [
-		{
-			id: "nfs-local-repo",
-			volume: {
-				backend: "nfs",
-				server: host,
-				exportPath: "/srv/zerobyte-backend-integration/fixtures",
-				port: 2049,
-				version: "4.1",
-				readOnly: true,
-			},
-			repository: { backend: "local", path: "repo-nfs" },
-			fixtureRoot: "case-a",
-			expectedEntries: nfsEntries,
-		},
-		{
-			id: "smb-local-repo",
-			volume: {
-				backend: "smb",
-				server: host,
-				share: "zerobyte-backend-integration",
-				username: "zerobyte-smb",
-				password: smbPassword,
-				mapToContainerUidGid: false,
-				vers: "3.0",
-				port: 445,
-				readOnly: true,
-			},
-			repository: { backend: "local", path: "repo-smb" },
-			fixtureRoot: "case-a",
-			expectedEntries: smbEntries,
-		},
-		{
-			id: "sftp-local-repo",
-			volume: {
-				backend: "sftp",
-				host,
-				port: 22,
-				username: "zerobyte-sftp",
-				privateKey,
-				path: "/srv/zerobyte-backend-integration/fixtures",
-				readOnly: true,
-				skipHostKeyCheck: false,
-				knownHosts,
-			},
-			repository: { backend: "local", path: "repo-sftp-volume" },
-			fixtureRoot: "case-a",
-			expectedEntries: contentOnlyEntries,
-		},
-		{
-			id: "sftp-password-local-repo",
-			volume: {
-				backend: "sftp",
-				host,
-				port: 22,
-				username: "zerobyte-sftp",
-				password: sftpPassword,
-				path: "/srv/zerobyte-backend-integration/fixtures",
-				readOnly: true,
-				skipHostKeyCheck: false,
-				knownHosts,
-			},
-			repository: { backend: "local", path: "repo-sftp-password-volume" },
-			fixtureRoot: "case-a",
-			expectedEntries: contentOnlyEntries,
-		},
 		{
 			id: "sftp-legacy-rsa-hostkey-local-repo",
 			volume: {
@@ -126,47 +39,6 @@ const config = {
 			fixtureRoot: "case-a",
 			expectedEntries: contentOnlyEntries,
 		},
-		{
-			id: "webdav-local-repo",
-			volume: {
-				backend: "webdav",
-				server: host,
-				path: "/zerobyte-backend-integration",
-				username: "zerobyte-webdav",
-				password: webdavPassword,
-				port: 80,
-				readOnly: true,
-				ssl: false,
-			},
-			repository: { backend: "local", path: "repo-webdav" },
-			fixtureRoot: "case-a",
-			expectedEntries: contentOnlyEntries,
-		},
-		{
-			id: "nfs-sftp-repository",
-			volume: {
-				backend: "nfs",
-				server: host,
-				exportPath: "/srv/zerobyte-backend-integration/fixtures",
-				port: 2049,
-				version: "4.1",
-				readOnly: true,
-			},
-			repository: {
-				backend: "sftp",
-				host,
-				port: 22,
-				user: "zerobyte-sftp",
-				path: "/srv/zerobyte-backend-integration/restic-repo",
-				privateKey,
-				skipHostKeyCheck: false,
-				knownHosts,
-				isExistingRepository: true,
-				customPassword: resticPassword,
-			},
-			fixtureRoot: "case-a",
-			expectedEntries: nfsEntries,
-		},
 	],
 };
 

app/test/helpers/agent-mock.ts

@@ -1,7 +1,7 @@
 import { vi } from "vitest";
 import { fromAny, fromPartial } from "@total-typescript/shoehorn";
 import type { SafeSpawnParams } from "@zerobyte/core/node";
-import type { BackupExecutionResult } from "~/server/modules/agents/agents-manager";
+import type { BackupExecutionProgress, BackupExecutionResult } from "~/server/modules/agents/agents-manager";
 
 export const createAgentBackupMocks = (
 	resticBackupMock: (params: SafeSpawnParams) => Promise<{
@@ -14,7 +14,15 @@ export const createAgentBackupMocks = (
 	const runningBackups = new Map<number, { resolve: (result: BackupExecutionResult) => void; cancelled: boolean }>();
 
 	const runBackupMock = vi.fn(
-		async (_agentId: string, request: { scheduleId: number; payload: { jobId: string }; signal: AbortSignal }) => {
+		async (
+			_agentId: string,
+			request: {
+				scheduleId: number;
+				payload: { jobId: string };
+				signal: AbortSignal;
+				onProgress: (progress: BackupExecutionProgress) => void;
+			},
+		) => {
 			return new Promise<BackupExecutionResult>((resolve) => {
 				runningBackups.set(request.scheduleId, { resolve, cancelled: false });
 

app/test/integration/.gitignore

@@ -0,0 +1,2 @@
+artifacts/*
+!artifacts/.gitignore

app/test/integration/artifacts/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

app/test/integration/infra/Dockerfile

@@ -0,0 +1,18 @@
+ARG BASE_IMAGE=zerobyte-integration-runtime-base:latest
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+
+ENV NODE_ENV=test
+
+COPY ./package.json ./bun.lock ./
+COPY ./packages/core/package.json ./packages/core/package.json
+COPY ./packages/contracts/package.json ./packages/contracts/package.json
+COPY ./apps/agent/package.json ./apps/agent/package.json
+COPY ./apps/docs/package.json ./apps/docs/package.json
+
+RUN VITE_GIT_HOOKS=0 bun install --frozen-lockfile
+
+COPY . .
+
+CMD ["sh", "app/test/integration/infra/entrypoint.sh"]

app/test/integration/infra/docker-compose.yml

@@ -0,0 +1,128 @@
+services:
+  rustfs:
+    image: rustfs/rustfs:latest
+    command: ["/data"]
+    environment:
+      RUSTFS_ADDRESS: ":9000"
+      RUSTFS_ACCESS_KEY: "rustfsadmin"
+      RUSTFS_SECRET_KEY: "rustfsadmin"
+      RUSTFS_CONSOLE_ENABLE: "false"
+    volumes:
+      - rustfs-data:/data
+
+  rustfs-setup:
+    image: minio/mc:latest
+    depends_on:
+      - rustfs
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        until mc alias set rustfs http://rustfs:9000 rustfsadmin rustfsadmin --api S3v4 --path on; do
+          echo "Waiting for RustFS..."
+          sleep 1
+        done
+        mc mb --ignore-existing --region us-east-1 rustfs/zerobyte-integration
+
+  sftp:
+    build:
+      context: ./sftp
+    environment:
+      SFTP_USER: "zerobyte-sftp"
+      SFTP_PASSWORD: "zerobyte-sftp-password"
+      SFTP_PUBLIC_KEY_PATH: "/run/zerobyte/sftp/id_ed25519.pub"
+    volumes:
+      - ../artifacts/sftp/id_ed25519.pub:/run/zerobyte/sftp/id_ed25519.pub:ro
+      - sftp-data:/srv/zerobyte-integration
+    healthcheck:
+      test: ["CMD", "ssh-keyscan", "-T", "2", "localhost"]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+
+  webdav:
+    build:
+      context: ./webdav
+    environment:
+      WEBDAV_USER: "zerobyte-webdav"
+      WEBDAV_PASSWORD: "zerobyte-webdav-password"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-fsS",
+          "-u",
+          "zerobyte-webdav:zerobyte-webdav-password",
+          "http://localhost/zerobyte-integration/case-a/hello.txt",
+        ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+
+  smb:
+    build:
+      context: ./smb
+    environment:
+      SMB_USER: "zerobyte-smb"
+      SMB_PASSWORD: "zerobyte-smb-password"
+      SMB_SHARE: "zerobyte-integration"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "smbclient",
+          "//localhost/zerobyte-integration",
+          "-U",
+          "zerobyte-smb%zerobyte-smb-password",
+          "-c",
+          "get case-a/hello.txt /tmp/zerobyte-smb-healthcheck.txt",
+        ]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+
+  nfs:
+    build:
+      context: ./nfs
+    privileged: true
+    volumes:
+      - nfs-data:/srv/zerobyte-integration
+    healthcheck:
+      test: ["CMD", "sh", "-c", "exportfs -v | grep -q /srv/zerobyte-integration/fixtures"]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+
+  integration:
+    build:
+      context: ../../../..
+      dockerfile: app/test/integration/infra/Dockerfile
+      args:
+        BASE_IMAGE: zerobyte-integration-runtime-base:latest
+    depends_on:
+      rustfs-setup:
+        condition: service_completed_successfully
+      sftp:
+        condition: service_healthy
+      webdav:
+        condition: service_healthy
+      smb:
+        condition: service_healthy
+      nfs:
+        condition: service_healthy
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - /dev/fuse:/dev/fuse
+    volumes:
+      - ../artifacts:/app/app/test/integration/artifacts
+      - ../artifacts/sftp/id_ed25519:/run/zerobyte/sftp/id_ed25519:ro
+    environment:
+      LOG_LEVEL: "error"
+      SFTP_PRIVATE_KEY_PATH: "/run/zerobyte/sftp/id_ed25519"
+      SKIP_VOLUME_MOUNT_INTEGRATION_TESTS: "${SKIP_VOLUME_MOUNT_INTEGRATION_TESTS:-false}"
+
+volumes:
+  rustfs-data:
+  sftp-data:
+  nfs-data:

app/test/integration/infra/entrypoint.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+set -eu
+
+if [ -w /etc/fuse.conf ] && ! grep -q '^user_allow_other$' /etc/fuse.conf; then
+	printf '\nuser_allow_other\n' >>/etc/fuse.conf
+fi
+
+bun app/test/integration/src/write-rclone-config.ts
+exec bunx --bun vitest run --config app/test/integration/vitest.config.ts

app/test/integration/infra/nfs/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.21
+
+RUN apk add --no-cache nfs-utils
+
+COPY entrypoint.sh /usr/local/bin/zerobyte-nfs-entrypoint
+RUN chmod +x /usr/local/bin/zerobyte-nfs-entrypoint
+
+EXPOSE 2049
+
+ENTRYPOINT ["/usr/local/bin/zerobyte-nfs-entrypoint"]

app/test/integration/infra/nfs/entrypoint.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+set -eu
+
+FIXTURE_ROOT="/srv/zerobyte-integration/fixtures"
+
+install -d -m 0755 "$FIXTURE_ROOT/case-a/docs"
+printf 'hello from zerobyte integration\n' >"$FIXTURE_ROOT/case-a/hello.txt"
+printf 'fixture documentation\n' >"$FIXTURE_ROOT/case-a/docs/readme.md"
+find "$FIXTURE_ROOT" -type d -exec chmod 0755 {} +
+find "$FIXTURE_ROOT" -type f -exec chmod 0644 {} +
+
+mkdir -p /run/rpc_pipefs /proc/fs/nfsd
+mountpoint -q /proc/fs/nfsd || mount -t nfsd nfsd /proc/fs/nfsd
+mountpoint -q /run/rpc_pipefs || mount -t rpc_pipefs rpc_pipefs /run/rpc_pipefs
+printf '1\n' >/proc/fs/nfsd/nfsv4gracetime
+printf '1\n' >/proc/fs/nfsd/nfsv4leasetime
+
+cat >/etc/exports <<EOF
+$FIXTURE_ROOT *(ro,sync,no_subtree_check,insecure,fsid=0)
+EOF
+
+rpcbind -w
+rpc.mountd --no-udp --foreground &
+exportfs -ra
+rpc.nfsd -V 4 -V 4.1 8
+
+trap 'exportfs -ua; rpc.nfsd 0' INT TERM
+while kill -0 "$(pidof rpc.mountd)" 2>/dev/null; do
+	sleep 1
+done

app/test/integration/infra/sftp/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.21
+
+RUN apk add --no-cache openssh-client openssh-server
+
+COPY entrypoint.sh /usr/local/bin/zerobyte-sftp-entrypoint
+RUN chmod +x /usr/local/bin/zerobyte-sftp-entrypoint
+
+EXPOSE 22
+
+ENTRYPOINT ["/usr/local/bin/zerobyte-sftp-entrypoint"]

app/test/integration/infra/sftp/entrypoint.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -eu
+
+SFTP_USER="${SFTP_USER:-zerobyte-sftp}"
+SFTP_PASSWORD="${SFTP_PASSWORD:-zerobyte-sftp-password}"
+SFTP_PUBLIC_KEY_PATH="${SFTP_PUBLIC_KEY_PATH:-/run/zerobyte/sftp/id_ed25519.pub}"
+SERVICE_ROOT="/srv/zerobyte-integration"
+
+ssh-keygen -A
+install -d -m 0755 /run/sshd
+
+if ! id "$SFTP_USER" >/dev/null 2>&1; then
+	addgroup -S "$SFTP_USER"
+	adduser -S -D -h "/home/$SFTP_USER" -s /bin/sh -G "$SFTP_USER" "$SFTP_USER"
+fi
+
+printf '%s:%s\n' "$SFTP_USER" "$SFTP_PASSWORD" | chpasswd
+
+install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0700 "/home/$SFTP_USER/.ssh"
+install -o "$SFTP_USER" -g "$SFTP_USER" -m 0600 "$SFTP_PUBLIC_KEY_PATH" "/home/$SFTP_USER/.ssh/authorized_keys"
+
+install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0755 "$SERVICE_ROOT/fixtures/case-a/docs"
+install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0755 "$SERVICE_ROOT/repos/sftp"
+printf 'hello from zerobyte integration\n' >"$SERVICE_ROOT/fixtures/case-a/hello.txt"
+printf 'fixture documentation\n' >"$SERVICE_ROOT/fixtures/case-a/docs/readme.md"
+chown -R "$SFTP_USER:$SFTP_USER" "$SERVICE_ROOT"
+
+cat >/etc/ssh/sshd_config <<EOF
+Port 22
+HostKey /etc/ssh/ssh_host_ed25519_key
+HostKey /etc/ssh/ssh_host_rsa_key
+PermitRootLogin no
+PasswordAuthentication yes
+PubkeyAuthentication yes
+KbdInteractiveAuthentication no
+Subsystem sftp internal-sftp
+
+Match User $SFTP_USER
+	ForceCommand internal-sftp
+	AllowTcpForwarding no
+	X11Forwarding no
+	PasswordAuthentication yes
+	PubkeyAuthentication yes
+EOF
+
+exec /usr/sbin/sshd -D -e

app/test/integration/infra/smb/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.21
+
+RUN apk add --no-cache samba samba-common-tools samba-client
+
+COPY entrypoint.sh /usr/local/bin/zerobyte-smb-entrypoint
+RUN chmod +x /usr/local/bin/zerobyte-smb-entrypoint
+
+EXPOSE 445
+
+ENTRYPOINT ["/usr/local/bin/zerobyte-smb-entrypoint"]

app/test/integration/infra/smb/entrypoint.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+set -eu
+
+SMB_USER="${SMB_USER:-zerobyte-smb}"
+SMB_PASSWORD="${SMB_PASSWORD:-zerobyte-smb-password}"
+SMB_SHARE="${SMB_SHARE:-zerobyte-integration}"
+FIXTURE_ROOT="/srv/zerobyte-integration/fixtures"
+
+adduser -D -H -s /sbin/nologin "$SMB_USER" 2>/dev/null || true
+
+install -d -m 0755 "$FIXTURE_ROOT/case-a/docs"
+printf 'hello from zerobyte integration\n' >"$FIXTURE_ROOT/case-a/hello.txt"
+printf 'fixture documentation\n' >"$FIXTURE_ROOT/case-a/docs/readme.md"
+chown -R "$SMB_USER:$SMB_USER" "$FIXTURE_ROOT"
+find "$FIXTURE_ROOT" -type d -exec chmod 0755 {} +
+find "$FIXTURE_ROOT" -type f -exec chmod 0644 {} +
+
+printf '%s\n%s\n' "$SMB_PASSWORD" "$SMB_PASSWORD" | smbpasswd -a -s "$SMB_USER" >/dev/null
+smbpasswd -e "$SMB_USER" >/dev/null
+
+cat >/etc/samba/smb.conf <<EOF
+[global]
+	server role = standalone server
+	server string = Zerobyte Integration SMB
+	map to guest = Never
+	log file = /dev/stdout
+	max log size = 0
+	load printers = no
+	printing = bsd
+	disable spoolss = yes
+	bind interfaces only = yes
+	interfaces = lo eth0
+	smb ports = 445
+
+[$SMB_SHARE]
+	path = $FIXTURE_ROOT
+	browseable = yes
+	read only = yes
+	guest ok = no
+	valid users = $SMB_USER
+EOF
+
+testparm -s >/dev/null
+exec smbd --foreground --no-process-group --debug-stdout

app/test/integration/infra/webdav/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.21
+
+RUN apk add --no-cache apache2 apache2-utils apache2-webdav curl
+
+COPY entrypoint.sh /usr/local/bin/zerobyte-webdav-entrypoint
+RUN chmod +x /usr/local/bin/zerobyte-webdav-entrypoint
+
+EXPOSE 80
+
+ENTRYPOINT ["/usr/local/bin/zerobyte-webdav-entrypoint"]

app/test/integration/infra/webdav/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+set -eu
+
+WEBDAV_USER="${WEBDAV_USER:-zerobyte-webdav}"
+WEBDAV_PASSWORD="${WEBDAV_PASSWORD:-zerobyte-webdav-password}"
+SERVICE_ROOT="/srv/zerobyte-integration"
+FIXTURE_ROOT="$SERVICE_ROOT/fixtures"
+LOCK_ROOT="/var/lib/zerobyte-webdav"
+
+install -d -m 0755 "$FIXTURE_ROOT/case-a/docs"
+printf 'hello from zerobyte integration\n' >"$FIXTURE_ROOT/case-a/hello.txt"
+printf 'fixture documentation\n' >"$FIXTURE_ROOT/case-a/docs/readme.md"
+chown -R apache:apache "$SERVICE_ROOT"
+
+install -d -o apache -g apache -m 0755 "$LOCK_ROOT"
+htpasswd -bc /etc/apache2/zerobyte-webdav.htpasswd "$WEBDAV_USER" "$WEBDAV_PASSWORD" >/dev/null
+
+cat >/etc/apache2/conf.d/zerobyte-webdav.conf <<EOF
+ServerName localhost
+ErrorLog /proc/self/fd/2
+CustomLog /proc/self/fd/1 combined
+
+DAVLockDB $LOCK_ROOT/lockdb
+Alias /zerobyte-integration $FIXTURE_ROOT
+
+<Location /zerobyte-integration>
+	DAV On
+	AuthType Basic
+	AuthName "Zerobyte Integration WebDAV"
+	AuthUserFile /etc/apache2/zerobyte-webdav.htpasswd
+	Require valid-user
+</Location>
+
+<Directory $FIXTURE_ROOT>
+	Options Indexes FollowSymLinks
+	AllowOverride None
+	Require all granted
+</Directory>
+EOF
+
+httpd -t
+exec httpd -D FOREGROUND

app/test/integration/run.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$script_dir/../../.." && pwd)"
+base_image="zerobyte-integration-runtime-base:latest"
+compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9_-')"
+artifacts_dir="$script_dir/artifacts"
+sftp_artifacts_dir="$artifacts_dir/sftp"
+compose_file="$script_dir/infra/docker-compose.yml"
+docker_output_log="$artifacts_dir/docker-output.log"
+compose=(docker compose -f "$compose_file" -p "$compose_project")
+
+mkdir -p "$artifacts_dir"
+if [[ -d "$artifacts_dir/runs" ]]; then
+	chmod -R u+rwX "$artifacts_dir/runs" || true
+fi
+rm -rf "$artifacts_dir/runs"
+rm -rf "$sftp_artifacts_dir"
+rm -f "$artifacts_dir/compose.log"
+rm -f "$docker_output_log"
+mkdir -p "$artifacts_dir/runs"
+mkdir -p "$sftp_artifacts_dir"
+
+ssh-keygen -q -t ed25519 -N "" -f "$sftp_artifacts_dir/id_ed25519"
+chmod 600 "$sftp_artifacts_dir/id_ed25519"
+chmod 644 "$sftp_artifacts_dir/id_ed25519.pub"
+
+docker build --progress quiet --target runtime-tools -t "$base_image" "$repo_root" >"$docker_output_log" 2>&1
+
+exit_code=0
+"${compose[@]}" up --build --no-color --detach rustfs >>"$docker_output_log" 2>&1 || exit_code=$?
+
+if [[ "$exit_code" -eq 0 ]]; then
+	"${compose[@]}" up --build --no-color --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup >>"$docker_output_log" 2>&1 || exit_code=$?
+fi
+
+if [[ "$exit_code" -eq 0 ]]; then
+	volume_services=(sftp webdav smb)
+	if [[ "${SKIP_VOLUME_MOUNT_INTEGRATION_TESTS:-false}" != "true" ]]; then
+		volume_services+=(nfs)
+	fi
+	"${compose[@]}" up --build --no-color --detach --wait "${volume_services[@]}" >>"$docker_output_log" 2>&1 || exit_code=$?
+fi
+
+if [[ "$exit_code" -eq 0 ]]; then
+	"${compose[@]}" run --rm --no-deps --build integration 2>>"$docker_output_log" || exit_code=$?
+fi
+
+if [[ "$exit_code" -ne 0 ]]; then
+	"${compose[@]}" logs --no-color >"$artifacts_dir/compose.log" || true
+	echo "Integration Docker logs: $artifacts_dir/compose.log" >&2
+	echo "Integration Docker command output: $docker_output_log" >&2
+fi
+
+"${compose[@]}" down --volumes --remove-orphans >>"$docker_output_log" 2>&1 || true
+
+exit "$exit_code"

app/test/integration/src/constants.ts

@@ -0,0 +1,14 @@
+import path from "node:path";
+
+export const RUSTFS_ENDPOINT = "http://rustfs:9000";
+export const RUSTFS_ACCESS_KEY_ID = "rustfsadmin";
+export const RUSTFS_SECRET_ACCESS_KEY = "rustfsadmin";
+export const RUSTFS_BUCKET = "zerobyte-integration";
+
+export const RCLONE_REMOTE = "e2e-rustfs";
+export const RCLONE_CONFIG_DIR = "/root/.config/rclone";
+export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf");
+
+export const INTEGRATION_ORGANIZATION_ID = "integration-suite";
+export const INTEGRATION_ARTIFACTS_DIR = path.resolve(import.meta.dirname, "../artifacts");
+export const INTEGRATION_RUNS_DIR = path.join(INTEGRATION_ARTIFACTS_DIR, "runs");

app/test/integration/src/helpers/assertions.ts

@@ -0,0 +1,42 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { expect } from "vitest";
+import type { ScenarioFixture } from "./fixture";
+
+type ResticLsNode = {
+	path: string;
+	type: string;
+};
+
+const nodeByPath = (nodes: ResticLsNode[]) => new Map(nodes.map((node) => [path.normalize(node.path), node]));
+
+export const assertSnapshotContainsFixture = (sourceRoot: string, nodes: ResticLsNode[], fixture: ScenarioFixture) => {
+	const nodesByPath = nodeByPath(nodes);
+
+	for (const entry of fixture.entries) {
+		const node = nodesByPath.get(path.join(sourceRoot, entry.relativePath));
+		expect(node?.type).toBe(entry.type === "directory" ? "dir" : entry.type);
+	}
+};
+
+export const assertRestoredFixture = async (restoreRoot: string, fixture: ScenarioFixture) => {
+	for (const entry of fixture.entries) {
+		const entryPath = path.join(restoreRoot, entry.relativePath);
+		if (entry.type === "file") {
+			await expect(fs.readFile(entryPath, "utf8")).resolves.toBe(entry.content);
+		}
+		if (entry.type === "directory") {
+			const stats = await fs.stat(entryPath);
+			expect(stats.isDirectory()).toBe(true);
+		}
+		if (entry.type === "symlink") {
+			await expect(fs.readlink(entryPath)).resolves.toBe(entry.target);
+		}
+	}
+};
+
+export const assertFixtureSourceExists = async (fixture: ScenarioFixture) => {
+	const stats = await fs.stat(fixture.sourceRoot);
+	expect(stats.isDirectory()).toBe(true);
+	await assertRestoredFixture(fixture.sourceRoot, fixture);
+};

app/test/integration/src/helpers/fixture.ts

@@ -0,0 +1,91 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export type FixtureEntry =
+	| {
+			type: "file";
+			relativePath: string;
+			content: string;
+	  }
+	| {
+			type: "directory";
+			relativePath: string;
+	  }
+	| {
+			type: "symlink";
+			relativePath: string;
+			target: string;
+	  };
+
+export type ScenarioFixture = {
+	sourceRoot: string;
+	entries: FixtureEntry[];
+};
+
+export const createScenarioFixture = async (workspace: string, scenarioId: string): Promise<ScenarioFixture> => {
+	const sourceRoot = path.join(workspace, "source");
+	const randomSuffix = crypto.randomBytes(8).toString("hex");
+
+	const fixture: ScenarioFixture = {
+		sourceRoot,
+		entries: [
+			{
+				type: "file",
+				relativePath: "regular.txt",
+				content: `regular fixture for ${scenarioId} (${randomSuffix})\n`,
+			},
+			{
+				type: "directory",
+				relativePath: "nested",
+			},
+			{
+				type: "file",
+				relativePath: "nested/deep.txt",
+				content: `nested fixture for ${scenarioId} (${randomSuffix})\n`,
+			},
+			{
+				type: "symlink",
+				relativePath: "regular-link",
+				target: "regular.txt",
+			},
+		],
+	};
+
+	for (const entry of fixture.entries) {
+		const entryPath = path.join(sourceRoot, entry.relativePath);
+		if (entry.type === "file") {
+			await fs.mkdir(path.dirname(entryPath), { recursive: true });
+			await fs.writeFile(entryPath, entry.content);
+		}
+		if (entry.type === "directory") {
+			await fs.mkdir(entryPath, { recursive: true });
+		}
+		if (entry.type === "symlink") {
+			await fs.mkdir(path.dirname(entryPath), { recursive: true });
+			await fs.symlink(entry.target, entryPath);
+		}
+	}
+
+	return fixture;
+};
+
+export const createStaticVolumeFixture = (sourceRoot: string): ScenarioFixture => ({
+	sourceRoot,
+	entries: [
+		{
+			type: "file",
+			relativePath: "hello.txt",
+			content: "hello from zerobyte integration\n",
+		},
+		{
+			type: "directory",
+			relativePath: "docs",
+		},
+		{
+			type: "file",
+			relativePath: "docs/readme.md",
+			content: "fixture documentation\n",
+		},
+	],
+});

app/test/integration/src/helpers/nfs.ts

@@ -0,0 +1,14 @@
+import type { BackendConfig } from "@zerobyte/contracts/volumes";
+
+export const NFS_HOST = "nfs";
+export const NFS_PORT = 2049;
+export const NFS_EXPORT_PATH = "/";
+
+export const buildNfsVolumeConfig = (): BackendConfig => ({
+	backend: "nfs",
+	server: NFS_HOST,
+	exportPath: NFS_EXPORT_PATH,
+	port: NFS_PORT,
+	version: "4.1",
+	readOnly: true,
+});

app/test/integration/src/helpers/restic.ts

@@ -0,0 +1,14 @@
+import path from "node:path";
+import { createRestic } from "@zerobyte/core/restic/server";
+import { RCLONE_CONFIG_FILE } from "../constants";
+
+export const createIntegrationRestic = (workspace: string, resticPassword: string) => {
+	return createRestic({
+		resolveSecret: async (value: string) => value,
+		getOrganizationResticPassword: async () => resticPassword,
+		resticCacheDir: path.join(workspace, "restic-cache"),
+		resticPassFile: path.join(workspace, "restic.pass"),
+		defaultExcludes: [],
+		rcloneConfigFile: RCLONE_CONFIG_FILE,
+	});
+};

app/test/integration/src/helpers/sftp.ts

@@ -0,0 +1,86 @@
+import fs from "node:fs/promises";
+import type { BackendConfig } from "@zerobyte/contracts/volumes";
+import { safeExec } from "@zerobyte/core/node";
+import type { RepositoryConfig } from "@zerobyte/core/restic";
+
+export const SFTP_HOST = "sftp";
+export const SFTP_PORT = 22;
+export const SFTP_USERNAME = "zerobyte-sftp";
+export const SFTP_PASSWORD = "zerobyte-sftp-password";
+export const SFTP_FIXTURE_ROOT = "/srv/zerobyte-integration/fixtures";
+export const SFTP_REPOSITORY_ROOT = "/srv/zerobyte-integration/repos";
+
+export const readSftpPrivateKey = async () => {
+	const privateKeyPath = process.env.SFTP_PRIVATE_KEY_PATH;
+	if (!privateKeyPath) {
+		throw new Error("SFTP_PRIVATE_KEY_PATH is required for SFTP integration tests");
+	}
+
+	return fs.readFile(privateKeyPath, "utf8");
+};
+
+export const scanSftpKnownHosts = async () => {
+	const result = await safeExec({
+		command: "ssh-keyscan",
+		args: ["-T", "5", SFTP_HOST],
+		timeout: 10_000,
+	});
+
+	if (result.exitCode !== 0 || result.stdout.trim().length === 0) {
+		throw new Error(`Failed to scan SFTP known hosts: ${result.stderr || result.stdout}`);
+	}
+
+	return result.stdout;
+};
+
+export const buildSftpPrivateKeyVolumeConfig = ({
+	privateKey,
+	knownHosts,
+}: {
+	privateKey: string;
+	knownHosts: string;
+}): BackendConfig => ({
+	backend: "sftp",
+	host: SFTP_HOST,
+	port: SFTP_PORT,
+	username: SFTP_USERNAME,
+	privateKey,
+	path: SFTP_FIXTURE_ROOT,
+	readOnly: true,
+	skipHostKeyCheck: false,
+	knownHosts,
+	allowLegacySshRsa: false,
+});
+
+export const buildSftpPasswordVolumeConfig = ({ knownHosts }: { knownHosts: string }): BackendConfig => ({
+	backend: "sftp",
+	host: SFTP_HOST,
+	port: SFTP_PORT,
+	username: SFTP_USERNAME,
+	password: SFTP_PASSWORD,
+	path: SFTP_FIXTURE_ROOT,
+	readOnly: true,
+	skipHostKeyCheck: false,
+	knownHosts,
+	allowLegacySshRsa: false,
+});
+
+export const buildSftpRepositoryConfig = ({
+	privateKey,
+	knownHosts,
+	path,
+}: {
+	privateKey: string;
+	knownHosts: string;
+	path: string;
+}): RepositoryConfig => ({
+	backend: "sftp",
+	host: SFTP_HOST,
+	port: SFTP_PORT,
+	user: SFTP_USERNAME,
+	path,
+	privateKey,
+	skipHostKeyCheck: false,
+	knownHosts,
+	allowLegacySshRsa: false,
+});

app/test/integration/src/helpers/smb.ts

@@ -0,0 +1,19 @@
+import type { BackendConfig } from "@zerobyte/contracts/volumes";
+
+export const SMB_HOST = "smb";
+export const SMB_PORT = 445;
+export const SMB_SHARE = "zerobyte-integration";
+export const SMB_USERNAME = "zerobyte-smb";
+export const SMB_PASSWORD = "zerobyte-smb-password";
+
+export const buildSmbVolumeConfig = (): BackendConfig => ({
+	backend: "smb",
+	server: SMB_HOST,
+	share: SMB_SHARE,
+	username: SMB_USERNAME,
+	password: SMB_PASSWORD,
+	mapToContainerUidGid: false,
+	vers: "3.0",
+	port: SMB_PORT,
+	readOnly: true,
+});

app/test/integration/src/helpers/webdav.ts

@@ -0,0 +1,18 @@
+import type { BackendConfig } from "@zerobyte/contracts/volumes";
+
+export const WEBDAV_HOST = "webdav";
+export const WEBDAV_PORT = 80;
+export const WEBDAV_USERNAME = "zerobyte-webdav";
+export const WEBDAV_PASSWORD = "zerobyte-webdav-password";
+export const WEBDAV_FIXTURE_ROOT = "/zerobyte-integration";
+
+export const buildWebdavVolumeConfig = (): BackendConfig => ({
+	backend: "webdav",
+	server: WEBDAV_HOST,
+	path: WEBDAV_FIXTURE_ROOT,
+	username: WEBDAV_USERNAME,
+	password: WEBDAV_PASSWORD,
+	port: WEBDAV_PORT,
+	readOnly: true,
+	ssl: false,
+});

app/test/integration/src/repositories.test.ts

@@ -0,0 +1,145 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { Effect } from "effect";
+import type { RepositoryConfig } from "@zerobyte/core/restic";
+import { expect, test } from "vitest";
+import {
+	INTEGRATION_ORGANIZATION_ID,
+	INTEGRATION_RUNS_DIR,
+	RCLONE_REMOTE,
+	RUSTFS_ACCESS_KEY_ID,
+	RUSTFS_BUCKET,
+	RUSTFS_ENDPOINT,
+	RUSTFS_SECRET_ACCESS_KEY,
+} from "./constants";
+import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
+import { createScenarioFixture } from "./helpers/fixture";
+import { createIntegrationRestic } from "./helpers/restic";
+import {
+	buildSftpRepositoryConfig,
+	readSftpPrivateKey,
+	scanSftpKnownHosts,
+	SFTP_REPOSITORY_ROOT,
+} from "./helpers/sftp";
+
+type RepositoryScenario = {
+	id: string;
+	name: string;
+	createRepositoryConfig: (prefix: string) => RepositoryConfig | Promise<RepositoryConfig>;
+};
+
+const scenarios: RepositoryScenario[] = [
+	{
+		id: "direct-s3",
+		name: "direct S3 repository",
+		createRepositoryConfig: (prefix) => ({
+			backend: "s3",
+			endpoint: RUSTFS_ENDPOINT,
+			bucket: `${RUSTFS_BUCKET}/${prefix}`,
+			accessKeyId: RUSTFS_ACCESS_KEY_ID,
+			secretAccessKey: RUSTFS_SECRET_ACCESS_KEY,
+		}),
+	},
+	{
+		id: "rclone-s3",
+		name: "rclone repository over RustFS S3",
+		createRepositoryConfig: (prefix) => ({
+			backend: "rclone",
+			remote: RCLONE_REMOTE,
+			path: `${RUSTFS_BUCKET}/${prefix}`,
+		}),
+	},
+	{
+		id: "sftp",
+		name: "SFTP repository",
+		createRepositoryConfig: async (prefix) =>
+			buildSftpRepositoryConfig({
+				privateKey: await readSftpPrivateKey(),
+				knownHosts: await scanSftpKnownHosts(),
+				path: `${SFTP_REPOSITORY_ROOT}/${prefix}`,
+			}),
+	},
+];
+
+test.concurrent.each(scenarios)("$name can backup, list, and restore fixture data", async (scenario) => {
+	const runId = crypto.randomUUID();
+	const repositoryPrefix = `${scenario.id}/${runId}`;
+	const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`);
+	const restoreTarget = path.join(workspace, "restore");
+	const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
+	const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
+	const repositoryConfig = await scenario.createRepositoryConfig(repositoryPrefix);
+	const restic = createIntegrationRestic(workspace, resticPassword);
+
+	let passed = false;
+
+	try {
+		await fs.mkdir(workspace, { recursive: true });
+		const fixture = await createScenarioFixture(workspace, scenario.id);
+		await assertFixtureSourceExists(fixture);
+
+		const initResult = await Effect.runPromise(
+			restic.init(repositoryConfig, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				timeoutMs: 120_000,
+			}),
+		);
+		expect(initResult.success).toBe(true);
+		expect(initResult.error).toBeNull();
+
+		const backupResult = await Effect.runPromise(
+			restic.backup(repositoryConfig, fixture.sourceRoot, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				tags: [backupTag],
+			}),
+		);
+
+		expect(backupResult.exitCode).toBe(0);
+		expect(backupResult.warningDetails).toBeNull();
+		expect(backupResult.result?.snapshot_id).toEqual(expect.any(String));
+
+		const snapshotId = backupResult.result?.snapshot_id;
+		if (!snapshotId) {
+			throw new Error("Restic backup completed without a snapshot id");
+		}
+
+		const snapshots = await Effect.runPromise(
+			restic.snapshots(repositoryConfig, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				tags: [backupTag],
+			}),
+		);
+		const snapshot = snapshots.find(
+			(candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId,
+		);
+		expect(snapshot).toBeDefined();
+		expect(snapshot?.paths).toContain(fixture.sourceRoot);
+
+		const lsResult = await Effect.runPromise(
+			restic.ls(repositoryConfig, snapshotId, undefined, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				limit: 100,
+			}),
+		);
+		assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
+
+		await fs.rm(fixture.sourceRoot, { recursive: true, force: true });
+
+		await Effect.runPromise(
+			restic.restore(repositoryConfig, snapshotId, restoreTarget, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				basePath: fixture.sourceRoot,
+			}),
+		);
+		await assertRestoredFixture(restoreTarget, fixture);
+
+		passed = true;
+	} finally {
+		if (passed) {
+			await fs.rm(workspace, { recursive: true, force: true });
+		} else {
+			console.error(`Integration scenario artifacts retained in ${workspace}`);
+		}
+	}
+});

app/test/integration/src/volume-backends.test.ts

@@ -0,0 +1,200 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { RepositoryConfig } from "@zerobyte/core/restic";
+import { Effect } from "effect";
+import { expect, test } from "vitest";
+import { makeNfsBackend } from "../../../../apps/agent/src/volume-host/backends/nfs";
+import { makeSmbBackend } from "../../../../apps/agent/src/volume-host/backends/smb";
+import { makeSftpBackend } from "../../../../apps/agent/src/volume-host/backends/sftp";
+import { makeWebdavBackend } from "../../../../apps/agent/src/volume-host/backends/webdav";
+import type { VolumeBackend } from "../../../../apps/agent/src/volume-host/types";
+import { INTEGRATION_ORGANIZATION_ID, INTEGRATION_RUNS_DIR } from "./constants";
+import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
+import { createStaticVolumeFixture } from "./helpers/fixture";
+import { buildNfsVolumeConfig } from "./helpers/nfs";
+import { createIntegrationRestic } from "./helpers/restic";
+import {
+	buildSftpPasswordVolumeConfig,
+	buildSftpPrivateKeyVolumeConfig,
+	readSftpPrivateKey,
+	scanSftpKnownHosts,
+} from "./helpers/sftp";
+import { buildSmbVolumeConfig } from "./helpers/smb";
+import { buildWebdavVolumeConfig } from "./helpers/webdav";
+
+type VolumeScenario = {
+	id: string;
+	name: string;
+	createBackend: (mountPath: string) => Promise<VolumeBackend>;
+};
+
+const scenarios: VolumeScenario[] = [
+	{
+		id: "sftp-local-repo",
+		name: "SFTP volume with private key auth and local repository",
+		createBackend: async (mountPath) => {
+			const knownHosts = await scanSftpKnownHosts();
+			const privateKey = await readSftpPrivateKey();
+			const config = buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts });
+			return makeSftpBackend(config, mountPath);
+		},
+	},
+	{
+		id: "sftp-password-local-repo",
+		name: "SFTP volume with password auth and local repository",
+		createBackend: async (mountPath) => {
+			const knownHosts = await scanSftpKnownHosts();
+			const config = buildSftpPasswordVolumeConfig({ knownHosts });
+			return makeSftpBackend(config, mountPath);
+		},
+	},
+	{
+		id: "webdav-local-repo",
+		name: "WebDAV volume with local repository",
+		createBackend: async (mountPath) => makeWebdavBackend(buildWebdavVolumeConfig(), mountPath),
+	},
+	{
+		id: "smb-local-repo",
+		name: "SMB volume with local repository",
+		createBackend: async (mountPath) => makeSmbBackend(buildSmbVolumeConfig(), mountPath),
+	},
+	{
+		id: "nfs-local-repo",
+		name: "NFS volume with local repository",
+		createBackend: async (mountPath) => makeNfsBackend(buildNfsVolumeConfig(), mountPath),
+	},
+];
+
+const volumeMountTest = process.env.SKIP_VOLUME_MOUNT_INTEGRATION_TESTS === "true" ? test.skip : test;
+
+const makeDirectoriesWritable = async (root: string): Promise<void> => {
+	await fs.chmod(root, 0o700).catch(() => {});
+
+	const entries = await fs.readdir(root, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
+		if (error.code === "ENOENT") return [];
+		throw error;
+	});
+
+	await Promise.all(
+		entries.map(async (entry) => {
+			if (entry.isDirectory()) {
+				await makeDirectoriesWritable(path.join(root, entry.name));
+			}
+		}),
+	);
+};
+
+volumeMountTest.concurrent.each(scenarios)("$name can backup and restore static fixture data", async (scenario) => {
+	const runId = crypto.randomUUID();
+	const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`);
+	const mountPath = path.join(workspace, "mount");
+	const repositoryPath = path.join(workspace, "repo");
+	const restoreTarget = path.join(workspace, "restore");
+	const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
+	const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
+	const repositoryConfig: RepositoryConfig = { backend: "local", path: repositoryPath };
+	const restic = createIntegrationRestic(workspace, resticPassword);
+
+	let backend: VolumeBackend | undefined;
+	let passed = false;
+	let unmountFailed = false;
+	let cleanupError: Error | undefined;
+
+	try {
+		await fs.mkdir(workspace, { recursive: true });
+
+		const initResult = await Effect.runPromise(
+			restic.init(repositoryConfig, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				timeoutMs: 120_000,
+			}),
+		);
+		expect(initResult.success).toBe(true);
+		expect(initResult.error).toBeNull();
+
+		backend = await scenario.createBackend(mountPath);
+
+		const mountResult = await backend.mount();
+		expect(mountResult.status).toBe("mounted");
+
+		const healthResult = await backend.checkHealth();
+		expect(healthResult.status).toBe("mounted");
+
+		const fixture = createStaticVolumeFixture(path.join(mountPath, "case-a"));
+		await assertFixtureSourceExists(fixture);
+
+		const backupResult = await Effect.runPromise(
+			restic.backup(repositoryConfig, fixture.sourceRoot, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				tags: [backupTag],
+			}),
+		);
+
+		expect(backupResult.exitCode).toBe(0);
+		expect(backupResult.warningDetails).toBeNull();
+		expect(backupResult.result?.snapshot_id).toEqual(expect.any(String));
+
+		const snapshotId = backupResult.result?.snapshot_id;
+		if (!snapshotId) {
+			throw new Error("Restic backup completed without a snapshot id");
+		}
+
+		const snapshots = await Effect.runPromise(
+			restic.snapshots(repositoryConfig, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				tags: [backupTag],
+			}),
+		);
+		const snapshot = snapshots.find(
+			(candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId,
+		);
+		expect(snapshot).toBeDefined();
+		expect(snapshot?.paths).toContain(fixture.sourceRoot);
+
+		const lsResult = await Effect.runPromise(
+			restic.ls(repositoryConfig, snapshotId, undefined, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				limit: 100,
+			}),
+		);
+		assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
+
+		const unmountResult = await backend.unmount();
+		expect(unmountResult.status).toBe("unmounted");
+		backend = undefined;
+
+		await Effect.runPromise(
+			restic.restore(repositoryConfig, snapshotId, restoreTarget, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				basePath: fixture.sourceRoot,
+			}),
+		);
+		await assertRestoredFixture(restoreTarget, fixture);
+
+		passed = true;
+	} finally {
+		if (backend) {
+			const unmountResult = await backend.unmount();
+			if (unmountResult.status === "error") {
+				unmountFailed = true;
+				console.error(`Failed to unmount ${scenario.id} volume at ${mountPath}: ${unmountResult.error}`);
+			}
+		}
+
+		if (passed && !unmountFailed) {
+			await makeDirectoriesWritable(workspace);
+			await fs.rm(workspace, { recursive: true, force: true });
+		} else {
+			console.error(`Integration scenario artifacts retained in ${workspace}`);
+		}
+
+		if (passed && unmountFailed) {
+			cleanupError = new Error(`Failed to unmount ${scenario.id} volume at ${mountPath}`);
+		}
+	}
+
+	if (cleanupError) {
+		throw cleanupError;
+	}
+});

app/test/integration/src/write-rclone-config.ts

@@ -0,0 +1,24 @@
+import fs from "node:fs/promises";
+import {
+	RCLONE_CONFIG_DIR,
+	RCLONE_CONFIG_FILE,
+	RCLONE_REMOTE,
+	RUSTFS_ACCESS_KEY_ID,
+	RUSTFS_ENDPOINT,
+	RUSTFS_SECRET_ACCESS_KEY,
+} from "./constants";
+
+const rcloneConfig = `[${RCLONE_REMOTE}]
+type = s3
+provider = Minio
+env_auth = false
+access_key_id = ${RUSTFS_ACCESS_KEY_ID}
+secret_access_key = ${RUSTFS_SECRET_ACCESS_KEY}
+endpoint = ${RUSTFS_ENDPOINT}
+force_path_style = true
+acl = private
+`;
+
+await fs.mkdir(RCLONE_CONFIG_DIR, { recursive: true });
+await fs.writeFile(RCLONE_CONFIG_FILE, rcloneConfig, { mode: 0o600 });
+await fs.chmod(RCLONE_CONFIG_FILE, 0o600);

app/test/integration/vitest.config.ts

@@ -0,0 +1,23 @@
+import path from "node:path";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+	resolve: {
+		alias: {
+			"~": path.resolve(import.meta.dirname, "../.."),
+		},
+	},
+	test: {
+		server: {
+			deps: {
+				inline: ["zod"],
+			},
+		},
+		name: "integration",
+		environment: "node",
+		include: ["app/test/integration/src/**/*.test.ts"],
+		testTimeout: 300_000,
+		hookTimeout: 300_000,
+		maxConcurrency: 4,
+	},
+});

apps/agent/src/commands/__tests__/restore.test.ts

@@ -0,0 +1,122 @@
+import { Effect } from "effect";
+import { afterEach, expect, test, vi } from "vitest";
+import waitForExpect from "wait-for-expect";
+import { fromPartial } from "@total-typescript/shoehorn";
+import { parseAgentMessage, type RestoreRunPayload } from "@zerobyte/contracts/agent-protocol";
+import * as resticServer from "@zerobyte/core/restic/server";
+import { handleRestoreCancelCommand } from "../restore-cancel";
+import { handleRestoreRunCommand } from "../restore";
+import type { ControllerCommandContext, RunningJob } from "../../context";
+
+afterEach(() => {
+	vi.restoreAllMocks();
+});
+
+const createRunPayload = (overrides: Partial<RestoreRunPayload> = {}) =>
+	fromPartial<RestoreRunPayload>({
+		restoreId: "restore-1",
+		organizationId: "org-1",
+		repositoryId: "repo-1",
+		snapshotId: "snapshot-1",
+		target: `${process.cwd()}/restore-target`,
+		repositoryConfig: { backend: "local", path: "/tmp/repository" },
+		runtime: { password: "password" },
+		options: { organizationId: "org-1", basePath: "/" },
+		...overrides,
+	});
+
+const createContext = () => {
+	const outboundMessages: string[] = [];
+	const runningJobs = new Map<string, RunningJob>();
+
+	const context: ControllerCommandContext = {
+		getRunningJob: (jobId) => Effect.succeed(runningJobs.get(jobId)),
+		setRunningJob: (jobId, job) =>
+			Effect.sync(() => {
+				runningJobs.set(jobId, job);
+			}),
+		deleteRunningJob: (jobId) =>
+			Effect.sync(() => {
+				runningJobs.delete(jobId);
+			}),
+		offerOutbound: (message) =>
+			Effect.sync(() => {
+				outboundMessages.push(message);
+				return true;
+			}),
+	};
+
+	return { context, runningJobs, messages: () => outboundMessages.map((message) => parseAgentMessage(message)) };
+};
+
+test("forks restore execution and emits lifecycle events", async () => {
+	vi.spyOn(resticServer, "createRestic").mockReturnValue(
+		fromPartial({
+			restore: () =>
+				Effect.succeed({
+					message_type: "summary" as const,
+					files_restored: 2,
+					files_skipped: 1,
+					bytes_skipped: 0,
+				}),
+		}),
+	);
+	const payload = createRunPayload();
+	const { context, runningJobs, messages } = createContext();
+
+	await Effect.runPromise(
+		Effect.gen(function* () {
+			yield* handleRestoreRunCommand(context, payload);
+			yield* Effect.promise(() =>
+				waitForExpect(() => {
+					expect(runningJobs.has(payload.restoreId)).toBe(false);
+				}),
+			);
+		}),
+	);
+
+	expect(messages().flatMap((message) => (message?.success ? [message.data.type] : []))).toEqual([
+		"restore.started",
+		"restore.completed",
+	]);
+});
+
+test("cancels a running restore with the shared running job registry", async () => {
+	vi.spyOn(resticServer, "createRestic").mockReturnValue(
+		fromPartial({
+			restore: (_config: unknown, _snapshotId: string, _target: string, options: { signal?: AbortSignal }) =>
+				Effect.tryPromise(
+					() =>
+						new Promise<never>((_resolve, reject) => {
+							options.signal?.addEventListener("abort", () => reject(new Error("aborted")), {
+								once: true,
+							});
+						}),
+				),
+		}),
+	);
+	const payload = createRunPayload();
+	const { context, runningJobs, messages } = createContext();
+
+	await Effect.runPromise(
+		Effect.gen(function* () {
+			yield* handleRestoreRunCommand(context, payload);
+			yield* Effect.promise(() =>
+				waitForExpect(() => {
+					expect(runningJobs.get(payload.restoreId)?.kind).toBe("restore");
+				}),
+			);
+
+			yield* handleRestoreCancelCommand(context, { restoreId: payload.restoreId });
+
+			yield* Effect.promise(() =>
+				waitForExpect(() => {
+					expect(runningJobs.has(payload.restoreId)).toBe(false);
+					expect(
+						messages().some((message) => message?.success && message.data.type === "restore.cancelled"),
+					).toBe(true);
+				}),
+			);
+		}),
+	);
+});

apps/agent/src/commands/backup-cancel.ts

@@ -11,7 +11,7 @@ export const handleBackupCancelCommand = (context: ControllerCommandContext, pay
 			return;
 		}
 
-		if (running.scheduleId !== payload.scheduleId) {
+		if (running.kind !== "backup" || running.scheduleId !== payload.scheduleId) {
 			logger.warn(`Ignoring cancel for backup ${payload.jobId} due to schedule mismatch ${payload.scheduleId}`);
 			return;
 		}

apps/agent/src/commands/backup-run.ts

@@ -64,7 +64,11 @@ export const handleBackupRunCommand = (context: ControllerCommandContext, payloa
 
 		yield* logger.effect.info(`Starting backup ${payload.jobId} for schedule ${payload.scheduleId}`);
 		const abortController = new AbortController();
-		yield* context.setRunningJob(payload.jobId, { scheduleId: payload.scheduleId, abortController });
+		yield* context.setRunningJob(payload.jobId, {
+			kind: "backup",
+			scheduleId: payload.scheduleId,
+			abortController,
+		});
 
 		yield* Effect.fork(
 			Effect.gen(function* () {

apps/agent/src/commands/helpers/__tests__/backup.helpers.test.ts

@@ -130,6 +130,37 @@ describe("backup path options", () => {
 		).toThrow("Include pattern escapes volume root");
 	});
 
+	test("rejects unsupported characters in selected include paths", () => {
+		const volumePath = "/var/lib/zerobyte/volumes/vol123/_data";
+
+		expect(() => createOptions(createPathOptions({ includePaths: ["/Photos\0/etc/passwd"] }), volumePath)).toThrow(
+			"Include path contains an unsupported path character",
+		);
+	});
+
+	test("allows line breaks in selected include paths", () => {
+		const volumePath = "/var/lib/zerobyte/volumes/vol123/_data";
+		const options = createOptions(
+			createPathOptions({ includePaths: ["/Photos\n2026", "/Photos\r2025"] }),
+			volumePath,
+		);
+
+		expect(options.includePaths).toEqual([
+			path.join(volumePath, "Photos\n2026"),
+			path.join(volumePath, "Photos\r2025"),
+		]);
+	});
+
+	test("rejects unsupported characters in include patterns", () => {
+		const volumePath = "/var/lib/zerobyte/volumes/vol123/_data";
+
+		for (const includePattern of ["/Photos\0/etc/passwd", "/Photos\n/etc/passwd", "/Photos\r/etc/passwd"]) {
+			expect(() => createOptions(createPathOptions({ includePatterns: [includePattern] }), volumePath)).toThrow(
+				"Include pattern contains an unsupported path character",
+			);
+		}
+	});
+
 	test("anchors generated include patterns under the volume path", () => {
 		const volumePath = "/var/lib/zerobyte/volumes/vol123/_data";
 

apps/agent/src/commands/helpers/backup.helpers.ts

@@ -1,8 +1,15 @@
 import path from "node:path";
 import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
+import { hasPathListSeparator } from "@zerobyte/core/utils";
 
 type BackupOptions = BackupRunPayload["options"];
 
+const validateIncludeEntry = (entry: string, name: string, format: "raw" | "text") => {
+	if (hasPathListSeparator(entry, format)) {
+		throw new Error(`${name} contains an unsupported path character: ${entry}`);
+	}
+};
+
 export const processPattern = (pattern: string, volumePath: string, relative = false) => {
 	const isNegated = pattern.startsWith("!");
 	const p = isNegated ? pattern.slice(1) : pattern;
@@ -41,8 +48,16 @@ export const createBackupOptions = (
 	signal,
 	exclude: params.options.excludePatterns?.map((p) => processPattern(p, volumePath)) ?? undefined,
 	excludeIfPresent: params.options.excludeIfPresent ?? undefined,
-	includePaths: params.options.includePaths?.map((p) => processPattern(p, volumePath, true)) ?? undefined,
-	includePatterns: params.options.includePatterns?.map((p) => processPattern(p, volumePath, true)) ?? undefined,
+	includePaths:
+		params.options.includePaths?.map((p) => {
+			validateIncludeEntry(p, "Include path", "raw");
+			return processPattern(p, volumePath, true);
+		}) ?? undefined,
+	includePatterns:
+		params.options.includePatterns?.map((p) => {
+			validateIncludeEntry(p, "Include pattern", "text");
+			return processPattern(p, volumePath, true);
+		}) ?? undefined,
 	customResticParams: params.options.customResticParams ?? [],
 	compressionMode: params.options.compressionMode,
 });

apps/agent/src/commands/index.ts

@@ -3,7 +3,8 @@ import { handleBackupCancelCommand } from "./backup-cancel";
 import { handleBackupRunCommand } from "./backup-run";
 import type { ControllerCommandContext } from "../context";
 import { handleHeartbeatPingCommand } from "./heartbeat-ping";
-import { handleRestoreCommand } from "./restore";
+import { handleRestoreCancelCommand } from "./restore-cancel";
+import { handleRestoreRunCommand } from "./restore";
 import { handleVolumeCommand } from "./volume";
 
 export const handleControllerCommand = (context: ControllerCommandContext, message: ControllerMessage) => {
@@ -17,8 +18,11 @@ export const handleControllerCommand = (context: ControllerCommandContext, messa
 		case "volume.command": {
 			return handleVolumeCommand(context, message.payload);
 		}
-		case "restore.command": {
-			return handleRestoreCommand(context, message.payload);
+		case "restore.run": {
+			return handleRestoreRunCommand(context, message.payload);
+		}
+		case "restore.cancel": {
+			return handleRestoreCancelCommand(context, message.payload);
 		}
 		case "heartbeat.ping": {
 			return handleHeartbeatPingCommand(context, message.payload);

apps/agent/src/commands/restore-cancel.ts

@@ -0,0 +1,21 @@
+import { Effect } from "effect";
+import { type RestoreCancelPayload } from "@zerobyte/contracts/agent-protocol";
+import { logger } from "@zerobyte/core/node";
+import type { ControllerCommandContext } from "../context";
+
+export const handleRestoreCancelCommand = (context: ControllerCommandContext, payload: RestoreCancelPayload) => {
+	return Effect.gen(function* () {
+		const running = yield* context.getRunningJob(payload.restoreId);
+		if (!running) {
+			logger.warn(`Restore ${payload.restoreId} is not running`);
+			return;
+		}
+
+		if (running.kind !== "restore") {
+			logger.warn(`Ignoring restore cancel for non-restore job ${payload.restoreId}`);
+			return;
+		}
+
+		running.abortController.abort();
+	});
+};

apps/agent/src/commands/restore.ts

@@ -1,7 +1,7 @@
 import os from "node:os";
 import path from "node:path";
 import { Effect, Runtime } from "effect";
-import { createAgentMessage, type RestoreCommandPayload } from "@zerobyte/contracts/agent-protocol";
+import { createAgentMessage, type RestoreRunPayload } from "@zerobyte/contracts/agent-protocol";
 import { createRestic } from "@zerobyte/core/restic/server";
 import { isPathWithin, toMessage } from "@zerobyte/core/utils";
 import { logger } from "@zerobyte/core/node";
@@ -26,48 +26,85 @@ const assertAllowedRestoreTarget = (target: string) => {
 	}
 };
 
-export const handleRestoreCommand = (context: ControllerCommandContext, payload: RestoreCommandPayload) => {
+export const handleRestoreRunCommand = (context: ControllerCommandContext, payload: RestoreRunPayload) => {
 	return Effect.gen(function* () {
-		assertAllowedRestoreTarget(payload.target);
+		const restoreContext = {
+			restoreId: payload.restoreId,
+			organizationId: payload.organizationId,
+			repositoryId: payload.repositoryId,
+			snapshotId: payload.snapshotId,
+		};
 
-		const runtime = yield* Effect.runtime<never>();
-		const restic = createRestic(resticDeps(payload.runtime.password));
-		const result = yield* restic.restore(payload.repositoryConfig, payload.snapshotId, payload.target, {
-			...payload.options,
-			onProgress: (progress) => {
-				void Runtime.runPromise(
-					runtime,
-					context.offerOutbound(
-						createAgentMessage("restore.progress", {
-							commandId: payload.commandId,
-							organizationId: payload.organizationId,
-							repositoryId: payload.repositoryId,
-							snapshotId: payload.snapshotId,
-							progress,
-						}),
-					),
-				).catch((error) => {
-					logger.error(`Failed to send restore progress update: ${toMessage(error)}`);
-				});
-			},
-		});
-
-		yield* context.offerOutbound(
-			createAgentMessage("restore.commandResult", {
-				commandId: payload.commandId,
-				status: "success",
-				result,
-			}),
-		);
-	}).pipe(
-		Effect.catchAll((error) =>
-			context.offerOutbound(
-				createAgentMessage("restore.commandResult", {
-					commandId: payload.commandId,
-					status: "error",
-					error: toMessage(error),
+		const existing = yield* context.getRunningJob(payload.restoreId);
+		if (existing) {
+			yield* context.offerOutbound(
+				createAgentMessage("restore.failed", {
+					...restoreContext,
+					error: "Restore job is already running",
 				}),
+			);
+			return;
+		}
+
+		logger.info(`Starting restore ${payload.restoreId} for snapshot ${payload.snapshotId}`);
+		const abortController = new AbortController();
+		yield* context.setRunningJob(payload.restoreId, { kind: "restore", abortController });
+
+		yield* Effect.fork(
+			Effect.gen(function* () {
+				assertAllowedRestoreTarget(payload.target);
+
+				const runtime = yield* Effect.runtime<never>();
+				const restic = createRestic(resticDeps(payload.runtime.password));
+
+				yield* context.offerOutbound(createAgentMessage("restore.started", restoreContext));
+
+				const result = yield* restic.restore(payload.repositoryConfig, payload.snapshotId, payload.target, {
+					...payload.options,
+					signal: abortController.signal,
+					onProgress: (progress) => {
+						void Runtime.runPromise(
+							runtime,
+							context.offerOutbound(
+								createAgentMessage("restore.progress", {
+									...restoreContext,
+									progress,
+								}),
+							),
+						).catch((error) => {
+							logger.error(`Failed to send restore progress update: ${toMessage(error)}`);
+						});
+					},
+				});
+
+				yield* context.offerOutbound(
+					createAgentMessage("restore.completed", {
+						...restoreContext,
+						result,
+					}),
+				);
+			}).pipe(
+				Effect.catchAll((error) => {
+					if (abortController.signal.aborted) {
+						return context.offerOutbound(
+							createAgentMessage("restore.cancelled", {
+								...restoreContext,
+								message: "Restore was cancelled",
+							}),
+						);
+					}
+
+					const errorMessage = toMessage(error);
+					return context.offerOutbound(
+						createAgentMessage("restore.failed", {
+							...restoreContext,
+							error: errorMessage,
+							errorDetails: errorMessage,
+						}),
+					);
+				}),
+				Effect.ensuring(context.deleteRunningJob(payload.restoreId)),
 			),
-		),
-	);
+		);
+	}).pipe(Effect.asVoid);
 };

apps/agent/src/context.ts

@@ -1,10 +1,9 @@
 import type { AgentWireMessage } from "@zerobyte/contracts/agent-protocol";
 import type { Effect } from "effect";
 
-export type RunningJob = {
-	scheduleId: string;
-	abortController: AbortController;
-};
+export type RunningJob =
+	| { kind: "backup"; scheduleId: string; abortController: AbortController }
+	| { kind: "restore"; abortController: AbortController };
 
 export type ControllerCommandContext = {
 	getRunningJob: (jobId: string) => Effect.Effect<RunningJob | undefined, never, never>;

apps/agent/src/controller-session.ts

@@ -1,5 +1,6 @@
 import { Effect, Fiber, Queue, Ref } from "effect";
 import {
+	AGENT_PROTOCOL_VERSION,
 	createAgentMessage,
 	parseControllerMessage,
 	type AgentWireMessage,
@@ -141,10 +142,10 @@ export const createControllerSession = (ws: WebSocket): ControllerSession => {
 				offerOutbound(
 					createAgentMessage("agent.ready", {
 						agentId: "",
-						protocolVersion: 1,
+						protocolVersion: AGENT_PROTOCOL_VERSION,
 						hostname: resolveResticHostname(),
 						platform: process.platform,
-						capabilities: { backup: true, volume: true, restic: true },
+						capabilities: { backup: true, restore: true, volume: true, restic: true },
 					}),
 				),
 			).catch((error) => {