refactor: stream restic ls result

app/client/api-client/@tanstack/react-query.gen.ts

@@ -801,6 +801,46 @@ export const listSnapshotFilesOptions = (options: Options<ListSnapshotFilesData>
 		queryKey: listSnapshotFilesQueryKey(options),
 	});
 
+export const listSnapshotFilesInfiniteQueryKey = (
+	options: Options<ListSnapshotFilesData>,
+): QueryKey<Options<ListSnapshotFilesData>> => createQueryKey("listSnapshotFiles", options, true);
+
+/**
+ * List files and directories in a snapshot
+ */
+export const listSnapshotFilesInfiniteOptions = (options: Options<ListSnapshotFilesData>) =>
+	infiniteQueryOptions<
+		ListSnapshotFilesResponse,
+		DefaultError,
+		InfiniteData<ListSnapshotFilesResponse>,
+		QueryKey<Options<ListSnapshotFilesData>>,
+		number | Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query">
+	>(
+		// @ts-ignore
+		{
+			queryFn: async ({ pageParam, queryKey, signal }) => {
+				// @ts-ignore
+				const page: Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query"> =
+					typeof pageParam === "object"
+						? pageParam
+						: {
+								query: {
+									offset: pageParam,
+								},
+							};
+				const params = createInfiniteParams(queryKey, page);
+				const { data } = await listSnapshotFiles({
+					...options,
+					...params,
+					signal,
+					throwOnError: true,
+				});
+				return data;
+			},
+			queryKey: listSnapshotFilesInfiniteQueryKey(options),
+		},
+	);
+
 /**
  * Restore a snapshot to a target path on the filesystem
  */

app/client/api-client/types.gen.ts

@@ -1737,6 +1737,14 @@ export type ListSnapshotFilesData = {
 	};
 	query?: {
 		path?: string;
+		/**
+		 * Offset for pagination (default: 0)
+		 */
+		offset?: number;
+		/**
+		 * Maximum number of files to return (default: 500, max: 1000)
+		 */
+		limit?: number;
 	};
 	url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files";
 };
@@ -1758,6 +1766,9 @@ export type ListSnapshotFilesResponses = {
 			size?: number;
 			uid?: number;
 		}>;
+		hasMore: boolean;
+		limit: number;
+		offset: number;
 		snapshot: {
 			hostname: string;
 			id: string;
@@ -1765,6 +1776,7 @@ export type ListSnapshotFilesResponses = {
 			short_id: string;
 			time: string;
 		};
+		total: number;
 	};
 };
 

app/client/components/restore-form.tsx

@@ -74,11 +74,11 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
 	const fileBrowser = useFileBrowser({
 		initialData: filesData,
 		isLoading: filesLoading,
-		fetchFolder: async (path) => {
+		fetchFolder: async (path, offset = 0) => {
 			return await queryClient.ensureQueryData(
 				listSnapshotFilesOptions({
 					path: { id: repository.id, snapshotId },
-					query: { path },
+					query: { path, offset, limit: 500 },
 				}),
 			);
 		},
@@ -86,7 +86,7 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
 			void queryClient.prefetchQuery(
 				listSnapshotFilesOptions({
 					path: { id: repository.id, snapshotId },
-					query: { path },
+					query: { path, offset: 0, limit: 500 },
 				}),
 			);
 		},
@@ -306,6 +306,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
 									onFolderHover={fileBrowser.handleFolderHover}
 									expandedFolders={fileBrowser.expandedFolders}
 									loadingFolders={fileBrowser.loadingFolders}
+									onLoadMore={fileBrowser.handleLoadMore}
+									getFolderPagination={fileBrowser.getFolderPagination}
 									className="px-2 py-2"
 									withCheckboxes={true}
 									selectedPaths={selectedPaths}

app/client/modules/backups/components/snapshot-file-browser.tsx

@@ -60,11 +60,11 @@ export const SnapshotFileBrowser = (props: Props) => {
 	const fileBrowser = useFileBrowser({
 		initialData: filesData,
 		isLoading: filesLoading,
-		fetchFolder: async (path) => {
+		fetchFolder: async (path, offset = 0) => {
 			return await queryClient.ensureQueryData(
 				listSnapshotFilesOptions({
 					path: { id: repositoryId, snapshotId: snapshot.short_id },
-					query: { path },
+					query: { path, offset, limit: 500 },
 				}),
 			);
 		},
@@ -72,7 +72,7 @@ export const SnapshotFileBrowser = (props: Props) => {
 			void queryClient.prefetchQuery(
 				listSnapshotFilesOptions({
 					path: { id: repositoryId, snapshotId: snapshot.short_id },
-					query: { path },
+					query: { path, offset: 0, limit: 500 },
 				}),
 			);
 		},
@@ -142,6 +142,8 @@ export const SnapshotFileBrowser = (props: Props) => {
 								onFolderHover={fileBrowser.handleFolderHover}
 								expandedFolders={fileBrowser.expandedFolders}
 								loadingFolders={fileBrowser.loadingFolders}
+								onLoadMore={fileBrowser.handleLoadMore}
+								getFolderPagination={fileBrowser.getFolderPagination}
 								className="px-2 py-2"
 							/>
 						</div>

app/server/modules/repositories/repositories.controller.ts

@@ -146,7 +146,12 @@ export const repositoriesController = new Hono()
 			const { path } = c.req.valid("query");
 
 			const decodedPath = path ? decodeURIComponent(path) : undefined;
-			const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath);
+
+			const query = c.req.query();
+			const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0);
+			const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500));
+
+			const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath, { offset, limit });
 
 			c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");
 

app/server/modules/repositories/repositories.dto.ts

@@ -252,6 +252,10 @@ export const listSnapshotFilesResponse = type({
 		paths: "string[]",
 	}),
 	files: snapshotFileNodeSchema.array(),
+	offset: "number",
+	limit: "number",
+	total: "number",
+	hasMore: "boolean",
 });
 
 export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer;
@@ -264,6 +268,37 @@ export const listSnapshotFilesDto = describeRoute({
 	description: "List files and directories in a snapshot",
 	tags: ["Repositories"],
 	operationId: "listSnapshotFiles",
+	parameters: [
+		{
+			in: "query",
+			name: "path",
+			required: false,
+			schema: {
+				type: "string",
+			},
+			description: "Subdirectory path to list",
+		},
+		{
+			in: "query",
+			name: "offset",
+			required: false,
+			schema: {
+				type: "integer",
+				default: 0,
+			},
+			description: "Offset for pagination (default: 0)",
+		},
+		{
+			in: "query",
+			name: "limit",
+			required: false,
+			schema: {
+				type: "integer",
+				default: 500,
+			},
+			description: "Maximum number of files to return (default: 500, max: 1000)",
+		},
+	],
 	responses: {
 		200: {
 			description: "List of files and directories in the snapshot",

app/server/modules/repositories/repositories.service.ts

@@ -210,7 +210,12 @@ const listSnapshots = async (id: string, backupId?: string) => {
 	}
 };
 
-const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) => {
+const listSnapshotFiles = async (
+	id: string,
+	snapshotId: string,
+	path?: string,
+	options?: { offset?: number; limit?: number },
+) => {
 	const organizationId = getOrganizationId();
 	const repository = await findRepository(id);
 
@@ -218,18 +223,30 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
 		throw new NotFoundError("Repository not found");
 	}
 
-	const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}`;
+	const offset = options?.offset ?? 0;
-	const cached = cache.get<Awaited<ReturnType<typeof restic.ls>>>(cacheKey);
+	const limit = options?.limit ?? 500;
+
+	const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`;
+	type LsResult = {
+		snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null;
+		nodes: { name: string; type: string; path: string; size?: number; mode?: number }[];
+		pagination: { offset: number; limit: number; total: number; hasMore: boolean };
+	};
+	const cached = cache.get<LsResult>(cacheKey);
 	if (cached?.snapshot) {
 		return {
 			snapshot: cached.snapshot,
 			files: cached.nodes,
+			offset: cached.pagination.offset,
+			limit: cached.pagination.limit,
+			total: cached.pagination.total,
+			hasMore: cached.pagination.hasMore,
 		};
 	}
 
 	const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
 	try {
-		const result = await restic.ls(repository.config, snapshotId, organizationId, path);
+		const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit });
 
 		if (!result.snapshot) {
 			throw new NotFoundError("Snapshot not found or empty");
@@ -244,6 +261,10 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
 				paths: result.snapshot.paths,
 			},
 			files: result.nodes,
+			offset: result.pagination.offset,
+			limit: result.pagination.limit,
+			total: result.pagination.total,
+			hasMore: result.pagination.hasMore,
 		};
 
 		cache.set(cacheKey, result);

app/server/modules/volumes/volume.service.ts

@@ -18,7 +18,6 @@ import { serverEvents } from "../../core/events";
 import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
 import { type } from "arktype";
 import { getOrganizationId } from "~/server/core/request-context";
-import { exec } from "../../utils/spawn";
 
 async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
 	switch (config.backend) {
@@ -322,11 +321,8 @@ const listFiles = async (
 		throw new InternalServerError("Volume is not mounted");
 	}
 
-	// For directory volumes, use the configured path directly
 	const volumePath = getVolumePath(volume);
-
 	const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
-
 	const normalizedPath = path.normalize(requestedPath);
 	const relative = path.relative(volumePath, normalizedPath);
 
@@ -338,58 +334,43 @@ const listFiles = async (
 	const startOffset = Math.max(offset, 0);
 
 	try {
-		const endLine = startOffset + pageSize;
+		const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
-		const listScript = `
-			find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | while IFS= read -r line; do
-				name=$(basename "$line")
-				if [ -d "$line" ]; then
-					echo "0|$name|$line"
-				else
-					echo "1|$name|$line"
-				fi
-			done | sort -t'|' -k1,1n -k2,2 | cut -d'|' -f3- | sed -n '${startOffset + 1},${endLine}p'
-		`;
 
-		const { exitCode: listExitCode, stdout: listStdout } = await exec({
+		dirents.sort((a, b) => {
-			command: "sh",
+			const aIsDir = a.isDirectory();
-			args: ["-c", listScript],
+			const bIsDir = b.isDirectory();
-			timeout: 30000,
-			maxBuffer: 1024 * 1024 * 10,
-		});
 
-		if (listExitCode !== 0) {
+			if (aIsDir === bIsDir) {
-			throw new Error(`Failed to list directory: ${normalizedPath}`);
+				return a.name.localeCompare(b.name);
-		}
-
-		const filePaths = listStdout.split("\n").filter((p) => p.length > 0);
-
-		const entries: DirEntry[] = [];
-		for (const fullPath of filePaths) {
-			try {
-				const stats = await fs.stat(fullPath);
-				const relativePath = path.relative(volumePath, fullPath);
-				const name = path.basename(fullPath);
-
-				entries.push({
-					name,
-					path: `/${relativePath}`,
-					type: stats.isDirectory() ? "directory" : "file",
-					size: stats.isFile() ? stats.size : undefined,
-					modifiedAt: stats.mtimeMs,
-				});
-			} catch {
-				// File may have been deleted between listing and stat
 			}
-		}
+			return aIsDir ? -1 : 1;
-
-		const countScript = `find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | wc -l`;
-		const { exitCode: countExitCode, stdout: countStdout } = await exec({
-			command: "sh",
-			args: ["-c", countScript],
-			timeout: 10000,
 		});
 
-		const total = countExitCode === 0 ? parseInt(countStdout.trim(), 10) || 0 : entries.length;
+		const total = dirents.length;
+		const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
+
+		const entries = (
+			await Promise.all(
+				paginatedDirents.map(async (dirent) => {
+					const fullPath = path.join(normalizedPath, dirent.name);
+
+					try {
+						const stats = await fs.stat(fullPath);
+						const relativePath = path.relative(volumePath, fullPath);
+
+						return {
+							name: dirent.name,
+							path: `/${relativePath}`,
+							type: dirent.isDirectory() ? "directory" : "file",
+							size: dirent.isFile() ? stats.size : undefined,
+							modifiedAt: stats.mtimeMs,
+						};
+					} catch {
+						return null;
+					}
+				}),
+			)
+		).filter((e) => e !== null);
 
 		return {
 			files: entries,
@@ -400,6 +381,9 @@ const listFiles = async (
 			hasMore: startOffset + entries.length < total,
 		};
 	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+			throw new NotFoundError("Directory not found");
+		}
 		throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
 	}
 };

app/server/utils/restic.ts

@@ -701,7 +701,13 @@ const lsSnapshotInfoSchema = type({
 	message_type: "'snapshot'",
 });
 
-const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: string, path?: string) => {
+const ls = async (
+	config: RepositoryConfig,
+	snapshotId: string,
+	organizationId: string,
+	path?: string,
+	options?: { offset?: number; limit?: number },
+) => {
 	const repoUrl = buildRepoUrl(config);
 	const env = await buildEnv(config, organizationId);
 
@@ -713,48 +719,71 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId:
 
 	addCommonArgs(args, env, config);
 
-	const res = await exec({ command: "restic", args, env });
+	let snapshot: typeof lsSnapshotInfoSchema.infer | null = null;
+	const nodes: Array<typeof lsNodeSchema.infer> = [];
+	let totalNodes = 0;
+	let isFirstLine = true;
+	let hasMore = false;
+
+	const offset = options?.offset ?? 0;
+	const limit = options?.limit ?? 500;
+
+	await safeSpawn({
+		command: "restic",
+		args,
+		env,
+		onStdout: (line) => {
+			const trimmedLine = line.trim();
+			if (!trimmedLine) return;
+
+			try {
+				const data = JSON.parse(trimmedLine);
+
+				if (isFirstLine) {
+					isFirstLine = false;
+					const snapshotValidation = lsSnapshotInfoSchema(data);
+					if (!(snapshotValidation instanceof type.errors)) {
+						snapshot = snapshotValidation;
+					}
+					return;
+				}
+
+				const nodeValidation = lsNodeSchema(data);
+				if (nodeValidation instanceof type.errors) {
+					logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
+					return;
+				}
+
+				if (totalNodes >= offset && totalNodes < offset + limit) {
+					nodes.push(nodeValidation);
+				}
+				totalNodes++;
+
+				if (totalNodes >= offset + limit + 1) {
+					hasMore = true;
+				}
+			} catch (_) {
+				// Ignore JSON parse errors for non-JSON lines
+			}
+		},
+	});
+
 	await cleanupTemporaryKeys(env);
 
-	if (res.exitCode !== 0) {
+	if (totalNodes > offset + limit) {
-		logger.error(`Restic ls failed: ${res.stderr}`);
+		hasMore = true;
-		throw new ResticError(res.exitCode, res.stderr);
 	}
 
-	// The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes
+	return {
-	const stdout = res.stdout;
+		snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null,
-	const lines = stdout
+		nodes,
-		.trim()
+		pagination: {
-		.split("\n")
+			offset,
-		.filter((line) => line.trim());
+			limit,
-
+			total: totalNodes,
-	if (lines.length === 0) {
+			hasMore,
-		return { snapshot: null, nodes: [] };
+		},
-	}
+	};
-
-	// First line is snapshot info
-	const snapshotLine = JSON.parse(lines[0] ?? "{}");
-	const snapshot = lsSnapshotInfoSchema(snapshotLine);
-
-	if (snapshot instanceof type.errors) {
-		logger.error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
-		throw new Error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
-	}
-
-	const nodes: Array<typeof lsNodeSchema.infer> = [];
-	for (let i = 1; i < lines.length; i++) {
-		const nodeLine = JSON.parse(lines[i] ?? "{}");
-		const nodeValidation = lsNodeSchema(nodeLine);
-
-		if (nodeValidation instanceof type.errors) {
-			logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
-			continue;
-		}
-
-		nodes.push(nodeValidation);
-	}
-
-	return { snapshot, nodes };
 };
 
 const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {