From 03aa1a7e7e4ee0acd49701c8078b76c77bac09fe Mon Sep 17 00:00:00 2001 From: Nicolas Meienberger Date: Sat, 31 Jan 2026 14:48:34 +0100 Subject: [PATCH] refactor: stream restic ls result --- .../api-client/@tanstack/react-query.gen.ts | 40 +++++++ app/client/api-client/types.gen.ts | 12 ++ app/client/components/restore-form.tsx | 8 +- .../components/snapshot-file-browser.tsx | 8 +- .../repositories/repositories.controller.ts | 7 +- .../modules/repositories/repositories.dto.ts | 35 ++++++ .../repositories/repositories.service.ts | 29 ++++- app/server/modules/volumes/volume.service.ts | 86 ++++++-------- app/server/utils/restic.ts | 107 +++++++++++------- 9 files changed, 231 insertions(+), 101 deletions(-) diff --git a/app/client/api-client/@tanstack/react-query.gen.ts b/app/client/api-client/@tanstack/react-query.gen.ts index 4a609671..2120ef7b 100644 --- a/app/client/api-client/@tanstack/react-query.gen.ts +++ b/app/client/api-client/@tanstack/react-query.gen.ts @@ -801,6 +801,46 @@ export const listSnapshotFilesOptions = (options: Options queryKey: listSnapshotFilesQueryKey(options), }); +export const listSnapshotFilesInfiniteQueryKey = ( + options: Options, +): QueryKey> => createQueryKey("listSnapshotFiles", options, true); + +/** + * List files and directories in a snapshot + */ +export const listSnapshotFilesInfiniteOptions = (options: Options) => + infiniteQueryOptions< + ListSnapshotFilesResponse, + DefaultError, + InfiniteData, + QueryKey>, + number | Pick>[0], "body" | "headers" | "path" | "query"> + >( + // @ts-ignore + { + queryFn: async ({ pageParam, queryKey, signal }) => { + // @ts-ignore + const page: Pick>[0], "body" | "headers" | "path" | "query"> = + typeof pageParam === "object" + ? pageParam + : { + query: { + offset: pageParam, + }, + }; + const params = createInfiniteParams(queryKey, page); + const { data } = await listSnapshotFiles({ + ...options, + ...params, + signal, + throwOnError: true, + }); + return data; + }, + queryKey: listSnapshotFilesInfiniteQueryKey(options), + }, + ); + /** * Restore a snapshot to a target path on the filesystem */ diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index d6421698..5629575b 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -1737,6 +1737,14 @@ export type ListSnapshotFilesData = { }; query?: { path?: string; + /** + * Offset for pagination (default: 0) + */ + offset?: number; + /** + * Maximum number of files to return (default: 500, max: 1000) + */ + limit?: number; }; url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files"; }; @@ -1758,6 +1766,9 @@ export type ListSnapshotFilesResponses = { size?: number; uid?: number; }>; + hasMore: boolean; + limit: number; + offset: number; snapshot: { hostname: string; id: string; @@ -1765,6 +1776,7 @@ export type ListSnapshotFilesResponses = { short_id: string; time: string; }; + total: number; }; }; diff --git a/app/client/components/restore-form.tsx b/app/client/components/restore-form.tsx index e2594232..67b72935 100644 --- a/app/client/components/restore-form.tsx +++ b/app/client/components/restore-form.tsx @@ -74,11 +74,11 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re const fileBrowser = useFileBrowser({ initialData: filesData, isLoading: filesLoading, - fetchFolder: async (path) => { + fetchFolder: async (path, offset = 0) => { return await queryClient.ensureQueryData( listSnapshotFilesOptions({ path: { id: repository.id, snapshotId }, - query: { path }, + query: { path, offset, limit: 500 }, }), ); }, @@ -86,7 +86,7 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re void queryClient.prefetchQuery( listSnapshotFilesOptions({ path: { id: repository.id, snapshotId }, - query: { path }, + query: { path, offset: 0, limit: 500 }, }), ); }, @@ -306,6 +306,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re onFolderHover={fileBrowser.handleFolderHover} expandedFolders={fileBrowser.expandedFolders} loadingFolders={fileBrowser.loadingFolders} + onLoadMore={fileBrowser.handleLoadMore} + getFolderPagination={fileBrowser.getFolderPagination} className="px-2 py-2" withCheckboxes={true} selectedPaths={selectedPaths} diff --git a/app/client/modules/backups/components/snapshot-file-browser.tsx b/app/client/modules/backups/components/snapshot-file-browser.tsx index e2efbb37..f1a0519f 100644 --- a/app/client/modules/backups/components/snapshot-file-browser.tsx +++ b/app/client/modules/backups/components/snapshot-file-browser.tsx @@ -60,11 +60,11 @@ export const SnapshotFileBrowser = (props: Props) => { const fileBrowser = useFileBrowser({ initialData: filesData, isLoading: filesLoading, - fetchFolder: async (path) => { + fetchFolder: async (path, offset = 0) => { return await queryClient.ensureQueryData( listSnapshotFilesOptions({ path: { id: repositoryId, snapshotId: snapshot.short_id }, - query: { path }, + query: { path, offset, limit: 500 }, }), ); }, @@ -72,7 +72,7 @@ export const SnapshotFileBrowser = (props: Props) => { void queryClient.prefetchQuery( listSnapshotFilesOptions({ path: { id: repositoryId, snapshotId: snapshot.short_id }, - query: { path }, + query: { path, offset: 0, limit: 500 }, }), ); }, @@ -142,6 +142,8 @@ export const SnapshotFileBrowser = (props: Props) => { onFolderHover={fileBrowser.handleFolderHover} expandedFolders={fileBrowser.expandedFolders} loadingFolders={fileBrowser.loadingFolders} + onLoadMore={fileBrowser.handleLoadMore} + getFolderPagination={fileBrowser.getFolderPagination} className="px-2 py-2" /> diff --git a/app/server/modules/repositories/repositories.controller.ts b/app/server/modules/repositories/repositories.controller.ts index 4bfcc345..b697eb55 100644 --- a/app/server/modules/repositories/repositories.controller.ts +++ b/app/server/modules/repositories/repositories.controller.ts @@ -146,7 +146,12 @@ export const repositoriesController = new Hono() const { path } = c.req.valid("query"); const decodedPath = path ? decodeURIComponent(path) : undefined; - const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath); + + const query = c.req.query(); + const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0); + const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500)); + + const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath, { offset, limit }); c.header("Cache-Control", "max-age=300, stale-while-revalidate=600"); diff --git a/app/server/modules/repositories/repositories.dto.ts b/app/server/modules/repositories/repositories.dto.ts index 7ceb5e67..d62a0844 100644 --- a/app/server/modules/repositories/repositories.dto.ts +++ b/app/server/modules/repositories/repositories.dto.ts @@ -252,6 +252,10 @@ export const listSnapshotFilesResponse = type({ paths: "string[]", }), files: snapshotFileNodeSchema.array(), + offset: "number", + limit: "number", + total: "number", + hasMore: "boolean", }); export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer; @@ -264,6 +268,37 @@ export const listSnapshotFilesDto = describeRoute({ description: "List files and directories in a snapshot", tags: ["Repositories"], operationId: "listSnapshotFiles", + parameters: [ + { + in: "query", + name: "path", + required: false, + schema: { + type: "string", + }, + description: "Subdirectory path to list", + }, + { + in: "query", + name: "offset", + required: false, + schema: { + type: "integer", + default: 0, + }, + description: "Offset for pagination (default: 0)", + }, + { + in: "query", + name: "limit", + required: false, + schema: { + type: "integer", + default: 500, + }, + description: "Maximum number of files to return (default: 500, max: 1000)", + }, + ], responses: { 200: { description: "List of files and directories in the snapshot", diff --git a/app/server/modules/repositories/repositories.service.ts b/app/server/modules/repositories/repositories.service.ts index ec2ecf56..c513f75a 100644 --- a/app/server/modules/repositories/repositories.service.ts +++ b/app/server/modules/repositories/repositories.service.ts @@ -210,7 +210,12 @@ const listSnapshots = async (id: string, backupId?: string) => { } }; -const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) => { +const listSnapshotFiles = async ( + id: string, + snapshotId: string, + path?: string, + options?: { offset?: number; limit?: number }, +) => { const organizationId = getOrganizationId(); const repository = await findRepository(id); @@ -218,18 +223,30 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) throw new NotFoundError("Repository not found"); } - const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}`; - const cached = cache.get>>(cacheKey); + const offset = options?.offset ?? 0; + const limit = options?.limit ?? 500; + + const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`; + type LsResult = { + snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null; + nodes: { name: string; type: string; path: string; size?: number; mode?: number }[]; + pagination: { offset: number; limit: number; total: number; hasMore: boolean }; + }; + const cached = cache.get(cacheKey); if (cached?.snapshot) { return { snapshot: cached.snapshot, files: cached.nodes, + offset: cached.pagination.offset, + limit: cached.pagination.limit, + total: cached.pagination.total, + hasMore: cached.pagination.hasMore, }; } const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`); try { - const result = await restic.ls(repository.config, snapshotId, organizationId, path); + const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit }); if (!result.snapshot) { throw new NotFoundError("Snapshot not found or empty"); @@ -244,6 +261,10 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) paths: result.snapshot.paths, }, files: result.nodes, + offset: result.pagination.offset, + limit: result.pagination.limit, + total: result.pagination.total, + hasMore: result.pagination.hasMore, }; cache.set(cacheKey, result); diff --git a/app/server/modules/volumes/volume.service.ts b/app/server/modules/volumes/volume.service.ts index 227a6fdf..34651328 100644 --- a/app/server/modules/volumes/volume.service.ts +++ b/app/server/modules/volumes/volume.service.ts @@ -18,7 +18,6 @@ import { serverEvents } from "../../core/events"; import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { type } from "arktype"; import { getOrganizationId } from "~/server/core/request-context"; -import { exec } from "../../utils/spawn"; async function encryptSensitiveFields(config: BackendConfig): Promise { switch (config.backend) { @@ -322,11 +321,8 @@ const listFiles = async ( throw new InternalServerError("Volume is not mounted"); } - // For directory volumes, use the configured path directly const volumePath = getVolumePath(volume); - const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath; - const normalizedPath = path.normalize(requestedPath); const relative = path.relative(volumePath, normalizedPath); @@ -338,58 +334,43 @@ const listFiles = async ( const startOffset = Math.max(offset, 0); try { - const endLine = startOffset + pageSize; - const listScript = ` - find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | while IFS= read -r line; do - name=$(basename "$line") - if [ -d "$line" ]; then - echo "0|$name|$line" - else - echo "1|$name|$line" - fi - done | sort -t'|' -k1,1n -k2,2 | cut -d'|' -f3- | sed -n '${startOffset + 1},${endLine}p' - `; + const dirents = await fs.readdir(normalizedPath, { withFileTypes: true }); - const { exitCode: listExitCode, stdout: listStdout } = await exec({ - command: "sh", - args: ["-c", listScript], - timeout: 30000, - maxBuffer: 1024 * 1024 * 10, - }); + dirents.sort((a, b) => { + const aIsDir = a.isDirectory(); + const bIsDir = b.isDirectory(); - if (listExitCode !== 0) { - throw new Error(`Failed to list directory: ${normalizedPath}`); - } - - const filePaths = listStdout.split("\n").filter((p) => p.length > 0); - - const entries: DirEntry[] = []; - for (const fullPath of filePaths) { - try { - const stats = await fs.stat(fullPath); - const relativePath = path.relative(volumePath, fullPath); - const name = path.basename(fullPath); - - entries.push({ - name, - path: `/${relativePath}`, - type: stats.isDirectory() ? "directory" : "file", - size: stats.isFile() ? stats.size : undefined, - modifiedAt: stats.mtimeMs, - }); - } catch { - // File may have been deleted between listing and stat + if (aIsDir === bIsDir) { + return a.name.localeCompare(b.name); } - } - - const countScript = `find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | wc -l`; - const { exitCode: countExitCode, stdout: countStdout } = await exec({ - command: "sh", - args: ["-c", countScript], - timeout: 10000, + return aIsDir ? -1 : 1; }); - const total = countExitCode === 0 ? parseInt(countStdout.trim(), 10) || 0 : entries.length; + const total = dirents.length; + const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize); + + const entries = ( + await Promise.all( + paginatedDirents.map(async (dirent) => { + const fullPath = path.join(normalizedPath, dirent.name); + + try { + const stats = await fs.stat(fullPath); + const relativePath = path.relative(volumePath, fullPath); + + return { + name: dirent.name, + path: `/${relativePath}`, + type: dirent.isDirectory() ? "directory" : "file", + size: dirent.isFile() ? stats.size : undefined, + modifiedAt: stats.mtimeMs, + }; + } catch { + return null; + } + }), + ) + ).filter((e) => e !== null); return { files: entries, @@ -400,6 +381,9 @@ const listFiles = async ( hasMore: startOffset + entries.length < total, }; } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new NotFoundError("Directory not found"); + } throw new InternalServerError(`Failed to list files: ${toMessage(error)}`); } }; diff --git a/app/server/utils/restic.ts b/app/server/utils/restic.ts index 53b52b79..2b5905ee 100644 --- a/app/server/utils/restic.ts +++ b/app/server/utils/restic.ts @@ -701,7 +701,13 @@ const lsSnapshotInfoSchema = type({ message_type: "'snapshot'", }); -const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: string, path?: string) => { +const ls = async ( + config: RepositoryConfig, + snapshotId: string, + organizationId: string, + path?: string, + options?: { offset?: number; limit?: number }, +) => { const repoUrl = buildRepoUrl(config); const env = await buildEnv(config, organizationId); @@ -713,48 +719,71 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: addCommonArgs(args, env, config); - const res = await exec({ command: "restic", args, env }); + let snapshot: typeof lsSnapshotInfoSchema.infer | null = null; + const nodes: Array = []; + let totalNodes = 0; + let isFirstLine = true; + let hasMore = false; + + const offset = options?.offset ?? 0; + const limit = options?.limit ?? 500; + + await safeSpawn({ + command: "restic", + args, + env, + onStdout: (line) => { + const trimmedLine = line.trim(); + if (!trimmedLine) return; + + try { + const data = JSON.parse(trimmedLine); + + if (isFirstLine) { + isFirstLine = false; + const snapshotValidation = lsSnapshotInfoSchema(data); + if (!(snapshotValidation instanceof type.errors)) { + snapshot = snapshotValidation; + } + return; + } + + const nodeValidation = lsNodeSchema(data); + if (nodeValidation instanceof type.errors) { + logger.warn(`Skipping invalid node: ${nodeValidation.summary}`); + return; + } + + if (totalNodes >= offset && totalNodes < offset + limit) { + nodes.push(nodeValidation); + } + totalNodes++; + + if (totalNodes >= offset + limit + 1) { + hasMore = true; + } + } catch (_) { + // Ignore JSON parse errors for non-JSON lines + } + }, + }); + await cleanupTemporaryKeys(env); - if (res.exitCode !== 0) { - logger.error(`Restic ls failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + if (totalNodes > offset + limit) { + hasMore = true; } - // The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes - const stdout = res.stdout; - const lines = stdout - .trim() - .split("\n") - .filter((line) => line.trim()); - - if (lines.length === 0) { - return { snapshot: null, nodes: [] }; - } - - // First line is snapshot info - const snapshotLine = JSON.parse(lines[0] ?? "{}"); - const snapshot = lsSnapshotInfoSchema(snapshotLine); - - if (snapshot instanceof type.errors) { - logger.error(`Restic ls snapshot info validation failed: ${snapshot.summary}`); - throw new Error(`Restic ls snapshot info validation failed: ${snapshot.summary}`); - } - - const nodes: Array = []; - for (let i = 1; i < lines.length; i++) { - const nodeLine = JSON.parse(lines[i] ?? "{}"); - const nodeValidation = lsNodeSchema(nodeLine); - - if (nodeValidation instanceof type.errors) { - logger.warn(`Skipping invalid node: ${nodeValidation.summary}`); - continue; - } - - nodes.push(nodeValidation); - } - - return { snapshot, nodes }; + return { + snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null, + nodes, + pagination: { + offset, + limit, + total: totalNodes, + hasMore, + }, + }; }; const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {