danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor: stream restic ls result

Browse source
This commit is contained in:
Nicolas Meienberger 2026-01-31 14:48:34 +01:00
parent 2da055e519
commit 03aa1a7e7e
9 changed files with 231 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
app/client/api-client/@tanstack/react-query.gen.ts
View file

@ -801,6 +801,46 @@ export const listSnapshotFilesOptions = (options: Options<ListSnapshotFilesData>
queryKey: listSnapshotFilesQueryKey(options),
});
export const listSnapshotFilesInfiniteQueryKey = (
options: Options<ListSnapshotFilesData>,
): QueryKey<Options<ListSnapshotFilesData>> => createQueryKey("listSnapshotFiles", options, true);
/**
* List files and directories in a snapshot
*/
export const listSnapshotFilesInfiniteOptions = (options: Options<ListSnapshotFilesData>) =>
infiniteQueryOptions<
ListSnapshotFilesResponse,
DefaultError,
InfiniteData<ListSnapshotFilesResponse>,
QueryKey<Options<ListSnapshotFilesData>>,
number | Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query">
>(
// @ts-ignore
{
queryFn: async ({ pageParam, queryKey, signal }) => {
// @ts-ignore
const page: Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query"> =
typeof pageParam === "object"
? pageParam
: {
query: {
offset: pageParam,
},
};
const params = createInfiniteParams(queryKey, page);
const { data } = await listSnapshotFiles({
...options,
...params,
signal,
throwOnError: true,
});
return data;
},
queryKey: listSnapshotFilesInfiniteQueryKey(options),
},
);
/**
* Restore a snapshot to a target path on the filesystem
*/

12
app/client/api-client/types.gen.ts
View file

@ -1737,6 +1737,14 @@ export type ListSnapshotFilesData = {
};
query?: {
path?: string;
/**
* Offset for pagination (default: 0)
*/
offset?: number;
/**
* Maximum number of files to return (default: 500, max: 1000)
*/
limit?: number;
};
url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files";
};
@ -1758,6 +1766,9 @@ export type ListSnapshotFilesResponses = {
size?: number;
uid?: number;
}>;
hasMore: boolean;
limit: number;
offset: number;
snapshot: {
hostname: string;
id: string;
@ -1765,6 +1776,7 @@ export type ListSnapshotFilesResponses = {
short_id: string;
time: string;
};
total: number;
};
};

8
app/client/components/restore-form.tsx
View file

@ -74,11 +74,11 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
const fileBrowser = useFileBrowser({
initialData: filesData,
isLoading: filesLoading,
fetchFolder: async (path) => {
fetchFolder: async (path, offset = 0) => {
return await queryClient.ensureQueryData(
listSnapshotFilesOptions({
path: { id: repository.id, snapshotId },
query: { path },
query: { path, offset, limit: 500 },
}),
);
},
@ -86,7 +86,7 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
void queryClient.prefetchQuery(
listSnapshotFilesOptions({
path: { id: repository.id, snapshotId },
query: { path },
query: { path, offset: 0, limit: 500 },
}),
);
},
@ -306,6 +306,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
onFolderHover={fileBrowser.handleFolderHover}
expandedFolders={fileBrowser.expandedFolders}
loadingFolders={fileBrowser.loadingFolders}
onLoadMore={fileBrowser.handleLoadMore}
getFolderPagination={fileBrowser.getFolderPagination}
className="px-2 py-2"
withCheckboxes={true}
selectedPaths={selectedPaths}

8
app/client/modules/backups/components/snapshot-file-browser.tsx
View file

@ -60,11 +60,11 @@ export const SnapshotFileBrowser = (props: Props) => {
const fileBrowser = useFileBrowser({
initialData: filesData,
isLoading: filesLoading,
fetchFolder: async (path) => {
fetchFolder: async (path, offset = 0) => {
return await queryClient.ensureQueryData(
listSnapshotFilesOptions({
path: { id: repositoryId, snapshotId: snapshot.short_id },
query: { path },
query: { path, offset, limit: 500 },
}),
);
},
@ -72,7 +72,7 @@ export const SnapshotFileBrowser = (props: Props) => {
void queryClient.prefetchQuery(
listSnapshotFilesOptions({
path: { id: repositoryId, snapshotId: snapshot.short_id },
query: { path },
query: { path, offset: 0, limit: 500 },
}),
);
},
@ -142,6 +142,8 @@ export const SnapshotFileBrowser = (props: Props) => {
onFolderHover={fileBrowser.handleFolderHover}
expandedFolders={fileBrowser.expandedFolders}
loadingFolders={fileBrowser.loadingFolders}
onLoadMore={fileBrowser.handleLoadMore}
getFolderPagination={fileBrowser.getFolderPagination}
className="px-2 py-2"
/>
</div>

7
app/server/modules/repositories/repositories.controller.ts
View file

@ -146,7 +146,12 @@ export const repositoriesController = new Hono()
const { path } = c.req.valid("query");
const decodedPath = path ? decodeURIComponent(path) : undefined;
const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath);
const query = c.req.query();
const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0);
const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500));
const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath, { offset, limit });
c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");

35
app/server/modules/repositories/repositories.dto.ts
View file

@ -252,6 +252,10 @@ export const listSnapshotFilesResponse = type({
paths: "string[]",
}),
files: snapshotFileNodeSchema.array(),
offset: "number",
limit: "number",
total: "number",
hasMore: "boolean",
});
export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer;
@ -264,6 +268,37 @@ export const listSnapshotFilesDto = describeRoute({
description: "List files and directories in a snapshot",
tags: ["Repositories"],
operationId: "listSnapshotFiles",
parameters: [
{
in: "query",
name: "path",
required: false,
schema: {
type: "string",
},
description: "Subdirectory path to list",
},
{
in: "query",
name: "offset",
required: false,
schema: {
type: "integer",
default: 0,
},
description: "Offset for pagination (default: 0)",
},
{
in: "query",
name: "limit",
required: false,
schema: {
type: "integer",
default: 500,
},
description: "Maximum number of files to return (default: 500, max: 1000)",
},
],
responses: {
200: {
description: "List of files and directories in the snapshot",

29
app/server/modules/repositories/repositories.service.ts
View file

@ -210,7 +210,12 @@ const listSnapshots = async (id: string, backupId?: string) => {
}
};
const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) => {
const listSnapshotFiles = async (
id: string,
snapshotId: string,
path?: string,
options?: { offset?: number; limit?: number },
) => {
const organizationId = getOrganizationId();
const repository = await findRepository(id);
@ -218,18 +223,30 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
throw new NotFoundError("Repository not found");
}
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}`;
const cached = cache.get<Awaited<ReturnType<typeof restic.ls>>>(cacheKey);
const offset = options?.offset ?? 0;
const limit = options?.limit ?? 500;
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`;
type LsResult = {
snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null;
nodes: { name: string; type: string; path: string; size?: number; mode?: number }[];
pagination: { offset: number; limit: number; total: number; hasMore: boolean };
};
const cached = cache.get<LsResult>(cacheKey);
if (cached?.snapshot) {
return {
snapshot: cached.snapshot,
files: cached.nodes,
offset: cached.pagination.offset,
limit: cached.pagination.limit,
total: cached.pagination.total,
hasMore: cached.pagination.hasMore,
};
}
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
try {
const result = await restic.ls(repository.config, snapshotId, organizationId, path);
const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit });
if (!result.snapshot) {
throw new NotFoundError("Snapshot not found or empty");
@ -244,6 +261,10 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
paths: result.snapshot.paths,
},
files: result.nodes,
offset: result.pagination.offset,
limit: result.pagination.limit,
total: result.pagination.total,
hasMore: result.pagination.hasMore,
};
cache.set(cacheKey, result);

86
app/server/modules/volumes/volume.service.ts
View file

@ -18,7 +18,6 @@ import { serverEvents } from "../../core/events";
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
import { type } from "arktype";
import { getOrganizationId } from "~/server/core/request-context";
import { exec } from "../../utils/spawn";
async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
switch (config.backend) {
@ -322,11 +321,8 @@ const listFiles = async (
throw new InternalServerError("Volume is not mounted");
}
// For directory volumes, use the configured path directly
const volumePath = getVolumePath(volume);
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
const normalizedPath = path.normalize(requestedPath);
const relative = path.relative(volumePath, normalizedPath);
@ -338,58 +334,43 @@ const listFiles = async (
const startOffset = Math.max(offset, 0);
try {
const endLine = startOffset + pageSize;
const listScript = `
find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | while IFS= read -r line; do
name=$(basename "$line")
if [ -d "$line" ]; then
echo "0|$name|$line"
else
echo "1|$name|$line"
fi
done | sort -t'|' -k1,1n -k2,2 | cut -d'|' -f3- | sed -n '${startOffset + 1},${endLine}p'
`;
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
const { exitCode: listExitCode, stdout: listStdout } = await exec({
command: "sh",
args: ["-c", listScript],
timeout: 30000,
maxBuffer: 1024 * 1024 * 10,
});
dirents.sort((a, b) => {
const aIsDir = a.isDirectory();
const bIsDir = b.isDirectory();
if (listExitCode !== 0) {
throw new Error(`Failed to list directory: ${normalizedPath}`);
}
const filePaths = listStdout.split("\n").filter((p) => p.length > 0);
const entries: DirEntry[] = [];
for (const fullPath of filePaths) {
try {
const stats = await fs.stat(fullPath);
const relativePath = path.relative(volumePath, fullPath);
const name = path.basename(fullPath);
entries.push({
name,
path: `/${relativePath}`,
type: stats.isDirectory() ? "directory" : "file",
size: stats.isFile() ? stats.size : undefined,
modifiedAt: stats.mtimeMs,
});
} catch {
// File may have been deleted between listing and stat
if (aIsDir === bIsDir) {
return a.name.localeCompare(b.name);
}
}
const countScript = `find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | wc -l`;
const { exitCode: countExitCode, stdout: countStdout } = await exec({
command: "sh",
args: ["-c", countScript],
timeout: 10000,
return aIsDir ? -1 : 1;
});
const total = countExitCode === 0 ? parseInt(countStdout.trim(), 10) || 0 : entries.length;
const total = dirents.length;
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
const entries = (
await Promise.all(
paginatedDirents.map(async (dirent) => {
const fullPath = path.join(normalizedPath, dirent.name);
try {
const stats = await fs.stat(fullPath);
const relativePath = path.relative(volumePath, fullPath);
return {
name: dirent.name,
path: `/${relativePath}`,
type: dirent.isDirectory() ? "directory" : "file",
size: dirent.isFile() ? stats.size : undefined,
modifiedAt: stats.mtimeMs,
};
} catch {
return null;
}
}),
)
).filter((e) => e !== null);
return {
files: entries,
@ -400,6 +381,9 @@ const listFiles = async (
hasMore: startOffset + entries.length < total,
};
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
throw new NotFoundError("Directory not found");
}
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
}
};

107
app/server/utils/restic.ts
View file

@ -701,7 +701,13 @@ const lsSnapshotInfoSchema = type({
message_type: "'snapshot'",
});
const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: string, path?: string) => {
const ls = async (
config: RepositoryConfig,
snapshotId: string,
organizationId: string,
path?: string,
options?: { offset?: number; limit?: number },
) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, organizationId);
@ -713,48 +719,71 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId:
addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env });
let snapshot: typeof lsSnapshotInfoSchema.infer | null = null;
const nodes: Array<typeof lsNodeSchema.infer> = [];
let totalNodes = 0;
let isFirstLine = true;
let hasMore = false;
const offset = options?.offset ?? 0;
const limit = options?.limit ?? 500;
await safeSpawn({
command: "restic",
args,
env,
onStdout: (line) => {
const trimmedLine = line.trim();
if (!trimmedLine) return;
try {
const data = JSON.parse(trimmedLine);
if (isFirstLine) {
isFirstLine = false;
const snapshotValidation = lsSnapshotInfoSchema(data);
if (!(snapshotValidation instanceof type.errors)) {
snapshot = snapshotValidation;
}
return;
}
const nodeValidation = lsNodeSchema(data);
if (nodeValidation instanceof type.errors) {
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
return;
}
if (totalNodes >= offset && totalNodes < offset + limit) {
nodes.push(nodeValidation);
}
totalNodes++;
if (totalNodes >= offset + limit + 1) {
hasMore = true;
}
} catch (_) {
// Ignore JSON parse errors for non-JSON lines
}
},
});
await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) {
logger.error(`Restic ls failed: ${res.stderr}`);
throw new ResticError(res.exitCode, res.stderr);
if (totalNodes > offset + limit) {
hasMore = true;
}
// The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes
const stdout = res.stdout;
const lines = stdout
.trim()
.split("\n")
.filter((line) => line.trim());
if (lines.length === 0) {
return { snapshot: null, nodes: [] };
}
// First line is snapshot info
const snapshotLine = JSON.parse(lines[0] ?? "{}");
const snapshot = lsSnapshotInfoSchema(snapshotLine);
if (snapshot instanceof type.errors) {
logger.error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
throw new Error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
}
const nodes: Array<typeof lsNodeSchema.infer> = [];
for (let i = 1; i < lines.length; i++) {
const nodeLine = JSON.parse(lines[i] ?? "{}");
const nodeValidation = lsNodeSchema(nodeLine);
if (nodeValidation instanceof type.errors) {
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
continue;
}
nodes.push(nodeValidation);
}
return { snapshot, nodes };
return {
snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null,
nodes,
pagination: {
offset,
limit,
total: totalNodes,
hasMore,
},
};
};
const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {