From 5bb5fcd09c396a84835fc3c6ba0f2279fda2a77a Mon Sep 17 00:00:00 2001 From: Nico <47644445+nicotsx@users.noreply.github.com> Date: Sat, 31 Jan 2026 16:05:42 +0100 Subject: [PATCH] refactor: paginate large file counts (#441) * refactor: add pagination to handle volume folders with extremely large folder counts * refactor: stream restic ls result * test: file-tree load more * refactor: string params * fix(tsc): string pagination params * chore: pr feedbacks --- .../api-client/@tanstack/react-query.gen.ts | 164 +++++++++++++++--- app/client/api-client/index.ts | 8 +- app/client/api-client/sdk.gen.ts | 26 +-- app/client/api-client/types.gen.ts | 57 +++--- .../components/__test__/file-tree.test.tsx | 108 ++++++++++++ app/client/components/file-tree.tsx | 104 +++++++++-- app/client/components/restore-form.tsx | 8 +- app/client/components/volume-file-browser.tsx | 8 +- app/client/hooks/use-file-browser.ts | 105 ++++++++++- .../components/create-schedule-form.tsx | 8 +- .../components/snapshot-file-browser.tsx | 8 +- .../repositories/repositories.controller.ts | 8 +- .../modules/repositories/repositories.dto.ts | 6 + .../repositories/repositories.service.ts | 29 +++- .../modules/volumes/volume.controller.ts | 15 +- app/server/modules/volumes/volume.dto.ts | 21 ++- app/server/modules/volumes/volume.service.ts | 91 ++++++---- app/server/utils/restic.ts | 106 +++++++---- scripts/create-test-files.ts | 2 +- 19 files changed, 701 insertions(+), 181 deletions(-) diff --git a/app/client/api-client/@tanstack/react-query.gen.ts b/app/client/api-client/@tanstack/react-query.gen.ts index 60f0ad71..f81bfbcb 100644 --- a/app/client/api-client/@tanstack/react-query.gen.ts +++ b/app/client/api-client/@tanstack/react-query.gen.ts @@ -1,6 +1,12 @@ // This file is auto-generated by @hey-api/openapi-ts -import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanstack/react-query"; +import { + type DefaultError, + type InfiniteData, + infiniteQueryOptions, + queryOptions, + type UseMutationOptions, +} from "@tanstack/react-query"; import { client } from "../client.gen"; import { @@ -31,7 +37,6 @@ import { getUpdates, getUserDeletionImpact, getVolume, - refreshSnapshots, healthCheckVolume, listBackupSchedules, listFiles, @@ -43,6 +48,7 @@ import { listVolumes, mountVolume, type Options, + refreshSnapshots, reorderBackupSchedules, restoreSnapshot, runBackupNow, @@ -116,8 +122,6 @@ import type { GetUserDeletionImpactResponse, GetVolumeData, GetVolumeResponse, - RefreshSnapshotsData, - RefreshSnapshotsResponse, HealthCheckVolumeData, HealthCheckVolumeResponse, ListBackupSchedulesData, @@ -138,6 +142,8 @@ import type { ListVolumesResponse, MountVolumeData, MountVolumeResponse, + RefreshSnapshotsData, + RefreshSnapshotsResponse, ReorderBackupSchedulesData, ReorderBackupSchedulesResponse, RestoreSnapshotData, @@ -447,6 +453,77 @@ export const listFilesOptions = (options: Options) => queryKey: listFilesQueryKey(options), }); +const createInfiniteParams = [0], "body" | "headers" | "path" | "query">>( + queryKey: QueryKey, + page: K, +) => { + const params = { ...queryKey[0] }; + if (page.body) { + params.body = { + ...(queryKey[0].body as any), + ...(page.body as any), + }; + } + if (page.headers) { + params.headers = { + ...queryKey[0].headers, + ...page.headers, + }; + } + if (page.path) { + params.path = { + ...(queryKey[0].path as any), + ...(page.path as any), + }; + } + if (page.query) { + params.query = { + ...(queryKey[0].query as any), + ...(page.query as any), + }; + } + return params as unknown as typeof page; +}; + +export const listFilesInfiniteQueryKey = (options: Options): QueryKey> => + createQueryKey("listFiles", options, true); + +/** + * List files in a volume directory + */ +export const listFilesInfiniteOptions = (options: Options) => + infiniteQueryOptions< + ListFilesResponse, + DefaultError, + InfiniteData, + QueryKey>, + string | Pick>[0], "body" | "headers" | "path" | "query"> + >( + // @ts-ignore + { + queryFn: async ({ pageParam, queryKey, signal }) => { + // @ts-ignore + const page: Pick>[0], "body" | "headers" | "path" | "query"> = + typeof pageParam === "object" + ? pageParam + : { + query: { + offset: pageParam, + }, + }; + const params = createInfiniteParams(queryKey, page); + const { data } = await listFiles({ + ...options, + ...params, + signal, + throwOnError: true, + }); + return data; + }, + queryKey: listFilesInfiniteQueryKey(options), + }, + ); + export const browseFilesystemQueryKey = (options?: Options) => createQueryKey("browseFilesystem", options); @@ -636,6 +713,25 @@ export const listSnapshotsOptions = (options: Options) => queryKey: listSnapshotsQueryKey(options), }); +/** + * Clear snapshot cache and force refresh from repository + */ +export const refreshSnapshotsMutation = ( + options?: Partial>, +): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await refreshSnapshots({ + ...options, + ...fnOptions, + throwOnError: true, + }); + return data; + }, + }; + return mutationOptions; +}; + /** * Delete a specific snapshot from a repository */ @@ -705,6 +801,46 @@ export const listSnapshotFilesOptions = (options: Options queryKey: listSnapshotFilesQueryKey(options), }); +export const listSnapshotFilesInfiniteQueryKey = ( + options: Options, +): QueryKey> => createQueryKey("listSnapshotFiles", options, true); + +/** + * List files and directories in a snapshot + */ +export const listSnapshotFilesInfiniteOptions = (options: Options) => + infiniteQueryOptions< + ListSnapshotFilesResponse, + DefaultError, + InfiniteData, + QueryKey>, + string | Pick>[0], "body" | "headers" | "path" | "query"> + >( + // @ts-ignore + { + queryFn: async ({ pageParam, queryKey, signal }) => { + // @ts-ignore + const page: Pick>[0], "body" | "headers" | "path" | "query"> = + typeof pageParam === "object" + ? pageParam + : { + query: { + offset: pageParam, + }, + }; + const params = createInfiniteParams(queryKey, page); + const { data } = await listSnapshotFiles({ + ...options, + ...params, + signal, + throwOnError: true, + }); + return data; + }, + queryKey: listSnapshotFilesInfiniteQueryKey(options), + }, + ); + /** * Restore a snapshot to a target path on the filesystem */ @@ -781,26 +917,6 @@ export const tagSnapshotsMutation = ( return mutationOptions; }; -export const refreshSnapshotsMutation = ( - options?: Partial>, -): UseMutationOptions> => { - const mutationOptions: UseMutationOptions< - RefreshSnapshotsResponse, - DefaultError, - Options - > = { - mutationFn: async (fnOptions) => { - const { data } = await refreshSnapshots({ - ...options, - ...fnOptions, - throwOnError: true, - }); - return data; - }, - }; - return mutationOptions; -}; - export const listBackupSchedulesQueryKey = (options?: Options) => createQueryKey("listBackupSchedules", options); diff --git a/app/client/api-client/index.ts b/app/client/api-client/index.ts index d5ae8aab..fedb701c 100644 --- a/app/client/api-client/index.ts +++ b/app/client/api-client/index.ts @@ -28,7 +28,6 @@ export { getUpdates, getUserDeletionImpact, getVolume, - refreshSnapshots, healthCheckVolume, listBackupSchedules, listFiles, @@ -40,6 +39,7 @@ export { listVolumes, mountVolume, type Options, + refreshSnapshots, reorderBackupSchedules, restoreSnapshot, runBackupNow, @@ -145,9 +145,6 @@ export type { GetVolumeErrors, GetVolumeResponse, GetVolumeResponses, - RefreshSnapshotsData, - RefreshSnapshotsResponse, - RefreshSnapshotsResponses, HealthCheckVolumeData, HealthCheckVolumeErrors, HealthCheckVolumeResponse, @@ -179,6 +176,9 @@ export type { MountVolumeData, MountVolumeResponse, MountVolumeResponses, + RefreshSnapshotsData, + RefreshSnapshotsResponse, + RefreshSnapshotsResponses, ReorderBackupSchedulesData, ReorderBackupSchedulesResponse, ReorderBackupSchedulesResponses, diff --git a/app/client/api-client/sdk.gen.ts b/app/client/api-client/sdk.gen.ts index 265943d9..98a1f117 100644 --- a/app/client/api-client/sdk.gen.ts +++ b/app/client/api-client/sdk.gen.ts @@ -61,8 +61,6 @@ import type { GetVolumeData, GetVolumeErrors, GetVolumeResponses, - RefreshSnapshotsData, - RefreshSnapshotsResponses, HealthCheckVolumeData, HealthCheckVolumeErrors, HealthCheckVolumeResponses, @@ -84,6 +82,8 @@ import type { ListVolumesResponses, MountVolumeData, MountVolumeResponses, + RefreshSnapshotsData, + RefreshSnapshotsResponses, ReorderBackupSchedulesData, ReorderBackupSchedulesResponses, RestoreSnapshotData, @@ -379,6 +379,17 @@ export const listSnapshots = ( ...options, }); +/** + * Clear snapshot cache and force refresh from repository + */ +export const refreshSnapshots = ( + options: Options, +) => + (options.client ?? client).post({ + url: "/api/v1/repositories/{id}/snapshots/refresh", + ...options, + }); + /** * Delete a specific snapshot from a repository */ @@ -458,17 +469,6 @@ export const tagSnapshots = (options: Opti }, }); -/** - * Clear snapshot cache and force refresh from repository - */ -export const refreshSnapshots = ( - options: Options, -) => - (options.client ?? client).post({ - url: "/api/v1/repositories/{id}/snapshots/refresh", - ...options, - }); - /** * List all backup schedules */ diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index 07a9d067..74fbae0e 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -682,9 +682,8 @@ export type ListFilesData = { id: string; }; query?: { - /** - * Subdirectory path to list (relative to volume root) - */ + limit?: string; + offset?: string; path?: string; }; url: "/api/v1/volumes/{id}/files"; @@ -702,7 +701,11 @@ export type ListFilesResponses = { modifiedAt?: number; size?: number; }>; + hasMore: boolean; + limit: number; + offset: number; path: string; + total: number; }; }; @@ -1649,6 +1652,27 @@ export type ListSnapshotsResponses = { export type ListSnapshotsResponse = ListSnapshotsResponses[keyof ListSnapshotsResponses]; +export type RefreshSnapshotsData = { + body?: never; + path: { + id: string; + }; + query?: never; + url: "/api/v1/repositories/{id}/snapshots/refresh"; +}; + +export type RefreshSnapshotsResponses = { + /** + * Snapshot cache cleared and refreshed + */ + 200: { + count: number; + message: string; + }; +}; + +export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses]; + export type DeleteSnapshotData = { body?: never; path: { @@ -1703,6 +1727,8 @@ export type ListSnapshotFilesData = { snapshotId: string; }; query?: { + limit?: string; + offset?: string; path?: string; }; url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files"; @@ -1725,6 +1751,9 @@ export type ListSnapshotFilesResponses = { size?: number; uid?: number; }>; + hasMore: boolean; + limit: number; + offset: number; snapshot: { hostname: string; id: string; @@ -1732,6 +1761,7 @@ export type ListSnapshotFilesResponses = { short_id: string; time: string; }; + total: number; }; }; @@ -1848,27 +1878,6 @@ export type TagSnapshotsResponses = { export type TagSnapshotsResponse = TagSnapshotsResponses[keyof TagSnapshotsResponses]; -export type RefreshSnapshotsData = { - body?: never; - path: { - id: string; - }; - query?: never; - url: "/api/v1/repositories/{id}/snapshots/refresh"; -}; - -export type RefreshSnapshotsResponses = { - /** - * Snapshot cache cleared and refreshed - */ - 200: { - message: string; - count: number; - }; -}; - -export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses]; - export type ListBackupSchedulesData = { body?: never; path?: never; diff --git a/app/client/components/__test__/file-tree.test.tsx b/app/client/components/__test__/file-tree.test.tsx index 7589fe63..f2aa43a6 100644 --- a/app/client/components/__test__/file-tree.test.tsx +++ b/app/client/components/__test__/file-tree.test.tsx @@ -3,6 +3,114 @@ import { expect, test, describe } from "bun:test"; import { render, screen, fireEvent } from "@testing-library/react"; import { FileTree, type FileEntry } from "../file-tree"; +describe("FileTree Pagination", () => { + const testFiles: FileEntry[] = [ + { name: "root", path: "/root", type: "folder" }, + { name: "file1", path: "/root/file1", type: "file" }, + { name: "file2", path: "/root/file2", type: "file" }, + ]; + + test("shows load more button when hasMore is true", () => { + render( + ({ hasMore: true, isLoadingMore: false })} + />, + ); + + expect(screen.getByText("Load more files")).toBeTruthy(); + }); + + test("does not show load more button when hasMore is false", () => { + render( + ({ hasMore: false, isLoadingMore: false })} + />, + ); + + expect(screen.queryByText("Load more files")).toBeNull(); + }); + + test("calls onLoadMore with folder path when load more button is clicked", () => { + let loadMoreCalled = false; + let loadMorePath = ""; + + render( + { + if (path === "/root") { + return { hasMore: true, isLoadingMore: false }; + } + return { hasMore: false, isLoadingMore: false }; + }} + onLoadMore={(path) => { + loadMoreCalled = true; + loadMorePath = path; + }} + />, + ); + + const loadMoreButton = screen.getByText("Load more files"); + fireEvent.click(loadMoreButton); + + expect(loadMoreCalled).toBe(true); + expect(loadMorePath).toBe("/root"); + }); + + test("shows loading state when isLoadingMore is true", () => { + render( + ({ hasMore: true, isLoadingMore: true })} + />, + ); + + expect(screen.getByText("Loading more...")).toBeTruthy(); + }); + + test("load more button appears for nested folders with hasMore", () => { + const nestedFiles: FileEntry[] = [ + { name: "root", path: "/root", type: "folder" }, + { name: "child", path: "/root/child", type: "folder" }, + { name: "file1", path: "/root/child/file1", type: "file" }, + ]; + + render( + { + if (path === "/root/child") { + return { hasMore: true, isLoadingMore: false }; + } + return { hasMore: false, isLoadingMore: false }; + }} + onLoadMore={() => {}} + />, + ); + + expect(screen.getByText("Load more files")).toBeTruthy(); + }); + + test("load more button does not appear when folder is collapsed", () => { + render( + ({ hasMore: true, isLoadingMore: false })} + />, + ); + + expect(screen.queryByText("Load more files")).toBeNull(); + }); +}); + describe("FileTree Selection Logic", () => { const testFiles: FileEntry[] = [ { name: "root", path: "/root", type: "folder" }, diff --git a/app/client/components/file-tree.tsx b/app/client/components/file-tree.tsx index ba101598..9a39ca7b 100644 --- a/app/client/components/file-tree.tsx +++ b/app/client/components/file-tree.tsx @@ -8,7 +8,15 @@ * Original source: https://github.com/stackblitz/bolt.new */ -import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2 } from "lucide-react"; +import { + ChevronDown, + ChevronRight, + File as FileIcon, + Folder as FolderIcon, + FolderOpen, + Loader2, + MoreHorizontal, +} from "lucide-react"; import { memo, type ReactNode, useCallback, useEffect, useMemo, useState } from "react"; import { cn } from "~/client/lib/utils"; import { Checkbox } from "~/client/components/ui/checkbox"; @@ -24,12 +32,19 @@ export interface FileEntry { modifiedAt?: number; } +interface PaginationState { + hasMore: boolean; + isLoadingMore: boolean; +} + interface Props { files?: FileEntry[]; selectedFile?: string; onFileSelect?: (filePath: string) => void; onFolderExpand?: (folderPath: string) => void; onFolderHover?: (folderPath: string) => void; + onLoadMore?: (folderPath: string) => void; + getFolderPagination?: (folderPath: string) => PaginationState; expandedFolders?: Set; loadingFolders?: Set; className?: string; @@ -49,6 +64,8 @@ export const FileTree = memo((props: Props) => { selectedFile, onFolderExpand, onFolderHover, + onLoadMore, + getFolderPagination, expandedFolders = new Set(), loadingFolders = new Set(), className, @@ -292,12 +309,35 @@ export const FileTree = memo((props: Props) => { [selectedPaths], ); + // Build a map of folder paths that need pagination to their last child's index + const folderPaginationMap = useMemo(() => { + const map = new Map(); + + for (let i = 0; i < filteredFileList.length; i++) { + const item = filteredFileList[i]; + const parentPath = item.fullPath.slice(0, item.fullPath.lastIndexOf("/")) || "/"; + + if (parentPath !== "/") { + const pagination = getFolderPagination?.(parentPath); + if (pagination?.hasMore && !collapsedFolders.has(parentPath)) { + // Update the last index for this parent + map.set(parentPath, i); + } + } + } + + return map; + }, [filteredFileList, getFolderPagination, collapsedFolders]); + return (
- {filteredFileList.map((fileOrFolder) => { + {filteredFileList.map((fileOrFolder, index) => { + const elements: React.ReactNode[] = []; + + // Render the current file or folder switch (fileOrFolder.kind) { case "file": { - return ( + elements.push( { withCheckbox={withCheckboxes} checked={isPathSelected(fileOrFolder.fullPath)} onCheckboxChange={handleSelectionChange} - /> + />, ); + break; } case "folder": { - return ( + elements.push( { selectableMode={selectableFolders} onFolderSelect={handleFolderSelect} selected={selectedFolder === fileOrFolder.fullPath} - /> + />, ); - } - default: { - return undefined; + break; } } + + // Check if this is the last child of any folder with more files to load + for (const [folderPath, lastIndex] of folderPaginationMap.entries()) { + if (lastIndex === index) { + // This is the last loaded child of folderPath + const pagination = getFolderPagination?.(folderPath); + if (pagination?.hasMore) { + elements.push( + onLoadMore?.(folderPath)} + isLoading={pagination.isLoadingMore} + />, + ); + } + } + } + + return elements; })}
); @@ -477,6 +536,31 @@ const File = memo(({ file, onFileSelect, selected, withCheckbox, checked, onChec ); }); +interface LoadMoreButtonProps { + depth: number; + onClick: () => void; + isLoading?: boolean; +} + +const LoadMoreButton = memo(({ depth, onClick, isLoading }: LoadMoreButtonProps) => { + return ( + + ) : ( + + ) + } + onClick={onClick} + > + {isLoading ? "Loading more..." : "Load more files"} + + ); +}); + interface ButtonProps { depth: number; icon: ReactNode; @@ -621,5 +705,5 @@ function compareNodes(a: Node, b: Node): number { return a.kind === "folder" ? -1 : 1; } - return a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: "base" }); + return a.name.localeCompare(b.name); } diff --git a/app/client/components/restore-form.tsx b/app/client/components/restore-form.tsx index e2594232..3c900a46 100644 --- a/app/client/components/restore-form.tsx +++ b/app/client/components/restore-form.tsx @@ -74,11 +74,11 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re const fileBrowser = useFileBrowser({ initialData: filesData, isLoading: filesLoading, - fetchFolder: async (path) => { + fetchFolder: async (path, offset = 0) => { return await queryClient.ensureQueryData( listSnapshotFilesOptions({ path: { id: repository.id, snapshotId }, - query: { path }, + query: { path, offset: offset.toString(), limit: "500" }, }), ); }, @@ -86,7 +86,7 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re void queryClient.prefetchQuery( listSnapshotFilesOptions({ path: { id: repository.id, snapshotId }, - query: { path }, + query: { path, offset: "0", limit: "500" }, }), ); }, @@ -306,6 +306,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re onFolderHover={fileBrowser.handleFolderHover} expandedFolders={fileBrowser.expandedFolders} loadingFolders={fileBrowser.loadingFolders} + onLoadMore={fileBrowser.handleLoadMore} + getFolderPagination={fileBrowser.getFolderPagination} className="px-2 py-2" withCheckboxes={true} selectedPaths={selectedPaths} diff --git a/app/client/components/volume-file-browser.tsx b/app/client/components/volume-file-browser.tsx index 9b1dbf83..3ed3a156 100644 --- a/app/client/components/volume-file-browser.tsx +++ b/app/client/components/volume-file-browser.tsx @@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { FolderOpen } from "lucide-react"; import { FileTree } from "~/client/components/file-tree"; import { listFilesOptions } from "../api-client/@tanstack/react-query.gen"; -import { useFileBrowser } from "../hooks/use-file-browser"; +import { useFileBrowser, type FetchFolderResult } from "../hooks/use-file-browser"; import { parseError } from "../lib/errors"; type VolumeFileBrowserProps = { @@ -38,11 +38,11 @@ export const VolumeFileBrowser = ({ const fileBrowser = useFileBrowser({ initialData: data, isLoading, - fetchFolder: async (path) => { + fetchFolder: async (path, offset): Promise => { return await queryClient.ensureQueryData( listFilesOptions({ path: { id: volumeId }, - query: { path }, + query: { path, offset: offset?.toString() }, }), ); }, @@ -88,6 +88,8 @@ export const VolumeFileBrowser = ({ files={fileBrowser.fileArray} onFolderExpand={fileBrowser.handleFolderExpand} onFolderHover={fileBrowser.handleFolderHover} + onLoadMore={fileBrowser.handleLoadMore} + getFolderPagination={fileBrowser.getFolderPagination} expandedFolders={fileBrowser.expandedFolders} loadingFolders={fileBrowser.loadingFolders} withCheckboxes={withCheckboxes} diff --git a/app/client/hooks/use-file-browser.ts b/app/client/hooks/use-file-browser.ts index 937a177d..bdb8103d 100644 --- a/app/client/hooks/use-file-browser.ts +++ b/app/client/hooks/use-file-browser.ts @@ -1,9 +1,16 @@ import { useCallback, useMemo, useState } from "react"; import type { FileEntry } from "../components/file-tree"; -type FetchFolderFn = ( - path: string, -) => Promise<{ files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }>; +export type FetchFolderResult = { + files?: FileEntry[]; + directories?: Array<{ name: string; path: string }>; + offset?: number; + limit?: number; + total?: number; + hasMore?: boolean; +}; + +type FetchFolderFn = (path: string, offset?: number) => Promise; type PathTransformFns = { strip?: (path: string) => string; @@ -11,7 +18,7 @@ type PathTransformFns = { }; type UseFileBrowserOptions = { - initialData?: { files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }; + initialData?: FetchFolderResult; isLoading?: boolean; fetchFolder: FetchFolderFn; prefetchFolder?: (path: string) => void; @@ -19,12 +26,20 @@ type UseFileBrowserOptions = { rootPath?: string; }; +type FolderPaginationState = { + currentOffset: number; + limit: number; + hasMore: boolean; + isLoadingMore: boolean; +}; + export const useFileBrowser = (props: UseFileBrowserOptions) => { const { initialData, isLoading, fetchFolder, prefetchFolder, pathTransform, rootPath = "/" } = props; const [expandedFolders, setExpandedFolders] = useState>(new Set()); const [fetchedFolders, setFetchedFolders] = useState>(new Set([rootPath])); const [loadingFolders, setLoadingFolders] = useState>(new Set()); const [allFiles, setAllFiles] = useState>(new Map()); + const [folderPagination, setFolderPagination] = useState>(new Map()); const stripPath = pathTransform?.strip; const addPath = pathTransform?.add; @@ -44,6 +59,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { }); if (rootPath) { setFetchedFolders((prev) => new Set(prev).add(rootPath)); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(rootPath, { + currentOffset: initialData.offset ?? 0, + limit: initialData.limit ?? 100, + hasMore: initialData.hasMore ?? false, + isLoadingMore: false, + }); + return next; + }); } } else if (initialData?.directories) { const directories = initialData.directories; @@ -87,6 +112,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { } return next; }); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { + currentOffset: result.offset ?? 0, + limit: result.limit ?? 100, + hasMore: result.hasMore ?? false, + isLoadingMore: false, + }); + return next; + }); } else if (result.directories) { const directories = result.directories; setAllFiles((prev) => { @@ -113,6 +148,59 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { [fetchedFolders, fetchFolder, stripPath, addPath], ); + const handleLoadMore = useCallback( + async (folderPath: string) => { + const pagination = folderPagination.get(folderPath); + if (!pagination?.hasMore || pagination?.isLoadingMore) { + return; + } + + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { ...pagination, isLoadingMore: true }); + return next; + }); + + try { + const pathToFetch = addPath ? addPath(folderPath) : folderPath; + const nextOffset = pagination.currentOffset + pagination.limit; + const result = await fetchFolder(pathToFetch, nextOffset); + + if (result.files) { + const files = result.files; + setAllFiles((prev) => { + const next = new Map(prev); + for (const file of files) { + const strippedPath = stripPath ? stripPath(file.path) : file.path; + if (strippedPath !== folderPath) { + next.set(strippedPath, { ...file, path: strippedPath }); + } + } + return next; + }); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { + currentOffset: result.offset ?? nextOffset, + limit: result.limit ?? pagination.limit, + hasMore: result.hasMore ?? false, + isLoadingMore: false, + }); + return next; + }); + } + } catch (error) { + console.error("Failed to load more files:", error); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { ...pagination, isLoadingMore: false }); + return next; + }); + } + }, + [folderPagination, fetchFolder, stripPath, addPath], + ); + const handleFolderHover = useCallback( (folderPath: string) => { if (!fetchedFolders.has(folderPath) && !loadingFolders.has(folderPath) && prefetchFolder) { @@ -123,12 +211,21 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { [fetchedFolders, loadingFolders, prefetchFolder, addPath], ); + const getFolderPagination = useCallback( + (folderPath: string) => { + return folderPagination.get(folderPath) ?? { hasMore: false, isLoadingMore: false }; + }, + [folderPagination], + ); + return { fileArray, expandedFolders, loadingFolders, handleFolderExpand, handleFolderHover, + handleLoadMore, + getFolderPagination, isLoading: isLoading && fileArray.length === 0, isEmpty: fileArray.length === 0 && !isLoading, }; diff --git a/app/client/modules/backups/components/create-schedule-form.tsx b/app/client/modules/backups/components/create-schedule-form.tsx index 17b1f0f0..718061f8 100644 --- a/app/client/modules/backups/components/create-schedule-form.tsx +++ b/app/client/modules/backups/components/create-schedule-form.tsx @@ -671,14 +671,20 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:

Include paths/patterns

- {formValues.includePatterns?.map((path) => ( + {formValues.includePatterns?.slice(0, 20).map((path) => ( {path} ))} + {formValues.includePatterns && formValues.includePatterns.length > 20 && ( + + + {formValues.includePatterns.length - 20} more + + )} {formValues.includePatternsText ?.split("\n") .filter(Boolean) + .slice(0, 20 - (formValues.includePatterns?.length || 0)) .map((pattern) => ( {pattern.trim()} diff --git a/app/client/modules/backups/components/snapshot-file-browser.tsx b/app/client/modules/backups/components/snapshot-file-browser.tsx index e2efbb37..9e62a816 100644 --- a/app/client/modules/backups/components/snapshot-file-browser.tsx +++ b/app/client/modules/backups/components/snapshot-file-browser.tsx @@ -60,11 +60,11 @@ export const SnapshotFileBrowser = (props: Props) => { const fileBrowser = useFileBrowser({ initialData: filesData, isLoading: filesLoading, - fetchFolder: async (path) => { + fetchFolder: async (path, offset = 0) => { return await queryClient.ensureQueryData( listSnapshotFilesOptions({ path: { id: repositoryId, snapshotId: snapshot.short_id }, - query: { path }, + query: { path, offset: offset.toString(), limit: "500" }, }), ); }, @@ -72,7 +72,7 @@ export const SnapshotFileBrowser = (props: Props) => { void queryClient.prefetchQuery( listSnapshotFilesOptions({ path: { id: repositoryId, snapshotId: snapshot.short_id }, - query: { path }, + query: { path, offset: "0", limit: "500" }, }), ); }, @@ -142,6 +142,8 @@ export const SnapshotFileBrowser = (props: Props) => { onFolderHover={fileBrowser.handleFolderHover} expandedFolders={fileBrowser.expandedFolders} loadingFolders={fileBrowser.loadingFolders} + onLoadMore={fileBrowser.handleLoadMore} + getFolderPagination={fileBrowser.getFolderPagination} className="px-2 py-2" />
diff --git a/app/server/modules/repositories/repositories.controller.ts b/app/server/modules/repositories/repositories.controller.ts index 4bfcc345..ba562d02 100644 --- a/app/server/modules/repositories/repositories.controller.ts +++ b/app/server/modules/repositories/repositories.controller.ts @@ -143,10 +143,14 @@ export const repositoriesController = new Hono() validator("query", listSnapshotFilesQuery), async (c) => { const { id, snapshotId } = c.req.param(); - const { path } = c.req.valid("query"); + const { path, ...query } = c.req.valid("query"); const decodedPath = path ? decodeURIComponent(path) : undefined; - const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath); + + const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0); + const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500)); + + const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath, { offset, limit }); c.header("Cache-Control", "max-age=300, stale-while-revalidate=600"); diff --git a/app/server/modules/repositories/repositories.dto.ts b/app/server/modules/repositories/repositories.dto.ts index 7ceb5e67..485db506 100644 --- a/app/server/modules/repositories/repositories.dto.ts +++ b/app/server/modules/repositories/repositories.dto.ts @@ -252,12 +252,18 @@ export const listSnapshotFilesResponse = type({ paths: "string[]", }), files: snapshotFileNodeSchema.array(), + offset: "number", + limit: "number", + total: "number", + hasMore: "boolean", }); export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer; export const listSnapshotFilesQuery = type({ path: "string?", + offset: "string.integer?", + limit: "string.integer?", }); export const listSnapshotFilesDto = describeRoute({ diff --git a/app/server/modules/repositories/repositories.service.ts b/app/server/modules/repositories/repositories.service.ts index ec2ecf56..c513f75a 100644 --- a/app/server/modules/repositories/repositories.service.ts +++ b/app/server/modules/repositories/repositories.service.ts @@ -210,7 +210,12 @@ const listSnapshots = async (id: string, backupId?: string) => { } }; -const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) => { +const listSnapshotFiles = async ( + id: string, + snapshotId: string, + path?: string, + options?: { offset?: number; limit?: number }, +) => { const organizationId = getOrganizationId(); const repository = await findRepository(id); @@ -218,18 +223,30 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) throw new NotFoundError("Repository not found"); } - const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}`; - const cached = cache.get>>(cacheKey); + const offset = options?.offset ?? 0; + const limit = options?.limit ?? 500; + + const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`; + type LsResult = { + snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null; + nodes: { name: string; type: string; path: string; size?: number; mode?: number }[]; + pagination: { offset: number; limit: number; total: number; hasMore: boolean }; + }; + const cached = cache.get(cacheKey); if (cached?.snapshot) { return { snapshot: cached.snapshot, files: cached.nodes, + offset: cached.pagination.offset, + limit: cached.pagination.limit, + total: cached.pagination.total, + hasMore: cached.pagination.hasMore, }; } const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`); try { - const result = await restic.ls(repository.config, snapshotId, organizationId, path); + const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit }); if (!result.snapshot) { throw new NotFoundError("Snapshot not found or empty"); @@ -244,6 +261,10 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) paths: result.snapshot.paths, }, files: result.nodes, + offset: result.pagination.offset, + limit: result.pagination.limit, + total: result.pagination.total, + hasMore: result.pagination.hasMore, }; cache.set(cacheKey, result); diff --git a/app/server/modules/volumes/volume.controller.ts b/app/server/modules/volumes/volume.controller.ts index d8fbf10b..7df2ed16 100644 --- a/app/server/modules/volumes/volume.controller.ts +++ b/app/server/modules/volumes/volume.controller.ts @@ -21,6 +21,7 @@ import { type ListFilesDto, browseFilesystemDto, type BrowseFilesystemDto, + listFilesQuery, } from "./volume.dto"; import { volumeService } from "./volume.service"; import { getVolumePath } from "./helpers"; @@ -104,14 +105,22 @@ export const volumeController = new Hono() return c.json({ error, status }, 200); }) - .get("/:id/files", listFilesDto, async (c) => { + .get("/:id/files", validator("query", listFilesQuery), listFilesDto, async (c) => { const { id } = c.req.param(); - const subPath = c.req.query("path"); - const result = await volumeService.listFiles(id, subPath); + const { path, ...query } = c.req.valid("query"); + + const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0); + const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500)); + + const result = await volumeService.listFiles(id, path, offset, limit); const response = { files: result.files, path: result.path, + offset: result.offset, + limit: result.limit, + total: result.total, + hasMore: result.hasMore, }; c.header("Cache-Control", "public, max-age=10, stale-while-revalidate=60"); diff --git a/app/server/modules/volumes/volume.dto.ts b/app/server/modules/volumes/volume.dto.ts index bb76f4c8..01214f95 100644 --- a/app/server/modules/volumes/volume.dto.ts +++ b/app/server/modules/volumes/volume.dto.ts @@ -276,24 +276,23 @@ const fileEntrySchema = type({ export const listFilesResponse = type({ files: fileEntrySchema.array(), path: "string", + offset: "number", + limit: "number", + total: "number", + hasMore: "boolean", }); export type ListFilesDto = typeof listFilesResponse.infer; +export const listFilesQuery = type({ + path: "string?", + offset: "string.integer?", + limit: "string.integer?", +}); + export const listFilesDto = describeRoute({ description: "List files in a volume directory", operationId: "listFiles", tags: ["Volumes"], - parameters: [ - { - in: "query", - name: "path", - required: false, - schema: { - type: "string", - }, - description: "Subdirectory path to list (relative to volume root)", - }, - ], responses: { 200: { description: "List of files in the volume", diff --git a/app/server/modules/volumes/volume.service.ts b/app/server/modules/volumes/volume.service.ts index ef869683..401966d6 100644 --- a/app/server/modules/volumes/volume.service.ts +++ b/app/server/modules/volumes/volume.service.ts @@ -18,6 +18,7 @@ import { serverEvents } from "../../core/events"; import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { type } from "arktype"; import { getOrganizationId } from "~/server/core/request-context"; +import { isNodeJSErrnoException } from "~/server/utils/fs"; async function encryptSensitiveFields(config: BackendConfig): Promise { switch (config.backend) { @@ -294,7 +295,15 @@ const checkHealth = async (idOrShortId: string | number) => { return { status, error }; }; -const listFiles = async (idOrShortId: string | number, subPath?: string) => { +const DEFAULT_PAGE_SIZE = 500; +const MAX_PAGE_SIZE = 500; + +const listFiles = async ( + idOrShortId: string | number, + subPath?: string, + offset: number = 0, + limit: number = DEFAULT_PAGE_SIZE, +) => { const volume = await findVolume(idOrShortId); if (!volume) { @@ -305,11 +314,8 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => { throw new InternalServerError("Volume is not mounted"); } - // For directory volumes, use the configured path directly const volumePath = getVolumePath(volume); - const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath; - const normalizedPath = path.normalize(requestedPath); const relative = path.relative(volumePath, normalizedPath); @@ -317,45 +323,60 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => { throw new BadRequestError("Invalid path"); } + const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE); + const startOffset = Math.max(offset, 0); + try { - const entries = await fs.readdir(normalizedPath, { withFileTypes: true }); + const dirents = await fs.readdir(normalizedPath, { withFileTypes: true }); - const files = await Promise.all( - entries.map(async (entry) => { - const fullPath = path.join(normalizedPath, entry.name); - const relativePath = path.relative(volumePath, fullPath); + dirents.sort((a, b) => { + const aIsDir = a.isDirectory(); + const bIsDir = b.isDirectory(); - try { - const stats = await fs.stat(fullPath); - return { - name: entry.name, - path: `/${relativePath}`, - type: entry.isDirectory() ? ("directory" as const) : ("file" as const), - size: entry.isFile() ? stats.size : undefined, - modifiedAt: stats.mtimeMs, - }; - } catch { - return { - name: entry.name, - path: `/${relativePath}`, - type: entry.isDirectory() ? ("directory" as const) : ("file" as const), - size: undefined, - modifiedAt: undefined, - }; - } - }), - ); + if (aIsDir === bIsDir) { + return a.name.localeCompare(b.name); + } + return aIsDir ? -1 : 1; + }); + + const total = dirents.length; + const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize); + + const entries = ( + await Promise.all( + paginatedDirents.map(async (dirent) => { + const fullPath = path.join(normalizedPath, dirent.name); + + try { + const stats = await fs.stat(fullPath); + const relativePath = path.relative(volumePath, fullPath); + + return { + name: dirent.name, + path: `/${relativePath}`, + type: dirent.isDirectory() ? ("directory" as const) : ("file" as const), + size: dirent.isFile() ? stats.size : undefined, + modifiedAt: stats.mtimeMs, + }; + } catch { + return null; + } + }), + ) + ).filter((e) => e !== null); return { - files: files.sort((a, b) => { - if (a.type !== b.type) { - return a.type === "directory" ? -1 : 1; - } - return a.name.localeCompare(b.name); - }), + files: entries, path: subPath || "/", + offset: startOffset, + limit: pageSize, + total, + hasMore: startOffset + pageSize < total, }; } catch (error) { + if (isNodeJSErrnoException(error) && error.code === "ENOENT") { + throw new NotFoundError("Directory not found"); + } throw new InternalServerError(`Failed to list files: ${toMessage(error)}`); } }; diff --git a/app/server/utils/restic.ts b/app/server/utils/restic.ts index 53b52b79..4f5b4874 100644 --- a/app/server/utils/restic.ts +++ b/app/server/utils/restic.ts @@ -701,7 +701,13 @@ const lsSnapshotInfoSchema = type({ message_type: "'snapshot'", }); -const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: string, path?: string) => { +const ls = async ( + config: RepositoryConfig, + snapshotId: string, + organizationId: string, + path?: string, + options?: { offset?: number; limit?: number }, +) => { const repoUrl = buildRepoUrl(config); const env = await buildEnv(config, organizationId); @@ -713,48 +719,76 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: addCommonArgs(args, env, config); - const res = await exec({ command: "restic", args, env }); + let snapshot: typeof lsSnapshotInfoSchema.infer | null = null; + const nodes: Array = []; + let totalNodes = 0; + let isFirstLine = true; + let hasMore = false; + + const offset = Math.max(options?.offset ?? 0, 0); + const limit = Math.min(Math.max(options?.limit ?? 500, 1), 500); + + const res = await safeSpawn({ + command: "restic", + args, + env, + onStdout: (line) => { + const trimmedLine = line.trim(); + if (!trimmedLine) return; + + try { + const data = JSON.parse(trimmedLine); + + if (isFirstLine) { + isFirstLine = false; + const snapshotValidation = lsSnapshotInfoSchema(data); + if (!(snapshotValidation instanceof type.errors)) { + snapshot = snapshotValidation; + } + return; + } + + const nodeValidation = lsNodeSchema(data); + if (nodeValidation instanceof type.errors) { + logger.warn(`Skipping invalid node: ${nodeValidation.summary}`); + return; + } + + if (totalNodes >= offset && totalNodes < offset + limit) { + nodes.push(nodeValidation); + } + totalNodes++; + + if (totalNodes >= offset + limit + 1) { + hasMore = true; + } + } catch (_) { + // Ignore JSON parse errors for non-JSON lines + } + }, + }); + await cleanupTemporaryKeys(env); if (res.exitCode !== 0) { - logger.error(`Restic ls failed: ${res.stderr}`); - throw new ResticError(res.exitCode, res.stderr); + logger.error(`Restic ls failed: ${res.error}`); + throw new ResticError(res.exitCode, res.error); } - // The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes - const stdout = res.stdout; - const lines = stdout - .trim() - .split("\n") - .filter((line) => line.trim()); - - if (lines.length === 0) { - return { snapshot: null, nodes: [] }; + if (totalNodes > offset + limit) { + hasMore = true; } - // First line is snapshot info - const snapshotLine = JSON.parse(lines[0] ?? "{}"); - const snapshot = lsSnapshotInfoSchema(snapshotLine); - - if (snapshot instanceof type.errors) { - logger.error(`Restic ls snapshot info validation failed: ${snapshot.summary}`); - throw new Error(`Restic ls snapshot info validation failed: ${snapshot.summary}`); - } - - const nodes: Array = []; - for (let i = 1; i < lines.length; i++) { - const nodeLine = JSON.parse(lines[i] ?? "{}"); - const nodeValidation = lsNodeSchema(nodeLine); - - if (nodeValidation instanceof type.errors) { - logger.warn(`Skipping invalid node: ${nodeValidation.summary}`); - continue; - } - - nodes.push(nodeValidation); - } - - return { snapshot, nodes }; + return { + snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null, + nodes, + pagination: { + offset, + limit, + total: totalNodes, + hasMore, + }, + }; }; const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => { diff --git a/scripts/create-test-files.ts b/scripts/create-test-files.ts index 27e194df..e5c15713 100755 --- a/scripts/create-test-files.ts +++ b/scripts/create-test-files.ts @@ -170,4 +170,4 @@ async function main(): Promise { } } -main(); +void main();