From 2da055e519ca500f5a922a8c5d0e32f2b1859bbe Mon Sep 17 00:00:00 2001 From: Nicolas Meienberger Date: Sat, 31 Jan 2026 14:04:01 +0100 Subject: [PATCH] refactor: add pagination to handle volume folders with extremely large folder counts --- .../api-client/@tanstack/react-query.gen.ts | 124 ++++++++++++++---- app/client/api-client/index.ts | 8 +- app/client/api-client/sdk.gen.ts | 26 ++-- app/client/api-client/types.gen.ts | 54 +++++--- app/client/components/file-tree.tsx | 90 +++++++++++-- app/client/components/volume-file-browser.tsx | 8 +- app/client/hooks/use-file-browser.ts | 105 ++++++++++++++- .../modules/volumes/volume.controller.ts | 11 +- app/server/modules/volumes/volume.dto.ts | 24 ++++ app/server/modules/volumes/volume.service.ts | 106 ++++++++++----- 10 files changed, 445 insertions(+), 111 deletions(-) diff --git a/app/client/api-client/@tanstack/react-query.gen.ts b/app/client/api-client/@tanstack/react-query.gen.ts index 60f0ad71..4a609671 100644 --- a/app/client/api-client/@tanstack/react-query.gen.ts +++ b/app/client/api-client/@tanstack/react-query.gen.ts @@ -1,6 +1,12 @@ // This file is auto-generated by @hey-api/openapi-ts -import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanstack/react-query"; +import { + type DefaultError, + type InfiniteData, + infiniteQueryOptions, + queryOptions, + type UseMutationOptions, +} from "@tanstack/react-query"; import { client } from "../client.gen"; import { @@ -31,7 +37,6 @@ import { getUpdates, getUserDeletionImpact, getVolume, - refreshSnapshots, healthCheckVolume, listBackupSchedules, listFiles, @@ -43,6 +48,7 @@ import { listVolumes, mountVolume, type Options, + refreshSnapshots, reorderBackupSchedules, restoreSnapshot, runBackupNow, @@ -116,8 +122,6 @@ import type { GetUserDeletionImpactResponse, GetVolumeData, GetVolumeResponse, - RefreshSnapshotsData, - RefreshSnapshotsResponse, HealthCheckVolumeData, HealthCheckVolumeResponse, ListBackupSchedulesData, @@ -138,6 +142,8 @@ import type { ListVolumesResponse, MountVolumeData, MountVolumeResponse, + RefreshSnapshotsData, + RefreshSnapshotsResponse, ReorderBackupSchedulesData, ReorderBackupSchedulesResponse, RestoreSnapshotData, @@ -447,6 +453,77 @@ export const listFilesOptions = (options: Options) => queryKey: listFilesQueryKey(options), }); +const createInfiniteParams = [0], "body" | "headers" | "path" | "query">>( + queryKey: QueryKey, + page: K, +) => { + const params = { ...queryKey[0] }; + if (page.body) { + params.body = { + ...(queryKey[0].body as any), + ...(page.body as any), + }; + } + if (page.headers) { + params.headers = { + ...queryKey[0].headers, + ...page.headers, + }; + } + if (page.path) { + params.path = { + ...(queryKey[0].path as any), + ...(page.path as any), + }; + } + if (page.query) { + params.query = { + ...(queryKey[0].query as any), + ...(page.query as any), + }; + } + return params as unknown as typeof page; +}; + +export const listFilesInfiniteQueryKey = (options: Options): QueryKey> => + createQueryKey("listFiles", options, true); + +/** + * List files in a volume directory + */ +export const listFilesInfiniteOptions = (options: Options) => + infiniteQueryOptions< + ListFilesResponse, + DefaultError, + InfiniteData, + QueryKey>, + number | Pick>[0], "body" | "headers" | "path" | "query"> + >( + // @ts-ignore + { + queryFn: async ({ pageParam, queryKey, signal }) => { + // @ts-ignore + const page: Pick>[0], "body" | "headers" | "path" | "query"> = + typeof pageParam === "object" + ? pageParam + : { + query: { + offset: pageParam, + }, + }; + const params = createInfiniteParams(queryKey, page); + const { data } = await listFiles({ + ...options, + ...params, + signal, + throwOnError: true, + }); + return data; + }, + queryKey: listFilesInfiniteQueryKey(options), + }, + ); + export const browseFilesystemQueryKey = (options?: Options) => createQueryKey("browseFilesystem", options); @@ -636,6 +713,25 @@ export const listSnapshotsOptions = (options: Options) => queryKey: listSnapshotsQueryKey(options), }); +/** + * Clear snapshot cache and force refresh from repository + */ +export const refreshSnapshotsMutation = ( + options?: Partial>, +): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await refreshSnapshots({ + ...options, + ...fnOptions, + throwOnError: true, + }); + return data; + }, + }; + return mutationOptions; +}; + /** * Delete a specific snapshot from a repository */ @@ -781,26 +877,6 @@ export const tagSnapshotsMutation = ( return mutationOptions; }; -export const refreshSnapshotsMutation = ( - options?: Partial>, -): UseMutationOptions> => { - const mutationOptions: UseMutationOptions< - RefreshSnapshotsResponse, - DefaultError, - Options - > = { - mutationFn: async (fnOptions) => { - const { data } = await refreshSnapshots({ - ...options, - ...fnOptions, - throwOnError: true, - }); - return data; - }, - }; - return mutationOptions; -}; - export const listBackupSchedulesQueryKey = (options?: Options) => createQueryKey("listBackupSchedules", options); diff --git a/app/client/api-client/index.ts b/app/client/api-client/index.ts index d5ae8aab..fedb701c 100644 --- a/app/client/api-client/index.ts +++ b/app/client/api-client/index.ts @@ -28,7 +28,6 @@ export { getUpdates, getUserDeletionImpact, getVolume, - refreshSnapshots, healthCheckVolume, listBackupSchedules, listFiles, @@ -40,6 +39,7 @@ export { listVolumes, mountVolume, type Options, + refreshSnapshots, reorderBackupSchedules, restoreSnapshot, runBackupNow, @@ -145,9 +145,6 @@ export type { GetVolumeErrors, GetVolumeResponse, GetVolumeResponses, - RefreshSnapshotsData, - RefreshSnapshotsResponse, - RefreshSnapshotsResponses, HealthCheckVolumeData, HealthCheckVolumeErrors, HealthCheckVolumeResponse, @@ -179,6 +176,9 @@ export type { MountVolumeData, MountVolumeResponse, MountVolumeResponses, + RefreshSnapshotsData, + RefreshSnapshotsResponse, + RefreshSnapshotsResponses, ReorderBackupSchedulesData, ReorderBackupSchedulesResponse, ReorderBackupSchedulesResponses, diff --git a/app/client/api-client/sdk.gen.ts b/app/client/api-client/sdk.gen.ts index 265943d9..98a1f117 100644 --- a/app/client/api-client/sdk.gen.ts +++ b/app/client/api-client/sdk.gen.ts @@ -61,8 +61,6 @@ import type { GetVolumeData, GetVolumeErrors, GetVolumeResponses, - RefreshSnapshotsData, - RefreshSnapshotsResponses, HealthCheckVolumeData, HealthCheckVolumeErrors, HealthCheckVolumeResponses, @@ -84,6 +82,8 @@ import type { ListVolumesResponses, MountVolumeData, MountVolumeResponses, + RefreshSnapshotsData, + RefreshSnapshotsResponses, ReorderBackupSchedulesData, ReorderBackupSchedulesResponses, RestoreSnapshotData, @@ -379,6 +379,17 @@ export const listSnapshots = ( ...options, }); +/** + * Clear snapshot cache and force refresh from repository + */ +export const refreshSnapshots = ( + options: Options, +) => + (options.client ?? client).post({ + url: "/api/v1/repositories/{id}/snapshots/refresh", + ...options, + }); + /** * Delete a specific snapshot from a repository */ @@ -458,17 +469,6 @@ export const tagSnapshots = (options: Opti }, }); -/** - * Clear snapshot cache and force refresh from repository - */ -export const refreshSnapshots = ( - options: Options, -) => - (options.client ?? client).post({ - url: "/api/v1/repositories/{id}/snapshots/refresh", - ...options, - }); - /** * List all backup schedules */ diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index 07a9d067..d6421698 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -686,6 +686,14 @@ export type ListFilesData = { * Subdirectory path to list (relative to volume root) */ path?: string; + /** + * Offset for pagination (default: 0) + */ + offset?: number; + /** + * Maximum number of files to return (default: 100, max: 1000) + */ + limit?: number; }; url: "/api/v1/volumes/{id}/files"; }; @@ -702,7 +710,11 @@ export type ListFilesResponses = { modifiedAt?: number; size?: number; }>; + hasMore: boolean; + limit: number; + offset: number; path: string; + total: number; }; }; @@ -1649,6 +1661,27 @@ export type ListSnapshotsResponses = { export type ListSnapshotsResponse = ListSnapshotsResponses[keyof ListSnapshotsResponses]; +export type RefreshSnapshotsData = { + body?: never; + path: { + id: string; + }; + query?: never; + url: "/api/v1/repositories/{id}/snapshots/refresh"; +}; + +export type RefreshSnapshotsResponses = { + /** + * Snapshot cache cleared and refreshed + */ + 200: { + count: number; + message: string; + }; +}; + +export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses]; + export type DeleteSnapshotData = { body?: never; path: { @@ -1848,27 +1881,6 @@ export type TagSnapshotsResponses = { export type TagSnapshotsResponse = TagSnapshotsResponses[keyof TagSnapshotsResponses]; -export type RefreshSnapshotsData = { - body?: never; - path: { - id: string; - }; - query?: never; - url: "/api/v1/repositories/{id}/snapshots/refresh"; -}; - -export type RefreshSnapshotsResponses = { - /** - * Snapshot cache cleared and refreshed - */ - 200: { - message: string; - count: number; - }; -}; - -export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses]; - export type ListBackupSchedulesData = { body?: never; path?: never; diff --git a/app/client/components/file-tree.tsx b/app/client/components/file-tree.tsx index ba101598..5731faa0 100644 --- a/app/client/components/file-tree.tsx +++ b/app/client/components/file-tree.tsx @@ -8,7 +8,7 @@ * Original source: https://github.com/stackblitz/bolt.new */ -import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2 } from "lucide-react"; +import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2, MoreHorizontal } from "lucide-react"; import { memo, type ReactNode, useCallback, useEffect, useMemo, useState } from "react"; import { cn } from "~/client/lib/utils"; import { Checkbox } from "~/client/components/ui/checkbox"; @@ -24,12 +24,19 @@ export interface FileEntry { modifiedAt?: number; } +interface PaginationState { + hasMore: boolean; + isLoadingMore: boolean; +} + interface Props { files?: FileEntry[]; selectedFile?: string; onFileSelect?: (filePath: string) => void; onFolderExpand?: (folderPath: string) => void; onFolderHover?: (folderPath: string) => void; + onLoadMore?: (folderPath: string) => void; + getFolderPagination?: (folderPath: string) => PaginationState; expandedFolders?: Set; loadingFolders?: Set; className?: string; @@ -49,6 +56,8 @@ export const FileTree = memo((props: Props) => { selectedFile, onFolderExpand, onFolderHover, + onLoadMore, + getFolderPagination, expandedFolders = new Set(), loadingFolders = new Set(), className, @@ -292,12 +301,35 @@ export const FileTree = memo((props: Props) => { [selectedPaths], ); + // Build a map of folder paths that need pagination to their last child's index + const folderPaginationMap = useMemo(() => { + const map = new Map(); + + for (let i = 0; i < filteredFileList.length; i++) { + const item = filteredFileList[i]; + const parentPath = item.fullPath.slice(0, item.fullPath.lastIndexOf("/")) || "/"; + + if (parentPath !== "/") { + const pagination = getFolderPagination?.(parentPath); + if (pagination?.hasMore && !collapsedFolders.has(parentPath)) { + // Update the last index for this parent + map.set(parentPath, i); + } + } + } + + return map; + }, [filteredFileList, getFolderPagination, collapsedFolders]); + return (
- {filteredFileList.map((fileOrFolder) => { + {filteredFileList.map((fileOrFolder, index) => { + const elements: React.ReactNode[] = []; + + // Render the current file or folder switch (fileOrFolder.kind) { case "file": { - return ( + elements.push( { withCheckbox={withCheckboxes} checked={isPathSelected(fileOrFolder.fullPath)} onCheckboxChange={handleSelectionChange} - /> + />, ); + break; } case "folder": { - return ( + elements.push( { selectableMode={selectableFolders} onFolderSelect={handleFolderSelect} selected={selectedFolder === fileOrFolder.fullPath} - /> + />, ); - } - default: { - return undefined; + break; } } + + // Check if this is the last child of any folder with more files to load + for (const [folderPath, lastIndex] of folderPaginationMap.entries()) { + if (lastIndex === index) { + // This is the last loaded child of folderPath + const pagination = getFolderPagination?.(folderPath); + if (pagination?.hasMore) { + elements.push( + onLoadMore?.(folderPath)} + isLoading={pagination.isLoadingMore} + />, + ); + } + } + } + + return elements; })}
); @@ -477,6 +528,25 @@ const File = memo(({ file, onFileSelect, selected, withCheckbox, checked, onChec ); }); +interface LoadMoreButtonProps { + depth: number; + onClick: () => void; + isLoading?: boolean; +} + +const LoadMoreButton = memo(({ depth, onClick, isLoading }: LoadMoreButtonProps) => { + return ( + : } + onClick={onClick} + > + {isLoading ? "Loading more..." : "Load more files"} + + ); +}); + interface ButtonProps { depth: number; icon: ReactNode; @@ -621,5 +691,5 @@ function compareNodes(a: Node, b: Node): number { return a.kind === "folder" ? -1 : 1; } - return a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: "base" }); + return a.name.localeCompare(b.name); } diff --git a/app/client/components/volume-file-browser.tsx b/app/client/components/volume-file-browser.tsx index 9b1dbf83..a5c6dd32 100644 --- a/app/client/components/volume-file-browser.tsx +++ b/app/client/components/volume-file-browser.tsx @@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { FolderOpen } from "lucide-react"; import { FileTree } from "~/client/components/file-tree"; import { listFilesOptions } from "../api-client/@tanstack/react-query.gen"; -import { useFileBrowser } from "../hooks/use-file-browser"; +import { useFileBrowser, type FetchFolderResult } from "../hooks/use-file-browser"; import { parseError } from "../lib/errors"; type VolumeFileBrowserProps = { @@ -38,11 +38,11 @@ export const VolumeFileBrowser = ({ const fileBrowser = useFileBrowser({ initialData: data, isLoading, - fetchFolder: async (path) => { + fetchFolder: async (path, offset): Promise => { return await queryClient.ensureQueryData( listFilesOptions({ path: { id: volumeId }, - query: { path }, + query: { path, offset }, }), ); }, @@ -88,6 +88,8 @@ export const VolumeFileBrowser = ({ files={fileBrowser.fileArray} onFolderExpand={fileBrowser.handleFolderExpand} onFolderHover={fileBrowser.handleFolderHover} + onLoadMore={fileBrowser.handleLoadMore} + getFolderPagination={fileBrowser.getFolderPagination} expandedFolders={fileBrowser.expandedFolders} loadingFolders={fileBrowser.loadingFolders} withCheckboxes={withCheckboxes} diff --git a/app/client/hooks/use-file-browser.ts b/app/client/hooks/use-file-browser.ts index 937a177d..bdb8103d 100644 --- a/app/client/hooks/use-file-browser.ts +++ b/app/client/hooks/use-file-browser.ts @@ -1,9 +1,16 @@ import { useCallback, useMemo, useState } from "react"; import type { FileEntry } from "../components/file-tree"; -type FetchFolderFn = ( - path: string, -) => Promise<{ files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }>; +export type FetchFolderResult = { + files?: FileEntry[]; + directories?: Array<{ name: string; path: string }>; + offset?: number; + limit?: number; + total?: number; + hasMore?: boolean; +}; + +type FetchFolderFn = (path: string, offset?: number) => Promise; type PathTransformFns = { strip?: (path: string) => string; @@ -11,7 +18,7 @@ type PathTransformFns = { }; type UseFileBrowserOptions = { - initialData?: { files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }; + initialData?: FetchFolderResult; isLoading?: boolean; fetchFolder: FetchFolderFn; prefetchFolder?: (path: string) => void; @@ -19,12 +26,20 @@ type UseFileBrowserOptions = { rootPath?: string; }; +type FolderPaginationState = { + currentOffset: number; + limit: number; + hasMore: boolean; + isLoadingMore: boolean; +}; + export const useFileBrowser = (props: UseFileBrowserOptions) => { const { initialData, isLoading, fetchFolder, prefetchFolder, pathTransform, rootPath = "/" } = props; const [expandedFolders, setExpandedFolders] = useState>(new Set()); const [fetchedFolders, setFetchedFolders] = useState>(new Set([rootPath])); const [loadingFolders, setLoadingFolders] = useState>(new Set()); const [allFiles, setAllFiles] = useState>(new Map()); + const [folderPagination, setFolderPagination] = useState>(new Map()); const stripPath = pathTransform?.strip; const addPath = pathTransform?.add; @@ -44,6 +59,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { }); if (rootPath) { setFetchedFolders((prev) => new Set(prev).add(rootPath)); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(rootPath, { + currentOffset: initialData.offset ?? 0, + limit: initialData.limit ?? 100, + hasMore: initialData.hasMore ?? false, + isLoadingMore: false, + }); + return next; + }); } } else if (initialData?.directories) { const directories = initialData.directories; @@ -87,6 +112,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { } return next; }); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { + currentOffset: result.offset ?? 0, + limit: result.limit ?? 100, + hasMore: result.hasMore ?? false, + isLoadingMore: false, + }); + return next; + }); } else if (result.directories) { const directories = result.directories; setAllFiles((prev) => { @@ -113,6 +148,59 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { [fetchedFolders, fetchFolder, stripPath, addPath], ); + const handleLoadMore = useCallback( + async (folderPath: string) => { + const pagination = folderPagination.get(folderPath); + if (!pagination?.hasMore || pagination?.isLoadingMore) { + return; + } + + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { ...pagination, isLoadingMore: true }); + return next; + }); + + try { + const pathToFetch = addPath ? addPath(folderPath) : folderPath; + const nextOffset = pagination.currentOffset + pagination.limit; + const result = await fetchFolder(pathToFetch, nextOffset); + + if (result.files) { + const files = result.files; + setAllFiles((prev) => { + const next = new Map(prev); + for (const file of files) { + const strippedPath = stripPath ? stripPath(file.path) : file.path; + if (strippedPath !== folderPath) { + next.set(strippedPath, { ...file, path: strippedPath }); + } + } + return next; + }); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { + currentOffset: result.offset ?? nextOffset, + limit: result.limit ?? pagination.limit, + hasMore: result.hasMore ?? false, + isLoadingMore: false, + }); + return next; + }); + } + } catch (error) { + console.error("Failed to load more files:", error); + setFolderPagination((prev) => { + const next = new Map(prev); + next.set(folderPath, { ...pagination, isLoadingMore: false }); + return next; + }); + } + }, + [folderPagination, fetchFolder, stripPath, addPath], + ); + const handleFolderHover = useCallback( (folderPath: string) => { if (!fetchedFolders.has(folderPath) && !loadingFolders.has(folderPath) && prefetchFolder) { @@ -123,12 +211,21 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => { [fetchedFolders, loadingFolders, prefetchFolder, addPath], ); + const getFolderPagination = useCallback( + (folderPath: string) => { + return folderPagination.get(folderPath) ?? { hasMore: false, isLoadingMore: false }; + }, + [folderPagination], + ); + return { fileArray, expandedFolders, loadingFolders, handleFolderExpand, handleFolderHover, + handleLoadMore, + getFolderPagination, isLoading: isLoading && fileArray.length === 0, isEmpty: fileArray.length === 0 && !isLoading, }; diff --git a/app/server/modules/volumes/volume.controller.ts b/app/server/modules/volumes/volume.controller.ts index d8fbf10b..b6acb95f 100644 --- a/app/server/modules/volumes/volume.controller.ts +++ b/app/server/modules/volumes/volume.controller.ts @@ -107,11 +107,20 @@ export const volumeController = new Hono() .get("/:id/files", listFilesDto, async (c) => { const { id } = c.req.param(); const subPath = c.req.query("path"); - const result = await volumeService.listFiles(id, subPath); + const offsetParam = c.req.query("offset"); + const offset = offsetParam ? parseInt(offsetParam, 10) : 0; + const limitParam = c.req.query("limit"); + const limit = limitParam ? parseInt(limitParam, 10) : undefined; + + const result = await volumeService.listFiles(id, subPath, offset, limit); const response = { files: result.files, path: result.path, + offset: result.offset, + limit: result.limit, + total: result.total, + hasMore: result.hasMore, }; c.header("Cache-Control", "public, max-age=10, stale-while-revalidate=60"); diff --git a/app/server/modules/volumes/volume.dto.ts b/app/server/modules/volumes/volume.dto.ts index bb76f4c8..6554f1b5 100644 --- a/app/server/modules/volumes/volume.dto.ts +++ b/app/server/modules/volumes/volume.dto.ts @@ -276,6 +276,10 @@ const fileEntrySchema = type({ export const listFilesResponse = type({ files: fileEntrySchema.array(), path: "string", + offset: "number", + limit: "number", + total: "number", + hasMore: "boolean", }); export type ListFilesDto = typeof listFilesResponse.infer; @@ -293,6 +297,26 @@ export const listFilesDto = describeRoute({ }, description: "Subdirectory path to list (relative to volume root)", }, + { + in: "query", + name: "offset", + required: false, + schema: { + type: "integer", + default: 0, + }, + description: "Offset for pagination (default: 0)", + }, + { + in: "query", + name: "limit", + required: false, + schema: { + type: "integer", + default: 100, + }, + description: "Maximum number of files to return (default: 100, max: 1000)", + }, ], responses: { 200: { diff --git a/app/server/modules/volumes/volume.service.ts b/app/server/modules/volumes/volume.service.ts index ef869683..227a6fdf 100644 --- a/app/server/modules/volumes/volume.service.ts +++ b/app/server/modules/volumes/volume.service.ts @@ -18,6 +18,7 @@ import { serverEvents } from "../../core/events"; import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { type } from "arktype"; import { getOrganizationId } from "~/server/core/request-context"; +import { exec } from "../../utils/spawn"; async function encryptSensitiveFields(config: BackendConfig): Promise { switch (config.backend) { @@ -294,7 +295,23 @@ const checkHealth = async (idOrShortId: string | number) => { return { status, error }; }; -const listFiles = async (idOrShortId: string | number, subPath?: string) => { +const DEFAULT_PAGE_SIZE = 500; +const MAX_PAGE_SIZE = 500; + +interface DirEntry { + name: string; + path: string; + type: "directory" | "file"; + size?: number; + modifiedAt?: number; +} + +const listFiles = async ( + idOrShortId: string | number, + subPath?: string, + offset: number = 0, + limit: number = DEFAULT_PAGE_SIZE, +) => { const volume = await findVolume(idOrShortId); if (!volume) { @@ -317,43 +334,70 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => { throw new BadRequestError("Invalid path"); } + const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE); + const startOffset = Math.max(offset, 0); + try { - const entries = await fs.readdir(normalizedPath, { withFileTypes: true }); + const endLine = startOffset + pageSize; + const listScript = ` + find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | while IFS= read -r line; do + name=$(basename "$line") + if [ -d "$line" ]; then + echo "0|$name|$line" + else + echo "1|$name|$line" + fi + done | sort -t'|' -k1,1n -k2,2 | cut -d'|' -f3- | sed -n '${startOffset + 1},${endLine}p' + `; - const files = await Promise.all( - entries.map(async (entry) => { - const fullPath = path.join(normalizedPath, entry.name); + const { exitCode: listExitCode, stdout: listStdout } = await exec({ + command: "sh", + args: ["-c", listScript], + timeout: 30000, + maxBuffer: 1024 * 1024 * 10, + }); + + if (listExitCode !== 0) { + throw new Error(`Failed to list directory: ${normalizedPath}`); + } + + const filePaths = listStdout.split("\n").filter((p) => p.length > 0); + + const entries: DirEntry[] = []; + for (const fullPath of filePaths) { + try { + const stats = await fs.stat(fullPath); const relativePath = path.relative(volumePath, fullPath); + const name = path.basename(fullPath); - try { - const stats = await fs.stat(fullPath); - return { - name: entry.name, - path: `/${relativePath}`, - type: entry.isDirectory() ? ("directory" as const) : ("file" as const), - size: entry.isFile() ? stats.size : undefined, - modifiedAt: stats.mtimeMs, - }; - } catch { - return { - name: entry.name, - path: `/${relativePath}`, - type: entry.isDirectory() ? ("directory" as const) : ("file" as const), - size: undefined, - modifiedAt: undefined, - }; - } - }), - ); + entries.push({ + name, + path: `/${relativePath}`, + type: stats.isDirectory() ? "directory" : "file", + size: stats.isFile() ? stats.size : undefined, + modifiedAt: stats.mtimeMs, + }); + } catch { + // File may have been deleted between listing and stat + } + } + + const countScript = `find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | wc -l`; + const { exitCode: countExitCode, stdout: countStdout } = await exec({ + command: "sh", + args: ["-c", countScript], + timeout: 10000, + }); + + const total = countExitCode === 0 ? parseInt(countStdout.trim(), 10) || 0 : entries.length; return { - files: files.sort((a, b) => { - if (a.type !== b.type) { - return a.type === "directory" ? -1 : 1; - } - return a.name.localeCompare(b.name); - }), + files: entries, path: subPath || "/", + offset: startOffset, + limit: pageSize, + total, + hasMore: startOffset + entries.length < total, }; } catch (error) { throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);