danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor: add pagination to handle volume folders with extremely large folder counts

Browse source
This commit is contained in:
Nicolas Meienberger 2026-01-31 14:04:01 +01:00
parent b8ad1ae41a
commit 2da055e519
10 changed files with 445 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
app/client/api-client/@tanstack/react-query.gen.ts
View file

@ -1,6 +1,12 @@
// This file is auto-generated by @hey-api/openapi-ts
import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanstack/react-query";
import {
type DefaultError,
type InfiniteData,
infiniteQueryOptions,
queryOptions,
type UseMutationOptions,
} from "@tanstack/react-query";
import { client } from "../client.gen";
import {
@ -31,7 +37,6 @@ import {
getUpdates,
getUserDeletionImpact,
getVolume,
refreshSnapshots,
healthCheckVolume,
listBackupSchedules,
listFiles,
@ -43,6 +48,7 @@ import {
listVolumes,
mountVolume,
type Options,
refreshSnapshots,
reorderBackupSchedules,
restoreSnapshot,
runBackupNow,
@ -116,8 +122,6 @@ import type {
GetUserDeletionImpactResponse,
GetVolumeData,
GetVolumeResponse,
RefreshSnapshotsData,
RefreshSnapshotsResponse,
HealthCheckVolumeData,
HealthCheckVolumeResponse,
ListBackupSchedulesData,
@ -138,6 +142,8 @@ import type {
ListVolumesResponse,
MountVolumeData,
MountVolumeResponse,
RefreshSnapshotsData,
RefreshSnapshotsResponse,
ReorderBackupSchedulesData,
ReorderBackupSchedulesResponse,
RestoreSnapshotData,
@ -447,6 +453,77 @@ export const listFilesOptions = (options: Options<ListFilesData>) =>
queryKey: listFilesQueryKey(options),
});
const createInfiniteParams = <K extends Pick<QueryKey<Options>[0], "body" | "headers" | "path" | "query">>(
queryKey: QueryKey<Options>,
page: K,
) => {
const params = { ...queryKey[0] };
if (page.body) {
params.body = {
...(queryKey[0].body as any),
...(page.body as any),
};
}
if (page.headers) {
params.headers = {
...queryKey[0].headers,
...page.headers,
};
}
if (page.path) {
params.path = {
...(queryKey[0].path as any),
...(page.path as any),
};
}
if (page.query) {
params.query = {
...(queryKey[0].query as any),
...(page.query as any),
};
}
return params as unknown as typeof page;
};
export const listFilesInfiniteQueryKey = (options: Options<ListFilesData>): QueryKey<Options<ListFilesData>> =>
createQueryKey("listFiles", options, true);
/**
* List files in a volume directory
*/
export const listFilesInfiniteOptions = (options: Options<ListFilesData>) =>
infiniteQueryOptions<
ListFilesResponse,
DefaultError,
InfiniteData<ListFilesResponse>,
QueryKey<Options<ListFilesData>>,
number | Pick<QueryKey<Options<ListFilesData>>[0], "body" | "headers" | "path" | "query">
>(
// @ts-ignore
{
queryFn: async ({ pageParam, queryKey, signal }) => {
// @ts-ignore
const page: Pick<QueryKey<Options<ListFilesData>>[0], "body" | "headers" | "path" | "query"> =
typeof pageParam === "object"
? pageParam
: {
query: {
offset: pageParam,
},
};
const params = createInfiniteParams(queryKey, page);
const { data } = await listFiles({
...options,
...params,
signal,
throwOnError: true,
});
return data;
},
queryKey: listFilesInfiniteQueryKey(options),
},
);
export const browseFilesystemQueryKey = (options?: Options<BrowseFilesystemData>) =>
createQueryKey("browseFilesystem", options);
@ -636,6 +713,25 @@ export const listSnapshotsOptions = (options: Options<ListSnapshotsData>) =>
queryKey: listSnapshotsQueryKey(options),
});
/**
* Clear snapshot cache and force refresh from repository
*/
export const refreshSnapshotsMutation = (
options?: Partial<Options<RefreshSnapshotsData>>,
): UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> => {
const mutationOptions: UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> = {
mutationFn: async (fnOptions) => {
const { data } = await refreshSnapshots({
...options,
...fnOptions,
throwOnError: true,
});
return data;
},
};
return mutationOptions;
};
/**
* Delete a specific snapshot from a repository
*/
@ -781,26 +877,6 @@ export const tagSnapshotsMutation = (
return mutationOptions;
};
export const refreshSnapshotsMutation = (
options?: Partial<Options<RefreshSnapshotsData>>,
): UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> => {
const mutationOptions: UseMutationOptions<
RefreshSnapshotsResponse,
DefaultError,
Options<RefreshSnapshotsData>
> = {
mutationFn: async (fnOptions) => {
const { data } = await refreshSnapshots({
...options,
...fnOptions,
throwOnError: true,
});
return data;
},
};
return mutationOptions;
};
export const listBackupSchedulesQueryKey = (options?: Options<ListBackupSchedulesData>) =>
createQueryKey("listBackupSchedules", options);

8
app/client/api-client/index.ts
View file

@ -28,7 +28,6 @@ export {
getUpdates,
getUserDeletionImpact,
getVolume,
refreshSnapshots,
healthCheckVolume,
listBackupSchedules,
listFiles,
@ -40,6 +39,7 @@ export {
listVolumes,
mountVolume,
type Options,
refreshSnapshots,
reorderBackupSchedules,
restoreSnapshot,
runBackupNow,
@ -145,9 +145,6 @@ export type {
GetVolumeErrors,
GetVolumeResponse,
GetVolumeResponses,
RefreshSnapshotsData,
RefreshSnapshotsResponse,
RefreshSnapshotsResponses,
HealthCheckVolumeData,
HealthCheckVolumeErrors,
HealthCheckVolumeResponse,
@ -179,6 +176,9 @@ export type {
MountVolumeData,
MountVolumeResponse,
MountVolumeResponses,
RefreshSnapshotsData,
RefreshSnapshotsResponse,
RefreshSnapshotsResponses,
ReorderBackupSchedulesData,
ReorderBackupSchedulesResponse,
ReorderBackupSchedulesResponses,

26
app/client/api-client/sdk.gen.ts
View file

@ -61,8 +61,6 @@ import type {
GetVolumeData,
GetVolumeErrors,
GetVolumeResponses,
RefreshSnapshotsData,
RefreshSnapshotsResponses,
HealthCheckVolumeData,
HealthCheckVolumeErrors,
HealthCheckVolumeResponses,
@ -84,6 +82,8 @@ import type {
ListVolumesResponses,
MountVolumeData,
MountVolumeResponses,
RefreshSnapshotsData,
RefreshSnapshotsResponses,
ReorderBackupSchedulesData,
ReorderBackupSchedulesResponses,
RestoreSnapshotData,
@ -379,6 +379,17 @@ export const listSnapshots = <ThrowOnError extends boolean = false>(
...options,
});
/**
* Clear snapshot cache and force refresh from repository
*/
export const refreshSnapshots = <ThrowOnError extends boolean = false>(
options: Options<RefreshSnapshotsData, ThrowOnError>,
) =>
(options.client ?? client).post<RefreshSnapshotsResponses, unknown, ThrowOnError>({
url: "/api/v1/repositories/{id}/snapshots/refresh",
...options,
});
/**
* Delete a specific snapshot from a repository
*/
@ -458,17 +469,6 @@ export const tagSnapshots = <ThrowOnError extends boolean = false>(options: Opti
},
});
/**
* Clear snapshot cache and force refresh from repository
*/
export const refreshSnapshots = <ThrowOnError extends boolean = false>(
options: Options<RefreshSnapshotsData, ThrowOnError>,
) =>
(options.client ?? client).post<RefreshSnapshotsResponses, unknown, ThrowOnError>({
url: "/api/v1/repositories/{id}/snapshots/refresh",
...options,
});
/**
* List all backup schedules
*/

54
app/client/api-client/types.gen.ts
View file

@ -686,6 +686,14 @@ export type ListFilesData = {
* Subdirectory path to list (relative to volume root)
*/
path?: string;
/**
* Offset for pagination (default: 0)
*/
offset?: number;
/**
* Maximum number of files to return (default: 100, max: 1000)
*/
limit?: number;
};
url: "/api/v1/volumes/{id}/files";
};
@ -702,7 +710,11 @@ export type ListFilesResponses = {
modifiedAt?: number;
size?: number;
}>;
hasMore: boolean;
limit: number;
offset: number;
path: string;
total: number;
};
};
@ -1649,6 +1661,27 @@ export type ListSnapshotsResponses = {
export type ListSnapshotsResponse = ListSnapshotsResponses[keyof ListSnapshotsResponses];
export type RefreshSnapshotsData = {
body?: never;
path: {
id: string;
};
query?: never;
url: "/api/v1/repositories/{id}/snapshots/refresh";
};
export type RefreshSnapshotsResponses = {
/**
* Snapshot cache cleared and refreshed
*/
200: {
count: number;
message: string;
};
};
export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses];
export type DeleteSnapshotData = {
body?: never;
path: {
@ -1848,27 +1881,6 @@ export type TagSnapshotsResponses = {
export type TagSnapshotsResponse = TagSnapshotsResponses[keyof TagSnapshotsResponses];
export type RefreshSnapshotsData = {
body?: never;
path: {
id: string;
};
query?: never;
url: "/api/v1/repositories/{id}/snapshots/refresh";
};
export type RefreshSnapshotsResponses = {
/**
* Snapshot cache cleared and refreshed
*/
200: {
message: string;
count: number;
};
};
export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses];
export type ListBackupSchedulesData = {
body?: never;
path?: never;

90
app/client/components/file-tree.tsx
View file

@ -8,7 +8,7 @@
* Original source: https://github.com/stackblitz/bolt.new
*/
import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2 } from "lucide-react";
import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2, MoreHorizontal } from "lucide-react";
import { memo, type ReactNode, useCallback, useEffect, useMemo, useState } from "react";
import { cn } from "~/client/lib/utils";
import { Checkbox } from "~/client/components/ui/checkbox";
@ -24,12 +24,19 @@ export interface FileEntry {
modifiedAt?: number;
}
interface PaginationState {
hasMore: boolean;
isLoadingMore: boolean;
}
interface Props {
files?: FileEntry[];
selectedFile?: string;
onFileSelect?: (filePath: string) => void;
onFolderExpand?: (folderPath: string) => void;
onFolderHover?: (folderPath: string) => void;
onLoadMore?: (folderPath: string) => void;
getFolderPagination?: (folderPath: string) => PaginationState;
expandedFolders?: Set<string>;
loadingFolders?: Set<string>;
className?: string;
@ -49,6 +56,8 @@ export const FileTree = memo((props: Props) => {
selectedFile,
onFolderExpand,
onFolderHover,
onLoadMore,
getFolderPagination,
expandedFolders = new Set(),
loadingFolders = new Set(),
className,
@ -292,12 +301,35 @@ export const FileTree = memo((props: Props) => {
[selectedPaths],
);
// Build a map of folder paths that need pagination to their last child's index
const folderPaginationMap = useMemo(() => {
const map = new Map<string, number>();
for (let i = 0; i < filteredFileList.length; i++) {
const item = filteredFileList[i];
const parentPath = item.fullPath.slice(0, item.fullPath.lastIndexOf("/")) || "/";
if (parentPath !== "/") {
const pagination = getFolderPagination?.(parentPath);
if (pagination?.hasMore && !collapsedFolders.has(parentPath)) {
// Update the last index for this parent
map.set(parentPath, i);
}
}
}
return map;
}, [filteredFileList, getFolderPagination, collapsedFolders]);
return (
<div className={cn("text-sm", className)}>
{filteredFileList.map((fileOrFolder) => {
{filteredFileList.map((fileOrFolder, index) => {
const elements: React.ReactNode[] = [];
// Render the current file or folder
switch (fileOrFolder.kind) {
case "file": {
return (
elements.push(
<File
key={fileOrFolder.id}
selected={selectedFile === fileOrFolder.fullPath}
@ -306,11 +338,12 @@ export const FileTree = memo((props: Props) => {
withCheckbox={withCheckboxes}
checked={isPathSelected(fileOrFolder.fullPath)}
onCheckboxChange={handleSelectionChange}
/>
/>,
);
break;
}
case "folder": {
return (
elements.push(
<Folder
key={fileOrFolder.id}
folder={fileOrFolder}
@ -325,13 +358,31 @@ export const FileTree = memo((props: Props) => {
selectableMode={selectableFolders}
onFolderSelect={handleFolderSelect}
selected={selectedFolder === fileOrFolder.fullPath}
/>
/>,
);
}
default: {
return undefined;
break;
}
}
// Check if this is the last child of any folder with more files to load
for (const [folderPath, lastIndex] of folderPaginationMap.entries()) {
if (lastIndex === index) {
// This is the last loaded child of folderPath
const pagination = getFolderPagination?.(folderPath);
if (pagination?.hasMore) {
elements.push(
<LoadMoreButton
key={`load-more-${folderPath}`}
depth={fileOrFolder.depth}
onClick={() => onLoadMore?.(folderPath)}
isLoading={pagination.isLoadingMore}
/>,
);
}
}
}
return elements;
})}
</div>
);
@ -477,6 +528,25 @@ const File = memo(({ file, onFileSelect, selected, withCheckbox, checked, onChec
);
});
interface LoadMoreButtonProps {
depth: number;
onClick: () => void;
isLoading?: boolean;
}
const LoadMoreButton = memo(({ depth, onClick, isLoading }: LoadMoreButtonProps) => {
return (
<NodeButton
depth={depth}
className="text-muted-foreground hover:bg-accent/50 cursor-pointer"
icon={isLoading ? <Loader2 className="w-4 h-4 shrink-0 animate-spin" /> : <MoreHorizontal className="w-4 h-4 shrink-0" />}
onClick={onClick}
>
<span className="text-xs">{isLoading ? "Loading more..." : "Load more files"}</span>
</NodeButton>
);
});
interface ButtonProps {
depth: number;
icon: ReactNode;
@ -621,5 +691,5 @@ function compareNodes(a: Node, b: Node): number {
return a.kind === "folder" ? -1 : 1;
}
return a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: "base" });
return a.name.localeCompare(b.name);
}

8
app/client/components/volume-file-browser.tsx
View file

@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query";
import { FolderOpen } from "lucide-react";
import { FileTree } from "~/client/components/file-tree";
import { listFilesOptions } from "../api-client/@tanstack/react-query.gen";
import { useFileBrowser } from "../hooks/use-file-browser";
import { useFileBrowser, type FetchFolderResult } from "../hooks/use-file-browser";
import { parseError } from "../lib/errors";
type VolumeFileBrowserProps = {
@ -38,11 +38,11 @@ export const VolumeFileBrowser = ({
const fileBrowser = useFileBrowser({
initialData: data,
isLoading,
fetchFolder: async (path) => {
fetchFolder: async (path, offset): Promise<FetchFolderResult> => {
return await queryClient.ensureQueryData(
listFilesOptions({
path: { id: volumeId },
query: { path },
query: { path, offset },
}),
);
},
@ -88,6 +88,8 @@ export const VolumeFileBrowser = ({
files={fileBrowser.fileArray}
onFolderExpand={fileBrowser.handleFolderExpand}
onFolderHover={fileBrowser.handleFolderHover}
onLoadMore={fileBrowser.handleLoadMore}
getFolderPagination={fileBrowser.getFolderPagination}
expandedFolders={fileBrowser.expandedFolders}
loadingFolders={fileBrowser.loadingFolders}
withCheckboxes={withCheckboxes}

105
app/client/hooks/use-file-browser.ts
View file

@ -1,9 +1,16 @@
import { useCallback, useMemo, useState } from "react";
import type { FileEntry } from "../components/file-tree";
type FetchFolderFn = (
path: string,
) => Promise<{ files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }>;
export type FetchFolderResult = {
files?: FileEntry[];
directories?: Array<{ name: string; path: string }>;
offset?: number;
limit?: number;
total?: number;
hasMore?: boolean;
};
type FetchFolderFn = (path: string, offset?: number) => Promise<FetchFolderResult>;
type PathTransformFns = {
strip?: (path: string) => string;
@ -11,7 +18,7 @@ type PathTransformFns = {
};
type UseFileBrowserOptions = {
initialData?: { files?: FileEntry[]; directories?: Array<{ name: string; path: string }> };
initialData?: FetchFolderResult;
isLoading?: boolean;
fetchFolder: FetchFolderFn;
prefetchFolder?: (path: string) => void;
@ -19,12 +26,20 @@ type UseFileBrowserOptions = {
rootPath?: string;
};
type FolderPaginationState = {
currentOffset: number;
limit: number;
hasMore: boolean;
isLoadingMore: boolean;
};
export const useFileBrowser = (props: UseFileBrowserOptions) => {
const { initialData, isLoading, fetchFolder, prefetchFolder, pathTransform, rootPath = "/" } = props;
const [expandedFolders, setExpandedFolders] = useState<Set<string>>(new Set());
const [fetchedFolders, setFetchedFolders] = useState<Set<string>>(new Set([rootPath]));
const [loadingFolders, setLoadingFolders] = useState<Set<string>>(new Set());
const [allFiles, setAllFiles] = useState<Map<string, FileEntry>>(new Map());
const [folderPagination, setFolderPagination] = useState<Map<string, FolderPaginationState>>(new Map());
const stripPath = pathTransform?.strip;
const addPath = pathTransform?.add;
@ -44,6 +59,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
});
if (rootPath) {
setFetchedFolders((prev) => new Set(prev).add(rootPath));
setFolderPagination((prev) => {
const next = new Map(prev);
next.set(rootPath, {
currentOffset: initialData.offset ?? 0,
limit: initialData.limit ?? 100,
hasMore: initialData.hasMore ?? false,
isLoadingMore: false,
});
return next;
});
}
} else if (initialData?.directories) {
const directories = initialData.directories;
@ -87,6 +112,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
}
return next;
});
setFolderPagination((prev) => {
const next = new Map(prev);
next.set(folderPath, {
currentOffset: result.offset ?? 0,
limit: result.limit ?? 100,
hasMore: result.hasMore ?? false,
isLoadingMore: false,
});
return next;
});
} else if (result.directories) {
const directories = result.directories;
setAllFiles((prev) => {
@ -113,6 +148,59 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
[fetchedFolders, fetchFolder, stripPath, addPath],
);
const handleLoadMore = useCallback(
async (folderPath: string) => {
const pagination = folderPagination.get(folderPath);
if (!pagination?.hasMore || pagination?.isLoadingMore) {
return;
}
setFolderPagination((prev) => {
const next = new Map(prev);
next.set(folderPath, { ...pagination, isLoadingMore: true });
return next;
});
try {
const pathToFetch = addPath ? addPath(folderPath) : folderPath;
const nextOffset = pagination.currentOffset + pagination.limit;
const result = await fetchFolder(pathToFetch, nextOffset);
if (result.files) {
const files = result.files;
setAllFiles((prev) => {
const next = new Map(prev);
for (const file of files) {
const strippedPath = stripPath ? stripPath(file.path) : file.path;
if (strippedPath !== folderPath) {
next.set(strippedPath, { ...file, path: strippedPath });
}
}
return next;
});
setFolderPagination((prev) => {
const next = new Map(prev);
next.set(folderPath, {
currentOffset: result.offset ?? nextOffset,
limit: result.limit ?? pagination.limit,
hasMore: result.hasMore ?? false,
isLoadingMore: false,
});
return next;
});
}
} catch (error) {
console.error("Failed to load more files:", error);
setFolderPagination((prev) => {
const next = new Map(prev);
next.set(folderPath, { ...pagination, isLoadingMore: false });
return next;
});
}
},
[folderPagination, fetchFolder, stripPath, addPath],
);
const handleFolderHover = useCallback(
(folderPath: string) => {
if (!fetchedFolders.has(folderPath) && !loadingFolders.has(folderPath) && prefetchFolder) {
@ -123,12 +211,21 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
[fetchedFolders, loadingFolders, prefetchFolder, addPath],
);
const getFolderPagination = useCallback(
(folderPath: string) => {
return folderPagination.get(folderPath) ?? { hasMore: false, isLoadingMore: false };
},
[folderPagination],
);
return {
fileArray,
expandedFolders,
loadingFolders,
handleFolderExpand,
handleFolderHover,
handleLoadMore,
getFolderPagination,
isLoading: isLoading && fileArray.length === 0,
isEmpty: fileArray.length === 0 && !isLoading,
};

11
app/server/modules/volumes/volume.controller.ts
View file

@ -107,11 +107,20 @@ export const volumeController = new Hono()
.get("/:id/files", listFilesDto, async (c) => {
const { id } = c.req.param();
const subPath = c.req.query("path");
const result = await volumeService.listFiles(id, subPath);
const offsetParam = c.req.query("offset");
const offset = offsetParam ? parseInt(offsetParam, 10) : 0;
const limitParam = c.req.query("limit");
const limit = limitParam ? parseInt(limitParam, 10) : undefined;
const result = await volumeService.listFiles(id, subPath, offset, limit);
const response = {
files: result.files,
path: result.path,
offset: result.offset,
limit: result.limit,
total: result.total,
hasMore: result.hasMore,
};
c.header("Cache-Control", "public, max-age=10, stale-while-revalidate=60");

24
app/server/modules/volumes/volume.dto.ts
View file

@ -276,6 +276,10 @@ const fileEntrySchema = type({
export const listFilesResponse = type({
files: fileEntrySchema.array(),
path: "string",
offset: "number",
limit: "number",
total: "number",
hasMore: "boolean",
});
export type ListFilesDto = typeof listFilesResponse.infer;
@ -293,6 +297,26 @@ export const listFilesDto = describeRoute({
},
description: "Subdirectory path to list (relative to volume root)",
},
{
in: "query",
name: "offset",
required: false,
schema: {
type: "integer",
default: 0,
},
description: "Offset for pagination (default: 0)",
},
{
in: "query",
name: "limit",
required: false,
schema: {
type: "integer",
default: 100,
},
description: "Maximum number of files to return (default: 100, max: 1000)",
},
],
responses: {
200: {

106
app/server/modules/volumes/volume.service.ts
View file

@ -18,6 +18,7 @@ import { serverEvents } from "../../core/events";
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
import { type } from "arktype";
import { getOrganizationId } from "~/server/core/request-context";
import { exec } from "../../utils/spawn";
async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
switch (config.backend) {
@ -294,7 +295,23 @@ const checkHealth = async (idOrShortId: string | number) => {
return { status, error };
};
const listFiles = async (idOrShortId: string | number, subPath?: string) => {
const DEFAULT_PAGE_SIZE = 500;
const MAX_PAGE_SIZE = 500;
interface DirEntry {
name: string;
path: string;
type: "directory" | "file";
size?: number;
modifiedAt?: number;
}
const listFiles = async (
idOrShortId: string | number,
subPath?: string,
offset: number = 0,
limit: number = DEFAULT_PAGE_SIZE,
) => {
const volume = await findVolume(idOrShortId);
if (!volume) {
@ -317,43 +334,70 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => {
throw new BadRequestError("Invalid path");
}
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
const startOffset = Math.max(offset, 0);
try {
const entries = await fs.readdir(normalizedPath, { withFileTypes: true });
const endLine = startOffset + pageSize;
const listScript = `
find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | while IFS= read -r line; do
name=$(basename "$line")
if [ -d "$line" ]; then
echo "0|$name|$line"
else
echo "1|$name|$line"
fi
done | sort -t'|' -k1,1n -k2,2 | cut -d'|' -f3- | sed -n '${startOffset + 1},${endLine}p'
`;
const files = await Promise.all(
entries.map(async (entry) => {
const fullPath = path.join(normalizedPath, entry.name);
const { exitCode: listExitCode, stdout: listStdout } = await exec({
command: "sh",
args: ["-c", listScript],
timeout: 30000,
maxBuffer: 1024 * 1024 * 10,
});
if (listExitCode !== 0) {
throw new Error(`Failed to list directory: ${normalizedPath}`);
}
const filePaths = listStdout.split("\n").filter((p) => p.length > 0);
const entries: DirEntry[] = [];
for (const fullPath of filePaths) {
try {
const stats = await fs.stat(fullPath);
const relativePath = path.relative(volumePath, fullPath);
const name = path.basename(fullPath);
try {
const stats = await fs.stat(fullPath);
return {
name: entry.name,
path: `/${relativePath}`,
type: entry.isDirectory() ? ("directory" as const) : ("file" as const),
size: entry.isFile() ? stats.size : undefined,
modifiedAt: stats.mtimeMs,
};
} catch {
return {
name: entry.name,
path: `/${relativePath}`,
type: entry.isDirectory() ? ("directory" as const) : ("file" as const),
size: undefined,
modifiedAt: undefined,
};
}
}),
);
entries.push({
name,
path: `/${relativePath}`,
type: stats.isDirectory() ? "directory" : "file",
size: stats.isFile() ? stats.size : undefined,
modifiedAt: stats.mtimeMs,
});
} catch {
// File may have been deleted between listing and stat
}
}
const countScript = `find "${normalizedPath.replace(/"/g, '\\"')}" -maxdepth 1 -mindepth 1 | wc -l`;
const { exitCode: countExitCode, stdout: countStdout } = await exec({
command: "sh",
args: ["-c", countScript],
timeout: 10000,
});
const total = countExitCode === 0 ? parseInt(countStdout.trim(), 10) || 0 : entries.length;
return {
files: files.sort((a, b) => {
if (a.type !== b.type) {
return a.type === "directory" ? -1 : 1;
}
return a.name.localeCompare(b.name);
}),
files: entries,
path: subPath || "/",
offset: startOffset,
limit: pageSize,
total,
hasMore: startOffset + entries.length < total,
};
} catch (error) {
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);