refactor: paginate large file counts (#441)
* refactor: add pagination to handle volume folders with extremely large folder counts * refactor: stream restic ls result * test: file-tree load more * refactor: string params * fix(tsc): string pagination params * chore: pr feedbacks
This commit is contained in:
Nico
GitHub
parent
b8ad1ae41a
commit
5bb5fcd09c
19 changed files with 701 additions and 181 deletions
|
|
@ -1,6 +1,12 @@
|
|||
// This file is auto-generated by @hey-api/openapi-ts
|
||||
|
||||
import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanstack/react-query";
|
||||
import {
|
||||
type DefaultError,
|
||||
type InfiniteData,
|
||||
infiniteQueryOptions,
|
||||
queryOptions,
|
||||
type UseMutationOptions,
|
||||
} from "@tanstack/react-query";
|
||||
|
||||
import { client } from "../client.gen";
|
||||
import {
|
||||
|
|
@ -31,7 +37,6 @@ import {
|
|||
getUpdates,
|
||||
getUserDeletionImpact,
|
||||
getVolume,
|
||||
refreshSnapshots,
|
||||
healthCheckVolume,
|
||||
listBackupSchedules,
|
||||
listFiles,
|
||||
|
|
@ -43,6 +48,7 @@ import {
|
|||
listVolumes,
|
||||
mountVolume,
|
||||
type Options,
|
||||
refreshSnapshots,
|
||||
reorderBackupSchedules,
|
||||
restoreSnapshot,
|
||||
runBackupNow,
|
||||
|
|
@ -116,8 +122,6 @@ import type {
|
|||
GetUserDeletionImpactResponse,
|
||||
GetVolumeData,
|
||||
GetVolumeResponse,
|
||||
RefreshSnapshotsData,
|
||||
RefreshSnapshotsResponse,
|
||||
HealthCheckVolumeData,
|
||||
HealthCheckVolumeResponse,
|
||||
ListBackupSchedulesData,
|
||||
|
|
@ -138,6 +142,8 @@ import type {
|
|||
ListVolumesResponse,
|
||||
MountVolumeData,
|
||||
MountVolumeResponse,
|
||||
RefreshSnapshotsData,
|
||||
RefreshSnapshotsResponse,
|
||||
ReorderBackupSchedulesData,
|
||||
ReorderBackupSchedulesResponse,
|
||||
RestoreSnapshotData,
|
||||
|
|
@ -447,6 +453,77 @@ export const listFilesOptions = (options: Options<ListFilesData>) =>
|
|||
queryKey: listFilesQueryKey(options),
|
||||
});
|
||||
|
||||
const createInfiniteParams = <K extends Pick<QueryKey<Options>[0], "body" | "headers" | "path" | "query">>(
|
||||
queryKey: QueryKey<Options>,
|
||||
page: K,
|
||||
) => {
|
||||
const params = { ...queryKey[0] };
|
||||
if (page.body) {
|
||||
params.body = {
|
||||
...(queryKey[0].body as any),
|
||||
...(page.body as any),
|
||||
};
|
||||
}
|
||||
if (page.headers) {
|
||||
params.headers = {
|
||||
...queryKey[0].headers,
|
||||
...page.headers,
|
||||
};
|
||||
}
|
||||
if (page.path) {
|
||||
params.path = {
|
||||
...(queryKey[0].path as any),
|
||||
...(page.path as any),
|
||||
};
|
||||
}
|
||||
if (page.query) {
|
||||
params.query = {
|
||||
...(queryKey[0].query as any),
|
||||
...(page.query as any),
|
||||
};
|
||||
}
|
||||
return params as unknown as typeof page;
|
||||
};
|
||||
|
||||
export const listFilesInfiniteQueryKey = (options: Options<ListFilesData>): QueryKey<Options<ListFilesData>> =>
|
||||
createQueryKey("listFiles", options, true);
|
||||
|
||||
/**
|
||||
* List files in a volume directory
|
||||
*/
|
||||
export const listFilesInfiniteOptions = (options: Options<ListFilesData>) =>
|
||||
infiniteQueryOptions<
|
||||
ListFilesResponse,
|
||||
DefaultError,
|
||||
InfiniteData<ListFilesResponse>,
|
||||
QueryKey<Options<ListFilesData>>,
|
||||
string | Pick<QueryKey<Options<ListFilesData>>[0], "body" | "headers" | "path" | "query">
|
||||
>(
|
||||
// @ts-ignore
|
||||
{
|
||||
queryFn: async ({ pageParam, queryKey, signal }) => {
|
||||
// @ts-ignore
|
||||
const page: Pick<QueryKey<Options<ListFilesData>>[0], "body" | "headers" | "path" | "query"> =
|
||||
typeof pageParam === "object"
|
||||
? pageParam
|
||||
: {
|
||||
query: {
|
||||
offset: pageParam,
|
||||
},
|
||||
};
|
||||
const params = createInfiniteParams(queryKey, page);
|
||||
const { data } = await listFiles({
|
||||
...options,
|
||||
...params,
|
||||
signal,
|
||||
throwOnError: true,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
queryKey: listFilesInfiniteQueryKey(options),
|
||||
},
|
||||
);
|
||||
|
||||
export const browseFilesystemQueryKey = (options?: Options<BrowseFilesystemData>) =>
|
||||
createQueryKey("browseFilesystem", options);
|
||||
|
||||
|
|
@ -636,6 +713,25 @@ export const listSnapshotsOptions = (options: Options<ListSnapshotsData>) =>
|
|||
queryKey: listSnapshotsQueryKey(options),
|
||||
});
|
||||
|
||||
/**
|
||||
* Clear snapshot cache and force refresh from repository
|
||||
*/
|
||||
export const refreshSnapshotsMutation = (
|
||||
options?: Partial<Options<RefreshSnapshotsData>>,
|
||||
): UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> => {
|
||||
const mutationOptions: UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> = {
|
||||
mutationFn: async (fnOptions) => {
|
||||
const { data } = await refreshSnapshots({
|
||||
...options,
|
||||
...fnOptions,
|
||||
throwOnError: true,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
};
|
||||
return mutationOptions;
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete a specific snapshot from a repository
|
||||
*/
|
||||
|
|
@ -705,6 +801,46 @@ export const listSnapshotFilesOptions = (options: Options<ListSnapshotFilesData>
|
|||
queryKey: listSnapshotFilesQueryKey(options),
|
||||
});
|
||||
|
||||
export const listSnapshotFilesInfiniteQueryKey = (
|
||||
options: Options<ListSnapshotFilesData>,
|
||||
): QueryKey<Options<ListSnapshotFilesData>> => createQueryKey("listSnapshotFiles", options, true);
|
||||
|
||||
/**
|
||||
* List files and directories in a snapshot
|
||||
*/
|
||||
export const listSnapshotFilesInfiniteOptions = (options: Options<ListSnapshotFilesData>) =>
|
||||
infiniteQueryOptions<
|
||||
ListSnapshotFilesResponse,
|
||||
DefaultError,
|
||||
InfiniteData<ListSnapshotFilesResponse>,
|
||||
QueryKey<Options<ListSnapshotFilesData>>,
|
||||
string | Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query">
|
||||
>(
|
||||
// @ts-ignore
|
||||
{
|
||||
queryFn: async ({ pageParam, queryKey, signal }) => {
|
||||
// @ts-ignore
|
||||
const page: Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query"> =
|
||||
typeof pageParam === "object"
|
||||
? pageParam
|
||||
: {
|
||||
query: {
|
||||
offset: pageParam,
|
||||
},
|
||||
};
|
||||
const params = createInfiniteParams(queryKey, page);
|
||||
const { data } = await listSnapshotFiles({
|
||||
...options,
|
||||
...params,
|
||||
signal,
|
||||
throwOnError: true,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
queryKey: listSnapshotFilesInfiniteQueryKey(options),
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* Restore a snapshot to a target path on the filesystem
|
||||
*/
|
||||
|
|
@ -781,26 +917,6 @@ export const tagSnapshotsMutation = (
|
|||
return mutationOptions;
|
||||
};
|
||||
|
||||
export const refreshSnapshotsMutation = (
|
||||
options?: Partial<Options<RefreshSnapshotsData>>,
|
||||
): UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> => {
|
||||
const mutationOptions: UseMutationOptions<
|
||||
RefreshSnapshotsResponse,
|
||||
DefaultError,
|
||||
Options<RefreshSnapshotsData>
|
||||
> = {
|
||||
mutationFn: async (fnOptions) => {
|
||||
const { data } = await refreshSnapshots({
|
||||
...options,
|
||||
...fnOptions,
|
||||
throwOnError: true,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
};
|
||||
return mutationOptions;
|
||||
};
|
||||
|
||||
export const listBackupSchedulesQueryKey = (options?: Options<ListBackupSchedulesData>) =>
|
||||
createQueryKey("listBackupSchedules", options);
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ export {
|
|||
getUpdates,
|
||||
getUserDeletionImpact,
|
||||
getVolume,
|
||||
refreshSnapshots,
|
||||
healthCheckVolume,
|
||||
listBackupSchedules,
|
||||
listFiles,
|
||||
|
|
@ -40,6 +39,7 @@ export {
|
|||
listVolumes,
|
||||
mountVolume,
|
||||
type Options,
|
||||
refreshSnapshots,
|
||||
reorderBackupSchedules,
|
||||
restoreSnapshot,
|
||||
runBackupNow,
|
||||
|
|
@ -145,9 +145,6 @@ export type {
|
|||
GetVolumeErrors,
|
||||
GetVolumeResponse,
|
||||
GetVolumeResponses,
|
||||
RefreshSnapshotsData,
|
||||
RefreshSnapshotsResponse,
|
||||
RefreshSnapshotsResponses,
|
||||
HealthCheckVolumeData,
|
||||
HealthCheckVolumeErrors,
|
||||
HealthCheckVolumeResponse,
|
||||
|
|
@ -179,6 +176,9 @@ export type {
|
|||
MountVolumeData,
|
||||
MountVolumeResponse,
|
||||
MountVolumeResponses,
|
||||
RefreshSnapshotsData,
|
||||
RefreshSnapshotsResponse,
|
||||
RefreshSnapshotsResponses,
|
||||
ReorderBackupSchedulesData,
|
||||
ReorderBackupSchedulesResponse,
|
||||
ReorderBackupSchedulesResponses,
|
||||
|
|
|
|||
|
|
@ -61,8 +61,6 @@ import type {
|
|||
GetVolumeData,
|
||||
GetVolumeErrors,
|
||||
GetVolumeResponses,
|
||||
RefreshSnapshotsData,
|
||||
RefreshSnapshotsResponses,
|
||||
HealthCheckVolumeData,
|
||||
HealthCheckVolumeErrors,
|
||||
HealthCheckVolumeResponses,
|
||||
|
|
@ -84,6 +82,8 @@ import type {
|
|||
ListVolumesResponses,
|
||||
MountVolumeData,
|
||||
MountVolumeResponses,
|
||||
RefreshSnapshotsData,
|
||||
RefreshSnapshotsResponses,
|
||||
ReorderBackupSchedulesData,
|
||||
ReorderBackupSchedulesResponses,
|
||||
RestoreSnapshotData,
|
||||
|
|
@ -379,6 +379,17 @@ export const listSnapshots = <ThrowOnError extends boolean = false>(
|
|||
...options,
|
||||
});
|
||||
|
||||
/**
|
||||
* Clear snapshot cache and force refresh from repository
|
||||
*/
|
||||
export const refreshSnapshots = <ThrowOnError extends boolean = false>(
|
||||
options: Options<RefreshSnapshotsData, ThrowOnError>,
|
||||
) =>
|
||||
(options.client ?? client).post<RefreshSnapshotsResponses, unknown, ThrowOnError>({
|
||||
url: "/api/v1/repositories/{id}/snapshots/refresh",
|
||||
...options,
|
||||
});
|
||||
|
||||
/**
|
||||
* Delete a specific snapshot from a repository
|
||||
*/
|
||||
|
|
@ -458,17 +469,6 @@ export const tagSnapshots = <ThrowOnError extends boolean = false>(options: Opti
|
|||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Clear snapshot cache and force refresh from repository
|
||||
*/
|
||||
export const refreshSnapshots = <ThrowOnError extends boolean = false>(
|
||||
options: Options<RefreshSnapshotsData, ThrowOnError>,
|
||||
) =>
|
||||
(options.client ?? client).post<RefreshSnapshotsResponses, unknown, ThrowOnError>({
|
||||
url: "/api/v1/repositories/{id}/snapshots/refresh",
|
||||
...options,
|
||||
});
|
||||
|
||||
/**
|
||||
* List all backup schedules
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -682,9 +682,8 @@ export type ListFilesData = {
|
|||
id: string;
|
||||
};
|
||||
query?: {
|
||||
/**
|
||||
* Subdirectory path to list (relative to volume root)
|
||||
*/
|
||||
limit?: string;
|
||||
offset?: string;
|
||||
path?: string;
|
||||
};
|
||||
url: "/api/v1/volumes/{id}/files";
|
||||
|
|
@ -702,7 +701,11 @@ export type ListFilesResponses = {
|
|||
modifiedAt?: number;
|
||||
size?: number;
|
||||
}>;
|
||||
hasMore: boolean;
|
||||
limit: number;
|
||||
offset: number;
|
||||
path: string;
|
||||
total: number;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -1649,6 +1652,27 @@ export type ListSnapshotsResponses = {
|
|||
|
||||
export type ListSnapshotsResponse = ListSnapshotsResponses[keyof ListSnapshotsResponses];
|
||||
|
||||
export type RefreshSnapshotsData = {
|
||||
body?: never;
|
||||
path: {
|
||||
id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: "/api/v1/repositories/{id}/snapshots/refresh";
|
||||
};
|
||||
|
||||
export type RefreshSnapshotsResponses = {
|
||||
/**
|
||||
* Snapshot cache cleared and refreshed
|
||||
*/
|
||||
200: {
|
||||
count: number;
|
||||
message: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses];
|
||||
|
||||
export type DeleteSnapshotData = {
|
||||
body?: never;
|
||||
path: {
|
||||
|
|
@ -1703,6 +1727,8 @@ export type ListSnapshotFilesData = {
|
|||
snapshotId: string;
|
||||
};
|
||||
query?: {
|
||||
limit?: string;
|
||||
offset?: string;
|
||||
path?: string;
|
||||
};
|
||||
url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files";
|
||||
|
|
@ -1725,6 +1751,9 @@ export type ListSnapshotFilesResponses = {
|
|||
size?: number;
|
||||
uid?: number;
|
||||
}>;
|
||||
hasMore: boolean;
|
||||
limit: number;
|
||||
offset: number;
|
||||
snapshot: {
|
||||
hostname: string;
|
||||
id: string;
|
||||
|
|
@ -1732,6 +1761,7 @@ export type ListSnapshotFilesResponses = {
|
|||
short_id: string;
|
||||
time: string;
|
||||
};
|
||||
total: number;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -1848,27 +1878,6 @@ export type TagSnapshotsResponses = {
|
|||
|
||||
export type TagSnapshotsResponse = TagSnapshotsResponses[keyof TagSnapshotsResponses];
|
||||
|
||||
export type RefreshSnapshotsData = {
|
||||
body?: never;
|
||||
path: {
|
||||
id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: "/api/v1/repositories/{id}/snapshots/refresh";
|
||||
};
|
||||
|
||||
export type RefreshSnapshotsResponses = {
|
||||
/**
|
||||
* Snapshot cache cleared and refreshed
|
||||
*/
|
||||
200: {
|
||||
message: string;
|
||||
count: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses];
|
||||
|
||||
export type ListBackupSchedulesData = {
|
||||
body?: never;
|
||||
path?: never;
|
||||
|
|
|
|||
|
|
@ -3,6 +3,114 @@ import { expect, test, describe } from "bun:test";
|
|||
import { render, screen, fireEvent } from "@testing-library/react";
|
||||
import { FileTree, type FileEntry } from "../file-tree";
|
||||
|
||||
describe("FileTree Pagination", () => {
|
||||
const testFiles: FileEntry[] = [
|
||||
{ name: "root", path: "/root", type: "folder" },
|
||||
{ name: "file1", path: "/root/file1", type: "file" },
|
||||
{ name: "file2", path: "/root/file2", type: "file" },
|
||||
];
|
||||
|
||||
test("shows load more button when hasMore is true", () => {
|
||||
render(
|
||||
<FileTree
|
||||
files={testFiles}
|
||||
expandedFolders={new Set(["/root"])}
|
||||
getFolderPagination={() => ({ hasMore: true, isLoadingMore: false })}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("Load more files")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("does not show load more button when hasMore is false", () => {
|
||||
render(
|
||||
<FileTree
|
||||
files={testFiles}
|
||||
expandedFolders={new Set(["/root"])}
|
||||
getFolderPagination={() => ({ hasMore: false, isLoadingMore: false })}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.queryByText("Load more files")).toBeNull();
|
||||
});
|
||||
|
||||
test("calls onLoadMore with folder path when load more button is clicked", () => {
|
||||
let loadMoreCalled = false;
|
||||
let loadMorePath = "";
|
||||
|
||||
render(
|
||||
<FileTree
|
||||
files={testFiles}
|
||||
expandedFolders={new Set(["/root"])}
|
||||
getFolderPagination={(path) => {
|
||||
if (path === "/root") {
|
||||
return { hasMore: true, isLoadingMore: false };
|
||||
}
|
||||
return { hasMore: false, isLoadingMore: false };
|
||||
}}
|
||||
onLoadMore={(path) => {
|
||||
loadMoreCalled = true;
|
||||
loadMorePath = path;
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
const loadMoreButton = screen.getByText("Load more files");
|
||||
fireEvent.click(loadMoreButton);
|
||||
|
||||
expect(loadMoreCalled).toBe(true);
|
||||
expect(loadMorePath).toBe("/root");
|
||||
});
|
||||
|
||||
test("shows loading state when isLoadingMore is true", () => {
|
||||
render(
|
||||
<FileTree
|
||||
files={testFiles}
|
||||
expandedFolders={new Set(["/root"])}
|
||||
getFolderPagination={() => ({ hasMore: true, isLoadingMore: true })}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("Loading more...")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("load more button appears for nested folders with hasMore", () => {
|
||||
const nestedFiles: FileEntry[] = [
|
||||
{ name: "root", path: "/root", type: "folder" },
|
||||
{ name: "child", path: "/root/child", type: "folder" },
|
||||
{ name: "file1", path: "/root/child/file1", type: "file" },
|
||||
];
|
||||
|
||||
render(
|
||||
<FileTree
|
||||
files={nestedFiles}
|
||||
expandedFolders={new Set(["/root", "/root/child"])}
|
||||
getFolderPagination={(path) => {
|
||||
if (path === "/root/child") {
|
||||
return { hasMore: true, isLoadingMore: false };
|
||||
}
|
||||
return { hasMore: false, isLoadingMore: false };
|
||||
}}
|
||||
onLoadMore={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("Load more files")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("load more button does not appear when folder is collapsed", () => {
|
||||
render(
|
||||
<FileTree
|
||||
files={testFiles}
|
||||
expandedFolders={new Set([])}
|
||||
getFolderPagination={() => ({ hasMore: true, isLoadingMore: false })}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.queryByText("Load more files")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("FileTree Selection Logic", () => {
|
||||
const testFiles: FileEntry[] = [
|
||||
{ name: "root", path: "/root", type: "folder" },
|
||||
|
|
|
|||
|
|
@ -8,7 +8,15 @@
|
|||
* Original source: https://github.com/stackblitz/bolt.new
|
||||
*/
|
||||
|
||||
import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2 } from "lucide-react";
|
||||
import {
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
File as FileIcon,
|
||||
Folder as FolderIcon,
|
||||
FolderOpen,
|
||||
Loader2,
|
||||
MoreHorizontal,
|
||||
} from "lucide-react";
|
||||
import { memo, type ReactNode, useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { cn } from "~/client/lib/utils";
|
||||
import { Checkbox } from "~/client/components/ui/checkbox";
|
||||
|
|
@ -24,12 +32,19 @@ export interface FileEntry {
|
|||
modifiedAt?: number;
|
||||
}
|
||||
|
||||
interface PaginationState {
|
||||
hasMore: boolean;
|
||||
isLoadingMore: boolean;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
files?: FileEntry[];
|
||||
selectedFile?: string;
|
||||
onFileSelect?: (filePath: string) => void;
|
||||
onFolderExpand?: (folderPath: string) => void;
|
||||
onFolderHover?: (folderPath: string) => void;
|
||||
onLoadMore?: (folderPath: string) => void;
|
||||
getFolderPagination?: (folderPath: string) => PaginationState;
|
||||
expandedFolders?: Set<string>;
|
||||
loadingFolders?: Set<string>;
|
||||
className?: string;
|
||||
|
|
@ -49,6 +64,8 @@ export const FileTree = memo((props: Props) => {
|
|||
selectedFile,
|
||||
onFolderExpand,
|
||||
onFolderHover,
|
||||
onLoadMore,
|
||||
getFolderPagination,
|
||||
expandedFolders = new Set(),
|
||||
loadingFolders = new Set(),
|
||||
className,
|
||||
|
|
@ -292,12 +309,35 @@ export const FileTree = memo((props: Props) => {
|
|||
[selectedPaths],
|
||||
);
|
||||
|
||||
// Build a map of folder paths that need pagination to their last child's index
|
||||
const folderPaginationMap = useMemo(() => {
|
||||
const map = new Map<string, number>();
|
||||
|
||||
for (let i = 0; i < filteredFileList.length; i++) {
|
||||
const item = filteredFileList[i];
|
||||
const parentPath = item.fullPath.slice(0, item.fullPath.lastIndexOf("/")) || "/";
|
||||
|
||||
if (parentPath !== "/") {
|
||||
const pagination = getFolderPagination?.(parentPath);
|
||||
if (pagination?.hasMore && !collapsedFolders.has(parentPath)) {
|
||||
// Update the last index for this parent
|
||||
map.set(parentPath, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}, [filteredFileList, getFolderPagination, collapsedFolders]);
|
||||
|
||||
return (
|
||||
<div className={cn("text-sm", className)}>
|
||||
{filteredFileList.map((fileOrFolder) => {
|
||||
{filteredFileList.map((fileOrFolder, index) => {
|
||||
const elements: React.ReactNode[] = [];
|
||||
|
||||
// Render the current file or folder
|
||||
switch (fileOrFolder.kind) {
|
||||
case "file": {
|
||||
return (
|
||||
elements.push(
|
||||
<File
|
||||
key={fileOrFolder.id}
|
||||
selected={selectedFile === fileOrFolder.fullPath}
|
||||
|
|
@ -306,11 +346,12 @@ export const FileTree = memo((props: Props) => {
|
|||
withCheckbox={withCheckboxes}
|
||||
checked={isPathSelected(fileOrFolder.fullPath)}
|
||||
onCheckboxChange={handleSelectionChange}
|
||||
/>
|
||||
/>,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case "folder": {
|
||||
return (
|
||||
elements.push(
|
||||
<Folder
|
||||
key={fileOrFolder.id}
|
||||
folder={fileOrFolder}
|
||||
|
|
@ -325,13 +366,31 @@ export const FileTree = memo((props: Props) => {
|
|||
selectableMode={selectableFolders}
|
||||
onFolderSelect={handleFolderSelect}
|
||||
selected={selectedFolder === fileOrFolder.fullPath}
|
||||
/>
|
||||
/>,
|
||||
);
|
||||
}
|
||||
default: {
|
||||
return undefined;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is the last child of any folder with more files to load
|
||||
for (const [folderPath, lastIndex] of folderPaginationMap.entries()) {
|
||||
if (lastIndex === index) {
|
||||
// This is the last loaded child of folderPath
|
||||
const pagination = getFolderPagination?.(folderPath);
|
||||
if (pagination?.hasMore) {
|
||||
elements.push(
|
||||
<LoadMoreButton
|
||||
key={`load-more-${folderPath}`}
|
||||
depth={fileOrFolder.depth}
|
||||
onClick={() => onLoadMore?.(folderPath)}
|
||||
isLoading={pagination.isLoadingMore}
|
||||
/>,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return elements;
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -477,6 +536,31 @@ const File = memo(({ file, onFileSelect, selected, withCheckbox, checked, onChec
|
|||
);
|
||||
});
|
||||
|
||||
interface LoadMoreButtonProps {
|
||||
depth: number;
|
||||
onClick: () => void;
|
||||
isLoading?: boolean;
|
||||
}
|
||||
|
||||
const LoadMoreButton = memo(({ depth, onClick, isLoading }: LoadMoreButtonProps) => {
|
||||
return (
|
||||
<NodeButton
|
||||
depth={depth}
|
||||
className="text-muted-foreground hover:bg-accent/50 cursor-pointer"
|
||||
icon={
|
||||
isLoading ? (
|
||||
<Loader2 className="w-4 h-4 shrink-0 animate-spin" />
|
||||
) : (
|
||||
<MoreHorizontal className="w-4 h-4 shrink-0" />
|
||||
)
|
||||
}
|
||||
onClick={onClick}
|
||||
>
|
||||
<span className="text-xs">{isLoading ? "Loading more..." : "Load more files"}</span>
|
||||
</NodeButton>
|
||||
);
|
||||
});
|
||||
|
||||
interface ButtonProps {
|
||||
depth: number;
|
||||
icon: ReactNode;
|
||||
|
|
@ -621,5 +705,5 @@ function compareNodes(a: Node, b: Node): number {
|
|||
return a.kind === "folder" ? -1 : 1;
|
||||
}
|
||||
|
||||
return a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: "base" });
|
||||
return a.name.localeCompare(b.name);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,11 +74,11 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
|
|||
const fileBrowser = useFileBrowser({
|
||||
initialData: filesData,
|
||||
isLoading: filesLoading,
|
||||
fetchFolder: async (path) => {
|
||||
fetchFolder: async (path, offset = 0) => {
|
||||
return await queryClient.ensureQueryData(
|
||||
listSnapshotFilesOptions({
|
||||
path: { id: repository.id, snapshotId },
|
||||
query: { path },
|
||||
query: { path, offset: offset.toString(), limit: "500" },
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -86,7 +86,7 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
|
|||
void queryClient.prefetchQuery(
|
||||
listSnapshotFilesOptions({
|
||||
path: { id: repository.id, snapshotId },
|
||||
query: { path },
|
||||
query: { path, offset: "0", limit: "500" },
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -306,6 +306,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
|
|||
onFolderHover={fileBrowser.handleFolderHover}
|
||||
expandedFolders={fileBrowser.expandedFolders}
|
||||
loadingFolders={fileBrowser.loadingFolders}
|
||||
onLoadMore={fileBrowser.handleLoadMore}
|
||||
getFolderPagination={fileBrowser.getFolderPagination}
|
||||
className="px-2 py-2"
|
||||
withCheckboxes={true}
|
||||
selectedPaths={selectedPaths}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query";
|
|||
import { FolderOpen } from "lucide-react";
|
||||
import { FileTree } from "~/client/components/file-tree";
|
||||
import { listFilesOptions } from "../api-client/@tanstack/react-query.gen";
|
||||
import { useFileBrowser } from "../hooks/use-file-browser";
|
||||
import { useFileBrowser, type FetchFolderResult } from "../hooks/use-file-browser";
|
||||
import { parseError } from "../lib/errors";
|
||||
|
||||
type VolumeFileBrowserProps = {
|
||||
|
|
@ -38,11 +38,11 @@ export const VolumeFileBrowser = ({
|
|||
const fileBrowser = useFileBrowser({
|
||||
initialData: data,
|
||||
isLoading,
|
||||
fetchFolder: async (path) => {
|
||||
fetchFolder: async (path, offset): Promise<FetchFolderResult> => {
|
||||
return await queryClient.ensureQueryData(
|
||||
listFilesOptions({
|
||||
path: { id: volumeId },
|
||||
query: { path },
|
||||
query: { path, offset: offset?.toString() },
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -88,6 +88,8 @@ export const VolumeFileBrowser = ({
|
|||
files={fileBrowser.fileArray}
|
||||
onFolderExpand={fileBrowser.handleFolderExpand}
|
||||
onFolderHover={fileBrowser.handleFolderHover}
|
||||
onLoadMore={fileBrowser.handleLoadMore}
|
||||
getFolderPagination={fileBrowser.getFolderPagination}
|
||||
expandedFolders={fileBrowser.expandedFolders}
|
||||
loadingFolders={fileBrowser.loadingFolders}
|
||||
withCheckboxes={withCheckboxes}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,16 @@
|
|||
import { useCallback, useMemo, useState } from "react";
|
||||
import type { FileEntry } from "../components/file-tree";
|
||||
|
||||
type FetchFolderFn = (
|
||||
path: string,
|
||||
) => Promise<{ files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }>;
|
||||
export type FetchFolderResult = {
|
||||
files?: FileEntry[];
|
||||
directories?: Array<{ name: string; path: string }>;
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
total?: number;
|
||||
hasMore?: boolean;
|
||||
};
|
||||
|
||||
type FetchFolderFn = (path: string, offset?: number) => Promise<FetchFolderResult>;
|
||||
|
||||
type PathTransformFns = {
|
||||
strip?: (path: string) => string;
|
||||
|
|
@ -11,7 +18,7 @@ type PathTransformFns = {
|
|||
};
|
||||
|
||||
type UseFileBrowserOptions = {
|
||||
initialData?: { files?: FileEntry[]; directories?: Array<{ name: string; path: string }> };
|
||||
initialData?: FetchFolderResult;
|
||||
isLoading?: boolean;
|
||||
fetchFolder: FetchFolderFn;
|
||||
prefetchFolder?: (path: string) => void;
|
||||
|
|
@ -19,12 +26,20 @@ type UseFileBrowserOptions = {
|
|||
rootPath?: string;
|
||||
};
|
||||
|
||||
type FolderPaginationState = {
|
||||
currentOffset: number;
|
||||
limit: number;
|
||||
hasMore: boolean;
|
||||
isLoadingMore: boolean;
|
||||
};
|
||||
|
||||
export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
||||
const { initialData, isLoading, fetchFolder, prefetchFolder, pathTransform, rootPath = "/" } = props;
|
||||
const [expandedFolders, setExpandedFolders] = useState<Set<string>>(new Set());
|
||||
const [fetchedFolders, setFetchedFolders] = useState<Set<string>>(new Set([rootPath]));
|
||||
const [loadingFolders, setLoadingFolders] = useState<Set<string>>(new Set());
|
||||
const [allFiles, setAllFiles] = useState<Map<string, FileEntry>>(new Map());
|
||||
const [folderPagination, setFolderPagination] = useState<Map<string, FolderPaginationState>>(new Map());
|
||||
|
||||
const stripPath = pathTransform?.strip;
|
||||
const addPath = pathTransform?.add;
|
||||
|
|
@ -44,6 +59,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
|||
});
|
||||
if (rootPath) {
|
||||
setFetchedFolders((prev) => new Set(prev).add(rootPath));
|
||||
setFolderPagination((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.set(rootPath, {
|
||||
currentOffset: initialData.offset ?? 0,
|
||||
limit: initialData.limit ?? 100,
|
||||
hasMore: initialData.hasMore ?? false,
|
||||
isLoadingMore: false,
|
||||
});
|
||||
return next;
|
||||
});
|
||||
}
|
||||
} else if (initialData?.directories) {
|
||||
const directories = initialData.directories;
|
||||
|
|
@ -87,6 +112,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
|||
}
|
||||
return next;
|
||||
});
|
||||
setFolderPagination((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.set(folderPath, {
|
||||
currentOffset: result.offset ?? 0,
|
||||
limit: result.limit ?? 100,
|
||||
hasMore: result.hasMore ?? false,
|
||||
isLoadingMore: false,
|
||||
});
|
||||
return next;
|
||||
});
|
||||
} else if (result.directories) {
|
||||
const directories = result.directories;
|
||||
setAllFiles((prev) => {
|
||||
|
|
@ -113,6 +148,59 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
|||
[fetchedFolders, fetchFolder, stripPath, addPath],
|
||||
);
|
||||
|
||||
const handleLoadMore = useCallback(
|
||||
async (folderPath: string) => {
|
||||
const pagination = folderPagination.get(folderPath);
|
||||
if (!pagination?.hasMore || pagination?.isLoadingMore) {
|
||||
return;
|
||||
}
|
||||
|
||||
setFolderPagination((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.set(folderPath, { ...pagination, isLoadingMore: true });
|
||||
return next;
|
||||
});
|
||||
|
||||
try {
|
||||
const pathToFetch = addPath ? addPath(folderPath) : folderPath;
|
||||
const nextOffset = pagination.currentOffset + pagination.limit;
|
||||
const result = await fetchFolder(pathToFetch, nextOffset);
|
||||
|
||||
if (result.files) {
|
||||
const files = result.files;
|
||||
setAllFiles((prev) => {
|
||||
const next = new Map(prev);
|
||||
for (const file of files) {
|
||||
const strippedPath = stripPath ? stripPath(file.path) : file.path;
|
||||
if (strippedPath !== folderPath) {
|
||||
next.set(strippedPath, { ...file, path: strippedPath });
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
setFolderPagination((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.set(folderPath, {
|
||||
currentOffset: result.offset ?? nextOffset,
|
||||
limit: result.limit ?? pagination.limit,
|
||||
hasMore: result.hasMore ?? false,
|
||||
isLoadingMore: false,
|
||||
});
|
||||
return next;
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load more files:", error);
|
||||
setFolderPagination((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.set(folderPath, { ...pagination, isLoadingMore: false });
|
||||
return next;
|
||||
});
|
||||
}
|
||||
},
|
||||
[folderPagination, fetchFolder, stripPath, addPath],
|
||||
);
|
||||
|
||||
const handleFolderHover = useCallback(
|
||||
(folderPath: string) => {
|
||||
if (!fetchedFolders.has(folderPath) && !loadingFolders.has(folderPath) && prefetchFolder) {
|
||||
|
|
@ -123,12 +211,21 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
|||
[fetchedFolders, loadingFolders, prefetchFolder, addPath],
|
||||
);
|
||||
|
||||
const getFolderPagination = useCallback(
|
||||
(folderPath: string) => {
|
||||
return folderPagination.get(folderPath) ?? { hasMore: false, isLoadingMore: false };
|
||||
},
|
||||
[folderPagination],
|
||||
);
|
||||
|
||||
return {
|
||||
fileArray,
|
||||
expandedFolders,
|
||||
loadingFolders,
|
||||
handleFolderExpand,
|
||||
handleFolderHover,
|
||||
handleLoadMore,
|
||||
getFolderPagination,
|
||||
isLoading: isLoading && fileArray.length === 0,
|
||||
isEmpty: fileArray.length === 0 && !isLoading,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -671,14 +671,20 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:
|
|||
<div>
|
||||
<p className="text-xs uppercase text-muted-foreground">Include paths/patterns</p>
|
||||
<div className="flex flex-col gap-1">
|
||||
{formValues.includePatterns?.map((path) => (
|
||||
{formValues.includePatterns?.slice(0, 20).map((path) => (
|
||||
<span key={path} className="text-xs font-mono bg-accent px-1.5 py-0.5 rounded">
|
||||
{path}
|
||||
</span>
|
||||
))}
|
||||
{formValues.includePatterns && formValues.includePatterns.length > 20 && (
|
||||
<span className="text-xs text-muted-foreground">
|
||||
+ {formValues.includePatterns.length - 20} more
|
||||
</span>
|
||||
)}
|
||||
{formValues.includePatternsText
|
||||
?.split("\n")
|
||||
.filter(Boolean)
|
||||
.slice(0, 20 - (formValues.includePatterns?.length || 0))
|
||||
.map((pattern) => (
|
||||
<span key={pattern} className="text-xs font-mono bg-accent px-1.5 py-0.5 rounded">
|
||||
{pattern.trim()}
|
||||
|
|
|
|||
|
|
@ -60,11 +60,11 @@ export const SnapshotFileBrowser = (props: Props) => {
|
|||
const fileBrowser = useFileBrowser({
|
||||
initialData: filesData,
|
||||
isLoading: filesLoading,
|
||||
fetchFolder: async (path) => {
|
||||
fetchFolder: async (path, offset = 0) => {
|
||||
return await queryClient.ensureQueryData(
|
||||
listSnapshotFilesOptions({
|
||||
path: { id: repositoryId, snapshotId: snapshot.short_id },
|
||||
query: { path },
|
||||
query: { path, offset: offset.toString(), limit: "500" },
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -72,7 +72,7 @@ export const SnapshotFileBrowser = (props: Props) => {
|
|||
void queryClient.prefetchQuery(
|
||||
listSnapshotFilesOptions({
|
||||
path: { id: repositoryId, snapshotId: snapshot.short_id },
|
||||
query: { path },
|
||||
query: { path, offset: "0", limit: "500" },
|
||||
}),
|
||||
);
|
||||
},
|
||||
|
|
@ -142,6 +142,8 @@ export const SnapshotFileBrowser = (props: Props) => {
|
|||
onFolderHover={fileBrowser.handleFolderHover}
|
||||
expandedFolders={fileBrowser.expandedFolders}
|
||||
loadingFolders={fileBrowser.loadingFolders}
|
||||
onLoadMore={fileBrowser.handleLoadMore}
|
||||
getFolderPagination={fileBrowser.getFolderPagination}
|
||||
className="px-2 py-2"
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -143,10 +143,14 @@ export const repositoriesController = new Hono()
|
|||
validator("query", listSnapshotFilesQuery),
|
||||
async (c) => {
|
||||
const { id, snapshotId } = c.req.param();
|
||||
const { path } = c.req.valid("query");
|
||||
const { path, ...query } = c.req.valid("query");
|
||||
|
||||
const decodedPath = path ? decodeURIComponent(path) : undefined;
|
||||
const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath);
|
||||
|
||||
const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0);
|
||||
const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500));
|
||||
|
||||
const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath, { offset, limit });
|
||||
|
||||
c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");
|
||||
|
||||
|
|
|
|||
|
|
@ -252,12 +252,18 @@ export const listSnapshotFilesResponse = type({
|
|||
paths: "string[]",
|
||||
}),
|
||||
files: snapshotFileNodeSchema.array(),
|
||||
offset: "number",
|
||||
limit: "number",
|
||||
total: "number",
|
||||
hasMore: "boolean",
|
||||
});
|
||||
|
||||
export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer;
|
||||
|
||||
export const listSnapshotFilesQuery = type({
|
||||
path: "string?",
|
||||
offset: "string.integer?",
|
||||
limit: "string.integer?",
|
||||
});
|
||||
|
||||
export const listSnapshotFilesDto = describeRoute({
|
||||
|
|
|
|||
|
|
@ -210,7 +210,12 @@ const listSnapshots = async (id: string, backupId?: string) => {
|
|||
}
|
||||
};
|
||||
|
||||
const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) => {
|
||||
const listSnapshotFiles = async (
|
||||
id: string,
|
||||
snapshotId: string,
|
||||
path?: string,
|
||||
options?: { offset?: number; limit?: number },
|
||||
) => {
|
||||
const organizationId = getOrganizationId();
|
||||
const repository = await findRepository(id);
|
||||
|
||||
|
|
@ -218,18 +223,30 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
|
|||
throw new NotFoundError("Repository not found");
|
||||
}
|
||||
|
||||
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}`;
|
||||
const cached = cache.get<Awaited<ReturnType<typeof restic.ls>>>(cacheKey);
|
||||
const offset = options?.offset ?? 0;
|
||||
const limit = options?.limit ?? 500;
|
||||
|
||||
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`;
|
||||
type LsResult = {
|
||||
snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null;
|
||||
nodes: { name: string; type: string; path: string; size?: number; mode?: number }[];
|
||||
pagination: { offset: number; limit: number; total: number; hasMore: boolean };
|
||||
};
|
||||
const cached = cache.get<LsResult>(cacheKey);
|
||||
if (cached?.snapshot) {
|
||||
return {
|
||||
snapshot: cached.snapshot,
|
||||
files: cached.nodes,
|
||||
offset: cached.pagination.offset,
|
||||
limit: cached.pagination.limit,
|
||||
total: cached.pagination.total,
|
||||
hasMore: cached.pagination.hasMore,
|
||||
};
|
||||
}
|
||||
|
||||
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
|
||||
try {
|
||||
const result = await restic.ls(repository.config, snapshotId, organizationId, path);
|
||||
const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit });
|
||||
|
||||
if (!result.snapshot) {
|
||||
throw new NotFoundError("Snapshot not found or empty");
|
||||
|
|
@ -244,6 +261,10 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
|
|||
paths: result.snapshot.paths,
|
||||
},
|
||||
files: result.nodes,
|
||||
offset: result.pagination.offset,
|
||||
limit: result.pagination.limit,
|
||||
total: result.pagination.total,
|
||||
hasMore: result.pagination.hasMore,
|
||||
};
|
||||
|
||||
cache.set(cacheKey, result);
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
type ListFilesDto,
|
||||
browseFilesystemDto,
|
||||
type BrowseFilesystemDto,
|
||||
listFilesQuery,
|
||||
} from "./volume.dto";
|
||||
import { volumeService } from "./volume.service";
|
||||
import { getVolumePath } from "./helpers";
|
||||
|
|
@ -104,14 +105,22 @@ export const volumeController = new Hono()
|
|||
|
||||
return c.json({ error, status }, 200);
|
||||
})
|
||||
.get("/:id/files", listFilesDto, async (c) => {
|
||||
.get("/:id/files", validator("query", listFilesQuery), listFilesDto, async (c) => {
|
||||
const { id } = c.req.param();
|
||||
const subPath = c.req.query("path");
|
||||
const result = await volumeService.listFiles(id, subPath);
|
||||
const { path, ...query } = c.req.valid("query");
|
||||
|
||||
const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0);
|
||||
const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500));
|
||||
|
||||
const result = await volumeService.listFiles(id, path, offset, limit);
|
||||
|
||||
const response = {
|
||||
files: result.files,
|
||||
path: result.path,
|
||||
offset: result.offset,
|
||||
limit: result.limit,
|
||||
total: result.total,
|
||||
hasMore: result.hasMore,
|
||||
};
|
||||
|
||||
c.header("Cache-Control", "public, max-age=10, stale-while-revalidate=60");
|
||||
|
|
|
|||
|
|
@ -276,24 +276,23 @@ const fileEntrySchema = type({
|
|||
export const listFilesResponse = type({
|
||||
files: fileEntrySchema.array(),
|
||||
path: "string",
|
||||
offset: "number",
|
||||
limit: "number",
|
||||
total: "number",
|
||||
hasMore: "boolean",
|
||||
});
|
||||
export type ListFilesDto = typeof listFilesResponse.infer;
|
||||
|
||||
export const listFilesQuery = type({
|
||||
path: "string?",
|
||||
offset: "string.integer?",
|
||||
limit: "string.integer?",
|
||||
});
|
||||
|
||||
export const listFilesDto = describeRoute({
|
||||
description: "List files in a volume directory",
|
||||
operationId: "listFiles",
|
||||
tags: ["Volumes"],
|
||||
parameters: [
|
||||
{
|
||||
in: "query",
|
||||
name: "path",
|
||||
required: false,
|
||||
schema: {
|
||||
type: "string",
|
||||
},
|
||||
description: "Subdirectory path to list (relative to volume root)",
|
||||
},
|
||||
],
|
||||
responses: {
|
||||
200: {
|
||||
description: "List of files in the volume",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import { serverEvents } from "../../core/events";
|
|||
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
||||
import { type } from "arktype";
|
||||
import { getOrganizationId } from "~/server/core/request-context";
|
||||
import { isNodeJSErrnoException } from "~/server/utils/fs";
|
||||
|
||||
async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
|
||||
switch (config.backend) {
|
||||
|
|
@ -294,7 +295,15 @@ const checkHealth = async (idOrShortId: string | number) => {
|
|||
return { status, error };
|
||||
};
|
||||
|
||||
const listFiles = async (idOrShortId: string | number, subPath?: string) => {
|
||||
const DEFAULT_PAGE_SIZE = 500;
|
||||
const MAX_PAGE_SIZE = 500;
|
||||
|
||||
const listFiles = async (
|
||||
idOrShortId: string | number,
|
||||
subPath?: string,
|
||||
offset: number = 0,
|
||||
limit: number = DEFAULT_PAGE_SIZE,
|
||||
) => {
|
||||
const volume = await findVolume(idOrShortId);
|
||||
|
||||
if (!volume) {
|
||||
|
|
@ -305,11 +314,8 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => {
|
|||
throw new InternalServerError("Volume is not mounted");
|
||||
}
|
||||
|
||||
// For directory volumes, use the configured path directly
|
||||
const volumePath = getVolumePath(volume);
|
||||
|
||||
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
|
||||
|
||||
const normalizedPath = path.normalize(requestedPath);
|
||||
const relative = path.relative(volumePath, normalizedPath);
|
||||
|
||||
|
|
@ -317,45 +323,60 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => {
|
|||
throw new BadRequestError("Invalid path");
|
||||
}
|
||||
|
||||
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
|
||||
const startOffset = Math.max(offset, 0);
|
||||
|
||||
try {
|
||||
const entries = await fs.readdir(normalizedPath, { withFileTypes: true });
|
||||
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
|
||||
|
||||
const files = await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
const fullPath = path.join(normalizedPath, entry.name);
|
||||
const relativePath = path.relative(volumePath, fullPath);
|
||||
dirents.sort((a, b) => {
|
||||
const aIsDir = a.isDirectory();
|
||||
const bIsDir = b.isDirectory();
|
||||
|
||||
try {
|
||||
const stats = await fs.stat(fullPath);
|
||||
return {
|
||||
name: entry.name,
|
||||
path: `/${relativePath}`,
|
||||
type: entry.isDirectory() ? ("directory" as const) : ("file" as const),
|
||||
size: entry.isFile() ? stats.size : undefined,
|
||||
modifiedAt: stats.mtimeMs,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
name: entry.name,
|
||||
path: `/${relativePath}`,
|
||||
type: entry.isDirectory() ? ("directory" as const) : ("file" as const),
|
||||
size: undefined,
|
||||
modifiedAt: undefined,
|
||||
};
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (aIsDir === bIsDir) {
|
||||
return a.name.localeCompare(b.name);
|
||||
}
|
||||
return aIsDir ? -1 : 1;
|
||||
});
|
||||
|
||||
const total = dirents.length;
|
||||
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
|
||||
|
||||
const entries = (
|
||||
await Promise.all(
|
||||
paginatedDirents.map(async (dirent) => {
|
||||
const fullPath = path.join(normalizedPath, dirent.name);
|
||||
|
||||
try {
|
||||
const stats = await fs.stat(fullPath);
|
||||
const relativePath = path.relative(volumePath, fullPath);
|
||||
|
||||
return {
|
||||
name: dirent.name,
|
||||
path: `/${relativePath}`,
|
||||
type: dirent.isDirectory() ? ("directory" as const) : ("file" as const),
|
||||
size: dirent.isFile() ? stats.size : undefined,
|
||||
modifiedAt: stats.mtimeMs,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
)
|
||||
).filter((e) => e !== null);
|
||||
|
||||
return {
|
||||
files: files.sort((a, b) => {
|
||||
if (a.type !== b.type) {
|
||||
return a.type === "directory" ? -1 : 1;
|
||||
}
|
||||
return a.name.localeCompare(b.name);
|
||||
}),
|
||||
files: entries,
|
||||
path: subPath || "/",
|
||||
offset: startOffset,
|
||||
limit: pageSize,
|
||||
total,
|
||||
hasMore: startOffset + pageSize < total,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isNodeJSErrnoException(error) && error.code === "ENOENT") {
|
||||
throw new NotFoundError("Directory not found");
|
||||
}
|
||||
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -701,7 +701,13 @@ const lsSnapshotInfoSchema = type({
|
|||
message_type: "'snapshot'",
|
||||
});
|
||||
|
||||
const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: string, path?: string) => {
|
||||
const ls = async (
|
||||
config: RepositoryConfig,
|
||||
snapshotId: string,
|
||||
organizationId: string,
|
||||
path?: string,
|
||||
options?: { offset?: number; limit?: number },
|
||||
) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config, organizationId);
|
||||
|
||||
|
|
@ -713,48 +719,76 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId:
|
|||
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env });
|
||||
let snapshot: typeof lsSnapshotInfoSchema.infer | null = null;
|
||||
const nodes: Array<typeof lsNodeSchema.infer> = [];
|
||||
let totalNodes = 0;
|
||||
let isFirstLine = true;
|
||||
let hasMore = false;
|
||||
|
||||
const offset = Math.max(options?.offset ?? 0, 0);
|
||||
const limit = Math.min(Math.max(options?.limit ?? 500, 1), 500);
|
||||
|
||||
const res = await safeSpawn({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
onStdout: (line) => {
|
||||
const trimmedLine = line.trim();
|
||||
if (!trimmedLine) return;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(trimmedLine);
|
||||
|
||||
if (isFirstLine) {
|
||||
isFirstLine = false;
|
||||
const snapshotValidation = lsSnapshotInfoSchema(data);
|
||||
if (!(snapshotValidation instanceof type.errors)) {
|
||||
snapshot = snapshotValidation;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const nodeValidation = lsNodeSchema(data);
|
||||
if (nodeValidation instanceof type.errors) {
|
||||
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (totalNodes >= offset && totalNodes < offset + limit) {
|
||||
nodes.push(nodeValidation);
|
||||
}
|
||||
totalNodes++;
|
||||
|
||||
if (totalNodes >= offset + limit + 1) {
|
||||
hasMore = true;
|
||||
}
|
||||
} catch (_) {
|
||||
// Ignore JSON parse errors for non-JSON lines
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic ls failed: ${res.stderr}`);
|
||||
throw new ResticError(res.exitCode, res.stderr);
|
||||
logger.error(`Restic ls failed: ${res.error}`);
|
||||
throw new ResticError(res.exitCode, res.error);
|
||||
}
|
||||
|
||||
// The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes
|
||||
const stdout = res.stdout;
|
||||
const lines = stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter((line) => line.trim());
|
||||
|
||||
if (lines.length === 0) {
|
||||
return { snapshot: null, nodes: [] };
|
||||
if (totalNodes > offset + limit) {
|
||||
hasMore = true;
|
||||
}
|
||||
|
||||
// First line is snapshot info
|
||||
const snapshotLine = JSON.parse(lines[0] ?? "{}");
|
||||
const snapshot = lsSnapshotInfoSchema(snapshotLine);
|
||||
|
||||
if (snapshot instanceof type.errors) {
|
||||
logger.error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
|
||||
throw new Error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
|
||||
}
|
||||
|
||||
const nodes: Array<typeof lsNodeSchema.infer> = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const nodeLine = JSON.parse(lines[i] ?? "{}");
|
||||
const nodeValidation = lsNodeSchema(nodeLine);
|
||||
|
||||
if (nodeValidation instanceof type.errors) {
|
||||
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
nodes.push(nodeValidation);
|
||||
}
|
||||
|
||||
return { snapshot, nodes };
|
||||
return {
|
||||
snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null,
|
||||
nodes,
|
||||
pagination: {
|
||||
offset,
|
||||
limit,
|
||||
total: totalNodes,
|
||||
hasMore,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {
|
||||
|
|
|
|||
|
|
@ -170,4 +170,4 @@ async function main(): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
main();
|
||||
void main();
|
||||
|
|
|
|||
Loading…
Reference in a new issue