refactor: paginate large file counts (#441)
* refactor: add pagination to handle volume folders with extremely large folder counts * refactor: stream restic ls result * test: file-tree load more * refactor: string params * fix(tsc): string pagination params * chore: pr feedbacks
This commit is contained in:
parent
b8ad1ae41a
commit
5bb5fcd09c
19 changed files with 701 additions and 181 deletions
|
|
@ -1,6 +1,12 @@
|
||||||
// This file is auto-generated by @hey-api/openapi-ts
|
// This file is auto-generated by @hey-api/openapi-ts
|
||||||
|
|
||||||
import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanstack/react-query";
|
import {
|
||||||
|
type DefaultError,
|
||||||
|
type InfiniteData,
|
||||||
|
infiniteQueryOptions,
|
||||||
|
queryOptions,
|
||||||
|
type UseMutationOptions,
|
||||||
|
} from "@tanstack/react-query";
|
||||||
|
|
||||||
import { client } from "../client.gen";
|
import { client } from "../client.gen";
|
||||||
import {
|
import {
|
||||||
|
|
@ -31,7 +37,6 @@ import {
|
||||||
getUpdates,
|
getUpdates,
|
||||||
getUserDeletionImpact,
|
getUserDeletionImpact,
|
||||||
getVolume,
|
getVolume,
|
||||||
refreshSnapshots,
|
|
||||||
healthCheckVolume,
|
healthCheckVolume,
|
||||||
listBackupSchedules,
|
listBackupSchedules,
|
||||||
listFiles,
|
listFiles,
|
||||||
|
|
@ -43,6 +48,7 @@ import {
|
||||||
listVolumes,
|
listVolumes,
|
||||||
mountVolume,
|
mountVolume,
|
||||||
type Options,
|
type Options,
|
||||||
|
refreshSnapshots,
|
||||||
reorderBackupSchedules,
|
reorderBackupSchedules,
|
||||||
restoreSnapshot,
|
restoreSnapshot,
|
||||||
runBackupNow,
|
runBackupNow,
|
||||||
|
|
@ -116,8 +122,6 @@ import type {
|
||||||
GetUserDeletionImpactResponse,
|
GetUserDeletionImpactResponse,
|
||||||
GetVolumeData,
|
GetVolumeData,
|
||||||
GetVolumeResponse,
|
GetVolumeResponse,
|
||||||
RefreshSnapshotsData,
|
|
||||||
RefreshSnapshotsResponse,
|
|
||||||
HealthCheckVolumeData,
|
HealthCheckVolumeData,
|
||||||
HealthCheckVolumeResponse,
|
HealthCheckVolumeResponse,
|
||||||
ListBackupSchedulesData,
|
ListBackupSchedulesData,
|
||||||
|
|
@ -138,6 +142,8 @@ import type {
|
||||||
ListVolumesResponse,
|
ListVolumesResponse,
|
||||||
MountVolumeData,
|
MountVolumeData,
|
||||||
MountVolumeResponse,
|
MountVolumeResponse,
|
||||||
|
RefreshSnapshotsData,
|
||||||
|
RefreshSnapshotsResponse,
|
||||||
ReorderBackupSchedulesData,
|
ReorderBackupSchedulesData,
|
||||||
ReorderBackupSchedulesResponse,
|
ReorderBackupSchedulesResponse,
|
||||||
RestoreSnapshotData,
|
RestoreSnapshotData,
|
||||||
|
|
@ -447,6 +453,77 @@ export const listFilesOptions = (options: Options<ListFilesData>) =>
|
||||||
queryKey: listFilesQueryKey(options),
|
queryKey: listFilesQueryKey(options),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const createInfiniteParams = <K extends Pick<QueryKey<Options>[0], "body" | "headers" | "path" | "query">>(
|
||||||
|
queryKey: QueryKey<Options>,
|
||||||
|
page: K,
|
||||||
|
) => {
|
||||||
|
const params = { ...queryKey[0] };
|
||||||
|
if (page.body) {
|
||||||
|
params.body = {
|
||||||
|
...(queryKey[0].body as any),
|
||||||
|
...(page.body as any),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (page.headers) {
|
||||||
|
params.headers = {
|
||||||
|
...queryKey[0].headers,
|
||||||
|
...page.headers,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (page.path) {
|
||||||
|
params.path = {
|
||||||
|
...(queryKey[0].path as any),
|
||||||
|
...(page.path as any),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (page.query) {
|
||||||
|
params.query = {
|
||||||
|
...(queryKey[0].query as any),
|
||||||
|
...(page.query as any),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return params as unknown as typeof page;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const listFilesInfiniteQueryKey = (options: Options<ListFilesData>): QueryKey<Options<ListFilesData>> =>
|
||||||
|
createQueryKey("listFiles", options, true);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List files in a volume directory
|
||||||
|
*/
|
||||||
|
export const listFilesInfiniteOptions = (options: Options<ListFilesData>) =>
|
||||||
|
infiniteQueryOptions<
|
||||||
|
ListFilesResponse,
|
||||||
|
DefaultError,
|
||||||
|
InfiniteData<ListFilesResponse>,
|
||||||
|
QueryKey<Options<ListFilesData>>,
|
||||||
|
string | Pick<QueryKey<Options<ListFilesData>>[0], "body" | "headers" | "path" | "query">
|
||||||
|
>(
|
||||||
|
// @ts-ignore
|
||||||
|
{
|
||||||
|
queryFn: async ({ pageParam, queryKey, signal }) => {
|
||||||
|
// @ts-ignore
|
||||||
|
const page: Pick<QueryKey<Options<ListFilesData>>[0], "body" | "headers" | "path" | "query"> =
|
||||||
|
typeof pageParam === "object"
|
||||||
|
? pageParam
|
||||||
|
: {
|
||||||
|
query: {
|
||||||
|
offset: pageParam,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const params = createInfiniteParams(queryKey, page);
|
||||||
|
const { data } = await listFiles({
|
||||||
|
...options,
|
||||||
|
...params,
|
||||||
|
signal,
|
||||||
|
throwOnError: true,
|
||||||
|
});
|
||||||
|
return data;
|
||||||
|
},
|
||||||
|
queryKey: listFilesInfiniteQueryKey(options),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
export const browseFilesystemQueryKey = (options?: Options<BrowseFilesystemData>) =>
|
export const browseFilesystemQueryKey = (options?: Options<BrowseFilesystemData>) =>
|
||||||
createQueryKey("browseFilesystem", options);
|
createQueryKey("browseFilesystem", options);
|
||||||
|
|
||||||
|
|
@ -636,6 +713,25 @@ export const listSnapshotsOptions = (options: Options<ListSnapshotsData>) =>
|
||||||
queryKey: listSnapshotsQueryKey(options),
|
queryKey: listSnapshotsQueryKey(options),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear snapshot cache and force refresh from repository
|
||||||
|
*/
|
||||||
|
export const refreshSnapshotsMutation = (
|
||||||
|
options?: Partial<Options<RefreshSnapshotsData>>,
|
||||||
|
): UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> => {
|
||||||
|
const mutationOptions: UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> = {
|
||||||
|
mutationFn: async (fnOptions) => {
|
||||||
|
const { data } = await refreshSnapshots({
|
||||||
|
...options,
|
||||||
|
...fnOptions,
|
||||||
|
throwOnError: true,
|
||||||
|
});
|
||||||
|
return data;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return mutationOptions;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a specific snapshot from a repository
|
* Delete a specific snapshot from a repository
|
||||||
*/
|
*/
|
||||||
|
|
@ -705,6 +801,46 @@ export const listSnapshotFilesOptions = (options: Options<ListSnapshotFilesData>
|
||||||
queryKey: listSnapshotFilesQueryKey(options),
|
queryKey: listSnapshotFilesQueryKey(options),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const listSnapshotFilesInfiniteQueryKey = (
|
||||||
|
options: Options<ListSnapshotFilesData>,
|
||||||
|
): QueryKey<Options<ListSnapshotFilesData>> => createQueryKey("listSnapshotFiles", options, true);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List files and directories in a snapshot
|
||||||
|
*/
|
||||||
|
export const listSnapshotFilesInfiniteOptions = (options: Options<ListSnapshotFilesData>) =>
|
||||||
|
infiniteQueryOptions<
|
||||||
|
ListSnapshotFilesResponse,
|
||||||
|
DefaultError,
|
||||||
|
InfiniteData<ListSnapshotFilesResponse>,
|
||||||
|
QueryKey<Options<ListSnapshotFilesData>>,
|
||||||
|
string | Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query">
|
||||||
|
>(
|
||||||
|
// @ts-ignore
|
||||||
|
{
|
||||||
|
queryFn: async ({ pageParam, queryKey, signal }) => {
|
||||||
|
// @ts-ignore
|
||||||
|
const page: Pick<QueryKey<Options<ListSnapshotFilesData>>[0], "body" | "headers" | "path" | "query"> =
|
||||||
|
typeof pageParam === "object"
|
||||||
|
? pageParam
|
||||||
|
: {
|
||||||
|
query: {
|
||||||
|
offset: pageParam,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const params = createInfiniteParams(queryKey, page);
|
||||||
|
const { data } = await listSnapshotFiles({
|
||||||
|
...options,
|
||||||
|
...params,
|
||||||
|
signal,
|
||||||
|
throwOnError: true,
|
||||||
|
});
|
||||||
|
return data;
|
||||||
|
},
|
||||||
|
queryKey: listSnapshotFilesInfiniteQueryKey(options),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Restore a snapshot to a target path on the filesystem
|
* Restore a snapshot to a target path on the filesystem
|
||||||
*/
|
*/
|
||||||
|
|
@ -781,26 +917,6 @@ export const tagSnapshotsMutation = (
|
||||||
return mutationOptions;
|
return mutationOptions;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const refreshSnapshotsMutation = (
|
|
||||||
options?: Partial<Options<RefreshSnapshotsData>>,
|
|
||||||
): UseMutationOptions<RefreshSnapshotsResponse, DefaultError, Options<RefreshSnapshotsData>> => {
|
|
||||||
const mutationOptions: UseMutationOptions<
|
|
||||||
RefreshSnapshotsResponse,
|
|
||||||
DefaultError,
|
|
||||||
Options<RefreshSnapshotsData>
|
|
||||||
> = {
|
|
||||||
mutationFn: async (fnOptions) => {
|
|
||||||
const { data } = await refreshSnapshots({
|
|
||||||
...options,
|
|
||||||
...fnOptions,
|
|
||||||
throwOnError: true,
|
|
||||||
});
|
|
||||||
return data;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
return mutationOptions;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listBackupSchedulesQueryKey = (options?: Options<ListBackupSchedulesData>) =>
|
export const listBackupSchedulesQueryKey = (options?: Options<ListBackupSchedulesData>) =>
|
||||||
createQueryKey("listBackupSchedules", options);
|
createQueryKey("listBackupSchedules", options);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,6 @@ export {
|
||||||
getUpdates,
|
getUpdates,
|
||||||
getUserDeletionImpact,
|
getUserDeletionImpact,
|
||||||
getVolume,
|
getVolume,
|
||||||
refreshSnapshots,
|
|
||||||
healthCheckVolume,
|
healthCheckVolume,
|
||||||
listBackupSchedules,
|
listBackupSchedules,
|
||||||
listFiles,
|
listFiles,
|
||||||
|
|
@ -40,6 +39,7 @@ export {
|
||||||
listVolumes,
|
listVolumes,
|
||||||
mountVolume,
|
mountVolume,
|
||||||
type Options,
|
type Options,
|
||||||
|
refreshSnapshots,
|
||||||
reorderBackupSchedules,
|
reorderBackupSchedules,
|
||||||
restoreSnapshot,
|
restoreSnapshot,
|
||||||
runBackupNow,
|
runBackupNow,
|
||||||
|
|
@ -145,9 +145,6 @@ export type {
|
||||||
GetVolumeErrors,
|
GetVolumeErrors,
|
||||||
GetVolumeResponse,
|
GetVolumeResponse,
|
||||||
GetVolumeResponses,
|
GetVolumeResponses,
|
||||||
RefreshSnapshotsData,
|
|
||||||
RefreshSnapshotsResponse,
|
|
||||||
RefreshSnapshotsResponses,
|
|
||||||
HealthCheckVolumeData,
|
HealthCheckVolumeData,
|
||||||
HealthCheckVolumeErrors,
|
HealthCheckVolumeErrors,
|
||||||
HealthCheckVolumeResponse,
|
HealthCheckVolumeResponse,
|
||||||
|
|
@ -179,6 +176,9 @@ export type {
|
||||||
MountVolumeData,
|
MountVolumeData,
|
||||||
MountVolumeResponse,
|
MountVolumeResponse,
|
||||||
MountVolumeResponses,
|
MountVolumeResponses,
|
||||||
|
RefreshSnapshotsData,
|
||||||
|
RefreshSnapshotsResponse,
|
||||||
|
RefreshSnapshotsResponses,
|
||||||
ReorderBackupSchedulesData,
|
ReorderBackupSchedulesData,
|
||||||
ReorderBackupSchedulesResponse,
|
ReorderBackupSchedulesResponse,
|
||||||
ReorderBackupSchedulesResponses,
|
ReorderBackupSchedulesResponses,
|
||||||
|
|
|
||||||
|
|
@ -61,8 +61,6 @@ import type {
|
||||||
GetVolumeData,
|
GetVolumeData,
|
||||||
GetVolumeErrors,
|
GetVolumeErrors,
|
||||||
GetVolumeResponses,
|
GetVolumeResponses,
|
||||||
RefreshSnapshotsData,
|
|
||||||
RefreshSnapshotsResponses,
|
|
||||||
HealthCheckVolumeData,
|
HealthCheckVolumeData,
|
||||||
HealthCheckVolumeErrors,
|
HealthCheckVolumeErrors,
|
||||||
HealthCheckVolumeResponses,
|
HealthCheckVolumeResponses,
|
||||||
|
|
@ -84,6 +82,8 @@ import type {
|
||||||
ListVolumesResponses,
|
ListVolumesResponses,
|
||||||
MountVolumeData,
|
MountVolumeData,
|
||||||
MountVolumeResponses,
|
MountVolumeResponses,
|
||||||
|
RefreshSnapshotsData,
|
||||||
|
RefreshSnapshotsResponses,
|
||||||
ReorderBackupSchedulesData,
|
ReorderBackupSchedulesData,
|
||||||
ReorderBackupSchedulesResponses,
|
ReorderBackupSchedulesResponses,
|
||||||
RestoreSnapshotData,
|
RestoreSnapshotData,
|
||||||
|
|
@ -379,6 +379,17 @@ export const listSnapshots = <ThrowOnError extends boolean = false>(
|
||||||
...options,
|
...options,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear snapshot cache and force refresh from repository
|
||||||
|
*/
|
||||||
|
export const refreshSnapshots = <ThrowOnError extends boolean = false>(
|
||||||
|
options: Options<RefreshSnapshotsData, ThrowOnError>,
|
||||||
|
) =>
|
||||||
|
(options.client ?? client).post<RefreshSnapshotsResponses, unknown, ThrowOnError>({
|
||||||
|
url: "/api/v1/repositories/{id}/snapshots/refresh",
|
||||||
|
...options,
|
||||||
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a specific snapshot from a repository
|
* Delete a specific snapshot from a repository
|
||||||
*/
|
*/
|
||||||
|
|
@ -458,17 +469,6 @@ export const tagSnapshots = <ThrowOnError extends boolean = false>(options: Opti
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear snapshot cache and force refresh from repository
|
|
||||||
*/
|
|
||||||
export const refreshSnapshots = <ThrowOnError extends boolean = false>(
|
|
||||||
options: Options<RefreshSnapshotsData, ThrowOnError>,
|
|
||||||
) =>
|
|
||||||
(options.client ?? client).post<RefreshSnapshotsResponses, unknown, ThrowOnError>({
|
|
||||||
url: "/api/v1/repositories/{id}/snapshots/refresh",
|
|
||||||
...options,
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List all backup schedules
|
* List all backup schedules
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -682,9 +682,8 @@ export type ListFilesData = {
|
||||||
id: string;
|
id: string;
|
||||||
};
|
};
|
||||||
query?: {
|
query?: {
|
||||||
/**
|
limit?: string;
|
||||||
* Subdirectory path to list (relative to volume root)
|
offset?: string;
|
||||||
*/
|
|
||||||
path?: string;
|
path?: string;
|
||||||
};
|
};
|
||||||
url: "/api/v1/volumes/{id}/files";
|
url: "/api/v1/volumes/{id}/files";
|
||||||
|
|
@ -702,7 +701,11 @@ export type ListFilesResponses = {
|
||||||
modifiedAt?: number;
|
modifiedAt?: number;
|
||||||
size?: number;
|
size?: number;
|
||||||
}>;
|
}>;
|
||||||
|
hasMore: boolean;
|
||||||
|
limit: number;
|
||||||
|
offset: number;
|
||||||
path: string;
|
path: string;
|
||||||
|
total: number;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -1649,6 +1652,27 @@ export type ListSnapshotsResponses = {
|
||||||
|
|
||||||
export type ListSnapshotsResponse = ListSnapshotsResponses[keyof ListSnapshotsResponses];
|
export type ListSnapshotsResponse = ListSnapshotsResponses[keyof ListSnapshotsResponses];
|
||||||
|
|
||||||
|
export type RefreshSnapshotsData = {
|
||||||
|
body?: never;
|
||||||
|
path: {
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
query?: never;
|
||||||
|
url: "/api/v1/repositories/{id}/snapshots/refresh";
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RefreshSnapshotsResponses = {
|
||||||
|
/**
|
||||||
|
* Snapshot cache cleared and refreshed
|
||||||
|
*/
|
||||||
|
200: {
|
||||||
|
count: number;
|
||||||
|
message: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses];
|
||||||
|
|
||||||
export type DeleteSnapshotData = {
|
export type DeleteSnapshotData = {
|
||||||
body?: never;
|
body?: never;
|
||||||
path: {
|
path: {
|
||||||
|
|
@ -1703,6 +1727,8 @@ export type ListSnapshotFilesData = {
|
||||||
snapshotId: string;
|
snapshotId: string;
|
||||||
};
|
};
|
||||||
query?: {
|
query?: {
|
||||||
|
limit?: string;
|
||||||
|
offset?: string;
|
||||||
path?: string;
|
path?: string;
|
||||||
};
|
};
|
||||||
url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files";
|
url: "/api/v1/repositories/{id}/snapshots/{snapshotId}/files";
|
||||||
|
|
@ -1725,6 +1751,9 @@ export type ListSnapshotFilesResponses = {
|
||||||
size?: number;
|
size?: number;
|
||||||
uid?: number;
|
uid?: number;
|
||||||
}>;
|
}>;
|
||||||
|
hasMore: boolean;
|
||||||
|
limit: number;
|
||||||
|
offset: number;
|
||||||
snapshot: {
|
snapshot: {
|
||||||
hostname: string;
|
hostname: string;
|
||||||
id: string;
|
id: string;
|
||||||
|
|
@ -1732,6 +1761,7 @@ export type ListSnapshotFilesResponses = {
|
||||||
short_id: string;
|
short_id: string;
|
||||||
time: string;
|
time: string;
|
||||||
};
|
};
|
||||||
|
total: number;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -1848,27 +1878,6 @@ export type TagSnapshotsResponses = {
|
||||||
|
|
||||||
export type TagSnapshotsResponse = TagSnapshotsResponses[keyof TagSnapshotsResponses];
|
export type TagSnapshotsResponse = TagSnapshotsResponses[keyof TagSnapshotsResponses];
|
||||||
|
|
||||||
export type RefreshSnapshotsData = {
|
|
||||||
body?: never;
|
|
||||||
path: {
|
|
||||||
id: string;
|
|
||||||
};
|
|
||||||
query?: never;
|
|
||||||
url: "/api/v1/repositories/{id}/snapshots/refresh";
|
|
||||||
};
|
|
||||||
|
|
||||||
export type RefreshSnapshotsResponses = {
|
|
||||||
/**
|
|
||||||
* Snapshot cache cleared and refreshed
|
|
||||||
*/
|
|
||||||
200: {
|
|
||||||
message: string;
|
|
||||||
count: number;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export type RefreshSnapshotsResponse = RefreshSnapshotsResponses[keyof RefreshSnapshotsResponses];
|
|
||||||
|
|
||||||
export type ListBackupSchedulesData = {
|
export type ListBackupSchedulesData = {
|
||||||
body?: never;
|
body?: never;
|
||||||
path?: never;
|
path?: never;
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,114 @@ import { expect, test, describe } from "bun:test";
|
||||||
import { render, screen, fireEvent } from "@testing-library/react";
|
import { render, screen, fireEvent } from "@testing-library/react";
|
||||||
import { FileTree, type FileEntry } from "../file-tree";
|
import { FileTree, type FileEntry } from "../file-tree";
|
||||||
|
|
||||||
|
describe("FileTree Pagination", () => {
|
||||||
|
const testFiles: FileEntry[] = [
|
||||||
|
{ name: "root", path: "/root", type: "folder" },
|
||||||
|
{ name: "file1", path: "/root/file1", type: "file" },
|
||||||
|
{ name: "file2", path: "/root/file2", type: "file" },
|
||||||
|
];
|
||||||
|
|
||||||
|
test("shows load more button when hasMore is true", () => {
|
||||||
|
render(
|
||||||
|
<FileTree
|
||||||
|
files={testFiles}
|
||||||
|
expandedFolders={new Set(["/root"])}
|
||||||
|
getFolderPagination={() => ({ hasMore: true, isLoadingMore: false })}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("Load more files")).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not show load more button when hasMore is false", () => {
|
||||||
|
render(
|
||||||
|
<FileTree
|
||||||
|
files={testFiles}
|
||||||
|
expandedFolders={new Set(["/root"])}
|
||||||
|
getFolderPagination={() => ({ hasMore: false, isLoadingMore: false })}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.queryByText("Load more files")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("calls onLoadMore with folder path when load more button is clicked", () => {
|
||||||
|
let loadMoreCalled = false;
|
||||||
|
let loadMorePath = "";
|
||||||
|
|
||||||
|
render(
|
||||||
|
<FileTree
|
||||||
|
files={testFiles}
|
||||||
|
expandedFolders={new Set(["/root"])}
|
||||||
|
getFolderPagination={(path) => {
|
||||||
|
if (path === "/root") {
|
||||||
|
return { hasMore: true, isLoadingMore: false };
|
||||||
|
}
|
||||||
|
return { hasMore: false, isLoadingMore: false };
|
||||||
|
}}
|
||||||
|
onLoadMore={(path) => {
|
||||||
|
loadMoreCalled = true;
|
||||||
|
loadMorePath = path;
|
||||||
|
}}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
const loadMoreButton = screen.getByText("Load more files");
|
||||||
|
fireEvent.click(loadMoreButton);
|
||||||
|
|
||||||
|
expect(loadMoreCalled).toBe(true);
|
||||||
|
expect(loadMorePath).toBe("/root");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("shows loading state when isLoadingMore is true", () => {
|
||||||
|
render(
|
||||||
|
<FileTree
|
||||||
|
files={testFiles}
|
||||||
|
expandedFolders={new Set(["/root"])}
|
||||||
|
getFolderPagination={() => ({ hasMore: true, isLoadingMore: true })}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("Loading more...")).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("load more button appears for nested folders with hasMore", () => {
|
||||||
|
const nestedFiles: FileEntry[] = [
|
||||||
|
{ name: "root", path: "/root", type: "folder" },
|
||||||
|
{ name: "child", path: "/root/child", type: "folder" },
|
||||||
|
{ name: "file1", path: "/root/child/file1", type: "file" },
|
||||||
|
];
|
||||||
|
|
||||||
|
render(
|
||||||
|
<FileTree
|
||||||
|
files={nestedFiles}
|
||||||
|
expandedFolders={new Set(["/root", "/root/child"])}
|
||||||
|
getFolderPagination={(path) => {
|
||||||
|
if (path === "/root/child") {
|
||||||
|
return { hasMore: true, isLoadingMore: false };
|
||||||
|
}
|
||||||
|
return { hasMore: false, isLoadingMore: false };
|
||||||
|
}}
|
||||||
|
onLoadMore={() => {}}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("Load more files")).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("load more button does not appear when folder is collapsed", () => {
|
||||||
|
render(
|
||||||
|
<FileTree
|
||||||
|
files={testFiles}
|
||||||
|
expandedFolders={new Set([])}
|
||||||
|
getFolderPagination={() => ({ hasMore: true, isLoadingMore: false })}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.queryByText("Load more files")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("FileTree Selection Logic", () => {
|
describe("FileTree Selection Logic", () => {
|
||||||
const testFiles: FileEntry[] = [
|
const testFiles: FileEntry[] = [
|
||||||
{ name: "root", path: "/root", type: "folder" },
|
{ name: "root", path: "/root", type: "folder" },
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,15 @@
|
||||||
* Original source: https://github.com/stackblitz/bolt.new
|
* Original source: https://github.com/stackblitz/bolt.new
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ChevronDown, ChevronRight, File as FileIcon, Folder as FolderIcon, FolderOpen, Loader2 } from "lucide-react";
|
import {
|
||||||
|
ChevronDown,
|
||||||
|
ChevronRight,
|
||||||
|
File as FileIcon,
|
||||||
|
Folder as FolderIcon,
|
||||||
|
FolderOpen,
|
||||||
|
Loader2,
|
||||||
|
MoreHorizontal,
|
||||||
|
} from "lucide-react";
|
||||||
import { memo, type ReactNode, useCallback, useEffect, useMemo, useState } from "react";
|
import { memo, type ReactNode, useCallback, useEffect, useMemo, useState } from "react";
|
||||||
import { cn } from "~/client/lib/utils";
|
import { cn } from "~/client/lib/utils";
|
||||||
import { Checkbox } from "~/client/components/ui/checkbox";
|
import { Checkbox } from "~/client/components/ui/checkbox";
|
||||||
|
|
@ -24,12 +32,19 @@ export interface FileEntry {
|
||||||
modifiedAt?: number;
|
modifiedAt?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface PaginationState {
|
||||||
|
hasMore: boolean;
|
||||||
|
isLoadingMore: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
files?: FileEntry[];
|
files?: FileEntry[];
|
||||||
selectedFile?: string;
|
selectedFile?: string;
|
||||||
onFileSelect?: (filePath: string) => void;
|
onFileSelect?: (filePath: string) => void;
|
||||||
onFolderExpand?: (folderPath: string) => void;
|
onFolderExpand?: (folderPath: string) => void;
|
||||||
onFolderHover?: (folderPath: string) => void;
|
onFolderHover?: (folderPath: string) => void;
|
||||||
|
onLoadMore?: (folderPath: string) => void;
|
||||||
|
getFolderPagination?: (folderPath: string) => PaginationState;
|
||||||
expandedFolders?: Set<string>;
|
expandedFolders?: Set<string>;
|
||||||
loadingFolders?: Set<string>;
|
loadingFolders?: Set<string>;
|
||||||
className?: string;
|
className?: string;
|
||||||
|
|
@ -49,6 +64,8 @@ export const FileTree = memo((props: Props) => {
|
||||||
selectedFile,
|
selectedFile,
|
||||||
onFolderExpand,
|
onFolderExpand,
|
||||||
onFolderHover,
|
onFolderHover,
|
||||||
|
onLoadMore,
|
||||||
|
getFolderPagination,
|
||||||
expandedFolders = new Set(),
|
expandedFolders = new Set(),
|
||||||
loadingFolders = new Set(),
|
loadingFolders = new Set(),
|
||||||
className,
|
className,
|
||||||
|
|
@ -292,12 +309,35 @@ export const FileTree = memo((props: Props) => {
|
||||||
[selectedPaths],
|
[selectedPaths],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Build a map of folder paths that need pagination to their last child's index
|
||||||
|
const folderPaginationMap = useMemo(() => {
|
||||||
|
const map = new Map<string, number>();
|
||||||
|
|
||||||
|
for (let i = 0; i < filteredFileList.length; i++) {
|
||||||
|
const item = filteredFileList[i];
|
||||||
|
const parentPath = item.fullPath.slice(0, item.fullPath.lastIndexOf("/")) || "/";
|
||||||
|
|
||||||
|
if (parentPath !== "/") {
|
||||||
|
const pagination = getFolderPagination?.(parentPath);
|
||||||
|
if (pagination?.hasMore && !collapsedFolders.has(parentPath)) {
|
||||||
|
// Update the last index for this parent
|
||||||
|
map.set(parentPath, i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return map;
|
||||||
|
}, [filteredFileList, getFolderPagination, collapsedFolders]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className={cn("text-sm", className)}>
|
<div className={cn("text-sm", className)}>
|
||||||
{filteredFileList.map((fileOrFolder) => {
|
{filteredFileList.map((fileOrFolder, index) => {
|
||||||
|
const elements: React.ReactNode[] = [];
|
||||||
|
|
||||||
|
// Render the current file or folder
|
||||||
switch (fileOrFolder.kind) {
|
switch (fileOrFolder.kind) {
|
||||||
case "file": {
|
case "file": {
|
||||||
return (
|
elements.push(
|
||||||
<File
|
<File
|
||||||
key={fileOrFolder.id}
|
key={fileOrFolder.id}
|
||||||
selected={selectedFile === fileOrFolder.fullPath}
|
selected={selectedFile === fileOrFolder.fullPath}
|
||||||
|
|
@ -306,11 +346,12 @@ export const FileTree = memo((props: Props) => {
|
||||||
withCheckbox={withCheckboxes}
|
withCheckbox={withCheckboxes}
|
||||||
checked={isPathSelected(fileOrFolder.fullPath)}
|
checked={isPathSelected(fileOrFolder.fullPath)}
|
||||||
onCheckboxChange={handleSelectionChange}
|
onCheckboxChange={handleSelectionChange}
|
||||||
/>
|
/>,
|
||||||
);
|
);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
case "folder": {
|
case "folder": {
|
||||||
return (
|
elements.push(
|
||||||
<Folder
|
<Folder
|
||||||
key={fileOrFolder.id}
|
key={fileOrFolder.id}
|
||||||
folder={fileOrFolder}
|
folder={fileOrFolder}
|
||||||
|
|
@ -325,13 +366,31 @@ export const FileTree = memo((props: Props) => {
|
||||||
selectableMode={selectableFolders}
|
selectableMode={selectableFolders}
|
||||||
onFolderSelect={handleFolderSelect}
|
onFolderSelect={handleFolderSelect}
|
||||||
selected={selectedFolder === fileOrFolder.fullPath}
|
selected={selectedFolder === fileOrFolder.fullPath}
|
||||||
/>
|
/>,
|
||||||
);
|
);
|
||||||
}
|
break;
|
||||||
default: {
|
|
||||||
return undefined;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if this is the last child of any folder with more files to load
|
||||||
|
for (const [folderPath, lastIndex] of folderPaginationMap.entries()) {
|
||||||
|
if (lastIndex === index) {
|
||||||
|
// This is the last loaded child of folderPath
|
||||||
|
const pagination = getFolderPagination?.(folderPath);
|
||||||
|
if (pagination?.hasMore) {
|
||||||
|
elements.push(
|
||||||
|
<LoadMoreButton
|
||||||
|
key={`load-more-${folderPath}`}
|
||||||
|
depth={fileOrFolder.depth}
|
||||||
|
onClick={() => onLoadMore?.(folderPath)}
|
||||||
|
isLoading={pagination.isLoadingMore}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return elements;
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
@ -477,6 +536,31 @@ const File = memo(({ file, onFileSelect, selected, withCheckbox, checked, onChec
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
interface LoadMoreButtonProps {
|
||||||
|
depth: number;
|
||||||
|
onClick: () => void;
|
||||||
|
isLoading?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const LoadMoreButton = memo(({ depth, onClick, isLoading }: LoadMoreButtonProps) => {
|
||||||
|
return (
|
||||||
|
<NodeButton
|
||||||
|
depth={depth}
|
||||||
|
className="text-muted-foreground hover:bg-accent/50 cursor-pointer"
|
||||||
|
icon={
|
||||||
|
isLoading ? (
|
||||||
|
<Loader2 className="w-4 h-4 shrink-0 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<MoreHorizontal className="w-4 h-4 shrink-0" />
|
||||||
|
)
|
||||||
|
}
|
||||||
|
onClick={onClick}
|
||||||
|
>
|
||||||
|
<span className="text-xs">{isLoading ? "Loading more..." : "Load more files"}</span>
|
||||||
|
</NodeButton>
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
interface ButtonProps {
|
interface ButtonProps {
|
||||||
depth: number;
|
depth: number;
|
||||||
icon: ReactNode;
|
icon: ReactNode;
|
||||||
|
|
@ -621,5 +705,5 @@ function compareNodes(a: Node, b: Node): number {
|
||||||
return a.kind === "folder" ? -1 : 1;
|
return a.kind === "folder" ? -1 : 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: "base" });
|
return a.name.localeCompare(b.name);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -74,11 +74,11 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
|
||||||
const fileBrowser = useFileBrowser({
|
const fileBrowser = useFileBrowser({
|
||||||
initialData: filesData,
|
initialData: filesData,
|
||||||
isLoading: filesLoading,
|
isLoading: filesLoading,
|
||||||
fetchFolder: async (path) => {
|
fetchFolder: async (path, offset = 0) => {
|
||||||
return await queryClient.ensureQueryData(
|
return await queryClient.ensureQueryData(
|
||||||
listSnapshotFilesOptions({
|
listSnapshotFilesOptions({
|
||||||
path: { id: repository.id, snapshotId },
|
path: { id: repository.id, snapshotId },
|
||||||
query: { path },
|
query: { path, offset: offset.toString(), limit: "500" },
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
@ -86,7 +86,7 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
|
||||||
void queryClient.prefetchQuery(
|
void queryClient.prefetchQuery(
|
||||||
listSnapshotFilesOptions({
|
listSnapshotFilesOptions({
|
||||||
path: { id: repository.id, snapshotId },
|
path: { id: repository.id, snapshotId },
|
||||||
query: { path },
|
query: { path, offset: "0", limit: "500" },
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
@ -306,6 +306,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
|
||||||
onFolderHover={fileBrowser.handleFolderHover}
|
onFolderHover={fileBrowser.handleFolderHover}
|
||||||
expandedFolders={fileBrowser.expandedFolders}
|
expandedFolders={fileBrowser.expandedFolders}
|
||||||
loadingFolders={fileBrowser.loadingFolders}
|
loadingFolders={fileBrowser.loadingFolders}
|
||||||
|
onLoadMore={fileBrowser.handleLoadMore}
|
||||||
|
getFolderPagination={fileBrowser.getFolderPagination}
|
||||||
className="px-2 py-2"
|
className="px-2 py-2"
|
||||||
withCheckboxes={true}
|
withCheckboxes={true}
|
||||||
selectedPaths={selectedPaths}
|
selectedPaths={selectedPaths}
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||||
import { FolderOpen } from "lucide-react";
|
import { FolderOpen } from "lucide-react";
|
||||||
import { FileTree } from "~/client/components/file-tree";
|
import { FileTree } from "~/client/components/file-tree";
|
||||||
import { listFilesOptions } from "../api-client/@tanstack/react-query.gen";
|
import { listFilesOptions } from "../api-client/@tanstack/react-query.gen";
|
||||||
import { useFileBrowser } from "../hooks/use-file-browser";
|
import { useFileBrowser, type FetchFolderResult } from "../hooks/use-file-browser";
|
||||||
import { parseError } from "../lib/errors";
|
import { parseError } from "../lib/errors";
|
||||||
|
|
||||||
type VolumeFileBrowserProps = {
|
type VolumeFileBrowserProps = {
|
||||||
|
|
@ -38,11 +38,11 @@ export const VolumeFileBrowser = ({
|
||||||
const fileBrowser = useFileBrowser({
|
const fileBrowser = useFileBrowser({
|
||||||
initialData: data,
|
initialData: data,
|
||||||
isLoading,
|
isLoading,
|
||||||
fetchFolder: async (path) => {
|
fetchFolder: async (path, offset): Promise<FetchFolderResult> => {
|
||||||
return await queryClient.ensureQueryData(
|
return await queryClient.ensureQueryData(
|
||||||
listFilesOptions({
|
listFilesOptions({
|
||||||
path: { id: volumeId },
|
path: { id: volumeId },
|
||||||
query: { path },
|
query: { path, offset: offset?.toString() },
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
@ -88,6 +88,8 @@ export const VolumeFileBrowser = ({
|
||||||
files={fileBrowser.fileArray}
|
files={fileBrowser.fileArray}
|
||||||
onFolderExpand={fileBrowser.handleFolderExpand}
|
onFolderExpand={fileBrowser.handleFolderExpand}
|
||||||
onFolderHover={fileBrowser.handleFolderHover}
|
onFolderHover={fileBrowser.handleFolderHover}
|
||||||
|
onLoadMore={fileBrowser.handleLoadMore}
|
||||||
|
getFolderPagination={fileBrowser.getFolderPagination}
|
||||||
expandedFolders={fileBrowser.expandedFolders}
|
expandedFolders={fileBrowser.expandedFolders}
|
||||||
loadingFolders={fileBrowser.loadingFolders}
|
loadingFolders={fileBrowser.loadingFolders}
|
||||||
withCheckboxes={withCheckboxes}
|
withCheckboxes={withCheckboxes}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,16 @@
|
||||||
import { useCallback, useMemo, useState } from "react";
|
import { useCallback, useMemo, useState } from "react";
|
||||||
import type { FileEntry } from "../components/file-tree";
|
import type { FileEntry } from "../components/file-tree";
|
||||||
|
|
||||||
type FetchFolderFn = (
|
export type FetchFolderResult = {
|
||||||
path: string,
|
files?: FileEntry[];
|
||||||
) => Promise<{ files?: FileEntry[]; directories?: Array<{ name: string; path: string }> }>;
|
directories?: Array<{ name: string; path: string }>;
|
||||||
|
offset?: number;
|
||||||
|
limit?: number;
|
||||||
|
total?: number;
|
||||||
|
hasMore?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type FetchFolderFn = (path: string, offset?: number) => Promise<FetchFolderResult>;
|
||||||
|
|
||||||
type PathTransformFns = {
|
type PathTransformFns = {
|
||||||
strip?: (path: string) => string;
|
strip?: (path: string) => string;
|
||||||
|
|
@ -11,7 +18,7 @@ type PathTransformFns = {
|
||||||
};
|
};
|
||||||
|
|
||||||
type UseFileBrowserOptions = {
|
type UseFileBrowserOptions = {
|
||||||
initialData?: { files?: FileEntry[]; directories?: Array<{ name: string; path: string }> };
|
initialData?: FetchFolderResult;
|
||||||
isLoading?: boolean;
|
isLoading?: boolean;
|
||||||
fetchFolder: FetchFolderFn;
|
fetchFolder: FetchFolderFn;
|
||||||
prefetchFolder?: (path: string) => void;
|
prefetchFolder?: (path: string) => void;
|
||||||
|
|
@ -19,12 +26,20 @@ type UseFileBrowserOptions = {
|
||||||
rootPath?: string;
|
rootPath?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type FolderPaginationState = {
|
||||||
|
currentOffset: number;
|
||||||
|
limit: number;
|
||||||
|
hasMore: boolean;
|
||||||
|
isLoadingMore: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
||||||
const { initialData, isLoading, fetchFolder, prefetchFolder, pathTransform, rootPath = "/" } = props;
|
const { initialData, isLoading, fetchFolder, prefetchFolder, pathTransform, rootPath = "/" } = props;
|
||||||
const [expandedFolders, setExpandedFolders] = useState<Set<string>>(new Set());
|
const [expandedFolders, setExpandedFolders] = useState<Set<string>>(new Set());
|
||||||
const [fetchedFolders, setFetchedFolders] = useState<Set<string>>(new Set([rootPath]));
|
const [fetchedFolders, setFetchedFolders] = useState<Set<string>>(new Set([rootPath]));
|
||||||
const [loadingFolders, setLoadingFolders] = useState<Set<string>>(new Set());
|
const [loadingFolders, setLoadingFolders] = useState<Set<string>>(new Set());
|
||||||
const [allFiles, setAllFiles] = useState<Map<string, FileEntry>>(new Map());
|
const [allFiles, setAllFiles] = useState<Map<string, FileEntry>>(new Map());
|
||||||
|
const [folderPagination, setFolderPagination] = useState<Map<string, FolderPaginationState>>(new Map());
|
||||||
|
|
||||||
const stripPath = pathTransform?.strip;
|
const stripPath = pathTransform?.strip;
|
||||||
const addPath = pathTransform?.add;
|
const addPath = pathTransform?.add;
|
||||||
|
|
@ -44,6 +59,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
||||||
});
|
});
|
||||||
if (rootPath) {
|
if (rootPath) {
|
||||||
setFetchedFolders((prev) => new Set(prev).add(rootPath));
|
setFetchedFolders((prev) => new Set(prev).add(rootPath));
|
||||||
|
setFolderPagination((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
next.set(rootPath, {
|
||||||
|
currentOffset: initialData.offset ?? 0,
|
||||||
|
limit: initialData.limit ?? 100,
|
||||||
|
hasMore: initialData.hasMore ?? false,
|
||||||
|
isLoadingMore: false,
|
||||||
|
});
|
||||||
|
return next;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
} else if (initialData?.directories) {
|
} else if (initialData?.directories) {
|
||||||
const directories = initialData.directories;
|
const directories = initialData.directories;
|
||||||
|
|
@ -87,6 +112,16 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
||||||
}
|
}
|
||||||
return next;
|
return next;
|
||||||
});
|
});
|
||||||
|
setFolderPagination((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
next.set(folderPath, {
|
||||||
|
currentOffset: result.offset ?? 0,
|
||||||
|
limit: result.limit ?? 100,
|
||||||
|
hasMore: result.hasMore ?? false,
|
||||||
|
isLoadingMore: false,
|
||||||
|
});
|
||||||
|
return next;
|
||||||
|
});
|
||||||
} else if (result.directories) {
|
} else if (result.directories) {
|
||||||
const directories = result.directories;
|
const directories = result.directories;
|
||||||
setAllFiles((prev) => {
|
setAllFiles((prev) => {
|
||||||
|
|
@ -113,6 +148,59 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
||||||
[fetchedFolders, fetchFolder, stripPath, addPath],
|
[fetchedFolders, fetchFolder, stripPath, addPath],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const handleLoadMore = useCallback(
|
||||||
|
async (folderPath: string) => {
|
||||||
|
const pagination = folderPagination.get(folderPath);
|
||||||
|
if (!pagination?.hasMore || pagination?.isLoadingMore) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setFolderPagination((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
next.set(folderPath, { ...pagination, isLoadingMore: true });
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const pathToFetch = addPath ? addPath(folderPath) : folderPath;
|
||||||
|
const nextOffset = pagination.currentOffset + pagination.limit;
|
||||||
|
const result = await fetchFolder(pathToFetch, nextOffset);
|
||||||
|
|
||||||
|
if (result.files) {
|
||||||
|
const files = result.files;
|
||||||
|
setAllFiles((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
for (const file of files) {
|
||||||
|
const strippedPath = stripPath ? stripPath(file.path) : file.path;
|
||||||
|
if (strippedPath !== folderPath) {
|
||||||
|
next.set(strippedPath, { ...file, path: strippedPath });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
setFolderPagination((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
next.set(folderPath, {
|
||||||
|
currentOffset: result.offset ?? nextOffset,
|
||||||
|
limit: result.limit ?? pagination.limit,
|
||||||
|
hasMore: result.hasMore ?? false,
|
||||||
|
isLoadingMore: false,
|
||||||
|
});
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to load more files:", error);
|
||||||
|
setFolderPagination((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
next.set(folderPath, { ...pagination, isLoadingMore: false });
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[folderPagination, fetchFolder, stripPath, addPath],
|
||||||
|
);
|
||||||
|
|
||||||
const handleFolderHover = useCallback(
|
const handleFolderHover = useCallback(
|
||||||
(folderPath: string) => {
|
(folderPath: string) => {
|
||||||
if (!fetchedFolders.has(folderPath) && !loadingFolders.has(folderPath) && prefetchFolder) {
|
if (!fetchedFolders.has(folderPath) && !loadingFolders.has(folderPath) && prefetchFolder) {
|
||||||
|
|
@ -123,12 +211,21 @@ export const useFileBrowser = (props: UseFileBrowserOptions) => {
|
||||||
[fetchedFolders, loadingFolders, prefetchFolder, addPath],
|
[fetchedFolders, loadingFolders, prefetchFolder, addPath],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const getFolderPagination = useCallback(
|
||||||
|
(folderPath: string) => {
|
||||||
|
return folderPagination.get(folderPath) ?? { hasMore: false, isLoadingMore: false };
|
||||||
|
},
|
||||||
|
[folderPagination],
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
fileArray,
|
fileArray,
|
||||||
expandedFolders,
|
expandedFolders,
|
||||||
loadingFolders,
|
loadingFolders,
|
||||||
handleFolderExpand,
|
handleFolderExpand,
|
||||||
handleFolderHover,
|
handleFolderHover,
|
||||||
|
handleLoadMore,
|
||||||
|
getFolderPagination,
|
||||||
isLoading: isLoading && fileArray.length === 0,
|
isLoading: isLoading && fileArray.length === 0,
|
||||||
isEmpty: fileArray.length === 0 && !isLoading,
|
isEmpty: fileArray.length === 0 && !isLoading,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -671,14 +671,20 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:
|
||||||
<div>
|
<div>
|
||||||
<p className="text-xs uppercase text-muted-foreground">Include paths/patterns</p>
|
<p className="text-xs uppercase text-muted-foreground">Include paths/patterns</p>
|
||||||
<div className="flex flex-col gap-1">
|
<div className="flex flex-col gap-1">
|
||||||
{formValues.includePatterns?.map((path) => (
|
{formValues.includePatterns?.slice(0, 20).map((path) => (
|
||||||
<span key={path} className="text-xs font-mono bg-accent px-1.5 py-0.5 rounded">
|
<span key={path} className="text-xs font-mono bg-accent px-1.5 py-0.5 rounded">
|
||||||
{path}
|
{path}
|
||||||
</span>
|
</span>
|
||||||
))}
|
))}
|
||||||
|
{formValues.includePatterns && formValues.includePatterns.length > 20 && (
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
+ {formValues.includePatterns.length - 20} more
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
{formValues.includePatternsText
|
{formValues.includePatternsText
|
||||||
?.split("\n")
|
?.split("\n")
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
|
.slice(0, 20 - (formValues.includePatterns?.length || 0))
|
||||||
.map((pattern) => (
|
.map((pattern) => (
|
||||||
<span key={pattern} className="text-xs font-mono bg-accent px-1.5 py-0.5 rounded">
|
<span key={pattern} className="text-xs font-mono bg-accent px-1.5 py-0.5 rounded">
|
||||||
{pattern.trim()}
|
{pattern.trim()}
|
||||||
|
|
|
||||||
|
|
@ -60,11 +60,11 @@ export const SnapshotFileBrowser = (props: Props) => {
|
||||||
const fileBrowser = useFileBrowser({
|
const fileBrowser = useFileBrowser({
|
||||||
initialData: filesData,
|
initialData: filesData,
|
||||||
isLoading: filesLoading,
|
isLoading: filesLoading,
|
||||||
fetchFolder: async (path) => {
|
fetchFolder: async (path, offset = 0) => {
|
||||||
return await queryClient.ensureQueryData(
|
return await queryClient.ensureQueryData(
|
||||||
listSnapshotFilesOptions({
|
listSnapshotFilesOptions({
|
||||||
path: { id: repositoryId, snapshotId: snapshot.short_id },
|
path: { id: repositoryId, snapshotId: snapshot.short_id },
|
||||||
query: { path },
|
query: { path, offset: offset.toString(), limit: "500" },
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
@ -72,7 +72,7 @@ export const SnapshotFileBrowser = (props: Props) => {
|
||||||
void queryClient.prefetchQuery(
|
void queryClient.prefetchQuery(
|
||||||
listSnapshotFilesOptions({
|
listSnapshotFilesOptions({
|
||||||
path: { id: repositoryId, snapshotId: snapshot.short_id },
|
path: { id: repositoryId, snapshotId: snapshot.short_id },
|
||||||
query: { path },
|
query: { path, offset: "0", limit: "500" },
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
@ -142,6 +142,8 @@ export const SnapshotFileBrowser = (props: Props) => {
|
||||||
onFolderHover={fileBrowser.handleFolderHover}
|
onFolderHover={fileBrowser.handleFolderHover}
|
||||||
expandedFolders={fileBrowser.expandedFolders}
|
expandedFolders={fileBrowser.expandedFolders}
|
||||||
loadingFolders={fileBrowser.loadingFolders}
|
loadingFolders={fileBrowser.loadingFolders}
|
||||||
|
onLoadMore={fileBrowser.handleLoadMore}
|
||||||
|
getFolderPagination={fileBrowser.getFolderPagination}
|
||||||
className="px-2 py-2"
|
className="px-2 py-2"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -143,10 +143,14 @@ export const repositoriesController = new Hono()
|
||||||
validator("query", listSnapshotFilesQuery),
|
validator("query", listSnapshotFilesQuery),
|
||||||
async (c) => {
|
async (c) => {
|
||||||
const { id, snapshotId } = c.req.param();
|
const { id, snapshotId } = c.req.param();
|
||||||
const { path } = c.req.valid("query");
|
const { path, ...query } = c.req.valid("query");
|
||||||
|
|
||||||
const decodedPath = path ? decodeURIComponent(path) : undefined;
|
const decodedPath = path ? decodeURIComponent(path) : undefined;
|
||||||
const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath);
|
|
||||||
|
const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0);
|
||||||
|
const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500));
|
||||||
|
|
||||||
|
const result = await repositoriesService.listSnapshotFiles(id, snapshotId, decodedPath, { offset, limit });
|
||||||
|
|
||||||
c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");
|
c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -252,12 +252,18 @@ export const listSnapshotFilesResponse = type({
|
||||||
paths: "string[]",
|
paths: "string[]",
|
||||||
}),
|
}),
|
||||||
files: snapshotFileNodeSchema.array(),
|
files: snapshotFileNodeSchema.array(),
|
||||||
|
offset: "number",
|
||||||
|
limit: "number",
|
||||||
|
total: "number",
|
||||||
|
hasMore: "boolean",
|
||||||
});
|
});
|
||||||
|
|
||||||
export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer;
|
export type ListSnapshotFilesDto = typeof listSnapshotFilesResponse.infer;
|
||||||
|
|
||||||
export const listSnapshotFilesQuery = type({
|
export const listSnapshotFilesQuery = type({
|
||||||
path: "string?",
|
path: "string?",
|
||||||
|
offset: "string.integer?",
|
||||||
|
limit: "string.integer?",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const listSnapshotFilesDto = describeRoute({
|
export const listSnapshotFilesDto = describeRoute({
|
||||||
|
|
|
||||||
|
|
@ -210,7 +210,12 @@ const listSnapshots = async (id: string, backupId?: string) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const listSnapshotFiles = async (id: string, snapshotId: string, path?: string) => {
|
const listSnapshotFiles = async (
|
||||||
|
id: string,
|
||||||
|
snapshotId: string,
|
||||||
|
path?: string,
|
||||||
|
options?: { offset?: number; limit?: number },
|
||||||
|
) => {
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
const repository = await findRepository(id);
|
const repository = await findRepository(id);
|
||||||
|
|
||||||
|
|
@ -218,18 +223,30 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
|
||||||
throw new NotFoundError("Repository not found");
|
throw new NotFoundError("Repository not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}`;
|
const offset = options?.offset ?? 0;
|
||||||
const cached = cache.get<Awaited<ReturnType<typeof restic.ls>>>(cacheKey);
|
const limit = options?.limit ?? 500;
|
||||||
|
|
||||||
|
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`;
|
||||||
|
type LsResult = {
|
||||||
|
snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null;
|
||||||
|
nodes: { name: string; type: string; path: string; size?: number; mode?: number }[];
|
||||||
|
pagination: { offset: number; limit: number; total: number; hasMore: boolean };
|
||||||
|
};
|
||||||
|
const cached = cache.get<LsResult>(cacheKey);
|
||||||
if (cached?.snapshot) {
|
if (cached?.snapshot) {
|
||||||
return {
|
return {
|
||||||
snapshot: cached.snapshot,
|
snapshot: cached.snapshot,
|
||||||
files: cached.nodes,
|
files: cached.nodes,
|
||||||
|
offset: cached.pagination.offset,
|
||||||
|
limit: cached.pagination.limit,
|
||||||
|
total: cached.pagination.total,
|
||||||
|
hasMore: cached.pagination.hasMore,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
|
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`);
|
||||||
try {
|
try {
|
||||||
const result = await restic.ls(repository.config, snapshotId, organizationId, path);
|
const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit });
|
||||||
|
|
||||||
if (!result.snapshot) {
|
if (!result.snapshot) {
|
||||||
throw new NotFoundError("Snapshot not found or empty");
|
throw new NotFoundError("Snapshot not found or empty");
|
||||||
|
|
@ -244,6 +261,10 @@ const listSnapshotFiles = async (id: string, snapshotId: string, path?: string)
|
||||||
paths: result.snapshot.paths,
|
paths: result.snapshot.paths,
|
||||||
},
|
},
|
||||||
files: result.nodes,
|
files: result.nodes,
|
||||||
|
offset: result.pagination.offset,
|
||||||
|
limit: result.pagination.limit,
|
||||||
|
total: result.pagination.total,
|
||||||
|
hasMore: result.pagination.hasMore,
|
||||||
};
|
};
|
||||||
|
|
||||||
cache.set(cacheKey, result);
|
cache.set(cacheKey, result);
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ import {
|
||||||
type ListFilesDto,
|
type ListFilesDto,
|
||||||
browseFilesystemDto,
|
browseFilesystemDto,
|
||||||
type BrowseFilesystemDto,
|
type BrowseFilesystemDto,
|
||||||
|
listFilesQuery,
|
||||||
} from "./volume.dto";
|
} from "./volume.dto";
|
||||||
import { volumeService } from "./volume.service";
|
import { volumeService } from "./volume.service";
|
||||||
import { getVolumePath } from "./helpers";
|
import { getVolumePath } from "./helpers";
|
||||||
|
|
@ -104,14 +105,22 @@ export const volumeController = new Hono()
|
||||||
|
|
||||||
return c.json({ error, status }, 200);
|
return c.json({ error, status }, 200);
|
||||||
})
|
})
|
||||||
.get("/:id/files", listFilesDto, async (c) => {
|
.get("/:id/files", validator("query", listFilesQuery), listFilesDto, async (c) => {
|
||||||
const { id } = c.req.param();
|
const { id } = c.req.param();
|
||||||
const subPath = c.req.query("path");
|
const { path, ...query } = c.req.valid("query");
|
||||||
const result = await volumeService.listFiles(id, subPath);
|
|
||||||
|
const offset = Math.max(0, Number.parseInt(query.offset ?? "0", 10) || 0);
|
||||||
|
const limit = Math.min(1000, Math.max(1, Number.parseInt(query.limit ?? "500", 10) || 500));
|
||||||
|
|
||||||
|
const result = await volumeService.listFiles(id, path, offset, limit);
|
||||||
|
|
||||||
const response = {
|
const response = {
|
||||||
files: result.files,
|
files: result.files,
|
||||||
path: result.path,
|
path: result.path,
|
||||||
|
offset: result.offset,
|
||||||
|
limit: result.limit,
|
||||||
|
total: result.total,
|
||||||
|
hasMore: result.hasMore,
|
||||||
};
|
};
|
||||||
|
|
||||||
c.header("Cache-Control", "public, max-age=10, stale-while-revalidate=60");
|
c.header("Cache-Control", "public, max-age=10, stale-while-revalidate=60");
|
||||||
|
|
|
||||||
|
|
@ -276,24 +276,23 @@ const fileEntrySchema = type({
|
||||||
export const listFilesResponse = type({
|
export const listFilesResponse = type({
|
||||||
files: fileEntrySchema.array(),
|
files: fileEntrySchema.array(),
|
||||||
path: "string",
|
path: "string",
|
||||||
|
offset: "number",
|
||||||
|
limit: "number",
|
||||||
|
total: "number",
|
||||||
|
hasMore: "boolean",
|
||||||
});
|
});
|
||||||
export type ListFilesDto = typeof listFilesResponse.infer;
|
export type ListFilesDto = typeof listFilesResponse.infer;
|
||||||
|
|
||||||
|
export const listFilesQuery = type({
|
||||||
|
path: "string?",
|
||||||
|
offset: "string.integer?",
|
||||||
|
limit: "string.integer?",
|
||||||
|
});
|
||||||
|
|
||||||
export const listFilesDto = describeRoute({
|
export const listFilesDto = describeRoute({
|
||||||
description: "List files in a volume directory",
|
description: "List files in a volume directory",
|
||||||
operationId: "listFiles",
|
operationId: "listFiles",
|
||||||
tags: ["Volumes"],
|
tags: ["Volumes"],
|
||||||
parameters: [
|
|
||||||
{
|
|
||||||
in: "query",
|
|
||||||
name: "path",
|
|
||||||
required: false,
|
|
||||||
schema: {
|
|
||||||
type: "string",
|
|
||||||
},
|
|
||||||
description: "Subdirectory path to list (relative to volume root)",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
responses: {
|
responses: {
|
||||||
200: {
|
200: {
|
||||||
description: "List of files in the volume",
|
description: "List of files in the volume",
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ import { serverEvents } from "../../core/events";
|
||||||
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
||||||
import { type } from "arktype";
|
import { type } from "arktype";
|
||||||
import { getOrganizationId } from "~/server/core/request-context";
|
import { getOrganizationId } from "~/server/core/request-context";
|
||||||
|
import { isNodeJSErrnoException } from "~/server/utils/fs";
|
||||||
|
|
||||||
async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
|
async function encryptSensitiveFields(config: BackendConfig): Promise<BackendConfig> {
|
||||||
switch (config.backend) {
|
switch (config.backend) {
|
||||||
|
|
@ -294,7 +295,15 @@ const checkHealth = async (idOrShortId: string | number) => {
|
||||||
return { status, error };
|
return { status, error };
|
||||||
};
|
};
|
||||||
|
|
||||||
const listFiles = async (idOrShortId: string | number, subPath?: string) => {
|
const DEFAULT_PAGE_SIZE = 500;
|
||||||
|
const MAX_PAGE_SIZE = 500;
|
||||||
|
|
||||||
|
const listFiles = async (
|
||||||
|
idOrShortId: string | number,
|
||||||
|
subPath?: string,
|
||||||
|
offset: number = 0,
|
||||||
|
limit: number = DEFAULT_PAGE_SIZE,
|
||||||
|
) => {
|
||||||
const volume = await findVolume(idOrShortId);
|
const volume = await findVolume(idOrShortId);
|
||||||
|
|
||||||
if (!volume) {
|
if (!volume) {
|
||||||
|
|
@ -305,11 +314,8 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => {
|
||||||
throw new InternalServerError("Volume is not mounted");
|
throw new InternalServerError("Volume is not mounted");
|
||||||
}
|
}
|
||||||
|
|
||||||
// For directory volumes, use the configured path directly
|
|
||||||
const volumePath = getVolumePath(volume);
|
const volumePath = getVolumePath(volume);
|
||||||
|
|
||||||
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
|
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
|
||||||
|
|
||||||
const normalizedPath = path.normalize(requestedPath);
|
const normalizedPath = path.normalize(requestedPath);
|
||||||
const relative = path.relative(volumePath, normalizedPath);
|
const relative = path.relative(volumePath, normalizedPath);
|
||||||
|
|
||||||
|
|
@ -317,45 +323,60 @@ const listFiles = async (idOrShortId: string | number, subPath?: string) => {
|
||||||
throw new BadRequestError("Invalid path");
|
throw new BadRequestError("Invalid path");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
|
||||||
|
const startOffset = Math.max(offset, 0);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const entries = await fs.readdir(normalizedPath, { withFileTypes: true });
|
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
|
||||||
|
|
||||||
const files = await Promise.all(
|
dirents.sort((a, b) => {
|
||||||
entries.map(async (entry) => {
|
const aIsDir = a.isDirectory();
|
||||||
const fullPath = path.join(normalizedPath, entry.name);
|
const bIsDir = b.isDirectory();
|
||||||
const relativePath = path.relative(volumePath, fullPath);
|
|
||||||
|
|
||||||
try {
|
if (aIsDir === bIsDir) {
|
||||||
const stats = await fs.stat(fullPath);
|
return a.name.localeCompare(b.name);
|
||||||
return {
|
}
|
||||||
name: entry.name,
|
return aIsDir ? -1 : 1;
|
||||||
path: `/${relativePath}`,
|
});
|
||||||
type: entry.isDirectory() ? ("directory" as const) : ("file" as const),
|
|
||||||
size: entry.isFile() ? stats.size : undefined,
|
const total = dirents.length;
|
||||||
modifiedAt: stats.mtimeMs,
|
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
|
||||||
};
|
|
||||||
} catch {
|
const entries = (
|
||||||
return {
|
await Promise.all(
|
||||||
name: entry.name,
|
paginatedDirents.map(async (dirent) => {
|
||||||
path: `/${relativePath}`,
|
const fullPath = path.join(normalizedPath, dirent.name);
|
||||||
type: entry.isDirectory() ? ("directory" as const) : ("file" as const),
|
|
||||||
size: undefined,
|
try {
|
||||||
modifiedAt: undefined,
|
const stats = await fs.stat(fullPath);
|
||||||
};
|
const relativePath = path.relative(volumePath, fullPath);
|
||||||
}
|
|
||||||
}),
|
return {
|
||||||
);
|
name: dirent.name,
|
||||||
|
path: `/${relativePath}`,
|
||||||
|
type: dirent.isDirectory() ? ("directory" as const) : ("file" as const),
|
||||||
|
size: dirent.isFile() ? stats.size : undefined,
|
||||||
|
modifiedAt: stats.mtimeMs,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
).filter((e) => e !== null);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
files: files.sort((a, b) => {
|
files: entries,
|
||||||
if (a.type !== b.type) {
|
|
||||||
return a.type === "directory" ? -1 : 1;
|
|
||||||
}
|
|
||||||
return a.name.localeCompare(b.name);
|
|
||||||
}),
|
|
||||||
path: subPath || "/",
|
path: subPath || "/",
|
||||||
|
offset: startOffset,
|
||||||
|
limit: pageSize,
|
||||||
|
total,
|
||||||
|
hasMore: startOffset + pageSize < total,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (isNodeJSErrnoException(error) && error.code === "ENOENT") {
|
||||||
|
throw new NotFoundError("Directory not found");
|
||||||
|
}
|
||||||
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
|
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -701,7 +701,13 @@ const lsSnapshotInfoSchema = type({
|
||||||
message_type: "'snapshot'",
|
message_type: "'snapshot'",
|
||||||
});
|
});
|
||||||
|
|
||||||
const ls = async (config: RepositoryConfig, snapshotId: string, organizationId: string, path?: string) => {
|
const ls = async (
|
||||||
|
config: RepositoryConfig,
|
||||||
|
snapshotId: string,
|
||||||
|
organizationId: string,
|
||||||
|
path?: string,
|
||||||
|
options?: { offset?: number; limit?: number },
|
||||||
|
) => {
|
||||||
const repoUrl = buildRepoUrl(config);
|
const repoUrl = buildRepoUrl(config);
|
||||||
const env = await buildEnv(config, organizationId);
|
const env = await buildEnv(config, organizationId);
|
||||||
|
|
||||||
|
|
@ -713,48 +719,76 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId:
|
||||||
|
|
||||||
addCommonArgs(args, env, config);
|
addCommonArgs(args, env, config);
|
||||||
|
|
||||||
const res = await exec({ command: "restic", args, env });
|
let snapshot: typeof lsSnapshotInfoSchema.infer | null = null;
|
||||||
|
const nodes: Array<typeof lsNodeSchema.infer> = [];
|
||||||
|
let totalNodes = 0;
|
||||||
|
let isFirstLine = true;
|
||||||
|
let hasMore = false;
|
||||||
|
|
||||||
|
const offset = Math.max(options?.offset ?? 0, 0);
|
||||||
|
const limit = Math.min(Math.max(options?.limit ?? 500, 1), 500);
|
||||||
|
|
||||||
|
const res = await safeSpawn({
|
||||||
|
command: "restic",
|
||||||
|
args,
|
||||||
|
env,
|
||||||
|
onStdout: (line) => {
|
||||||
|
const trimmedLine = line.trim();
|
||||||
|
if (!trimmedLine) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(trimmedLine);
|
||||||
|
|
||||||
|
if (isFirstLine) {
|
||||||
|
isFirstLine = false;
|
||||||
|
const snapshotValidation = lsSnapshotInfoSchema(data);
|
||||||
|
if (!(snapshotValidation instanceof type.errors)) {
|
||||||
|
snapshot = snapshotValidation;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const nodeValidation = lsNodeSchema(data);
|
||||||
|
if (nodeValidation instanceof type.errors) {
|
||||||
|
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalNodes >= offset && totalNodes < offset + limit) {
|
||||||
|
nodes.push(nodeValidation);
|
||||||
|
}
|
||||||
|
totalNodes++;
|
||||||
|
|
||||||
|
if (totalNodes >= offset + limit + 1) {
|
||||||
|
hasMore = true;
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
// Ignore JSON parse errors for non-JSON lines
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
await cleanupTemporaryKeys(env);
|
await cleanupTemporaryKeys(env);
|
||||||
|
|
||||||
if (res.exitCode !== 0) {
|
if (res.exitCode !== 0) {
|
||||||
logger.error(`Restic ls failed: ${res.stderr}`);
|
logger.error(`Restic ls failed: ${res.error}`);
|
||||||
throw new ResticError(res.exitCode, res.stderr);
|
throw new ResticError(res.exitCode, res.error);
|
||||||
}
|
}
|
||||||
|
|
||||||
// The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes
|
if (totalNodes > offset + limit) {
|
||||||
const stdout = res.stdout;
|
hasMore = true;
|
||||||
const lines = stdout
|
|
||||||
.trim()
|
|
||||||
.split("\n")
|
|
||||||
.filter((line) => line.trim());
|
|
||||||
|
|
||||||
if (lines.length === 0) {
|
|
||||||
return { snapshot: null, nodes: [] };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// First line is snapshot info
|
return {
|
||||||
const snapshotLine = JSON.parse(lines[0] ?? "{}");
|
snapshot: snapshot as typeof lsSnapshotInfoSchema.infer | null,
|
||||||
const snapshot = lsSnapshotInfoSchema(snapshotLine);
|
nodes,
|
||||||
|
pagination: {
|
||||||
if (snapshot instanceof type.errors) {
|
offset,
|
||||||
logger.error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
|
limit,
|
||||||
throw new Error(`Restic ls snapshot info validation failed: ${snapshot.summary}`);
|
total: totalNodes,
|
||||||
}
|
hasMore,
|
||||||
|
},
|
||||||
const nodes: Array<typeof lsNodeSchema.infer> = [];
|
};
|
||||||
for (let i = 1; i < lines.length; i++) {
|
|
||||||
const nodeLine = JSON.parse(lines[i] ?? "{}");
|
|
||||||
const nodeValidation = lsNodeSchema(nodeLine);
|
|
||||||
|
|
||||||
if (nodeValidation instanceof type.errors) {
|
|
||||||
logger.warn(`Skipping invalid node: ${nodeValidation.summary}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
nodes.push(nodeValidation);
|
|
||||||
}
|
|
||||||
|
|
||||||
return { snapshot, nodes };
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {
|
const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {
|
||||||
|
|
|
||||||
|
|
@ -170,4 +170,4 @@ async function main(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
main();
|
void main();
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue