feat: restore snapshot as tar (#550)
* feat: export snapshot as tar file chore(mutext): prevent double lock release * chore: pr feedbacks * fix: dump single file no tar * chore: pr feedbacks
This commit is contained in:
Nico
GitHub
parent
8681ebc0c0
commit
182d39a887
33 changed files with 1273 additions and 184 deletions
|
|
@ -25,6 +25,7 @@ import {
|
|||
deleteVolume,
|
||||
devPanelExec,
|
||||
downloadResticPassword,
|
||||
dumpSnapshot,
|
||||
getBackupSchedule,
|
||||
getBackupScheduleForVolume,
|
||||
getDevPanel,
|
||||
|
|
@ -100,6 +101,8 @@ import type {
|
|||
DevPanelExecResponse,
|
||||
DownloadResticPasswordData,
|
||||
DownloadResticPasswordResponse,
|
||||
DumpSnapshotData,
|
||||
DumpSnapshotResponse,
|
||||
GetBackupScheduleData,
|
||||
GetBackupScheduleForVolumeData,
|
||||
GetBackupScheduleForVolumeResponse,
|
||||
|
|
@ -851,6 +854,25 @@ export const listSnapshotFilesInfiniteOptions = (options: Options<ListSnapshotFi
|
|||
},
|
||||
);
|
||||
|
||||
export const dumpSnapshotQueryKey = (options: Options<DumpSnapshotData>) => createQueryKey("dumpSnapshot", options);
|
||||
|
||||
/**
|
||||
* Download a snapshot path as a tar archive (folders) or raw file stream (single files)
|
||||
*/
|
||||
export const dumpSnapshotOptions = (options: Options<DumpSnapshotData>) =>
|
||||
queryOptions<DumpSnapshotResponse, DefaultError, DumpSnapshotResponse, ReturnType<typeof dumpSnapshotQueryKey>>({
|
||||
queryFn: async ({ queryKey, signal }) => {
|
||||
const { data } = await dumpSnapshot({
|
||||
...options,
|
||||
...queryKey[0],
|
||||
signal,
|
||||
throwOnError: true,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
queryKey: dumpSnapshotQueryKey(options),
|
||||
});
|
||||
|
||||
/**
|
||||
* Restore a snapshot to a target path on the filesystem
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ export {
|
|||
deleteVolume,
|
||||
devPanelExec,
|
||||
downloadResticPassword,
|
||||
dumpSnapshot,
|
||||
getBackupSchedule,
|
||||
getBackupScheduleForVolume,
|
||||
getDevPanel,
|
||||
|
|
@ -109,6 +110,9 @@ export type {
|
|||
DownloadResticPasswordData,
|
||||
DownloadResticPasswordResponse,
|
||||
DownloadResticPasswordResponses,
|
||||
DumpSnapshotData,
|
||||
DumpSnapshotResponse,
|
||||
DumpSnapshotResponses,
|
||||
GetBackupScheduleData,
|
||||
GetBackupScheduleForVolumeData,
|
||||
GetBackupScheduleForVolumeResponse,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ import type {
|
|||
DevPanelExecResponses,
|
||||
DownloadResticPasswordData,
|
||||
DownloadResticPasswordResponses,
|
||||
DumpSnapshotData,
|
||||
DumpSnapshotResponses,
|
||||
GetBackupScheduleData,
|
||||
GetBackupScheduleForVolumeData,
|
||||
GetBackupScheduleForVolumeResponses,
|
||||
|
|
@ -431,6 +433,15 @@ export const listSnapshotFiles = <ThrowOnError extends boolean = false>(
|
|||
...options,
|
||||
});
|
||||
|
||||
/**
|
||||
* Download a snapshot path as a tar archive (folders) or raw file stream (single files)
|
||||
*/
|
||||
export const dumpSnapshot = <ThrowOnError extends boolean = false>(options: Options<DumpSnapshotData, ThrowOnError>) =>
|
||||
(options.client ?? client).get<DumpSnapshotResponses, unknown, ThrowOnError>({
|
||||
url: "/api/v1/repositories/{shortId}/snapshots/{snapshotId}/dump",
|
||||
...options,
|
||||
});
|
||||
|
||||
/**
|
||||
* Restore a snapshot to a target path on the filesystem
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1970,6 +1970,28 @@ export type ListSnapshotFilesResponses = {
|
|||
|
||||
export type ListSnapshotFilesResponse = ListSnapshotFilesResponses[keyof ListSnapshotFilesResponses];
|
||||
|
||||
export type DumpSnapshotData = {
|
||||
body?: never;
|
||||
path: {
|
||||
shortId: string;
|
||||
snapshotId: string;
|
||||
};
|
||||
query?: {
|
||||
kind?: "dir" | "file";
|
||||
path?: string;
|
||||
};
|
||||
url: "/api/v1/repositories/{shortId}/snapshots/{snapshotId}/dump";
|
||||
};
|
||||
|
||||
export type DumpSnapshotResponses = {
|
||||
/**
|
||||
* Snapshot content stream
|
||||
*/
|
||||
200: Blob | File;
|
||||
};
|
||||
|
||||
export type DumpSnapshotResponse = DumpSnapshotResponses[keyof DumpSnapshotResponses];
|
||||
|
||||
export type RestoreSnapshotData = {
|
||||
body?: {
|
||||
snapshotId: string;
|
||||
|
|
|
|||
|
|
@ -3,13 +3,7 @@ import { browseFilesystemOptions } from "~/client/api-client/@tanstack/react-que
|
|||
import { FileBrowser, type FileBrowserUiProps } from "~/client/components/file-browsers/file-browser";
|
||||
import { useFileBrowser } from "~/client/hooks/use-file-browser";
|
||||
import { parseError } from "~/client/lib/errors";
|
||||
|
||||
const normalizeAbsolutePath = (path?: string): string => {
|
||||
if (!path) return "/";
|
||||
const withLeadingSlash = path.startsWith("/") ? path : `/${path}`;
|
||||
const trimmed = withLeadingSlash.replace(/\/+$/, "");
|
||||
return trimmed || "/";
|
||||
};
|
||||
import { normalizeAbsolutePath } from "~/utils/path";
|
||||
|
||||
type LocalFileBrowserProps = FileBrowserUiProps & {
|
||||
initialPath?: string;
|
||||
|
|
|
|||
|
|
@ -4,13 +4,7 @@ import { listSnapshotFilesOptions } from "~/client/api-client/@tanstack/react-qu
|
|||
import { FileBrowser, type FileBrowserUiProps } from "~/client/components/file-browsers/file-browser";
|
||||
import { useFileBrowser } from "~/client/hooks/use-file-browser";
|
||||
import { parseError } from "~/client/lib/errors";
|
||||
|
||||
const normalizeAbsolutePath = (path?: string): string => {
|
||||
if (!path) return "/";
|
||||
const withLeadingSlash = path.startsWith("/") ? path : `/${path}`;
|
||||
const trimmed = withLeadingSlash.replace(/\/+$/, "");
|
||||
return trimmed || "/";
|
||||
};
|
||||
import { normalizeAbsolutePath } from "~/utils/path";
|
||||
|
||||
type SnapshotTreeBrowserProps = FileBrowserUiProps & {
|
||||
repositoryId: string;
|
||||
|
|
@ -18,6 +12,7 @@ type SnapshotTreeBrowserProps = FileBrowserUiProps & {
|
|||
basePath?: string;
|
||||
pageSize?: number;
|
||||
enabled?: boolean;
|
||||
onSingleSelectionKindChange?: (kind: "file" | "dir" | null) => void;
|
||||
};
|
||||
|
||||
export const SnapshotTreeBrowser = ({
|
||||
|
|
@ -28,7 +23,7 @@ export const SnapshotTreeBrowser = ({
|
|||
enabled = true,
|
||||
...uiProps
|
||||
}: SnapshotTreeBrowserProps) => {
|
||||
const { selectedPaths, onSelectionChange, ...fileBrowserUiProps } = uiProps;
|
||||
const { selectedPaths, onSelectionChange, onSingleSelectionKindChange, ...fileBrowserUiProps } = uiProps;
|
||||
const queryClient = useQueryClient();
|
||||
const normalizedBasePath = normalizeAbsolutePath(basePath);
|
||||
|
||||
|
|
@ -72,20 +67,6 @@ export const SnapshotTreeBrowser = ({
|
|||
return displayPaths;
|
||||
}, [selectedPaths, stripBasePath]);
|
||||
|
||||
const handleSelectionChange = useCallback(
|
||||
(nextDisplayPaths: Set<string>) => {
|
||||
if (!onSelectionChange) return;
|
||||
|
||||
const nextFullPaths = new Set<string>();
|
||||
for (const displayPath of nextDisplayPaths) {
|
||||
nextFullPaths.add(addBasePath(displayPath));
|
||||
}
|
||||
|
||||
onSelectionChange(nextFullPaths);
|
||||
},
|
||||
[onSelectionChange, addBasePath],
|
||||
);
|
||||
|
||||
const fileBrowser = useFileBrowser({
|
||||
initialData: data,
|
||||
isLoading,
|
||||
|
|
@ -119,6 +100,41 @@ export const SnapshotTreeBrowser = ({
|
|||
},
|
||||
});
|
||||
|
||||
const displayPathKinds = useMemo(() => {
|
||||
const kinds = new Map<string, "file" | "dir">();
|
||||
for (const entry of fileBrowser.fileArray) {
|
||||
kinds.set(entry.path, entry.type === "file" ? "file" : "dir");
|
||||
}
|
||||
return kinds;
|
||||
}, [fileBrowser.fileArray]);
|
||||
|
||||
const handleSelectionChange = useCallback(
|
||||
(nextDisplayPaths: Set<string>) => {
|
||||
if (!onSelectionChange) return;
|
||||
|
||||
const nextFullPaths = new Set<string>();
|
||||
for (const displayPath of nextDisplayPaths) {
|
||||
nextFullPaths.add(addBasePath(displayPath));
|
||||
}
|
||||
|
||||
if (onSingleSelectionKindChange) {
|
||||
if (nextDisplayPaths.size === 1) {
|
||||
const [selectedDisplayPath] = nextDisplayPaths;
|
||||
if (selectedDisplayPath) {
|
||||
onSingleSelectionKindChange(displayPathKinds.get(selectedDisplayPath) ?? null);
|
||||
} else {
|
||||
onSingleSelectionKindChange(null);
|
||||
}
|
||||
} else {
|
||||
onSingleSelectionKindChange(null);
|
||||
}
|
||||
}
|
||||
|
||||
onSelectionChange(nextFullPaths);
|
||||
},
|
||||
[onSelectionChange, addBasePath, onSingleSelectionKindChange, displayPathKinds],
|
||||
);
|
||||
|
||||
const errorDetails = parseError(error)?.message;
|
||||
const errorMessage = errorDetails
|
||||
? `Failed to load files: ${errorDetails}`
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { ChevronDown, FolderOpen, RotateCcw } from "lucide-react";
|
||||
import { ChevronDown, Download, FolderOpen, RotateCcw } from "lucide-react";
|
||||
import { Button } from "~/client/components/ui/button";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "~/client/components/ui/card";
|
||||
import { Input } from "~/client/components/ui/input";
|
||||
import { Label } from "~/client/components/ui/label";
|
||||
|
|
@ -24,6 +25,7 @@ import { OVERWRITE_MODES, type OverwriteMode } from "~/schemas/restic";
|
|||
import type { Repository } from "~/client/lib/types";
|
||||
import { handleRepositoryError } from "~/client/lib/errors";
|
||||
import { useNavigate } from "@tanstack/react-router";
|
||||
import { cn } from "~/client/lib/utils";
|
||||
|
||||
type RestoreLocation = "original" | "custom";
|
||||
|
||||
|
|
@ -51,41 +53,55 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
const restoreCompletedRef = useRef(false);
|
||||
|
||||
const [selectedPaths, setSelectedPaths] = useState<Set<string>>(new Set());
|
||||
const [selectedPathKind, setSelectedPathKind] = useState<"file" | "dir" | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribeStarted = addEventListener("restore:started", (startedData) => {
|
||||
if (startedData.repositoryId === repository.id && startedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = false;
|
||||
setIsRestoreActive(true);
|
||||
setRestoreResult(null);
|
||||
setShowRestoreResultAlert(false);
|
||||
}
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
const signal = abortController.signal;
|
||||
|
||||
const unsubscribeProgress = addEventListener("restore:progress", (progressData) => {
|
||||
if (progressData.repositoryId === repository.id && progressData.snapshotId === snapshotId) {
|
||||
if (restoreCompletedRef.current) {
|
||||
return;
|
||||
addEventListener(
|
||||
"restore:started",
|
||||
(startedData) => {
|
||||
if (startedData.repositoryId === repository.shortId && startedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = false;
|
||||
setIsRestoreActive(true);
|
||||
setRestoreResult(null);
|
||||
setShowRestoreResultAlert(false);
|
||||
}
|
||||
setIsRestoreActive(true);
|
||||
}
|
||||
});
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const unsubscribeCompleted = addEventListener("restore:completed", (completedData) => {
|
||||
if (completedData.repositoryId === repository.id && completedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = true;
|
||||
setIsRestoreActive(false);
|
||||
setRestoreResult(completedData);
|
||||
setShowRestoreResultAlert(true);
|
||||
}
|
||||
});
|
||||
addEventListener(
|
||||
"restore:progress",
|
||||
(progressData) => {
|
||||
if (progressData.repositoryId === repository.shortId && progressData.snapshotId === snapshotId) {
|
||||
if (restoreCompletedRef.current) {
|
||||
return;
|
||||
}
|
||||
setIsRestoreActive(true);
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
addEventListener(
|
||||
"restore:completed",
|
||||
(completedData) => {
|
||||
if (completedData.repositoryId === repository.shortId && completedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = true;
|
||||
setIsRestoreActive(false);
|
||||
setRestoreResult(completedData);
|
||||
setShowRestoreResultAlert(true);
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribeStarted();
|
||||
unsubscribeProgress();
|
||||
unsubscribeCompleted();
|
||||
abortController.abort();
|
||||
};
|
||||
}, [addEventListener, repository.id, snapshotId]);
|
||||
}, [addEventListener, repository.shortId, snapshotId]);
|
||||
|
||||
const { mutate: restoreSnapshot, isPending: isRestoring } = useMutation({
|
||||
...restoreSnapshotMutation(),
|
||||
|
|
@ -133,6 +149,33 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
restoreSnapshot,
|
||||
]);
|
||||
|
||||
const handleDownload = useCallback(() => {
|
||||
if (selectedPaths.size > 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
const dumpUrl = new URL(
|
||||
`/api/v1/repositories/${repository.shortId}/snapshots/${snapshotId}/dump`,
|
||||
window.location.origin,
|
||||
);
|
||||
|
||||
if (selectedPaths.size === 1) {
|
||||
const [selectedPath] = selectedPaths;
|
||||
if (selectedPath) {
|
||||
dumpUrl.searchParams.set("path", selectedPath);
|
||||
if (selectedPathKind) {
|
||||
dumpUrl.searchParams.set("kind", selectedPathKind);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const link = document.createElement("a");
|
||||
link.href = dumpUrl.toString();
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
}, [repository.shortId, snapshotId, selectedPathKind, selectedPaths]);
|
||||
|
||||
const acknowledgeRestoreResult = useCallback(() => {
|
||||
setShowRestoreResultAlert(false);
|
||||
setRestoreResult(null);
|
||||
|
|
@ -145,35 +188,62 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
}, []);
|
||||
|
||||
const canRestore = restoreLocation === "original" || customTargetPath.trim();
|
||||
const canDownload = selectedPaths.size <= 1;
|
||||
const isRestoreRunning = isRestoring || isRestoreActive;
|
||||
|
||||
function getDownloadButtonText(): string {
|
||||
if (selectedPaths.size > 0) {
|
||||
return `Download ${selectedPaths.size} ${selectedPaths.size === 1 ? "item" : "items"}`;
|
||||
}
|
||||
return "Download All";
|
||||
}
|
||||
|
||||
function getRestoreButtonText(): string {
|
||||
if (isRestoreRunning) {
|
||||
return "Restoring...";
|
||||
}
|
||||
if (selectedPaths.size > 0) {
|
||||
return `Restore ${selectedPaths.size} ${selectedPaths.size === 1 ? "item" : "items"}`;
|
||||
}
|
||||
return "Restore All";
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex flex-col gap-4 lg:flex-row lg:items-center lg:justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold">Restore Snapshot</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{repository.name} / {snapshotId}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Button variant="outline" onClick={() => navigate({ to: returnPath })}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<span className="inline-flex">
|
||||
<Button variant="outline" onClick={handleDownload} disabled={!canDownload}>
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
{getDownloadButtonText()}
|
||||
</Button>
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className={cn({ hidden: canDownload })}>
|
||||
<p>Download is available only for one selected item, or with no selection to download everything.</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<Button variant="primary" onClick={handleRestore} disabled={isRestoreRunning || !canRestore}>
|
||||
<RotateCcw className="h-4 w-4 mr-2" />
|
||||
{isRestoreRunning
|
||||
? "Restoring..."
|
||||
: selectedPaths.size > 0
|
||||
? `Restore ${selectedPaths.size} ${selectedPaths.size === 1 ? "item" : "items"}`
|
||||
: "Restore All"}
|
||||
{getRestoreButtonText()}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
|
||||
<div className="space-y-6">
|
||||
{isRestoreRunning && <RestoreProgress repositoryId={repository.id} snapshotId={snapshotId} />}
|
||||
{isRestoreRunning && <RestoreProgress repositoryId={repository.shortId} snapshotId={snapshotId} />}
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
|
|
@ -292,6 +362,7 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
withCheckboxes
|
||||
selectedPaths={selectedPaths}
|
||||
onSelectionChange={setSelectedPaths}
|
||||
onSingleSelectionKindChange={setSelectedPathKind}
|
||||
stateClassName="flex-1 min-h-0"
|
||||
/>
|
||||
</CardContent>
|
||||
|
|
|
|||
|
|
@ -16,22 +16,29 @@ export const RestoreProgress = ({ repositoryId, snapshotId }: Props) => {
|
|||
const [progress, setProgress] = useState<RestoreProgressEvent | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = addEventListener("restore:progress", (progressData) => {
|
||||
if (progressData.repositoryId === repositoryId && progressData.snapshotId === snapshotId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
|
||||
const unsubscribeComplete = addEventListener("restore:completed", (completedData) => {
|
||||
if (completedData.repositoryId === repositoryId && completedData.snapshotId === snapshotId) {
|
||||
setProgress(null);
|
||||
}
|
||||
});
|
||||
addEventListener(
|
||||
"restore:progress",
|
||||
(progressData) => {
|
||||
if (progressData.repositoryId === repositoryId && progressData.snapshotId === snapshotId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
unsubscribeComplete();
|
||||
};
|
||||
addEventListener(
|
||||
"restore:completed",
|
||||
(completedData) => {
|
||||
if (completedData.repositoryId === repositoryId && completedData.snapshotId === snapshotId) {
|
||||
setProgress(null);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => abortController.abort();
|
||||
}, [addEventListener, repositoryId, snapshotId]);
|
||||
|
||||
if (!progress) {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { Slot } from "@radix-ui/react-slot";
|
||||
import { cva, type VariantProps } from "class-variance-authority";
|
||||
import { Loader2 } from "lucide-react";
|
||||
import type * as React from "react";
|
||||
import * as React from "react";
|
||||
|
||||
import { cn } from "~/client/lib/utils";
|
||||
import { useMinimumDuration } from "~/client/hooks/useMinimumDuration";
|
||||
|
||||
const buttonVariants = cva(
|
||||
"inline-flex cursor-pointer uppercase rounded-sm items-center justify-center gap-2 whitespace-nowrap text-xs font-semibold tracking-wide transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-ring border-0",
|
||||
|
|
@ -33,28 +34,32 @@ const buttonVariants = cva(
|
|||
},
|
||||
);
|
||||
|
||||
const MINIMUM_LOADING_DURATION = 300;
|
||||
|
||||
function Button({
|
||||
className,
|
||||
variant,
|
||||
size,
|
||||
asChild = false,
|
||||
loading,
|
||||
disabled,
|
||||
...props
|
||||
}: React.ComponentProps<"button"> &
|
||||
VariantProps<typeof buttonVariants> & {
|
||||
asChild?: boolean;
|
||||
} & { loading?: boolean }) {
|
||||
const Comp = asChild ? Slot : "button";
|
||||
const isLoading = useMinimumDuration(loading ?? false, MINIMUM_LOADING_DURATION);
|
||||
|
||||
return (
|
||||
<Comp
|
||||
disabled={loading}
|
||||
{...props}
|
||||
data-slot="button"
|
||||
className={cn(buttonVariants({ variant, size, className }), "transition-all")}
|
||||
{...props}
|
||||
disabled={disabled || loading || isLoading}
|
||||
>
|
||||
<Loader2 className={cn("h-4 w-4 animate-spin absolute", { invisible: !loading })} />
|
||||
<div className={cn("flex items-center justify-center", { invisible: loading })}>{props.children}</div>
|
||||
<Loader2 className={cn("h-4 w-4 animate-spin absolute", { invisible: !isLoading })} />
|
||||
<div className={cn("flex items-center justify-center", { invisible: isLoading })}>{props.children}</div>
|
||||
</Comp>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,20 +98,36 @@ export function useServerEvents() {
|
|||
};
|
||||
}, [emit, queryClient]);
|
||||
|
||||
const addEventListener = useCallback(<T extends ServerEventType>(eventName: T, handler: EventHandler<T>) => {
|
||||
const existingHandlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
const eventHandlers = existingHandlers ?? new Set<EventHandler<T>>();
|
||||
eventHandlers.add(handler);
|
||||
handlersRef.current[eventName] = eventHandlers as EventHandlerMap[T];
|
||||
|
||||
return () => {
|
||||
const handlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
handlers?.delete(handler);
|
||||
if (handlers && handlers.size === 0) {
|
||||
delete handlersRef.current[eventName];
|
||||
const addEventListener = useCallback(
|
||||
<T extends ServerEventType>(eventName: T, handler: EventHandler<T>, options?: { signal?: AbortSignal }) => {
|
||||
if (options?.signal?.aborted) {
|
||||
return () => {};
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const existingHandlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
const eventHandlers = existingHandlers ?? new Set<EventHandler<T>>();
|
||||
eventHandlers.add(handler);
|
||||
handlersRef.current[eventName] = eventHandlers as EventHandlerMap[T];
|
||||
|
||||
const unsubscribe = () => {
|
||||
const handlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
handlers?.delete(handler);
|
||||
if (handlers && handlers.size === 0) {
|
||||
delete handlersRef.current[eventName];
|
||||
}
|
||||
if (options?.signal) {
|
||||
options.signal.removeEventListener("abort", unsubscribe);
|
||||
}
|
||||
};
|
||||
|
||||
if (options?.signal) {
|
||||
options.signal.addEventListener("abort", unsubscribe, { once: true });
|
||||
}
|
||||
|
||||
return unsubscribe;
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
return { addEventListener };
|
||||
}
|
||||
|
|
|
|||
39
app/client/hooks/useMinimumDuration.ts
Normal file
39
app/client/hooks/useMinimumDuration.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
export function useMinimumDuration(isActive: boolean, minimumDuration: number): boolean {
|
||||
const [displayActive, setDisplayActive] = useState(isActive);
|
||||
const startTimeRef = useRef<number | null>(null);
|
||||
const timeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (isActive) {
|
||||
if (timeoutRef.current) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
timeoutRef.current = null;
|
||||
}
|
||||
startTimeRef.current = Date.now();
|
||||
setDisplayActive(true);
|
||||
} else if (startTimeRef.current !== null) {
|
||||
const elapsed = Date.now() - startTimeRef.current;
|
||||
const remaining = Math.max(0, minimumDuration - elapsed);
|
||||
|
||||
if (remaining > 0) {
|
||||
timeoutRef.current = setTimeout(() => {
|
||||
setDisplayActive(false);
|
||||
startTimeRef.current = null;
|
||||
}, remaining);
|
||||
} else {
|
||||
setDisplayActive(false);
|
||||
startTimeRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (timeoutRef.current) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, [isActive, minimumDuration]);
|
||||
|
||||
return displayActive;
|
||||
}
|
||||
|
|
@ -15,22 +15,29 @@ export const BackupProgressCard = ({ scheduleShortId }: Props) => {
|
|||
const [progress, setProgress] = useState<BackupProgressEvent | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = addEventListener("backup:progress", (progressData) => {
|
||||
if (progressData.scheduleId === scheduleShortId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
|
||||
const unsubscribeComplete = addEventListener("backup:completed", (completedData) => {
|
||||
if (completedData.scheduleId === scheduleShortId) {
|
||||
setProgress(null);
|
||||
}
|
||||
});
|
||||
addEventListener(
|
||||
"backup:progress",
|
||||
(progressData) => {
|
||||
if (progressData.scheduleId === scheduleShortId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
unsubscribeComplete();
|
||||
};
|
||||
addEventListener(
|
||||
"backup:completed",
|
||||
(completedData) => {
|
||||
if (completedData.scheduleId === scheduleShortId) {
|
||||
setProgress(null);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => abortController.abort();
|
||||
}, [addEventListener, scheduleShortId]);
|
||||
|
||||
const percentDone = progress ? Math.round(progress.percent_done * 100) : 0;
|
||||
|
|
|
|||
|
|
@ -99,37 +99,44 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re
|
|||
}, [compatibility]);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribeStarted = addEventListener("mirror:started", (event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, { ...existing, lastCopyStatus: "in_progress", lastCopyError: null });
|
||||
return next;
|
||||
});
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
|
||||
const unsubscribeCompleted = addEventListener("mirror:completed", (event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, {
|
||||
...existing,
|
||||
lastCopyStatus: event.status ?? existing.lastCopyStatus,
|
||||
lastCopyError: event.error ?? null,
|
||||
lastCopyAt: Date.now(),
|
||||
addEventListener(
|
||||
"mirror:started",
|
||||
(event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, { ...existing, lastCopyStatus: "in_progress", lastCopyError: null });
|
||||
return next;
|
||||
});
|
||||
return next;
|
||||
});
|
||||
});
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribeStarted();
|
||||
unsubscribeCompleted();
|
||||
};
|
||||
addEventListener(
|
||||
"mirror:completed",
|
||||
(event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, {
|
||||
...existing,
|
||||
lastCopyStatus: event.status ?? existing.lastCopyStatus,
|
||||
lastCopyError: event.error ?? null,
|
||||
lastCopyAt: Date.now(),
|
||||
});
|
||||
return next;
|
||||
});
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => abortController.abort();
|
||||
}, [addEventListener, scheduleShortId]);
|
||||
|
||||
const addRepository = (repositoryId: string) => {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,13 @@ const restoreEventBaseSchema = type({
|
|||
snapshotId: "string",
|
||||
});
|
||||
|
||||
const dumpStartedEventSchema = type({
|
||||
repositoryId: "string",
|
||||
snapshotId: "string",
|
||||
path: "string",
|
||||
filename: "string",
|
||||
});
|
||||
|
||||
const restoreProgressMetricsSchema = type({
|
||||
seconds_elapsed: "number",
|
||||
percent_done: "number",
|
||||
|
|
@ -59,6 +66,8 @@ export const serverRestoreProgressEventSchema = organizationScopedSchema.and(res
|
|||
|
||||
export const serverRestoreCompletedEventSchema = organizationScopedSchema.and(restoreCompletedEventSchema);
|
||||
|
||||
export const serverDumpStartedEventSchema = organizationScopedSchema.and(dumpStartedEventSchema);
|
||||
|
||||
export type BackupEventStatusDto = typeof backupEventStatusSchema.infer;
|
||||
export type BackupStartedEventDto = typeof backupStartedEventSchema.infer;
|
||||
export type BackupProgressEventDto = typeof backupProgressEventSchema.infer;
|
||||
|
|
@ -66,9 +75,11 @@ export type BackupCompletedEventDto = typeof backupCompletedEventSchema.infer;
|
|||
export type RestoreStartedEventDto = typeof restoreStartedEventSchema.infer;
|
||||
export type RestoreProgressEventDto = typeof restoreProgressEventSchema.infer;
|
||||
export type RestoreCompletedEventDto = typeof restoreCompletedEventSchema.infer;
|
||||
export type DumpStartedEventDto = typeof dumpStartedEventSchema.infer;
|
||||
export type ServerBackupStartedEventDto = typeof serverBackupStartedEventSchema.infer;
|
||||
export type ServerBackupProgressEventDto = typeof serverBackupProgressEventSchema.infer;
|
||||
export type ServerBackupCompletedEventDto = typeof serverBackupCompletedEventSchema.infer;
|
||||
export type ServerRestoreStartedEventDto = typeof serverRestoreStartedEventSchema.infer;
|
||||
export type ServerRestoreProgressEventDto = typeof serverRestoreProgressEventSchema.infer;
|
||||
export type ServerRestoreCompletedEventDto = typeof serverRestoreCompletedEventSchema.infer;
|
||||
export type ServerDumpStartedEventDto = typeof serverDumpStartedEventSchema.infer;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import type {
|
|||
ServerBackupCompletedEventDto,
|
||||
ServerBackupProgressEventDto,
|
||||
ServerBackupStartedEventDto,
|
||||
ServerDumpStartedEventDto,
|
||||
ServerRestoreCompletedEventDto,
|
||||
ServerRestoreProgressEventDto,
|
||||
ServerRestoreStartedEventDto,
|
||||
|
|
@ -21,6 +22,7 @@ export const serverEventPayloads = {
|
|||
"restore:started": payload<ServerRestoreStartedEventDto>(),
|
||||
"restore:progress": payload<ServerRestoreProgressEventDto>(),
|
||||
"restore:completed": payload<ServerRestoreCompletedEventDto>(),
|
||||
"dump:started": payload<ServerDumpStartedEventDto>(),
|
||||
"mirror:started": payload<{
|
||||
organizationId: string;
|
||||
scheduleId: string;
|
||||
|
|
|
|||
|
|
@ -122,4 +122,163 @@ describe("RepositoryMutex", () => {
|
|||
release();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should allow concurrent shared locks", async () => {
|
||||
const repoId = "concurrent-shared";
|
||||
const release1 = await repoMutex.acquireShared(repoId, "op1");
|
||||
const release2 = await repoMutex.acquireShared(repoId, "op2");
|
||||
const release3 = await repoMutex.acquireShared(repoId, "op3");
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
release1();
|
||||
release2();
|
||||
release3();
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should block exclusive lock until all shared locks are released", async () => {
|
||||
const repoId = "shared-blocks-exclusive";
|
||||
let exclusiveAcquired = false;
|
||||
|
||||
const releaseShared1 = await repoMutex.acquireShared(repoId, "s1");
|
||||
const releaseShared2 = await repoMutex.acquireShared(repoId, "s2");
|
||||
|
||||
const exclusivePromise = repoMutex.acquireExclusive(repoId, "e1").then((release) => {
|
||||
exclusiveAcquired = true;
|
||||
return release;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(exclusiveAcquired).toBe(false);
|
||||
|
||||
releaseShared1();
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(exclusiveAcquired).toBe(false); // still waiting for s2
|
||||
|
||||
releaseShared2();
|
||||
const releaseExclusive = await exclusivePromise;
|
||||
expect(exclusiveAcquired).toBe(true);
|
||||
|
||||
releaseExclusive();
|
||||
});
|
||||
|
||||
test("should block all locks while exclusive lock is held", async () => {
|
||||
const repoId = "exclusive-blocks-all";
|
||||
const results: string[] = [];
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
results.push("e1-acquired");
|
||||
|
||||
const s1Promise = repoMutex.acquireShared(repoId, "s1").then((release) => {
|
||||
results.push("s1-acquired");
|
||||
return release;
|
||||
});
|
||||
const e2Promise = repoMutex.acquireExclusive(repoId, "e2").then((release) => {
|
||||
results.push("e2-acquired");
|
||||
return release;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(results).toEqual(["e1-acquired"]);
|
||||
|
||||
releaseExclusive();
|
||||
|
||||
const releaseS1 = await s1Promise;
|
||||
expect(results).toEqual(["e1-acquired", "s1-acquired"]);
|
||||
|
||||
releaseS1();
|
||||
|
||||
const releaseE2 = await e2Promise;
|
||||
expect(results).toEqual(["e1-acquired", "s1-acquired", "e2-acquired"]);
|
||||
|
||||
releaseE2();
|
||||
});
|
||||
|
||||
test("should grant all waiting shared locks at once when exclusive lock is released", async () => {
|
||||
const repoId = "batch-shared";
|
||||
const results: string[] = [];
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
|
||||
const s1Promise = repoMutex.acquireShared(repoId, "s1").then((release) => {
|
||||
results.push("s1");
|
||||
return release;
|
||||
});
|
||||
const s2Promise = repoMutex.acquireShared(repoId, "s2").then((release) => {
|
||||
results.push("s2");
|
||||
return release;
|
||||
});
|
||||
const s3Promise = repoMutex.acquireShared(repoId, "s3").then((release) => {
|
||||
results.push("s3");
|
||||
return release;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(results).toEqual([]);
|
||||
|
||||
releaseExclusive();
|
||||
|
||||
const [releaseS1, releaseS2, releaseS3] = await Promise.all([s1Promise, s2Promise, s3Promise]);
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
expect(results).toContain("s1");
|
||||
expect(results).toContain("s2");
|
||||
expect(results).toContain("s3");
|
||||
|
||||
releaseS1();
|
||||
releaseS2();
|
||||
releaseS3();
|
||||
});
|
||||
|
||||
test("should safely handle multiple calls to the release function", async () => {
|
||||
const repoId = "idempotent-release";
|
||||
|
||||
const releaseShared = await repoMutex.acquireShared(repoId, "s1");
|
||||
releaseShared();
|
||||
releaseShared(); // Should not throw or cause issues
|
||||
releaseShared();
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
releaseExclusive();
|
||||
releaseExclusive(); // Should not throw
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should immediately throw if AbortSignal is already aborted", async () => {
|
||||
const repoId = "already-aborted";
|
||||
const controller = new AbortController();
|
||||
controller.abort(new Error("pre-aborted"));
|
||||
|
||||
expect(repoMutex.acquireShared(repoId, "s1", controller.signal)).rejects.toThrow("pre-aborted");
|
||||
expect(repoMutex.acquireExclusive(repoId, "e1", controller.signal)).rejects.toThrow("pre-aborted");
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should accurately report isLocked status", async () => {
|
||||
const repoId = "is-locked-status";
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
|
||||
const releaseShared1 = await repoMutex.acquireShared(repoId, "s1");
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
const releaseShared2 = await repoMutex.acquireShared(repoId, "s2");
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
releaseShared1();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true); // still locked by s2
|
||||
|
||||
releaseShared2();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false); // all shared released
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
releaseExclusive();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ class RepositoryMutex {
|
|||
throw signal.reason || new Error("Operation aborted");
|
||||
}
|
||||
|
||||
return () => this.releaseShared(repositoryId, lockId!);
|
||||
return this.createRelease("shared", repositoryId, lockId!);
|
||||
}
|
||||
|
||||
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal): Promise<() => void> {
|
||||
|
|
@ -154,7 +154,8 @@ class RepositoryMutex {
|
|||
}
|
||||
|
||||
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation} (${lockId})`);
|
||||
return () => this.releaseExclusive(repositoryId, lockId!);
|
||||
|
||||
return this.createRelease("exclusive", repositoryId, lockId!);
|
||||
}
|
||||
|
||||
private releaseShared(repositoryId: string, lockId: string): void {
|
||||
|
|
@ -239,6 +240,19 @@ class RepositoryMutex {
|
|||
if (!state) return false;
|
||||
return state.exclusiveHolder !== null || state.sharedHolders.size > 0;
|
||||
}
|
||||
|
||||
private createRelease(type: LockType, repositoryId: string, lockId: string) {
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
if (type === "shared") {
|
||||
this.releaseShared(repositoryId, lockId);
|
||||
} else {
|
||||
this.releaseExclusive(repositoryId, lockId);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const repoMutex = new RepositoryMutex();
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ const broadcastEvents = [
|
|||
"restore:started",
|
||||
"restore:progress",
|
||||
"restore:completed",
|
||||
"dump:started",
|
||||
"doctor:started",
|
||||
"doctor:completed",
|
||||
"doctor:cancelled",
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ describe("repositories security", () => {
|
|||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots" },
|
||||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot" },
|
||||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot/files" },
|
||||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot/dump" },
|
||||
{ method: "POST", path: "/api/v1/repositories/test-repo/restore" },
|
||||
{ method: "POST", path: "/api/v1/repositories/test-repo/doctor" },
|
||||
{ method: "DELETE", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot" },
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
|
||||
import type { RepositoryConfig } from "~/schemas/restic";
|
||||
import { REPOSITORY_BASE } from "~/server/core/constants";
|
||||
import { serverEvents } from "~/server/core/events";
|
||||
import { withContext } from "~/server/core/request-context";
|
||||
import { db } from "~/server/db/db";
|
||||
import { repositoriesTable } from "~/server/db/schema";
|
||||
import { generateShortId } from "~/server/utils/id";
|
||||
import { restic } from "~/server/utils/restic";
|
||||
import { createTestSession } from "~/test/helpers/auth";
|
||||
import { repositoriesService } from "../repositories.service";
|
||||
|
|
@ -77,3 +81,225 @@ describe("repositoriesService.createRepository", () => {
|
|||
expect(created.status).toBe("healthy");
|
||||
});
|
||||
});
|
||||
|
||||
describe("repositoriesService.dumpSnapshot", () => {
|
||||
afterEach(() => {
|
||||
mock.restore();
|
||||
});
|
||||
|
||||
test("calls restic.dump with common-ancestor selector and stripped path", async () => {
|
||||
const { organizationId, user } = await createTestSession();
|
||||
const shortId = generateShortId();
|
||||
const basePath = "/var/lib/zerobyte/volumes/vol123/_data";
|
||||
|
||||
await db.insert(repositoriesTable).values({
|
||||
id: randomUUID(),
|
||||
shortId,
|
||||
name: `Repository-${randomUUID()}`,
|
||||
type: "local",
|
||||
config: {
|
||||
backend: "local",
|
||||
path: `/tmp/repository-${randomUUID()}`,
|
||||
isExistingRepository: true,
|
||||
},
|
||||
compressionMode: "off",
|
||||
organizationId,
|
||||
});
|
||||
|
||||
const snapshotsMock = mock(() =>
|
||||
Promise.resolve([
|
||||
{
|
||||
id: "snapshot-123",
|
||||
short_id: "snapshot-123",
|
||||
time: new Date().toISOString(),
|
||||
tree: "tree-1",
|
||||
paths: [basePath],
|
||||
hostname: "host",
|
||||
},
|
||||
]),
|
||||
);
|
||||
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
|
||||
|
||||
const dumpMock = mock(() =>
|
||||
Promise.resolve({
|
||||
stream: Readable.from([]),
|
||||
completion: Promise.resolve(),
|
||||
abort: () => {},
|
||||
}),
|
||||
);
|
||||
spyOn(restic, "dump").mockImplementation(dumpMock);
|
||||
const emitSpy = spyOn(serverEvents, "emit");
|
||||
|
||||
await withContext({ organizationId, userId: user.id }, () =>
|
||||
repositoriesService.dumpSnapshot(shortId, "snapshot-123", `${basePath}/documents`, "dir"),
|
||||
);
|
||||
|
||||
expect(dumpMock).toHaveBeenCalledTimes(1);
|
||||
expect(dumpMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
backend: "local",
|
||||
}),
|
||||
`snapshot-123:${basePath}`,
|
||||
{
|
||||
organizationId,
|
||||
path: "/documents",
|
||||
},
|
||||
);
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
"dump:started",
|
||||
expect.objectContaining({
|
||||
organizationId,
|
||||
repositoryId: shortId,
|
||||
snapshotId: "snapshot-123",
|
||||
path: "/documents",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("streams a single file directly when selected path is a file", async () => {
|
||||
const { organizationId, user } = await createTestSession();
|
||||
const shortId = generateShortId();
|
||||
const basePath = "/var/lib/zerobyte/volumes/vol123/_data";
|
||||
|
||||
await db.insert(repositoriesTable).values({
|
||||
id: randomUUID(),
|
||||
shortId,
|
||||
name: `Repository-${randomUUID()}`,
|
||||
type: "local",
|
||||
config: {
|
||||
backend: "local",
|
||||
path: `/tmp/repository-${randomUUID()}`,
|
||||
isExistingRepository: true,
|
||||
},
|
||||
compressionMode: "off",
|
||||
organizationId,
|
||||
});
|
||||
|
||||
const snapshotsMock = mock(() =>
|
||||
Promise.resolve([
|
||||
{
|
||||
id: "snapshot-file",
|
||||
short_id: "snapshot-file",
|
||||
time: new Date().toISOString(),
|
||||
tree: "tree-file",
|
||||
paths: [basePath],
|
||||
hostname: "host",
|
||||
},
|
||||
]),
|
||||
);
|
||||
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
|
||||
|
||||
const dumpMock = mock(() =>
|
||||
Promise.resolve({
|
||||
stream: Readable.from([]),
|
||||
completion: Promise.resolve(),
|
||||
abort: () => {},
|
||||
}),
|
||||
);
|
||||
spyOn(restic, "dump").mockImplementation(dumpMock);
|
||||
|
||||
const result = await withContext({ organizationId, userId: user.id }, () =>
|
||||
repositoriesService.dumpSnapshot(shortId, "snapshot-file", `${basePath}/documents/report.txt`, "file"),
|
||||
);
|
||||
|
||||
expect(dumpMock).toHaveBeenCalledWith(expect.anything(), `snapshot-file:${basePath}`, {
|
||||
organizationId,
|
||||
path: "/documents/report.txt",
|
||||
archive: false,
|
||||
});
|
||||
expect(result.filename).toBe("report.txt");
|
||||
expect(result.contentType).toBe("application/octet-stream");
|
||||
});
|
||||
|
||||
test("rejects path downloads without a kind", async () => {
|
||||
const { organizationId, user } = await createTestSession();
|
||||
const shortId = generateShortId();
|
||||
const basePath = "/var/lib/zerobyte/volumes/vol123/_data";
|
||||
|
||||
await db.insert(repositoriesTable).values({
|
||||
id: randomUUID(),
|
||||
shortId,
|
||||
name: `Repository-${randomUUID()}`,
|
||||
type: "local",
|
||||
config: {
|
||||
backend: "local",
|
||||
path: `/tmp/repository-${randomUUID()}`,
|
||||
isExistingRepository: true,
|
||||
},
|
||||
compressionMode: "off",
|
||||
organizationId,
|
||||
});
|
||||
|
||||
const snapshotsMock = mock(() =>
|
||||
Promise.resolve([
|
||||
{
|
||||
id: "snapshot-no-kind",
|
||||
short_id: "snapshot-no-kind",
|
||||
time: new Date().toISOString(),
|
||||
tree: "tree-no-kind",
|
||||
paths: [basePath],
|
||||
hostname: "host",
|
||||
},
|
||||
]),
|
||||
);
|
||||
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
|
||||
|
||||
await expect(
|
||||
withContext({ organizationId, userId: user.id }, () =>
|
||||
repositoriesService.dumpSnapshot(shortId, "snapshot-no-kind", `${basePath}/documents/report.txt`),
|
||||
),
|
||||
).rejects.toThrow("Path kind is required when downloading a specific snapshot path");
|
||||
});
|
||||
|
||||
test("downloads full snapshot relative to common ancestor when path is omitted", async () => {
|
||||
const { organizationId, user } = await createTestSession();
|
||||
const shortId = generateShortId();
|
||||
const basePath = "/var/lib/zerobyte/volumes/vol555/_data";
|
||||
|
||||
await db.insert(repositoriesTable).values({
|
||||
id: randomUUID(),
|
||||
shortId,
|
||||
name: `Repository-${randomUUID()}`,
|
||||
type: "local",
|
||||
config: {
|
||||
backend: "local",
|
||||
path: `/tmp/repository-${randomUUID()}`,
|
||||
isExistingRepository: true,
|
||||
},
|
||||
compressionMode: "off",
|
||||
organizationId,
|
||||
});
|
||||
|
||||
const snapshotsMock = mock(() =>
|
||||
Promise.resolve([
|
||||
{
|
||||
id: "snapshot-999",
|
||||
short_id: "snapshot-999",
|
||||
time: new Date().toISOString(),
|
||||
tree: "tree-9",
|
||||
paths: [basePath],
|
||||
hostname: "host",
|
||||
},
|
||||
]),
|
||||
);
|
||||
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
|
||||
|
||||
const dumpMock = mock(() =>
|
||||
Promise.resolve({
|
||||
stream: Readable.from([]),
|
||||
completion: Promise.resolve(),
|
||||
abort: () => {},
|
||||
}),
|
||||
);
|
||||
spyOn(restic, "dump").mockImplementation(dumpMock);
|
||||
|
||||
await withContext({ organizationId, userId: user.id }, () =>
|
||||
repositoriesService.dumpSnapshot(shortId, "snapshot-999"),
|
||||
);
|
||||
|
||||
expect(dumpMock).toHaveBeenCalledWith(expect.anything(), `snapshot-999:${basePath}`, {
|
||||
organizationId,
|
||||
path: "/",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../../db/db";
|
||||
import { repositoriesTable } from "../../db/schema";
|
||||
import { toMessage } from "../../utils/errors";
|
||||
import { restic } from "../../utils/restic";
|
||||
import { repoMutex } from "../../core/repository-mutex";
|
||||
import { type DoctorStep, type DoctorResult, type RepositoryConfig } from "~/schemas/restic";
|
||||
import { type } from "arktype";
|
||||
import { serverEvents } from "../../core/events";
|
||||
import { logger } from "../../utils/logger";
|
||||
import { safeJsonParse } from "../../utils/json";
|
||||
import { getOrganizationId } from "~/server/core/request-context";
|
||||
import { restic } from "~/server/utils/restic";
|
||||
import { toMessage } from "~/server/utils/errors";
|
||||
import { safeJsonParse } from "~/server/utils/json";
|
||||
import { logger } from "~/server/utils/logger";
|
||||
import { db } from "~/server/db/db";
|
||||
import { repositoriesTable } from "~/server/db/schema";
|
||||
import { repoMutex } from "~/server/core/repository-mutex";
|
||||
import { serverEvents } from "~/server/core/events";
|
||||
|
||||
class AbortError extends Error {
|
||||
name = "AbortError";
|
||||
50
app/server/modules/repositories/helpers/dump.ts
Normal file
50
app/server/modules/repositories/helpers/dump.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { BadRequestError } from "http-errors-enhanced";
|
||||
import path from "node:path";
|
||||
import { findCommonAncestor } from "~/utils/common-ancestor";
|
||||
import { normalizeAbsolutePath } from "~/utils/path";
|
||||
|
||||
const sanitizeFilenamePart = (value: string): string => {
|
||||
const sanitized = value.replace(/[^a-zA-Z0-9._-]/g, "_").replace(/^_+|_+$/g, "");
|
||||
return sanitized || "snapshot";
|
||||
};
|
||||
|
||||
export const prepareSnapshotDump = (params: {
|
||||
snapshotId: string;
|
||||
snapshotPaths: string[];
|
||||
requestedPath?: string;
|
||||
}) => {
|
||||
const { snapshotId, snapshotPaths, requestedPath } = params;
|
||||
|
||||
const archiveFilename = `snapshot-${sanitizeFilenamePart(snapshotId)}.tar`;
|
||||
const normalizedRequestedPath = normalizeAbsolutePath(requestedPath);
|
||||
const basePath = normalizeAbsolutePath(findCommonAncestor(snapshotPaths));
|
||||
|
||||
if (basePath === "/") {
|
||||
return {
|
||||
snapshotRef: snapshotId,
|
||||
path: normalizedRequestedPath,
|
||||
filename: archiveFilename,
|
||||
};
|
||||
}
|
||||
|
||||
if (normalizedRequestedPath === "/" || normalizedRequestedPath === basePath) {
|
||||
return {
|
||||
snapshotRef: `${snapshotId}:${basePath}`,
|
||||
path: "/",
|
||||
filename: archiveFilename,
|
||||
};
|
||||
}
|
||||
|
||||
const relativeFromBase = path.posix.relative(basePath, normalizedRequestedPath);
|
||||
if (relativeFromBase === ".." || relativeFromBase.startsWith("../")) {
|
||||
throw new BadRequestError("Requested path is outside the snapshot base path");
|
||||
}
|
||||
|
||||
const relativePath = relativeFromBase ? `/${relativeFromBase}` : "/";
|
||||
|
||||
return {
|
||||
snapshotRef: `${snapshotId}:${basePath}`,
|
||||
path: relativePath,
|
||||
filename: archiveFilename,
|
||||
};
|
||||
};
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
import { Readable } from "node:stream";
|
||||
import { Hono } from "hono";
|
||||
import { validator } from "hono-openapi";
|
||||
import { streamSSE } from "hono/streaming";
|
||||
import contentDisposition from "content-disposition";
|
||||
import {
|
||||
createRepositoryBody,
|
||||
createRepositoryDto,
|
||||
|
|
@ -19,6 +21,8 @@ import {
|
|||
listSnapshotFilesQuery,
|
||||
listSnapshotsDto,
|
||||
listSnapshotsFilters,
|
||||
dumpSnapshotDto,
|
||||
dumpSnapshotQuery,
|
||||
restoreSnapshotBody,
|
||||
restoreSnapshotDto,
|
||||
tagSnapshotsBody,
|
||||
|
|
@ -165,6 +169,30 @@ export const repositoriesController = new Hono()
|
|||
return c.json<ListSnapshotFilesDto>(result, 200);
|
||||
},
|
||||
)
|
||||
.get("/:shortId/snapshots/:snapshotId/dump", dumpSnapshotDto, validator("query", dumpSnapshotQuery), async (c) => {
|
||||
const { shortId, snapshotId } = c.req.param();
|
||||
const { path, kind } = c.req.valid("query");
|
||||
|
||||
const dumpStream = await repositoriesService.dumpSnapshot(shortId, snapshotId, path, kind);
|
||||
const signal = c.req.raw.signal;
|
||||
|
||||
if (signal.aborted) {
|
||||
dumpStream.abort();
|
||||
} else {
|
||||
signal.addEventListener("abort", () => dumpStream.abort(), { once: true });
|
||||
}
|
||||
|
||||
const webStream = Readable.toWeb(dumpStream.stream) as unknown as ReadableStream<Uint8Array>;
|
||||
|
||||
return new Response(webStream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": dumpStream.contentType,
|
||||
"Content-Disposition": contentDisposition(dumpStream.filename || "snapshot.tar"),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
});
|
||||
})
|
||||
.post("/:shortId/restore", restoreSnapshotDto, validator("json", restoreSnapshotBody), async (c) => {
|
||||
const { shortId } = c.req.param();
|
||||
const { snapshotId, ...options } = c.req.valid("json");
|
||||
|
|
|
|||
|
|
@ -290,6 +290,41 @@ export const listSnapshotFilesDto = describeRoute({
|
|||
},
|
||||
});
|
||||
|
||||
const DUMP_PATH_KINDS = {
|
||||
file: "file",
|
||||
dir: "dir",
|
||||
} as const;
|
||||
|
||||
export const dumpPathKindSchema = type.valueOf(DUMP_PATH_KINDS);
|
||||
export type DumpPathKind = typeof dumpPathKindSchema.infer;
|
||||
|
||||
/**
|
||||
* Download snapshot paths as tar archives (folders) or raw file streams (single files)
|
||||
*/
|
||||
export const dumpSnapshotQuery = type({
|
||||
path: "string?",
|
||||
kind: dumpPathKindSchema.optional(),
|
||||
});
|
||||
|
||||
export const dumpSnapshotDto = describeRoute({
|
||||
description: "Download a snapshot path as a tar archive (folders) or raw file stream (single files)",
|
||||
tags: ["Repositories"],
|
||||
operationId: "dumpSnapshot",
|
||||
responses: {
|
||||
200: {
|
||||
description: "Snapshot content stream",
|
||||
content: {
|
||||
"application/x-tar": {
|
||||
schema: { type: "string", format: "binary" },
|
||||
},
|
||||
"application/octet-stream": {
|
||||
schema: { type: "string", format: "binary" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Restore a snapshot
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import crypto from "node:crypto";
|
||||
import nodePath from "node:path";
|
||||
import { type } from "arktype";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { BadRequestError, ConflictError, InternalServerError, NotFoundError } from "http-errors-enhanced";
|
||||
|
|
@ -22,10 +23,11 @@ import { generateShortId } from "../../utils/id";
|
|||
import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys, restic } from "../../utils/restic";
|
||||
import { safeSpawn } from "../../utils/spawn";
|
||||
import { backupsService } from "../backups/backups.service";
|
||||
import type { UpdateRepositoryBody } from "./repositories.dto";
|
||||
import { executeDoctor } from "./doctor";
|
||||
import type { DumpPathKind, UpdateRepositoryBody } from "./repositories.dto";
|
||||
import { REPOSITORY_BASE } from "~/server/core/constants";
|
||||
import { findCommonAncestor } from "~/utils/common-ancestor";
|
||||
import { prepareSnapshotDump } from "./helpers/dump";
|
||||
import { executeDoctor } from "./helpers/doctor";
|
||||
|
||||
const runningDoctors = new Map<string, AbortController>();
|
||||
|
||||
|
|
@ -389,6 +391,66 @@ const restoreSnapshot = async (
|
|||
}
|
||||
};
|
||||
|
||||
const dumpSnapshot = async (shortId: string, snapshotId: string, path?: string, kind?: DumpPathKind) => {
|
||||
const organizationId = getOrganizationId();
|
||||
const repository = await findRepository(shortId);
|
||||
|
||||
if (!repository) {
|
||||
throw new NotFoundError("Repository not found");
|
||||
}
|
||||
|
||||
const releaseLock = await repoMutex.acquireShared(repository.id, `dump:${snapshotId}`);
|
||||
let dumpStream: Awaited<ReturnType<typeof restic.dump>> | undefined = undefined;
|
||||
|
||||
try {
|
||||
const snapshot = await getSnapshotDetails(repository.shortId, snapshotId);
|
||||
const preparedDump = prepareSnapshotDump({ snapshotId, snapshotPaths: snapshot.paths, requestedPath: path });
|
||||
const dumpOptions: Parameters<typeof restic.dump>[2] = {
|
||||
organizationId,
|
||||
path: preparedDump.path,
|
||||
};
|
||||
|
||||
let filename = preparedDump.filename;
|
||||
let contentType = "application/x-tar";
|
||||
|
||||
if (path && preparedDump.path !== "/") {
|
||||
if (!kind) {
|
||||
throw new BadRequestError("Path kind is required when downloading a specific snapshot path");
|
||||
}
|
||||
|
||||
if (kind === "file") {
|
||||
dumpOptions.archive = false;
|
||||
contentType = "application/octet-stream";
|
||||
const fileName = nodePath.posix.basename(preparedDump.path);
|
||||
if (fileName) {
|
||||
filename = fileName;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dumpStream = await restic.dump(repository.config, preparedDump.snapshotRef, dumpOptions);
|
||||
|
||||
serverEvents.emit("dump:started", {
|
||||
organizationId,
|
||||
repositoryId: repository.shortId,
|
||||
snapshotId,
|
||||
path: preparedDump.path,
|
||||
filename,
|
||||
});
|
||||
|
||||
const completion = dumpStream.completion.finally(releaseLock);
|
||||
void completion.catch(() => {});
|
||||
|
||||
return { ...dumpStream, completion, filename, contentType };
|
||||
} catch (error) {
|
||||
if (dumpStream) {
|
||||
dumpStream.abort();
|
||||
}
|
||||
releaseLock();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const getSnapshotDetails = async (shortId: string, snapshotId: string) => {
|
||||
const organizationId = getOrganizationId();
|
||||
const repository = await findRepository(shortId);
|
||||
|
|
@ -777,6 +839,7 @@ export const repositoriesService = {
|
|||
listSnapshots,
|
||||
listSnapshotFiles,
|
||||
restoreSnapshot,
|
||||
dumpSnapshot,
|
||||
getSnapshotDetails,
|
||||
checkHealth,
|
||||
startDoctor,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ const successfulRestoreSummary = JSON.stringify({
|
|||
|
||||
let lastSafeSpawnArgs: string[] = [];
|
||||
|
||||
const safeSpawnMock = mock((params: Parameters<typeof spawnModule.safeSpawn>[0]) => {
|
||||
const safeSpawnMock = mock((params: spawnModule.SafeSpawnParams) => {
|
||||
lastSafeSpawnArgs = params.args;
|
||||
|
||||
return Promise.resolve({
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import crypto from "node:crypto";
|
||||
import { normalizeAbsolutePath } from "~/utils/path";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import { type } from "arktype";
|
||||
import { throttle } from "es-toolkit";
|
||||
import type { BandwidthLimit, CompressionMode, OverwriteMode, RepositoryConfig } from "~/schemas/restic";
|
||||
|
|
@ -83,10 +85,14 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
const decryptedPassword = await cryptoUtils.resolveSecret(config.customPassword);
|
||||
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
|
||||
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env.RESTIC_PASSWORD_FILE = passwordFilePath;
|
||||
} else {
|
||||
const org = await db.query.organization.findFirst({ where: { id: organizationId } });
|
||||
const org = await db.query.organization.findFirst({
|
||||
where: { id: organizationId },
|
||||
});
|
||||
|
||||
if (!org) {
|
||||
throw new Error(`Organization ${organizationId} not found`);
|
||||
|
|
@ -98,7 +104,9 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
} else {
|
||||
const decryptedPassword = await cryptoUtils.resolveSecret(metadata.resticPassword);
|
||||
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env.RESTIC_PASSWORD_FILE = passwordFilePath;
|
||||
}
|
||||
}
|
||||
|
|
@ -122,7 +130,9 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
case "gcs": {
|
||||
const decryptedCredentials = await cryptoUtils.resolveSecret(config.credentialsJson);
|
||||
const credentialsPath = path.join("/tmp", `zerobyte-gcs-${crypto.randomBytes(8).toString("hex")}.json`);
|
||||
await fs.writeFile(credentialsPath, decryptedCredentials, { mode: 0o600 });
|
||||
await fs.writeFile(credentialsPath, decryptedCredentials, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env.GOOGLE_PROJECT_ID = config.projectId;
|
||||
env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;
|
||||
break;
|
||||
|
|
@ -193,7 +203,9 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
sshArgs.push("-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null");
|
||||
} else if (config.knownHosts) {
|
||||
const knownHostsPath = path.join("/tmp", `zerobyte-known-hosts-${crypto.randomBytes(8).toString("hex")}`);
|
||||
await fs.writeFile(knownHostsPath, config.knownHosts, { mode: 0o600 });
|
||||
await fs.writeFile(knownHostsPath, config.knownHosts, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env._SFTP_KNOWN_HOSTS_PATH = knownHostsPath;
|
||||
sshArgs.push("-o", "StrictHostKeyChecking=yes", "-o", `UserKnownHostsFile=${knownHostsPath}`);
|
||||
}
|
||||
|
|
@ -232,7 +244,12 @@ const init = async (config: RepositoryConfig, organizationId: string, options?:
|
|||
const args = ["init", "--repo", repoUrl];
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, timeout: options?.timeoutMs ?? 20000 });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
timeout: options?.timeoutMs ?? 20000,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
|
|
@ -398,6 +415,13 @@ const restoreProgressSchema = type({
|
|||
});
|
||||
|
||||
export type RestoreProgress = typeof restoreProgressSchema.infer;
|
||||
|
||||
export interface ResticDumpStream {
|
||||
stream: Readable;
|
||||
completion: Promise<void>;
|
||||
abort: () => void;
|
||||
}
|
||||
|
||||
const restore = async (
|
||||
config: RepositoryConfig,
|
||||
snapshotId: string,
|
||||
|
|
@ -533,6 +557,106 @@ const restore = async (
|
|||
return result;
|
||||
};
|
||||
|
||||
const normalizeDumpPath = (pathToDump?: string): string => {
|
||||
const trimmedPath = pathToDump?.trim();
|
||||
if (!trimmedPath) {
|
||||
return "/";
|
||||
}
|
||||
|
||||
return normalizeAbsolutePath(trimmedPath);
|
||||
};
|
||||
|
||||
const dump = async (
|
||||
config: RepositoryConfig,
|
||||
snapshotRef: string,
|
||||
options: {
|
||||
organizationId: string;
|
||||
path?: string;
|
||||
archive?: false;
|
||||
},
|
||||
): Promise<ResticDumpStream> => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config, options.organizationId);
|
||||
const pathToDump = normalizeDumpPath(options.path);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "dump", snapshotRef, pathToDump];
|
||||
|
||||
if (options.archive !== false) {
|
||||
args.push("--archive", "tar");
|
||||
}
|
||||
|
||||
addCommonArgs(args, env, config, { includeJson: false });
|
||||
|
||||
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||
|
||||
let didCleanup = false;
|
||||
const cleanup = async () => {
|
||||
if (didCleanup) {
|
||||
return;
|
||||
}
|
||||
|
||||
didCleanup = true;
|
||||
await cleanupTemporaryKeys(env);
|
||||
};
|
||||
|
||||
let stream: Readable | null = null;
|
||||
let abortController: AbortController | null = new AbortController();
|
||||
|
||||
const MAX_STDERR_CHARS = 64 * 1024;
|
||||
let stderrTail = "";
|
||||
|
||||
const completion = safeSpawn({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: abortController.signal,
|
||||
stdoutMode: "raw",
|
||||
onSpawn: (child) => {
|
||||
stream = child.stdout;
|
||||
},
|
||||
onStderr: (line) => {
|
||||
const chunk = line.trim();
|
||||
if (chunk) {
|
||||
stderrTail += `${line}\n`;
|
||||
if (stderrTail.length > MAX_STDERR_CHARS) {
|
||||
stderrTail = stderrTail.slice(-MAX_STDERR_CHARS);
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
.then((result) => {
|
||||
if (result.exitCode === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const stderr = stderrTail.trim() || result.error;
|
||||
logger.error(`Restic dump failed: ${stderr}`);
|
||||
throw new ResticError(result.exitCode, stderr);
|
||||
})
|
||||
.finally(async () => {
|
||||
abortController = null;
|
||||
await cleanup();
|
||||
});
|
||||
|
||||
completion.catch(() => {});
|
||||
const completionPromise = new Promise<void>((res, rej) => completion.then(res, rej));
|
||||
|
||||
if (!stream) {
|
||||
await cleanup();
|
||||
throw new Error("Failed to initialize restic dump stream");
|
||||
}
|
||||
|
||||
return {
|
||||
stream,
|
||||
completion: completionPromise,
|
||||
abort: () => {
|
||||
if (abortController) {
|
||||
abortController.abort();
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; organizationId: string }) => {
|
||||
const { tags, organizationId } = options;
|
||||
|
||||
|
|
@ -852,7 +976,12 @@ const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal;
|
|||
const args = ["unlock", "--repo", repoUrl, "--remove-all"];
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, signal: options?.signal });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
|
|
@ -871,7 +1000,11 @@ const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal;
|
|||
|
||||
const check = async (
|
||||
config: RepositoryConfig,
|
||||
options: { readData?: boolean; signal?: AbortSignal; organizationId: string },
|
||||
options: {
|
||||
readData?: boolean;
|
||||
signal?: AbortSignal;
|
||||
organizationId: string;
|
||||
},
|
||||
) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config, options.organizationId);
|
||||
|
|
@ -884,12 +1017,22 @@ const check = async (
|
|||
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, signal: options?.signal });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
logger.warn("Restic check was aborted by signal.");
|
||||
return { success: false, hasErrors: true, output: "", error: "Operation aborted" };
|
||||
return {
|
||||
success: false,
|
||||
hasErrors: true,
|
||||
output: "",
|
||||
error: "Operation aborted",
|
||||
};
|
||||
}
|
||||
|
||||
const { stdout, stderr } = res;
|
||||
|
|
@ -922,7 +1065,12 @@ const repairIndex = async (config: RepositoryConfig, options: { signal?: AbortSi
|
|||
const args = ["repair", "index", "--repo", repoUrl];
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, signal: options?.signal });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
|
|
@ -1041,9 +1189,11 @@ export const addCommonArgs = (
|
|||
args: string[],
|
||||
env: Record<string, string>,
|
||||
config?: RepositoryConfig,
|
||||
options?: { skipBandwidth?: boolean },
|
||||
options?: { skipBandwidth?: boolean; includeJson?: boolean },
|
||||
) => {
|
||||
args.push("--json");
|
||||
if (options?.includeJson !== false) {
|
||||
args.push("--json");
|
||||
}
|
||||
|
||||
if (env._SFTP_SSH_ARGS) {
|
||||
args.push("-o", `sftp.args=${env._SFTP_SSH_ARGS}`);
|
||||
|
|
@ -1078,6 +1228,7 @@ export const restic = {
|
|||
init,
|
||||
backup,
|
||||
restore,
|
||||
dump,
|
||||
snapshots,
|
||||
forget,
|
||||
deleteSnapshot,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,11 @@ export const exec = async ({ command, args = [], env = {}, ...rest }: ExecProps)
|
|||
};
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await promisify(execFile)(command, args, { ...options, ...rest, encoding: "utf8" });
|
||||
const { stdout, stderr } = await promisify(execFile)(command, args, {
|
||||
...options,
|
||||
...rest,
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
return { exitCode: 0, stdout, stderr };
|
||||
} catch (error) {
|
||||
|
|
@ -28,23 +32,39 @@ export const exec = async ({ command, args = [], env = {}, ...rest }: ExecProps)
|
|||
}
|
||||
};
|
||||
|
||||
export interface SafeSpawnParams {
|
||||
export interface SafeSpawnParamsBase {
|
||||
command: string;
|
||||
args: string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
signal?: AbortSignal;
|
||||
onStdout?: (line: string) => void;
|
||||
onStderr?: (error: string) => void;
|
||||
onSpawn?: (child: ReturnType<typeof spawn>) => void;
|
||||
}
|
||||
|
||||
type SpawnResult = {
|
||||
export interface SafeSpawnParamsLines extends SafeSpawnParamsBase {
|
||||
stdoutMode?: "lines";
|
||||
onStdout?: (line: string) => void;
|
||||
}
|
||||
|
||||
export interface SafeSpawnParamsRaw extends SafeSpawnParamsBase {
|
||||
stdoutMode: "raw";
|
||||
onStdout?: never;
|
||||
}
|
||||
|
||||
export type SafeSpawnParams = SafeSpawnParamsLines | SafeSpawnParamsRaw;
|
||||
|
||||
export type SpawnResult = {
|
||||
exitCode: number;
|
||||
summary: string;
|
||||
error: string;
|
||||
};
|
||||
|
||||
export const safeSpawn = (params: SafeSpawnParams) => {
|
||||
const { command, args, env = {}, signal, onStdout, onStderr } = params;
|
||||
export function safeSpawn(params: SafeSpawnParamsLines): Promise<SpawnResult>;
|
||||
export function safeSpawn(params: SafeSpawnParamsRaw): Promise<SpawnResult>;
|
||||
export function safeSpawn(params: SafeSpawnParams): Promise<SpawnResult> {
|
||||
const { command, args, env = {}, signal, onStderr, onSpawn } = params;
|
||||
const stdoutMode = params.stdoutMode ?? "lines";
|
||||
const onStdout = stdoutMode === "lines" ? params.onStdout : undefined;
|
||||
|
||||
let lastStdout = "";
|
||||
let lastStderr = "";
|
||||
|
|
@ -56,19 +76,25 @@ export const safeSpawn = (params: SafeSpawnParams) => {
|
|||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
onSpawn?.(child);
|
||||
|
||||
child.stderr.setEncoding("utf8");
|
||||
|
||||
const rl = createInterface({ input: child.stdout });
|
||||
const rlErr = createInterface({ input: child.stderr });
|
||||
|
||||
rl.on("line", (line) => {
|
||||
if (onStdout) onStdout(line);
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length > 0) {
|
||||
lastStdout = line;
|
||||
}
|
||||
});
|
||||
if (stdoutMode === "lines") {
|
||||
child.stdout.setEncoding("utf8");
|
||||
|
||||
const rl = createInterface({ input: child.stdout });
|
||||
|
||||
rl.on("line", (line) => {
|
||||
if (onStdout) onStdout(line);
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length > 0) {
|
||||
lastStdout = line;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
rlErr.on("line", (line) => {
|
||||
if (onStderr) onStderr(line);
|
||||
|
|
@ -79,11 +105,19 @@ export const safeSpawn = (params: SafeSpawnParams) => {
|
|||
});
|
||||
|
||||
child.on("error", (err) => {
|
||||
resolve({ exitCode: -1, summary: lastStdout, error: err.message || lastStderr });
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
summary: lastStdout,
|
||||
error: err.message || lastStderr,
|
||||
});
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
resolve({ exitCode: code ?? -1, summary: lastStdout, error: lastStderr });
|
||||
resolve({
|
||||
exitCode: code ?? -1,
|
||||
summary: lastStdout,
|
||||
error: lastStderr,
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
}
|
||||
|
|
|
|||
39
app/utils/__tests__/path.test.ts
Normal file
39
app/utils/__tests__/path.test.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
import { normalizeAbsolutePath } from "../path";
|
||||
|
||||
describe("normalizeAbsolutePath", () => {
|
||||
test("handles undefined and empty inputs", () => {
|
||||
expect(normalizeAbsolutePath()).toBe("/");
|
||||
expect(normalizeAbsolutePath("")).toBe("/");
|
||||
expect(normalizeAbsolutePath(" ")).toBe("/");
|
||||
});
|
||||
|
||||
test("normalizes posix paths", () => {
|
||||
expect(normalizeAbsolutePath("/foo/bar")).toBe("/foo/bar");
|
||||
expect(normalizeAbsolutePath("foo/bar")).toBe("/foo/bar");
|
||||
expect(normalizeAbsolutePath("/foo//bar")).toBe("/foo/bar");
|
||||
expect(normalizeAbsolutePath("/foo/./bar")).toBe("/foo/bar");
|
||||
expect(normalizeAbsolutePath("/foo/../bar")).toBe("/bar");
|
||||
});
|
||||
|
||||
test("trims trailing slashes", () => {
|
||||
expect(normalizeAbsolutePath("/foo/bar/")).toBe("/foo/bar");
|
||||
expect(normalizeAbsolutePath("/foo/bar//")).toBe("/foo/bar");
|
||||
});
|
||||
|
||||
test("handles windows style paths from URI", () => {
|
||||
expect(normalizeAbsolutePath("foo\\\\bar")).toBe("/foo/bar");
|
||||
expect(normalizeAbsolutePath("foo\\\\bar\\\\")).toBe("/foo/bar");
|
||||
});
|
||||
|
||||
test("handles URI encoded paths", () => {
|
||||
expect(normalizeAbsolutePath("/foo%20bar")).toBe("/foo bar");
|
||||
expect(normalizeAbsolutePath("foo%2Fbar")).toBe("/foo/bar");
|
||||
});
|
||||
|
||||
test("prevents parent traversal beyond root", () => {
|
||||
expect(normalizeAbsolutePath("..")).toBe("/");
|
||||
expect(normalizeAbsolutePath("/..")).toBe("/");
|
||||
expect(normalizeAbsolutePath("/foo/../../bar")).toBe("/bar");
|
||||
});
|
||||
});
|
||||
45
app/utils/path.ts
Normal file
45
app/utils/path.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
export const normalizeAbsolutePath = (value?: string): string => {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) return "/";
|
||||
|
||||
let normalizedInput: string;
|
||||
try {
|
||||
normalizedInput = decodeURIComponent(trimmed).replace(/\\+/g, "/");
|
||||
} catch {
|
||||
normalizedInput = trimmed.replace(/\\+/g, "/");
|
||||
}
|
||||
const withLeadingSlash = normalizedInput.startsWith("/") ? normalizedInput : `/${normalizedInput}`;
|
||||
|
||||
const parts = withLeadingSlash.split("/");
|
||||
const stack: string[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
if (part === "" || part === ".") {
|
||||
continue;
|
||||
}
|
||||
if (part === "..") {
|
||||
if (stack.length > 0) {
|
||||
stack.pop();
|
||||
}
|
||||
} else {
|
||||
stack.push(part);
|
||||
}
|
||||
}
|
||||
|
||||
let normalized = "/" + stack.join("/");
|
||||
|
||||
if (!normalized || normalized === "." || normalized.startsWith("..")) {
|
||||
return "/";
|
||||
}
|
||||
|
||||
const withoutTrailingSlash = normalized.replace(/\/+$/, "");
|
||||
if (!withoutTrailingSlash) {
|
||||
return "/";
|
||||
}
|
||||
|
||||
const withSingleLeadingSlash = withoutTrailingSlash.startsWith("/")
|
||||
? `/${withoutTrailingSlash.replace(/^\/+/, "")}`
|
||||
: `/${withoutTrailingSlash}`;
|
||||
|
||||
return withSingleLeadingSlash || "/";
|
||||
};
|
||||
6
bun.lock
6
bun.lock
|
|
@ -39,6 +39,7 @@
|
|||
"clsx": "^2.1.1",
|
||||
"commander": "^14.0.2",
|
||||
"consola": "^3.4.2",
|
||||
"content-disposition": "^1.0.1",
|
||||
"cron-parser": "^5.5.0",
|
||||
"date-fns": "^4.1.0",
|
||||
"dither-plugin": "^1.1.1",
|
||||
|
|
@ -81,6 +82,7 @@
|
|||
"@testing-library/react": "^16.3.2",
|
||||
"@total-typescript/shoehorn": "^0.1.2",
|
||||
"@types/bun": "^1.3.6",
|
||||
"@types/content-disposition": "^0.5.9",
|
||||
"@types/node": "^25.2.3",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
|
|
@ -831,6 +833,8 @@
|
|||
|
||||
"@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
|
||||
|
||||
"@types/content-disposition": ["@types/content-disposition@0.5.9", "", {}, "sha512-8uYXI3Gw35MhiVYhG3s295oihrxRyytcRHjSjqnqZVDDy/xcGBRny7+Xj1Wgfhv5QzRtN2hB2dVRBUX9XW3UcQ=="],
|
||||
|
||||
"@types/d3-array": ["@types/d3-array@3.2.2", "", {}, "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="],
|
||||
|
||||
"@types/d3-color": ["@types/d3-color@3.1.3", "", {}, "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="],
|
||||
|
|
@ -989,6 +993,8 @@
|
|||
|
||||
"consola": ["consola@3.4.2", "", {}, "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA=="],
|
||||
|
||||
"content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="],
|
||||
|
||||
"convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="],
|
||||
|
||||
"cookie-es": ["cookie-es@2.0.0", "", {}, "sha512-RAj4E421UYRgqokKUmotqAwuplYw15qtdXfY+hGzgCJ/MBjCVZcSoHK/kH9kocfjRjcDME7IiDWR/1WX1TM2Pg=="],
|
||||
|
|
|
|||
|
|
@ -46,8 +46,9 @@ services:
|
|||
- LOG_LEVEL=debug
|
||||
volumes:
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- /var/lib/zerobyte:/var/lib/zerobyte
|
||||
- ~/.config/rclone:/root/.config/rclone:ro
|
||||
- ./tmp:/test-data
|
||||
- ./data:/var/lib/zerobyte
|
||||
|
||||
zerobyte-e2e:
|
||||
build:
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@
|
|||
"clsx": "^2.1.1",
|
||||
"commander": "^14.0.2",
|
||||
"consola": "^3.4.2",
|
||||
"content-disposition": "^1.0.1",
|
||||
"cron-parser": "^5.5.0",
|
||||
"date-fns": "^4.1.0",
|
||||
"dither-plugin": "^1.1.1",
|
||||
|
|
@ -102,6 +103,7 @@
|
|||
"@testing-library/react": "^16.3.2",
|
||||
"@total-typescript/shoehorn": "^0.1.2",
|
||||
"@types/bun": "^1.3.6",
|
||||
"@types/content-disposition": "^0.5.9",
|
||||
"@types/node": "^25.2.3",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
|
|
|
|||
Loading…
Reference in a new issue