feat: export snapshot as tar file
chore(mutext): prevent double lock release
This commit is contained in:
parent
8681ebc0c0
commit
49abcc5a12
23 changed files with 967 additions and 147 deletions
|
|
@ -1,7 +1,8 @@
|
|||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { ChevronDown, FolderOpen, RotateCcw } from "lucide-react";
|
||||
import { ChevronDown, Download, FolderOpen, RotateCcw } from "lucide-react";
|
||||
import { Button } from "~/client/components/ui/button";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "~/client/components/ui/card";
|
||||
import { Input } from "~/client/components/ui/input";
|
||||
import { Label } from "~/client/components/ui/label";
|
||||
|
|
@ -24,6 +25,7 @@ import { OVERWRITE_MODES, type OverwriteMode } from "~/schemas/restic";
|
|||
import type { Repository } from "~/client/lib/types";
|
||||
import { handleRepositoryError } from "~/client/lib/errors";
|
||||
import { useNavigate } from "@tanstack/react-router";
|
||||
import { cn } from "~/client/lib/utils";
|
||||
|
||||
type RestoreLocation = "original" | "custom";
|
||||
|
||||
|
|
@ -40,6 +42,7 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
|
||||
const volumeBasePath = basePath ?? "/";
|
||||
|
||||
const [waitingForDownload, setWaitingForDownload] = useState(false);
|
||||
const [restoreLocation, setRestoreLocation] = useState<RestoreLocation>("original");
|
||||
const [customTargetPath, setCustomTargetPath] = useState("");
|
||||
const [overwriteMode, setOverwriteMode] = useState<OverwriteMode>("always");
|
||||
|
|
@ -53,39 +56,52 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
const [selectedPaths, setSelectedPaths] = useState<Set<string>>(new Set());
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribeStarted = addEventListener("restore:started", (startedData) => {
|
||||
if (startedData.repositoryId === repository.id && startedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = false;
|
||||
setIsRestoreActive(true);
|
||||
setRestoreResult(null);
|
||||
setShowRestoreResultAlert(false);
|
||||
}
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
const signal = abortController.signal;
|
||||
|
||||
const unsubscribeProgress = addEventListener("restore:progress", (progressData) => {
|
||||
if (progressData.repositoryId === repository.id && progressData.snapshotId === snapshotId) {
|
||||
if (restoreCompletedRef.current) {
|
||||
return;
|
||||
addEventListener(
|
||||
"restore:started",
|
||||
(startedData) => {
|
||||
if (startedData.repositoryId === repository.shortId && startedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = false;
|
||||
setIsRestoreActive(true);
|
||||
setRestoreResult(null);
|
||||
setShowRestoreResultAlert(false);
|
||||
}
|
||||
setIsRestoreActive(true);
|
||||
}
|
||||
});
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const unsubscribeCompleted = addEventListener("restore:completed", (completedData) => {
|
||||
if (completedData.repositoryId === repository.id && completedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = true;
|
||||
setIsRestoreActive(false);
|
||||
setRestoreResult(completedData);
|
||||
setShowRestoreResultAlert(true);
|
||||
}
|
||||
});
|
||||
addEventListener(
|
||||
"restore:progress",
|
||||
(progressData) => {
|
||||
if (progressData.repositoryId === repository.shortId && progressData.snapshotId === snapshotId) {
|
||||
if (restoreCompletedRef.current) {
|
||||
return;
|
||||
}
|
||||
setIsRestoreActive(true);
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
addEventListener(
|
||||
"restore:completed",
|
||||
(completedData) => {
|
||||
if (completedData.repositoryId === repository.shortId && completedData.snapshotId === snapshotId) {
|
||||
restoreCompletedRef.current = true;
|
||||
setIsRestoreActive(false);
|
||||
setRestoreResult(completedData);
|
||||
setShowRestoreResultAlert(true);
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribeStarted();
|
||||
unsubscribeProgress();
|
||||
unsubscribeCompleted();
|
||||
abortController.abort();
|
||||
};
|
||||
}, [addEventListener, repository.id, snapshotId]);
|
||||
}, [addEventListener, repository.shortId, snapshotId]);
|
||||
|
||||
const { mutate: restoreSnapshot, isPending: isRestoring } = useMutation({
|
||||
...restoreSnapshotMutation(),
|
||||
|
|
@ -133,6 +149,33 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
restoreSnapshot,
|
||||
]);
|
||||
|
||||
const handleDownload = useCallback(() => {
|
||||
setWaitingForDownload(true);
|
||||
|
||||
if (selectedPaths.size > 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
const dumpUrl = new URL(
|
||||
`/api/v1/repositories/${repository.shortId}/snapshots/${snapshotId}/dump`,
|
||||
window.location.origin,
|
||||
);
|
||||
|
||||
if (selectedPaths.size === 1) {
|
||||
const [selectedPath] = selectedPaths;
|
||||
if (selectedPath) {
|
||||
dumpUrl.searchParams.set("path", selectedPath);
|
||||
}
|
||||
}
|
||||
|
||||
const link = document.createElement("a");
|
||||
link.href = dumpUrl.toString();
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
setWaitingForDownload(false);
|
||||
}, [repository.shortId, snapshotId, selectedPaths]);
|
||||
|
||||
const acknowledgeRestoreResult = useCallback(() => {
|
||||
setShowRestoreResultAlert(false);
|
||||
setRestoreResult(null);
|
||||
|
|
@ -145,35 +188,62 @@ export function RestoreForm({ repository, snapshotId, returnPath, basePath }: Re
|
|||
}, []);
|
||||
|
||||
const canRestore = restoreLocation === "original" || customTargetPath.trim();
|
||||
const canDownload = selectedPaths.size <= 1;
|
||||
const isRestoreRunning = isRestoring || isRestoreActive;
|
||||
|
||||
function getDownloadButtonText(): string {
|
||||
if (selectedPaths.size > 0) {
|
||||
return `Download ${selectedPaths.size} ${selectedPaths.size === 1 ? "item" : "items"}`;
|
||||
}
|
||||
return "Download All";
|
||||
}
|
||||
|
||||
function getRestoreButtonText(): string {
|
||||
if (isRestoreRunning) {
|
||||
return "Restoring...";
|
||||
}
|
||||
if (selectedPaths.size > 0) {
|
||||
return `Restore ${selectedPaths.size} ${selectedPaths.size === 1 ? "item" : "items"}`;
|
||||
}
|
||||
return "Restore All";
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex flex-col gap-4 lg:flex-row lg:items-center lg:justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold">Restore Snapshot</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{repository.name} / {snapshotId}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Button variant="outline" onClick={() => navigate({ to: returnPath })}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<span className="inline-flex">
|
||||
<Button variant="outline" onClick={handleDownload} disabled={!canDownload} loading={waitingForDownload}>
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
{getDownloadButtonText()}
|
||||
</Button>
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className={cn({ hidden: canDownload })}>
|
||||
<p>Download is available only for one selected item, or with no selection to download everything.</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<Button variant="primary" onClick={handleRestore} disabled={isRestoreRunning || !canRestore}>
|
||||
<RotateCcw className="h-4 w-4 mr-2" />
|
||||
{isRestoreRunning
|
||||
? "Restoring..."
|
||||
: selectedPaths.size > 0
|
||||
? `Restore ${selectedPaths.size} ${selectedPaths.size === 1 ? "item" : "items"}`
|
||||
: "Restore All"}
|
||||
{getRestoreButtonText()}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
|
||||
<div className="space-y-6">
|
||||
{isRestoreRunning && <RestoreProgress repositoryId={repository.id} snapshotId={snapshotId} />}
|
||||
{isRestoreRunning && <RestoreProgress repositoryId={repository.shortId} snapshotId={snapshotId} />}
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
|
|
|
|||
|
|
@ -16,22 +16,29 @@ export const RestoreProgress = ({ repositoryId, snapshotId }: Props) => {
|
|||
const [progress, setProgress] = useState<RestoreProgressEvent | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = addEventListener("restore:progress", (progressData) => {
|
||||
if (progressData.repositoryId === repositoryId && progressData.snapshotId === snapshotId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
|
||||
const unsubscribeComplete = addEventListener("restore:completed", (completedData) => {
|
||||
if (completedData.repositoryId === repositoryId && completedData.snapshotId === snapshotId) {
|
||||
setProgress(null);
|
||||
}
|
||||
});
|
||||
addEventListener(
|
||||
"restore:progress",
|
||||
(progressData) => {
|
||||
if (progressData.repositoryId === repositoryId && progressData.snapshotId === snapshotId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
unsubscribeComplete();
|
||||
};
|
||||
addEventListener(
|
||||
"restore:completed",
|
||||
(completedData) => {
|
||||
if (completedData.repositoryId === repositoryId && completedData.snapshotId === snapshotId) {
|
||||
setProgress(null);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => abortController.abort();
|
||||
}, [addEventListener, repositoryId, snapshotId]);
|
||||
|
||||
if (!progress) {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { Slot } from "@radix-ui/react-slot";
|
||||
import { cva, type VariantProps } from "class-variance-authority";
|
||||
import { Loader2 } from "lucide-react";
|
||||
import type * as React from "react";
|
||||
import * as React from "react";
|
||||
|
||||
import { cn } from "~/client/lib/utils";
|
||||
import { useMinimumDuration } from "~/client/hooks/useMinimumDuration";
|
||||
|
||||
const buttonVariants = cva(
|
||||
"inline-flex cursor-pointer uppercase rounded-sm items-center justify-center gap-2 whitespace-nowrap text-xs font-semibold tracking-wide transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-ring border-0",
|
||||
|
|
@ -33,28 +34,32 @@ const buttonVariants = cva(
|
|||
},
|
||||
);
|
||||
|
||||
const MINIMUM_LOADING_DURATION = 300;
|
||||
|
||||
function Button({
|
||||
className,
|
||||
variant,
|
||||
size,
|
||||
asChild = false,
|
||||
loading,
|
||||
disabled,
|
||||
...props
|
||||
}: React.ComponentProps<"button"> &
|
||||
VariantProps<typeof buttonVariants> & {
|
||||
asChild?: boolean;
|
||||
} & { loading?: boolean }) {
|
||||
const Comp = asChild ? Slot : "button";
|
||||
const isLoading = useMinimumDuration(loading ?? false, MINIMUM_LOADING_DURATION);
|
||||
|
||||
return (
|
||||
<Comp
|
||||
disabled={loading}
|
||||
data-slot="button"
|
||||
className={cn(buttonVariants({ variant, size, className }), "transition-all")}
|
||||
disabled={disabled || loading || isLoading}
|
||||
{...props}
|
||||
>
|
||||
<Loader2 className={cn("h-4 w-4 animate-spin absolute", { invisible: !loading })} />
|
||||
<div className={cn("flex items-center justify-center", { invisible: loading })}>{props.children}</div>
|
||||
<Loader2 className={cn("h-4 w-4 animate-spin absolute", { invisible: !isLoading })} />
|
||||
<div className={cn("flex items-center justify-center", { invisible: isLoading })}>{props.children}</div>
|
||||
</Comp>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,20 +98,36 @@ export function useServerEvents() {
|
|||
};
|
||||
}, [emit, queryClient]);
|
||||
|
||||
const addEventListener = useCallback(<T extends ServerEventType>(eventName: T, handler: EventHandler<T>) => {
|
||||
const existingHandlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
const eventHandlers = existingHandlers ?? new Set<EventHandler<T>>();
|
||||
eventHandlers.add(handler);
|
||||
handlersRef.current[eventName] = eventHandlers as EventHandlerMap[T];
|
||||
|
||||
return () => {
|
||||
const handlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
handlers?.delete(handler);
|
||||
if (handlers && handlers.size === 0) {
|
||||
delete handlersRef.current[eventName];
|
||||
const addEventListener = useCallback(
|
||||
<T extends ServerEventType>(eventName: T, handler: EventHandler<T>, options?: { signal?: AbortSignal }) => {
|
||||
if (options?.signal?.aborted) {
|
||||
return () => {};
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const existingHandlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
const eventHandlers = existingHandlers ?? new Set<EventHandler<T>>();
|
||||
eventHandlers.add(handler);
|
||||
handlersRef.current[eventName] = eventHandlers as EventHandlerMap[T];
|
||||
|
||||
const unsubscribe = () => {
|
||||
const handlers = handlersRef.current[eventName] as EventHandlerSet<T> | undefined;
|
||||
handlers?.delete(handler);
|
||||
if (handlers && handlers.size === 0) {
|
||||
delete handlersRef.current[eventName];
|
||||
}
|
||||
if (options?.signal) {
|
||||
options.signal.removeEventListener("abort", unsubscribe);
|
||||
}
|
||||
};
|
||||
|
||||
if (options?.signal) {
|
||||
options.signal.addEventListener("abort", unsubscribe, { once: true });
|
||||
}
|
||||
|
||||
return unsubscribe;
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
return { addEventListener };
|
||||
}
|
||||
|
|
|
|||
39
app/client/hooks/useMinimumDuration.ts
Normal file
39
app/client/hooks/useMinimumDuration.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
export function useMinimumDuration(isActive: boolean, minimumDuration: number): boolean {
|
||||
const [displayActive, setDisplayActive] = useState(isActive);
|
||||
const startTimeRef = useRef<number | null>(null);
|
||||
const timeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (isActive) {
|
||||
if (timeoutRef.current) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
timeoutRef.current = null;
|
||||
}
|
||||
startTimeRef.current = Date.now();
|
||||
setDisplayActive(true);
|
||||
} else if (!isActive && startTimeRef.current !== null) {
|
||||
const elapsed = Date.now() - startTimeRef.current;
|
||||
const remaining = Math.max(0, minimumDuration - elapsed);
|
||||
|
||||
if (remaining > 0) {
|
||||
timeoutRef.current = setTimeout(() => {
|
||||
setDisplayActive(false);
|
||||
startTimeRef.current = null;
|
||||
}, remaining);
|
||||
} else {
|
||||
setDisplayActive(false);
|
||||
startTimeRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (timeoutRef.current) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, [isActive, minimumDuration]);
|
||||
|
||||
return displayActive;
|
||||
}
|
||||
|
|
@ -15,22 +15,29 @@ export const BackupProgressCard = ({ scheduleShortId }: Props) => {
|
|||
const [progress, setProgress] = useState<BackupProgressEvent | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribe = addEventListener("backup:progress", (progressData) => {
|
||||
if (progressData.scheduleId === scheduleShortId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
|
||||
const unsubscribeComplete = addEventListener("backup:completed", (completedData) => {
|
||||
if (completedData.scheduleId === scheduleShortId) {
|
||||
setProgress(null);
|
||||
}
|
||||
});
|
||||
addEventListener(
|
||||
"backup:progress",
|
||||
(progressData) => {
|
||||
if (progressData.scheduleId === scheduleShortId) {
|
||||
setProgress(progressData);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
unsubscribeComplete();
|
||||
};
|
||||
addEventListener(
|
||||
"backup:completed",
|
||||
(completedData) => {
|
||||
if (completedData.scheduleId === scheduleShortId) {
|
||||
setProgress(null);
|
||||
}
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => abortController.abort();
|
||||
}, [addEventListener, scheduleShortId]);
|
||||
|
||||
const percentDone = progress ? Math.round(progress.percent_done * 100) : 0;
|
||||
|
|
|
|||
|
|
@ -99,37 +99,44 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re
|
|||
}, [compatibility]);
|
||||
|
||||
useEffect(() => {
|
||||
const unsubscribeStarted = addEventListener("mirror:started", (event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, { ...existing, lastCopyStatus: "in_progress", lastCopyError: null });
|
||||
return next;
|
||||
});
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
|
||||
const unsubscribeCompleted = addEventListener("mirror:completed", (event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, {
|
||||
...existing,
|
||||
lastCopyStatus: event.status ?? existing.lastCopyStatus,
|
||||
lastCopyError: event.error ?? null,
|
||||
lastCopyAt: Date.now(),
|
||||
addEventListener(
|
||||
"mirror:started",
|
||||
(event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, { ...existing, lastCopyStatus: "in_progress", lastCopyError: null });
|
||||
return next;
|
||||
});
|
||||
return next;
|
||||
});
|
||||
});
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => {
|
||||
unsubscribeStarted();
|
||||
unsubscribeCompleted();
|
||||
};
|
||||
addEventListener(
|
||||
"mirror:completed",
|
||||
(event) => {
|
||||
if (event.scheduleId !== scheduleShortId) return;
|
||||
setAssignments((prev) => {
|
||||
const next = new Map(prev);
|
||||
const existing = next.get(event.repositoryId);
|
||||
if (!existing) return prev;
|
||||
next.set(event.repositoryId, {
|
||||
...existing,
|
||||
lastCopyStatus: event.status ?? existing.lastCopyStatus,
|
||||
lastCopyError: event.error ?? null,
|
||||
lastCopyAt: Date.now(),
|
||||
});
|
||||
return next;
|
||||
});
|
||||
},
|
||||
{ signal: abortController.signal },
|
||||
);
|
||||
|
||||
return () => abortController.abort();
|
||||
}, [addEventListener, scheduleShortId]);
|
||||
|
||||
const addRepository = (repositoryId: string) => {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,13 @@ const restoreEventBaseSchema = type({
|
|||
snapshotId: "string",
|
||||
});
|
||||
|
||||
const dumpStartedEventSchema = type({
|
||||
repositoryId: "string",
|
||||
snapshotId: "string",
|
||||
path: "string",
|
||||
filename: "string",
|
||||
});
|
||||
|
||||
const restoreProgressMetricsSchema = type({
|
||||
seconds_elapsed: "number",
|
||||
percent_done: "number",
|
||||
|
|
@ -59,6 +66,8 @@ export const serverRestoreProgressEventSchema = organizationScopedSchema.and(res
|
|||
|
||||
export const serverRestoreCompletedEventSchema = organizationScopedSchema.and(restoreCompletedEventSchema);
|
||||
|
||||
export const serverDumpStartedEventSchema = organizationScopedSchema.and(dumpStartedEventSchema);
|
||||
|
||||
export type BackupEventStatusDto = typeof backupEventStatusSchema.infer;
|
||||
export type BackupStartedEventDto = typeof backupStartedEventSchema.infer;
|
||||
export type BackupProgressEventDto = typeof backupProgressEventSchema.infer;
|
||||
|
|
@ -66,9 +75,11 @@ export type BackupCompletedEventDto = typeof backupCompletedEventSchema.infer;
|
|||
export type RestoreStartedEventDto = typeof restoreStartedEventSchema.infer;
|
||||
export type RestoreProgressEventDto = typeof restoreProgressEventSchema.infer;
|
||||
export type RestoreCompletedEventDto = typeof restoreCompletedEventSchema.infer;
|
||||
export type DumpStartedEventDto = typeof dumpStartedEventSchema.infer;
|
||||
export type ServerBackupStartedEventDto = typeof serverBackupStartedEventSchema.infer;
|
||||
export type ServerBackupProgressEventDto = typeof serverBackupProgressEventSchema.infer;
|
||||
export type ServerBackupCompletedEventDto = typeof serverBackupCompletedEventSchema.infer;
|
||||
export type ServerRestoreStartedEventDto = typeof serverRestoreStartedEventSchema.infer;
|
||||
export type ServerRestoreProgressEventDto = typeof serverRestoreProgressEventSchema.infer;
|
||||
export type ServerRestoreCompletedEventDto = typeof serverRestoreCompletedEventSchema.infer;
|
||||
export type ServerDumpStartedEventDto = typeof serverDumpStartedEventSchema.infer;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import type {
|
|||
ServerBackupCompletedEventDto,
|
||||
ServerBackupProgressEventDto,
|
||||
ServerBackupStartedEventDto,
|
||||
ServerDumpStartedEventDto,
|
||||
ServerRestoreCompletedEventDto,
|
||||
ServerRestoreProgressEventDto,
|
||||
ServerRestoreStartedEventDto,
|
||||
|
|
@ -21,6 +22,7 @@ export const serverEventPayloads = {
|
|||
"restore:started": payload<ServerRestoreStartedEventDto>(),
|
||||
"restore:progress": payload<ServerRestoreProgressEventDto>(),
|
||||
"restore:completed": payload<ServerRestoreCompletedEventDto>(),
|
||||
"dump:started": payload<ServerDumpStartedEventDto>(),
|
||||
"mirror:started": payload<{
|
||||
organizationId: string;
|
||||
scheduleId: string;
|
||||
|
|
|
|||
|
|
@ -122,4 +122,163 @@ describe("RepositoryMutex", () => {
|
|||
release();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should allow concurrent shared locks", async () => {
|
||||
const repoId = "concurrent-shared";
|
||||
const release1 = await repoMutex.acquireShared(repoId, "op1");
|
||||
const release2 = await repoMutex.acquireShared(repoId, "op2");
|
||||
const release3 = await repoMutex.acquireShared(repoId, "op3");
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
release1();
|
||||
release2();
|
||||
release3();
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should block exclusive lock until all shared locks are released", async () => {
|
||||
const repoId = "shared-blocks-exclusive";
|
||||
let exclusiveAcquired = false;
|
||||
|
||||
const releaseShared1 = await repoMutex.acquireShared(repoId, "s1");
|
||||
const releaseShared2 = await repoMutex.acquireShared(repoId, "s2");
|
||||
|
||||
const exclusivePromise = repoMutex.acquireExclusive(repoId, "e1").then((release) => {
|
||||
exclusiveAcquired = true;
|
||||
return release;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(exclusiveAcquired).toBe(false);
|
||||
|
||||
releaseShared1();
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(exclusiveAcquired).toBe(false); // still waiting for s2
|
||||
|
||||
releaseShared2();
|
||||
const releaseExclusive = await exclusivePromise;
|
||||
expect(exclusiveAcquired).toBe(true);
|
||||
|
||||
releaseExclusive();
|
||||
});
|
||||
|
||||
test("should block all locks while exclusive lock is held", async () => {
|
||||
const repoId = "exclusive-blocks-all";
|
||||
const results: string[] = [];
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
results.push("e1-acquired");
|
||||
|
||||
const s1Promise = repoMutex.acquireShared(repoId, "s1").then((release) => {
|
||||
results.push("s1-acquired");
|
||||
return release;
|
||||
});
|
||||
const e2Promise = repoMutex.acquireExclusive(repoId, "e2").then((release) => {
|
||||
results.push("e2-acquired");
|
||||
return release;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(results).toEqual(["e1-acquired"]);
|
||||
|
||||
releaseExclusive();
|
||||
|
||||
const releaseS1 = await s1Promise;
|
||||
expect(results).toEqual(["e1-acquired", "s1-acquired"]);
|
||||
|
||||
releaseS1();
|
||||
|
||||
const releaseE2 = await e2Promise;
|
||||
expect(results).toEqual(["e1-acquired", "s1-acquired", "e2-acquired"]);
|
||||
|
||||
releaseE2();
|
||||
});
|
||||
|
||||
test("should grant all waiting shared locks at once when exclusive lock is released", async () => {
|
||||
const repoId = "batch-shared";
|
||||
const results: string[] = [];
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
|
||||
const s1Promise = repoMutex.acquireShared(repoId, "s1").then((release) => {
|
||||
results.push("s1");
|
||||
return release;
|
||||
});
|
||||
const s2Promise = repoMutex.acquireShared(repoId, "s2").then((release) => {
|
||||
results.push("s2");
|
||||
return release;
|
||||
});
|
||||
const s3Promise = repoMutex.acquireShared(repoId, "s3").then((release) => {
|
||||
results.push("s3");
|
||||
return release;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(results).toEqual([]);
|
||||
|
||||
releaseExclusive();
|
||||
|
||||
const [releaseS1, releaseS2, releaseS3] = await Promise.all([s1Promise, s2Promise, s3Promise]);
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
expect(results).toContain("s1");
|
||||
expect(results).toContain("s2");
|
||||
expect(results).toContain("s3");
|
||||
|
||||
releaseS1();
|
||||
releaseS2();
|
||||
releaseS3();
|
||||
});
|
||||
|
||||
test("should safely handle multiple calls to the release function", async () => {
|
||||
const repoId = "idempotent-release";
|
||||
|
||||
const releaseShared = await repoMutex.acquireShared(repoId, "s1");
|
||||
releaseShared();
|
||||
releaseShared(); // Should not throw or cause issues
|
||||
releaseShared();
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
releaseExclusive();
|
||||
releaseExclusive(); // Should not throw
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should immediately throw if AbortSignal is already aborted", async () => {
|
||||
const repoId = "already-aborted";
|
||||
const controller = new AbortController();
|
||||
controller.abort(new Error("pre-aborted"));
|
||||
|
||||
expect(repoMutex.acquireShared(repoId, "s1", controller.signal)).rejects.toThrow("pre-aborted");
|
||||
expect(repoMutex.acquireExclusive(repoId, "e1", controller.signal)).rejects.toThrow("pre-aborted");
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
|
||||
test("should accurately report isLocked status", async () => {
|
||||
const repoId = "is-locked-status";
|
||||
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
|
||||
const releaseShared1 = await repoMutex.acquireShared(repoId, "s1");
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
const releaseShared2 = await repoMutex.acquireShared(repoId, "s2");
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
releaseShared1();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true); // still locked by s2
|
||||
|
||||
releaseShared2();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false); // all shared released
|
||||
|
||||
const releaseExclusive = await repoMutex.acquireExclusive(repoId, "e1");
|
||||
expect(repoMutex.isLocked(repoId)).toBe(true);
|
||||
|
||||
releaseExclusive();
|
||||
expect(repoMutex.isLocked(repoId)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ class RepositoryMutex {
|
|||
throw signal.reason || new Error("Operation aborted");
|
||||
}
|
||||
|
||||
return () => this.releaseShared(repositoryId, lockId!);
|
||||
return this.createRelease("shared", repositoryId, lockId!);
|
||||
}
|
||||
|
||||
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal): Promise<() => void> {
|
||||
|
|
@ -154,7 +154,8 @@ class RepositoryMutex {
|
|||
}
|
||||
|
||||
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation} (${lockId})`);
|
||||
return () => this.releaseExclusive(repositoryId, lockId!);
|
||||
|
||||
return this.createRelease("exclusive", repositoryId, lockId!);
|
||||
}
|
||||
|
||||
private releaseShared(repositoryId: string, lockId: string): void {
|
||||
|
|
@ -239,6 +240,19 @@ class RepositoryMutex {
|
|||
if (!state) return false;
|
||||
return state.exclusiveHolder !== null || state.sharedHolders.size > 0;
|
||||
}
|
||||
|
||||
private createRelease(type: LockType, repositoryId: string, lockId: string) {
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
if (type === "shared") {
|
||||
this.releaseShared(repositoryId, lockId);
|
||||
} else {
|
||||
this.releaseExclusive(repositoryId, lockId);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const repoMutex = new RepositoryMutex();
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ const broadcastEvents = [
|
|||
"restore:started",
|
||||
"restore:progress",
|
||||
"restore:completed",
|
||||
"dump:started",
|
||||
"doctor:started",
|
||||
"doctor:completed",
|
||||
"doctor:cancelled",
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ describe("repositories security", () => {
|
|||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots" },
|
||||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot" },
|
||||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot/files" },
|
||||
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot/dump" },
|
||||
{ method: "POST", path: "/api/v1/repositories/test-repo/restore" },
|
||||
{ method: "POST", path: "/api/v1/repositories/test-repo/doctor" },
|
||||
{ method: "DELETE", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot" },
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
|
||||
import type { RepositoryConfig } from "~/schemas/restic";
|
||||
import { REPOSITORY_BASE } from "~/server/core/constants";
|
||||
import { serverEvents } from "~/server/core/events";
|
||||
import { withContext } from "~/server/core/request-context";
|
||||
import { db } from "~/server/db/db";
|
||||
import { repositoriesTable } from "~/server/db/schema";
|
||||
import { generateShortId } from "~/server/utils/id";
|
||||
import { restic } from "~/server/utils/restic";
|
||||
import { createTestSession } from "~/test/helpers/auth";
|
||||
import { repositoriesService } from "../repositories.service";
|
||||
|
|
@ -77,3 +81,130 @@ describe("repositoriesService.createRepository", () => {
|
|||
expect(created.status).toBe("healthy");
|
||||
});
|
||||
});
|
||||
|
||||
describe("repositoriesService.dumpSnapshot", () => {
|
||||
afterEach(() => {
|
||||
mock.restore();
|
||||
});
|
||||
|
||||
test("calls restic.dump with common-ancestor selector and stripped path", async () => {
|
||||
const { organizationId, user } = await createTestSession();
|
||||
const shortId = generateShortId();
|
||||
const basePath = "/var/lib/zerobyte/volumes/vol123/_data";
|
||||
|
||||
await db.insert(repositoriesTable).values({
|
||||
id: randomUUID(),
|
||||
shortId,
|
||||
name: `Repository-${randomUUID()}`,
|
||||
type: "local",
|
||||
config: {
|
||||
backend: "local",
|
||||
path: `/tmp/repository-${randomUUID()}`,
|
||||
isExistingRepository: true,
|
||||
},
|
||||
compressionMode: "off",
|
||||
organizationId,
|
||||
});
|
||||
|
||||
const snapshotsMock = mock(() =>
|
||||
Promise.resolve([
|
||||
{
|
||||
id: "snapshot-123",
|
||||
short_id: "snapshot-123",
|
||||
time: new Date().toISOString(),
|
||||
tree: "tree-1",
|
||||
paths: [basePath],
|
||||
hostname: "host",
|
||||
},
|
||||
]),
|
||||
);
|
||||
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
|
||||
|
||||
const dumpMock = mock(() =>
|
||||
Promise.resolve({
|
||||
stream: Readable.from([]),
|
||||
completion: Promise.resolve(),
|
||||
abort: () => {},
|
||||
}),
|
||||
);
|
||||
spyOn(restic, "dump").mockImplementation(dumpMock);
|
||||
const emitSpy = spyOn(serverEvents, "emit");
|
||||
|
||||
await withContext({ organizationId, userId: user.id }, () =>
|
||||
repositoriesService.dumpSnapshot(shortId, "snapshot-123", `${basePath}/documents`),
|
||||
);
|
||||
|
||||
expect(dumpMock).toHaveBeenCalledTimes(1);
|
||||
expect(dumpMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
backend: "local",
|
||||
}),
|
||||
`snapshot-123:${basePath}`,
|
||||
{
|
||||
organizationId,
|
||||
path: "/documents",
|
||||
},
|
||||
);
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
"dump:started",
|
||||
expect.objectContaining({
|
||||
organizationId,
|
||||
repositoryId: shortId,
|
||||
snapshotId: "snapshot-123",
|
||||
path: "/documents",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("downloads full snapshot relative to common ancestor when path is omitted", async () => {
|
||||
const { organizationId, user } = await createTestSession();
|
||||
const shortId = generateShortId();
|
||||
const basePath = "/var/lib/zerobyte/volumes/vol555/_data";
|
||||
|
||||
await db.insert(repositoriesTable).values({
|
||||
id: randomUUID(),
|
||||
shortId,
|
||||
name: `Repository-${randomUUID()}`,
|
||||
type: "local",
|
||||
config: {
|
||||
backend: "local",
|
||||
path: `/tmp/repository-${randomUUID()}`,
|
||||
isExistingRepository: true,
|
||||
},
|
||||
compressionMode: "off",
|
||||
organizationId,
|
||||
});
|
||||
|
||||
const snapshotsMock = mock(() =>
|
||||
Promise.resolve([
|
||||
{
|
||||
id: "snapshot-999",
|
||||
short_id: "snapshot-999",
|
||||
time: new Date().toISOString(),
|
||||
tree: "tree-9",
|
||||
paths: [basePath],
|
||||
hostname: "host",
|
||||
},
|
||||
]),
|
||||
);
|
||||
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
|
||||
|
||||
const dumpMock = mock(() =>
|
||||
Promise.resolve({
|
||||
stream: Readable.from([]),
|
||||
completion: Promise.resolve(),
|
||||
abort: () => {},
|
||||
}),
|
||||
);
|
||||
spyOn(restic, "dump").mockImplementation(dumpMock);
|
||||
|
||||
await withContext({ organizationId, userId: user.id }, () =>
|
||||
repositoriesService.dumpSnapshot(shortId, "snapshot-999"),
|
||||
);
|
||||
|
||||
expect(dumpMock).toHaveBeenCalledWith(expect.anything(), `snapshot-999:${basePath}`, {
|
||||
organizationId,
|
||||
path: "/",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../../db/db";
|
||||
import { repositoriesTable } from "../../db/schema";
|
||||
import { toMessage } from "../../utils/errors";
|
||||
import { restic } from "../../utils/restic";
|
||||
import { repoMutex } from "../../core/repository-mutex";
|
||||
import { type DoctorStep, type DoctorResult, type RepositoryConfig } from "~/schemas/restic";
|
||||
import { type } from "arktype";
|
||||
import { serverEvents } from "../../core/events";
|
||||
import { logger } from "../../utils/logger";
|
||||
import { safeJsonParse } from "../../utils/json";
|
||||
import { getOrganizationId } from "~/server/core/request-context";
|
||||
import { restic } from "~/server/utils/restic";
|
||||
import { toMessage } from "~/server/utils/errors";
|
||||
import { safeJsonParse } from "~/server/utils/json";
|
||||
import { logger } from "~/server/utils/logger";
|
||||
import { db } from "~/server/db/db";
|
||||
import { repositoriesTable } from "~/server/db/schema";
|
||||
import { repoMutex } from "~/server/core/repository-mutex";
|
||||
import { serverEvents } from "~/server/core/events";
|
||||
|
||||
class AbortError extends Error {
|
||||
name = "AbortError";
|
||||
73
app/server/modules/repositories/helpers/dump.ts
Normal file
73
app/server/modules/repositories/helpers/dump.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import { BadRequestError } from "http-errors-enhanced";
|
||||
import path from "node:path";
|
||||
import { findCommonAncestor } from "~/utils/common-ancestor";
|
||||
|
||||
const normalizeAbsolutePath = (value?: string): string => {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) return "/";
|
||||
|
||||
const normalizedInput = decodeURIComponent(trimmed).replace(/\\+/g, "/");
|
||||
const withLeadingSlash = normalizedInput.startsWith("/") ? normalizedInput : `/${normalizedInput}`;
|
||||
const normalized = path.posix.normalize(withLeadingSlash);
|
||||
|
||||
if (!normalized || normalized === "." || normalized.startsWith("..")) {
|
||||
return "/";
|
||||
}
|
||||
|
||||
const withoutTrailingSlash = normalized.replace(/\/+$/, "");
|
||||
if (!withoutTrailingSlash) {
|
||||
return "/";
|
||||
}
|
||||
|
||||
const withSingleLeadingSlash = withoutTrailingSlash.startsWith("/")
|
||||
? `/${withoutTrailingSlash.replace(/^\/+/, "")}`
|
||||
: `/${withoutTrailingSlash}`;
|
||||
|
||||
return withSingleLeadingSlash || "/";
|
||||
};
|
||||
|
||||
const sanitizeFilenamePart = (value: string): string => {
|
||||
const sanitized = value.replace(/[^a-zA-Z0-9._-]/g, "_").replace(/^_+|_+$/g, "");
|
||||
return sanitized || "snapshot";
|
||||
};
|
||||
|
||||
export const prepareSnapshotDump = (params: {
|
||||
snapshotId: string;
|
||||
snapshotPaths: string[];
|
||||
requestedPath?: string;
|
||||
}) => {
|
||||
const { snapshotId, snapshotPaths, requestedPath } = params;
|
||||
|
||||
const archiveFilename = `snapshot-${sanitizeFilenamePart(snapshotId)}.tar`;
|
||||
const normalizedRequestedPath = normalizeAbsolutePath(requestedPath);
|
||||
const basePath = normalizeAbsolutePath(findCommonAncestor(snapshotPaths));
|
||||
|
||||
if (basePath === "/") {
|
||||
return {
|
||||
snapshotRef: snapshotId,
|
||||
path: normalizedRequestedPath,
|
||||
filename: archiveFilename,
|
||||
};
|
||||
}
|
||||
|
||||
if (normalizedRequestedPath === "/" || normalizedRequestedPath === basePath) {
|
||||
return {
|
||||
snapshotRef: `${snapshotId}:${basePath}`,
|
||||
path: "/",
|
||||
filename: archiveFilename,
|
||||
};
|
||||
}
|
||||
|
||||
const relativeFromBase = path.posix.relative(basePath, normalizedRequestedPath);
|
||||
if (relativeFromBase === ".." || relativeFromBase.startsWith("../")) {
|
||||
throw new BadRequestError("Requested path is outside the snapshot base path");
|
||||
}
|
||||
|
||||
const relativePath = relativeFromBase ? `/${relativeFromBase}` : "/";
|
||||
|
||||
return {
|
||||
snapshotRef: `${snapshotId}:${basePath}`,
|
||||
path: relativePath,
|
||||
filename: archiveFilename,
|
||||
};
|
||||
};
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { Readable } from "node:stream";
|
||||
import { Hono } from "hono";
|
||||
import { validator } from "hono-openapi";
|
||||
import { streamSSE } from "hono/streaming";
|
||||
|
|
@ -19,6 +20,8 @@ import {
|
|||
listSnapshotFilesQuery,
|
||||
listSnapshotsDto,
|
||||
listSnapshotsFilters,
|
||||
dumpSnapshotDto,
|
||||
dumpSnapshotQuery,
|
||||
restoreSnapshotBody,
|
||||
restoreSnapshotDto,
|
||||
tagSnapshotsBody,
|
||||
|
|
@ -48,6 +51,7 @@ import { repositoriesService } from "./repositories.service";
|
|||
import { getRcloneRemoteInfo, listRcloneRemotes } from "../../utils/rclone";
|
||||
import { requireAuth, requireOrgAdmin } from "../auth/auth.middleware";
|
||||
import { toMessage } from "~/server/utils/errors";
|
||||
import { sanitizeContentDispositionFilename } from "~/server/utils/sanitize";
|
||||
import { requireDevPanel } from "../auth/dev-panel.middleware";
|
||||
import { getSnapshotDuration } from "../../utils/snapshots";
|
||||
|
||||
|
|
@ -165,6 +169,32 @@ export const repositoriesController = new Hono()
|
|||
return c.json<ListSnapshotFilesDto>(result, 200);
|
||||
},
|
||||
)
|
||||
.get("/:shortId/snapshots/:snapshotId/dump", dumpSnapshotDto, validator("query", dumpSnapshotQuery), async (c) => {
|
||||
const { shortId, snapshotId } = c.req.param();
|
||||
const { path } = c.req.valid("query");
|
||||
|
||||
const dumpStream = await repositoriesService.dumpSnapshot(shortId, snapshotId, path);
|
||||
const signal = c.req.raw.signal;
|
||||
|
||||
if (signal.aborted) {
|
||||
dumpStream.abort();
|
||||
} else {
|
||||
signal.addEventListener("abort", () => dumpStream.abort(), { once: true });
|
||||
}
|
||||
|
||||
const safeFilename = sanitizeContentDispositionFilename(dumpStream.filename);
|
||||
|
||||
const webStream = Readable.toWeb(dumpStream.stream) as unknown as ReadableStream<Uint8Array>;
|
||||
|
||||
return new Response(webStream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "application/x-tar",
|
||||
"Content-Disposition": `attachment; filename="${safeFilename}"`,
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
});
|
||||
})
|
||||
.post("/:shortId/restore", restoreSnapshotDto, validator("json", restoreSnapshotBody), async (c) => {
|
||||
const { shortId } = c.req.param();
|
||||
const { snapshotId, ...options } = c.req.valid("json");
|
||||
|
|
|
|||
|
|
@ -290,6 +290,29 @@ export const listSnapshotFilesDto = describeRoute({
|
|||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Download snapshot or a specific path as tar archive
|
||||
*/
|
||||
export const dumpSnapshotQuery = type({
|
||||
path: "string?",
|
||||
});
|
||||
|
||||
export const dumpSnapshotDto = describeRoute({
|
||||
description: "Download a full snapshot or a single file/folder path as a tar archive",
|
||||
tags: ["Repositories"],
|
||||
operationId: "dumpSnapshot",
|
||||
responses: {
|
||||
200: {
|
||||
description: "Snapshot archive stream",
|
||||
content: {
|
||||
"application/x-tar": {
|
||||
schema: { type: "string", format: "binary" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Restore a snapshot
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -23,9 +23,10 @@ import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys, restic } f
|
|||
import { safeSpawn } from "../../utils/spawn";
|
||||
import { backupsService } from "../backups/backups.service";
|
||||
import type { UpdateRepositoryBody } from "./repositories.dto";
|
||||
import { executeDoctor } from "./doctor";
|
||||
import { REPOSITORY_BASE } from "~/server/core/constants";
|
||||
import { findCommonAncestor } from "~/utils/common-ancestor";
|
||||
import { prepareSnapshotDump } from "./helpers/dump";
|
||||
import { executeDoctor } from "./helpers/doctor";
|
||||
|
||||
const runningDoctors = new Map<string, AbortController>();
|
||||
|
||||
|
|
@ -389,6 +390,42 @@ const restoreSnapshot = async (
|
|||
}
|
||||
};
|
||||
|
||||
const dumpSnapshot = async (shortId: string, snapshotId: string, path?: string) => {
|
||||
const organizationId = getOrganizationId();
|
||||
const repository = await findRepository(shortId);
|
||||
|
||||
if (!repository) {
|
||||
throw new NotFoundError("Repository not found");
|
||||
}
|
||||
|
||||
const releaseLock = await repoMutex.acquireShared(repository.id, `dump:${snapshotId}`);
|
||||
|
||||
try {
|
||||
const snapshot = await getSnapshotDetails(repository.shortId, snapshotId);
|
||||
const preparedDump = prepareSnapshotDump({ snapshotId, snapshotPaths: snapshot.paths, requestedPath: path });
|
||||
const dumpStream = await restic.dump(repository.config, preparedDump.snapshotRef, {
|
||||
organizationId,
|
||||
path: preparedDump.path,
|
||||
});
|
||||
|
||||
serverEvents.emit("dump:started", {
|
||||
organizationId,
|
||||
repositoryId: repository.shortId,
|
||||
snapshotId,
|
||||
path: preparedDump.path,
|
||||
filename: preparedDump.filename,
|
||||
});
|
||||
|
||||
const completion = dumpStream.completion.finally(releaseLock);
|
||||
void completion.catch(() => {});
|
||||
|
||||
return { ...dumpStream, completion, filename: preparedDump.filename };
|
||||
} catch (error) {
|
||||
releaseLock();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const getSnapshotDetails = async (shortId: string, snapshotId: string) => {
|
||||
const organizationId = getOrganizationId();
|
||||
const repository = await findRepository(shortId);
|
||||
|
|
@ -777,6 +814,7 @@ export const repositoriesService = {
|
|||
listSnapshots,
|
||||
listSnapshotFiles,
|
||||
restoreSnapshot,
|
||||
dumpSnapshot,
|
||||
getSnapshotDetails,
|
||||
checkHealth,
|
||||
startDoctor,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import crypto from "node:crypto";
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import { type } from "arktype";
|
||||
import { throttle } from "es-toolkit";
|
||||
import type { BandwidthLimit, CompressionMode, OverwriteMode, RepositoryConfig } from "~/schemas/restic";
|
||||
|
|
@ -83,10 +84,14 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
const decryptedPassword = await cryptoUtils.resolveSecret(config.customPassword);
|
||||
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
|
||||
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env.RESTIC_PASSWORD_FILE = passwordFilePath;
|
||||
} else {
|
||||
const org = await db.query.organization.findFirst({ where: { id: organizationId } });
|
||||
const org = await db.query.organization.findFirst({
|
||||
where: { id: organizationId },
|
||||
});
|
||||
|
||||
if (!org) {
|
||||
throw new Error(`Organization ${organizationId} not found`);
|
||||
|
|
@ -98,7 +103,9 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
} else {
|
||||
const decryptedPassword = await cryptoUtils.resolveSecret(metadata.resticPassword);
|
||||
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
|
||||
await fs.writeFile(passwordFilePath, decryptedPassword, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env.RESTIC_PASSWORD_FILE = passwordFilePath;
|
||||
}
|
||||
}
|
||||
|
|
@ -122,7 +129,9 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
case "gcs": {
|
||||
const decryptedCredentials = await cryptoUtils.resolveSecret(config.credentialsJson);
|
||||
const credentialsPath = path.join("/tmp", `zerobyte-gcs-${crypto.randomBytes(8).toString("hex")}.json`);
|
||||
await fs.writeFile(credentialsPath, decryptedCredentials, { mode: 0o600 });
|
||||
await fs.writeFile(credentialsPath, decryptedCredentials, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env.GOOGLE_PROJECT_ID = config.projectId;
|
||||
env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;
|
||||
break;
|
||||
|
|
@ -193,7 +202,9 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
|||
sshArgs.push("-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null");
|
||||
} else if (config.knownHosts) {
|
||||
const knownHostsPath = path.join("/tmp", `zerobyte-known-hosts-${crypto.randomBytes(8).toString("hex")}`);
|
||||
await fs.writeFile(knownHostsPath, config.knownHosts, { mode: 0o600 });
|
||||
await fs.writeFile(knownHostsPath, config.knownHosts, {
|
||||
mode: 0o600,
|
||||
});
|
||||
env._SFTP_KNOWN_HOSTS_PATH = knownHostsPath;
|
||||
sshArgs.push("-o", "StrictHostKeyChecking=yes", "-o", `UserKnownHostsFile=${knownHostsPath}`);
|
||||
}
|
||||
|
|
@ -232,7 +243,12 @@ const init = async (config: RepositoryConfig, organizationId: string, options?:
|
|||
const args = ["init", "--repo", repoUrl];
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, timeout: options?.timeoutMs ?? 20000 });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
timeout: options?.timeoutMs ?? 20000,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
|
|
@ -398,6 +414,13 @@ const restoreProgressSchema = type({
|
|||
});
|
||||
|
||||
export type RestoreProgress = typeof restoreProgressSchema.infer;
|
||||
|
||||
export interface ResticDumpStream {
|
||||
stream: Readable;
|
||||
completion: Promise<void>;
|
||||
abort: () => void;
|
||||
}
|
||||
|
||||
const restore = async (
|
||||
config: RepositoryConfig,
|
||||
snapshotId: string,
|
||||
|
|
@ -533,6 +556,109 @@ const restore = async (
|
|||
return result;
|
||||
};
|
||||
|
||||
const normalizeDumpPath = (pathToDump?: string): string => {
|
||||
const trimmedPath = pathToDump?.trim();
|
||||
if (!trimmedPath) {
|
||||
return "/";
|
||||
}
|
||||
|
||||
const decodedPath = (() => {
|
||||
try {
|
||||
return decodeURIComponent(trimmedPath);
|
||||
} catch {
|
||||
return trimmedPath;
|
||||
}
|
||||
})();
|
||||
|
||||
const withLeadingSlash = decodedPath.startsWith("/") ? decodedPath : `/${decodedPath}`;
|
||||
const normalizedPath = withLeadingSlash.replace(/\/+$/, "");
|
||||
|
||||
return normalizedPath || "/";
|
||||
};
|
||||
|
||||
const dump = async (
|
||||
config: RepositoryConfig,
|
||||
snapshotRef: string,
|
||||
options: {
|
||||
organizationId: string;
|
||||
path?: string;
|
||||
},
|
||||
): Promise<ResticDumpStream> => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config, options.organizationId);
|
||||
const pathToDump = normalizeDumpPath(options.path);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "dump", snapshotRef, pathToDump, "--archive", "tar"];
|
||||
|
||||
addCommonArgs(args, env, config, { includeJson: false });
|
||||
|
||||
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||
|
||||
let didCleanup = false;
|
||||
const cleanup = async () => {
|
||||
if (didCleanup) {
|
||||
return;
|
||||
}
|
||||
|
||||
didCleanup = true;
|
||||
await cleanupTemporaryKeys(env);
|
||||
};
|
||||
|
||||
let stream: Readable | null = null;
|
||||
let abortController: AbortController | null = new AbortController();
|
||||
|
||||
const MAX_STDERR_CHARS = 64 * 1024;
|
||||
let stderrTail = "";
|
||||
|
||||
const completion = safeSpawn({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: abortController.signal,
|
||||
stdoutMode: "raw",
|
||||
onSpawn: (child) => {
|
||||
stream = child.stdout;
|
||||
},
|
||||
onStderr: (line) => {
|
||||
const chunk = line.trim();
|
||||
if (chunk) {
|
||||
stderrTail += `${line}\n`;
|
||||
if (stderrTail.length > MAX_STDERR_CHARS) {
|
||||
stderrTail = stderrTail.slice(-MAX_STDERR_CHARS);
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
.then((result) => {
|
||||
if (result.exitCode === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const stderr = stderrTail.trim() || result.error;
|
||||
logger.error(`Restic dump failed: ${stderr}`);
|
||||
throw new ResticError(result.exitCode, stderr);
|
||||
})
|
||||
.finally(async () => {
|
||||
abortController = null;
|
||||
await cleanup();
|
||||
});
|
||||
|
||||
if (!stream) {
|
||||
await cleanup();
|
||||
throw new Error("Failed to initialize restic dump stream");
|
||||
}
|
||||
|
||||
return {
|
||||
stream,
|
||||
completion,
|
||||
abort: () => {
|
||||
if (abortController) {
|
||||
abortController.abort();
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; organizationId: string }) => {
|
||||
const { tags, organizationId } = options;
|
||||
|
||||
|
|
@ -852,7 +978,12 @@ const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal;
|
|||
const args = ["unlock", "--repo", repoUrl, "--remove-all"];
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, signal: options?.signal });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
|
|
@ -871,7 +1002,11 @@ const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal;
|
|||
|
||||
const check = async (
|
||||
config: RepositoryConfig,
|
||||
options: { readData?: boolean; signal?: AbortSignal; organizationId: string },
|
||||
options: {
|
||||
readData?: boolean;
|
||||
signal?: AbortSignal;
|
||||
organizationId: string;
|
||||
},
|
||||
) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config, options.organizationId);
|
||||
|
|
@ -884,12 +1019,22 @@ const check = async (
|
|||
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, signal: options?.signal });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
logger.warn("Restic check was aborted by signal.");
|
||||
return { success: false, hasErrors: true, output: "", error: "Operation aborted" };
|
||||
return {
|
||||
success: false,
|
||||
hasErrors: true,
|
||||
output: "",
|
||||
error: "Operation aborted",
|
||||
};
|
||||
}
|
||||
|
||||
const { stdout, stderr } = res;
|
||||
|
|
@ -922,7 +1067,12 @@ const repairIndex = async (config: RepositoryConfig, options: { signal?: AbortSi
|
|||
const args = ["repair", "index", "--repo", repoUrl];
|
||||
addCommonArgs(args, env, config);
|
||||
|
||||
const res = await exec({ command: "restic", args, env, signal: options?.signal });
|
||||
const res = await exec({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
});
|
||||
await cleanupTemporaryKeys(env);
|
||||
|
||||
if (options?.signal?.aborted) {
|
||||
|
|
@ -1041,9 +1191,11 @@ export const addCommonArgs = (
|
|||
args: string[],
|
||||
env: Record<string, string>,
|
||||
config?: RepositoryConfig,
|
||||
options?: { skipBandwidth?: boolean },
|
||||
options?: { skipBandwidth?: boolean; includeJson?: boolean },
|
||||
) => {
|
||||
args.push("--json");
|
||||
if (options?.includeJson !== false) {
|
||||
args.push("--json");
|
||||
}
|
||||
|
||||
if (env._SFTP_SSH_ARGS) {
|
||||
args.push("-o", `sftp.args=${env._SFTP_SSH_ARGS}`);
|
||||
|
|
@ -1078,6 +1230,7 @@ export const restic = {
|
|||
init,
|
||||
backup,
|
||||
restore,
|
||||
dump,
|
||||
snapshots,
|
||||
forget,
|
||||
deleteSnapshot,
|
||||
|
|
|
|||
|
|
@ -20,3 +20,15 @@ export const sanitizeSensitiveData = (text: string): string => {
|
|||
|
||||
return sanitized;
|
||||
};
|
||||
|
||||
/**
|
||||
* Sanitizes a filename for use in HTTP Content-Disposition header
|
||||
* Removes control characters and replaces special characters to prevent header injection
|
||||
*/
|
||||
export const sanitizeContentDispositionFilename = (filename: string): string => {
|
||||
const sanitized = filename
|
||||
.replace(/[\r\n]/g, "")
|
||||
.replace(/["\\]/g, "_")
|
||||
.trim();
|
||||
return sanitized || "snapshot.tar";
|
||||
};
|
||||
|
|
|
|||
|
|
@ -14,7 +14,11 @@ export const exec = async ({ command, args = [], env = {}, ...rest }: ExecProps)
|
|||
};
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await promisify(execFile)(command, args, { ...options, ...rest, encoding: "utf8" });
|
||||
const { stdout, stderr } = await promisify(execFile)(command, args, {
|
||||
...options,
|
||||
...rest,
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
return { exitCode: 0, stdout, stderr };
|
||||
} catch (error) {
|
||||
|
|
@ -33,8 +37,10 @@ export interface SafeSpawnParams {
|
|||
args: string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
signal?: AbortSignal;
|
||||
stdoutMode?: "lines" | "raw";
|
||||
onStdout?: (line: string) => void;
|
||||
onStderr?: (error: string) => void;
|
||||
onSpawn?: (child: ReturnType<typeof spawn>) => void;
|
||||
}
|
||||
|
||||
type SpawnResult = {
|
||||
|
|
@ -44,7 +50,7 @@ type SpawnResult = {
|
|||
};
|
||||
|
||||
export const safeSpawn = (params: SafeSpawnParams) => {
|
||||
const { command, args, env = {}, signal, onStdout, onStderr } = params;
|
||||
const { command, args, env = {}, signal, stdoutMode = "lines", onStdout, onStderr, onSpawn } = params;
|
||||
|
||||
let lastStdout = "";
|
||||
let lastStderr = "";
|
||||
|
|
@ -56,19 +62,25 @@ export const safeSpawn = (params: SafeSpawnParams) => {
|
|||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
onSpawn?.(child);
|
||||
|
||||
child.stderr.setEncoding("utf8");
|
||||
|
||||
const rl = createInterface({ input: child.stdout });
|
||||
const rlErr = createInterface({ input: child.stderr });
|
||||
|
||||
rl.on("line", (line) => {
|
||||
if (onStdout) onStdout(line);
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length > 0) {
|
||||
lastStdout = line;
|
||||
}
|
||||
});
|
||||
if (stdoutMode === "lines") {
|
||||
child.stdout.setEncoding("utf8");
|
||||
|
||||
const rl = createInterface({ input: child.stdout });
|
||||
|
||||
rl.on("line", (line) => {
|
||||
if (onStdout) onStdout(line);
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length > 0) {
|
||||
lastStdout = line;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
rlErr.on("line", (line) => {
|
||||
if (onStderr) onStderr(line);
|
||||
|
|
@ -79,11 +91,19 @@ export const safeSpawn = (params: SafeSpawnParams) => {
|
|||
});
|
||||
|
||||
child.on("error", (err) => {
|
||||
resolve({ exitCode: -1, summary: lastStdout, error: err.message || lastStderr });
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
summary: lastStdout,
|
||||
error: err.message || lastStderr,
|
||||
});
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
resolve({ exitCode: code ?? -1, summary: lastStdout, error: lastStderr });
|
||||
resolve({
|
||||
exitCode: code ?? -1,
|
||||
summary: lastStdout,
|
||||
error: lastStderr,
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -46,8 +46,9 @@ services:
|
|||
- LOG_LEVEL=debug
|
||||
volumes:
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- /var/lib/zerobyte:/var/lib/zerobyte
|
||||
- ~/.config/rclone:/root/.config/rclone:ro
|
||||
- ./tmp:/test-data
|
||||
- ./data:/var/lib/zerobyte
|
||||
|
||||
zerobyte-e2e:
|
||||
build:
|
||||
|
|
|
|||
Loading…
Reference in a new issue