refactor: async doctor (#375)

* feat: background doctor operation

* refactor(repo-details): design layout

* refactor(doctor): support abort signal in all operations

* chore: fix linting issue

* chore: pr feedbacks

* chore: merge conflicts

* refactor: handle aborted signal in all operations

* chore: pr feedbacks

* chore: remove old migration
This commit is contained in:
Nico 2026-01-22 21:55:45 +01:00 committed by GitHub
parent da37b08fa0
commit 2ab37e6b67
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 2666 additions and 389 deletions

View file

@ -18,7 +18,7 @@ Zerobyte is a backup automation tool built on top of Restic that provides a web
- **Validation**: ArkType for runtime schema validation
- **Styling**: Tailwind CSS v4 + Radix UI components
- **Architecture**: Unified application structure (not a monorepo)
- **Code Quality**: Biome (formatter & linter)
- **Code Quality**: Oxfmt for formatting, Oxlint for linting
## Repository Structure
@ -75,8 +75,6 @@ bun run gen:api-client
### Code Quality
```bash
# Format and lint
# Format
bunx oxfmt format --write <path>
@ -238,8 +236,6 @@ On startup, the server detects available capabilities (see `core/capabilities.ts
## Important Notes
- **Code Style**: Uses Biome with tabs (not spaces), 120 char line width, double quotes
- **Imports**: Organize imports is disabled in Biome - do not auto-organize
- **TypeScript**: Uses `"type": "module"` - all imports must include extensions when targeting Node/Bun
- **Validation**: Prefer ArkType over Zod - it's used throughout the codebase
- **Visibility**: Prefer using the `cn` helper with `{ hidden: condition }` instead of conditional rendering with ternaries or `&&` for toggling element visibility in the DOM.

View file

@ -5,6 +5,7 @@ import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanst
import { client } from "../client.gen";
import {
browseFilesystem,
cancelDoctor,
createBackupSchedule,
createNotificationDestination,
createRepository,
@ -15,7 +16,6 @@ import {
deleteSnapshot,
deleteSnapshots,
deleteVolume,
doctorRepository,
downloadResticPassword,
getBackupSchedule,
getBackupScheduleForVolume,
@ -47,6 +47,7 @@ import {
runBackupNow,
runForget,
setRegistrationStatus,
startDoctor,
stopBackup,
tagSnapshots,
testConnection,
@ -62,6 +63,8 @@ import {
import type {
BrowseFilesystemData,
BrowseFilesystemResponse,
CancelDoctorData,
CancelDoctorResponse,
CreateBackupScheduleData,
CreateBackupScheduleResponse,
CreateNotificationDestinationData,
@ -82,8 +85,6 @@ import type {
DeleteSnapshotsResponse,
DeleteVolumeData,
DeleteVolumeResponse,
DoctorRepositoryData,
DoctorRepositoryResponse,
DownloadResticPasswordData,
DownloadResticPasswordResponse,
GetBackupScheduleData,
@ -144,6 +145,8 @@ import type {
RunForgetResponse,
SetRegistrationStatusData,
SetRegistrationStatusResponse,
StartDoctorData,
StartDoctorResponse,
StopBackupData,
StopBackupResponse,
TagSnapshotsData,
@ -719,14 +722,33 @@ export const restoreSnapshotMutation = (
};
/**
* Run doctor operations on a repository to fix common issues (unlock, check, repair index). Use this when the repository is locked or has errors.
* Cancel a running doctor operation on a repository
*/
export const doctorRepositoryMutation = (
options?: Partial<Options<DoctorRepositoryData>>,
): UseMutationOptions<DoctorRepositoryResponse, DefaultError, Options<DoctorRepositoryData>> => {
const mutationOptions: UseMutationOptions<DoctorRepositoryResponse, DefaultError, Options<DoctorRepositoryData>> = {
export const cancelDoctorMutation = (
options?: Partial<Options<CancelDoctorData>>,
): UseMutationOptions<CancelDoctorResponse, DefaultError, Options<CancelDoctorData>> => {
const mutationOptions: UseMutationOptions<CancelDoctorResponse, DefaultError, Options<CancelDoctorData>> = {
mutationFn: async (fnOptions) => {
const { data } = await doctorRepository({
const { data } = await cancelDoctor({
...options,
...fnOptions,
throwOnError: true,
});
return data;
},
};
return mutationOptions;
};
/**
* Start an asynchronous doctor operation on a repository to fix common issues (unlock, check, repair index). The operation runs in the background and sends results via SSE events.
*/
export const startDoctorMutation = (
options?: Partial<Options<StartDoctorData>>,
): UseMutationOptions<StartDoctorResponse, DefaultError, Options<StartDoctorData>> => {
const mutationOptions: UseMutationOptions<StartDoctorResponse, DefaultError, Options<StartDoctorData>> = {
mutationFn: async (fnOptions) => {
const { data } = await startDoctor({
...options,
...fnOptions,
throwOnError: true,

View file

@ -2,6 +2,7 @@
export {
browseFilesystem,
cancelDoctor,
createBackupSchedule,
createNotificationDestination,
createRepository,
@ -12,7 +13,6 @@ export {
deleteSnapshot,
deleteSnapshots,
deleteVolume,
doctorRepository,
downloadResticPassword,
getBackupSchedule,
getBackupScheduleForVolume,
@ -44,6 +44,7 @@ export {
runBackupNow,
runForget,
setRegistrationStatus,
startDoctor,
stopBackup,
tagSnapshots,
testConnection,
@ -60,6 +61,10 @@ export type {
BrowseFilesystemData,
BrowseFilesystemResponse,
BrowseFilesystemResponses,
CancelDoctorData,
CancelDoctorErrors,
CancelDoctorResponse,
CancelDoctorResponses,
ClientOptions,
CreateBackupScheduleData,
CreateBackupScheduleResponse,
@ -92,9 +97,6 @@ export type {
DeleteVolumeData,
DeleteVolumeResponse,
DeleteVolumeResponses,
DoctorRepositoryData,
DoctorRepositoryResponse,
DoctorRepositoryResponses,
DownloadResticPasswordData,
DownloadResticPasswordResponse,
DownloadResticPasswordResponses,
@ -188,6 +190,10 @@ export type {
SetRegistrationStatusData,
SetRegistrationStatusResponse,
SetRegistrationStatusResponses,
StartDoctorData,
StartDoctorErrors,
StartDoctorResponse,
StartDoctorResponses,
StopBackupData,
StopBackupErrors,
StopBackupResponse,

View file

@ -5,6 +5,9 @@ import { client } from "./client.gen";
import type {
BrowseFilesystemData,
BrowseFilesystemResponses,
CancelDoctorData,
CancelDoctorErrors,
CancelDoctorResponses,
CreateBackupScheduleData,
CreateBackupScheduleResponses,
CreateNotificationDestinationData,
@ -26,8 +29,6 @@ import type {
DeleteSnapshotsResponses,
DeleteVolumeData,
DeleteVolumeResponses,
DoctorRepositoryData,
DoctorRepositoryResponses,
DownloadResticPasswordData,
DownloadResticPasswordResponses,
GetBackupScheduleData,
@ -91,6 +92,9 @@ import type {
RunForgetResponses,
SetRegistrationStatusData,
SetRegistrationStatusResponses,
StartDoctorData,
StartDoctorErrors,
StartDoctorResponses,
StopBackupData,
StopBackupErrors,
StopBackupResponses,
@ -422,12 +426,19 @@ export const restoreSnapshot = <ThrowOnError extends boolean = false>(
});
/**
* Run doctor operations on a repository to fix common issues (unlock, check, repair index). Use this when the repository is locked or has errors.
* Cancel a running doctor operation on a repository
*/
export const doctorRepository = <ThrowOnError extends boolean = false>(
options: Options<DoctorRepositoryData, ThrowOnError>,
) =>
(options.client ?? client).post<DoctorRepositoryResponses, unknown, ThrowOnError>({
export const cancelDoctor = <ThrowOnError extends boolean = false>(options: Options<CancelDoctorData, ThrowOnError>) =>
(options.client ?? client).delete<CancelDoctorResponses, CancelDoctorErrors, ThrowOnError>({
url: "/api/v1/repositories/{id}/doctor",
...options,
});
/**
* Start an asynchronous doctor operation on a repository to fix common issues (unlock, check, repair index). The operation runs in the background and sends results via SSE events.
*/
export const startDoctor = <ThrowOnError extends boolean = false>(options: Options<StartDoctorData, ThrowOnError>) =>
(options.client ?? client).post<StartDoctorResponses, StartDoctorErrors, ThrowOnError>({
url: "/api/v1/repositories/{id}/doctor",
...options,
});

View file

@ -912,12 +912,22 @@ export type ListRepositoriesResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
}>;
@ -1340,12 +1350,22 @@ export type GetRepositoryResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};
@ -1550,12 +1570,22 @@ export type UpdateRepositoryResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};
@ -1731,7 +1761,7 @@ export type RestoreSnapshotResponses = {
export type RestoreSnapshotResponse = RestoreSnapshotResponses[keyof RestoreSnapshotResponses];
export type DoctorRepositoryData = {
export type CancelDoctorData = {
body?: never;
path: {
id: string;
@ -1740,22 +1770,51 @@ export type DoctorRepositoryData = {
url: "/api/v1/repositories/{id}/doctor";
};
export type DoctorRepositoryResponses = {
export type CancelDoctorErrors = {
/**
* Doctor operation completed
* No doctor operation is currently running
*/
409: unknown;
};
export type CancelDoctorResponses = {
/**
* Doctor operation cancelled
*/
200: {
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
message: string;
};
};
export type DoctorRepositoryResponse = DoctorRepositoryResponses[keyof DoctorRepositoryResponses];
export type CancelDoctorResponse = CancelDoctorResponses[keyof CancelDoctorResponses];
export type StartDoctorData = {
body?: never;
path: {
id: string;
};
query?: never;
url: "/api/v1/repositories/{id}/doctor";
};
export type StartDoctorErrors = {
/**
* Doctor operation already in progress
*/
409: unknown;
};
export type StartDoctorResponses = {
/**
* Doctor operation started
*/
202: {
message: string;
repositoryId: string;
};
};
export type StartDoctorResponse = StartDoctorResponses[keyof StartDoctorResponses];
export type TagSnapshotsData = {
body?: {
@ -1977,12 +2036,22 @@ export type ListBackupSchedulesResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};
@ -2349,12 +2418,22 @@ export type GetBackupScheduleResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};
@ -2702,12 +2781,22 @@ export type GetBackupScheduleForVolumeResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};
@ -3263,12 +3352,22 @@ export type GetScheduleMirrorsResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};
@ -3473,12 +3572,22 @@ export type UpdateScheduleMirrorsResponses = {
};
};
createdAt: number;
doctorResult: {
completedAt: number;
steps: Array<{
error: string | null;
output: string | null;
step: string;
success: boolean;
}>;
success: boolean;
} | null;
id: string;
lastChecked: number | null;
lastError: string | null;
name: string;
shortId: string;
status: "error" | "healthy" | "unknown" | null;
status: "cancelled" | "doctor" | "error" | "healthy" | "unknown" | null;
type: "azure" | "gcs" | "local" | "r2" | "rclone" | "rest" | "s3" | "sftp";
updatedAt: number;
};

View file

@ -7,7 +7,7 @@ function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
<textarea
data-slot="textarea"
className={cn(
"border-input placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 flex field-sizing-content min-h-16 w-full rounded-md border bg-transparent px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
"border-input placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 flex field-sizing-content min-h-16 w-full border bg-transparent px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
className,
)}
{...props}

View file

@ -11,7 +11,10 @@ type ServerEventType =
| "volume:unmounted"
| "volume:updated"
| "mirror:started"
| "mirror:completed";
| "mirror:completed"
| "doctor:started"
| "doctor:completed"
| "doctor:cancelled";
export interface BackupEvent {
scheduleId: number;
@ -45,6 +48,23 @@ export interface MirrorEvent {
error?: string;
}
export interface DoctorEvent {
repositoryId: string;
repositoryName: string;
error?: string;
}
export interface DoctorCompletedEvent extends DoctorEvent {
success: boolean;
completedAt: number;
steps: Array<{
step: string;
success: boolean;
output: string | null;
error: string | null;
}>;
}
type EventHandler = (data: unknown) => void;
/**
@ -156,6 +176,39 @@ export function useServerEvents() {
});
});
eventSource.addEventListener("doctor:started", (e) => {
const data = JSON.parse(e.data) as DoctorEvent;
console.info("[SSE] Doctor started:", data);
void queryClient.invalidateQueries();
handlersRef.current.get("doctor:started")?.forEach((handler) => {
handler(data);
});
});
eventSource.addEventListener("doctor:completed", (e) => {
const data = JSON.parse(e.data) as DoctorCompletedEvent;
console.info("[SSE] Doctor completed:", data);
void queryClient.invalidateQueries();
handlersRef.current.get("doctor:completed")?.forEach((handler) => {
handler(data);
});
});
eventSource.addEventListener("doctor:cancelled", (e) => {
const data = JSON.parse(e.data) as DoctorEvent;
console.info("[SSE] Doctor cancelled:", data);
void queryClient.invalidateQueries();
handlersRef.current.get("doctor:cancelled")?.forEach((handler) => {
handler(data);
});
});
eventSource.onerror = (error) => {
console.error("[SSE] Connection error:", error);
};

View file

@ -28,3 +28,11 @@ export function slugify(input: string): string {
.replace(/[_]{2,}/g, "_")
.trim();
}
export function safeJsonParse<T>(input: string): T | null {
try {
return JSON.parse(input) as T;
} catch {
return null;
}
}

View file

@ -0,0 +1,88 @@
import { AlertCircle, CheckCircle2 } from "lucide-react";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "~/client/components/ui/collapsible";
import { formatDateTime } from "~/client/lib/datetime";
import { cn, safeJsonParse } from "~/client/lib/utils";
type DoctorStep = {
step: string;
success: boolean;
output: string | null;
error: string | null;
};
type DoctorResult = {
success: boolean;
steps: DoctorStep[];
completedAt: number;
};
type Props = {
result?: DoctorResult | null;
repositoryStatus: string | null;
};
export const DoctorReport = ({ result, repositoryStatus }: Props) => {
return (
<div>
<h3 className="text-lg font-semibold">Doctor Report</h3>
{result && (
<div className="space-y-2">
<span className="text-xs text-muted-foreground">Completed {formatDateTime(result.completedAt)}</span>
<div className="space-y-2 mt-2">
{result.steps.map((step) => (
<Collapsible key={step.step} className="border rounded overflow-hidden bg-muted/30 group">
<CollapsibleTrigger className="w-full flex items-center justify-start p-3 hover:bg-muted/50 transition-colors">
<div className="flex items-center gap-3">
<span className="text-sm font-medium">{step.step.replaceAll("_", " ")}</span>
{step.success ? (
<CheckCircle2 className="h-4 w-4 text-green-500" />
) : (
<AlertCircle className="h-4 w-4 text-red-500" />
)}
</div>
</CollapsibleTrigger>
<CollapsibleContent className="border-t bg-muted/50">
<div className="p-2 space-y-3">
{step.output && (
<pre className="text-xs font-mono bg-background/50 p-3 border overflow-auto max-h-50 whitespace-pre-wrap">
{safeJsonParse(step.output) ? JSON.stringify(safeJsonParse(step.output), null, 2) : step.output}
</pre>
)}
{step.error && (
<div className="space-y-1.5">
<div className="text-[10px] uppercase font-bold text-red-500/70 px-1">Error</div>
<pre className="text-xs font-mono bg-red-500/5 text-red-500 p-3 border border-red-500/20 overflow-auto whitespace-pre-wrap">
{step.error}
</pre>
</div>
)}
{!step.output && !step.error && (
<div className="text-xs text-muted-foreground italic px-1">No output recorded</div>
)}
</div>
</CollapsibleContent>
</Collapsible>
))}
</div>
</div>
)}
<div
className={cn("mt-2 bg-muted/30 border p-6 text-center", {
hidden: result != null || repositoryStatus === "doctor",
})}
>
<p className="text-sm text-muted-foreground">No doctor report available.</p>
</div>
<div
className={cn("mt-2 border p-6 text-center", {
hidden: repositoryStatus !== "doctor",
})}
>
<div className="flex items-center justify-center gap-2">
<div className="h-4 w-4 animate-spin rounded-full border-2 border-primary border-t-transparent" />
<p className="text-sm ">Doctor operation running...</p>
</div>
</div>
</div>
);
};

View file

@ -29,11 +29,11 @@ export const AdvancedForm = ({ form }: Props) => {
return (
<Collapsible>
<CollapsibleTrigger className="">Advanced Settings</CollapsibleTrigger>
<CollapsibleTrigger>Advanced Settings</CollapsibleTrigger>
<CollapsibleContent className="pb-4 space-y-4">
<div className="space-y-4 mt-4">
<div className="grid gap-6">
<div className="space-y-4 rounded-lg border bg-background/50 p-4">
<div className="space-y-4 border p-4">
<FormField
control={form.control}
name="uploadLimit.enabled"
@ -109,7 +109,7 @@ export const AdvancedForm = ({ form }: Props) => {
</div>
</div>
<div className="rounded-lg border bg-background/50 p-4">
<div className="border bg-background/50 p-4">
<FormField
control={form.control}
name="downloadLimit.enabled"
@ -191,7 +191,7 @@ export const AdvancedForm = ({ form }: Props) => {
control={form.control}
name="insecureTls"
render={({ field }) => (
<FormItem className="flex flex-row items-start space-x-3 space-y-0 rounded-md border p-4">
<FormItem className="flex flex-row items-start space-x-3 space-y-0 border p-4">
<FormControl>
<Tooltip delayDuration={500}>
<TooltipTrigger asChild>

View file

@ -1,31 +1,12 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { redirect, useNavigate, useSearchParams } from "react-router";
import { toast } from "sonner";
import { useState, useEffect } from "react";
import {
deleteRepositoryMutation,
doctorRepositoryMutation,
getRepositoryOptions,
listSnapshotsOptions,
} from "~/client/api-client/@tanstack/react-query.gen";
import { Button } from "~/client/components/ui/button";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogHeader,
AlertDialogTitle,
} from "~/client/components/ui/alert-dialog";
import { parseError } from "~/client/lib/errors";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { redirect, useSearchParams } from "react-router";
import { useEffect } from "react";
import { getRepositoryOptions, listSnapshotsOptions } from "~/client/api-client/@tanstack/react-query.gen";
import { getRepository } from "~/client/api-client/sdk.gen";
import type { Route } from "./+types/repository-details";
import { cn } from "~/client/lib/utils";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "~/client/components/ui/tabs";
import { RepositoryInfoTabContent } from "../tabs/info";
import { RepositorySnapshotsTabContent } from "../tabs/snapshots";
import { Loader2, Stethoscope, Trash2 } from "lucide-react";
export const handle = {
breadcrumb: (match: Route.MetaArgs) => [
@ -52,11 +33,7 @@ export const clientLoader = async ({ params }: Route.ClientLoaderArgs) => {
};
export default function RepositoryDetailsPage({ loaderData }: Route.ComponentProps) {
const [showDoctorResults, setShowDoctorResults] = useState(false);
const navigate = useNavigate();
const queryClient = useQueryClient();
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
const [searchParams, setSearchParams] = useSearchParams();
const activeTab = searchParams.get("tab") || "info";
@ -70,101 +47,8 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
void queryClient.prefetchQuery(listSnapshotsOptions({ path: { id: data.id } }));
}, [queryClient, data.id]);
const deleteRepo = useMutation({
...deleteRepositoryMutation(),
onSuccess: () => {
toast.success("Repository deleted successfully");
void navigate("/repositories");
},
onError: (error) => {
toast.error("Failed to delete repository", {
description: parseError(error)?.message,
});
},
});
const doctorMutation = useMutation({
...doctorRepositoryMutation(),
onSuccess: (data) => {
if (data) {
setShowDoctorResults(true);
if (data.success) {
toast.success("Repository doctor completed successfully");
} else {
toast.warning("Doctor completed with some issues", {
description: "Check the details for more information",
richColors: true,
});
}
}
},
onError: (error) => {
toast.error("Failed to run doctor", {
description: parseError(error)?.message,
});
},
});
const handleConfirmDelete = () => {
setShowDeleteConfirm(false);
deleteRepo.mutate({ path: { id: data.id } });
};
const getStepLabel = (step: string) => {
switch (step) {
case "unlock":
return "Unlock Repository";
case "check":
return "Check Repository";
case "repair_index":
return "Repair Index";
case "recheck":
return "Re-check Repository";
default:
return step;
}
};
return (
<>
<div className="flex items-center justify-between mb-4">
<div className="text-sm font-semibold text-muted-foreground flex items-center gap-2">
<span
className={cn("inline-flex items-center gap-2 px-2 py-1 rounded-md text-xs bg-gray-500/10 text-gray-500", {
"bg-green-500/10 text-green-500": data.status === "healthy",
"bg-red-500/10 text-red-500": data.status === "error",
})}
>
{data.status || "unknown"}
</span>
<span className="text-xs bg-primary/10 rounded-md px-2 py-1">{data.type}</span>
</div>
<div className="flex gap-4">
<Button
onClick={() => doctorMutation.mutate({ path: { id: data.id } })}
disabled={doctorMutation.isPending}
variant={"outline"}
>
{doctorMutation.isPending ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Running doctor...
</>
) : (
<>
<Stethoscope className="h-4 w-4 mr-2" />
Run doctor
</>
)}
</Button>
<Button variant="destructive" onClick={() => setShowDeleteConfirm(true)} disabled={deleteRepo.isPending}>
<Trash2 className="h-4 w-4 mr-2" />
Delete
</Button>
</div>
</div>
<Tabs value={activeTab} onValueChange={(value) => setSearchParams({ tab: value })}>
<TabsList className="mb-2">
<TabsTrigger value="info">Configuration</TabsTrigger>
@ -177,71 +61,6 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
<RepositorySnapshotsTabContent repository={data} />
</TabsContent>
</Tabs>
<AlertDialog open={showDeleteConfirm} onOpenChange={setShowDeleteConfirm}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Delete repository?</AlertDialogTitle>
<AlertDialogDescription>
Are you sure you want to delete the repository <strong>{data.name}</strong>? This will not remove the
actual data from the backend storage, only the repository configuration will be deleted.
<br />
<br />
All backup schedules associated with this repository will also be removed.
</AlertDialogDescription>
</AlertDialogHeader>
<div className="flex gap-3 justify-end">
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleConfirmDelete}
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
>
<Trash2 className="h-4 w-4 mr-2" />
Delete repository
</AlertDialogAction>
</div>
</AlertDialogContent>
</AlertDialog>
<AlertDialog open={showDoctorResults} onOpenChange={setShowDoctorResults}>
<AlertDialogContent className="max-w-2xl">
<AlertDialogHeader>
<AlertDialogTitle>Doctor results</AlertDialogTitle>
<AlertDialogDescription>Repository doctor operation completed</AlertDialogDescription>
</AlertDialogHeader>
{doctorMutation.data && (
<div className="space-y-3 max-h-96 overflow-y-auto">
{doctorMutation.data.steps.map((step) => (
<div
key={step.step}
className={cn("border rounded-md p-3", {
"bg-green-500/10 border-green-500/20": step.success,
"bg-yellow-500/10 border-yellow-500/20": !step.success,
})}
>
<div className="flex items-center justify-between mb-2">
<span className="font-medium text-sm">{getStepLabel(step.step)}</span>
<span
className={cn("text-xs px-2 py-1 rounded", {
"bg-green-500/20 text-green-500": step.success,
"bg-yellow-500/20 text-yellow-500": !step.success,
})}
>
{step.success ? "Success" : "Warning"}
</span>
</div>
{step.error && <p className="text-xs text-red-500 mt-1">{step.error}</p>}
</div>
))}
</div>
)}
<div className="flex justify-end">
<Button onClick={() => setShowDoctorResults(false)}>Close</Button>
</div>
</AlertDialogContent>
</AlertDialog>
</>
);
}

View file

@ -1,7 +1,7 @@
import { useMutation } from "@tanstack/react-query";
import { useState } from "react";
import { toast } from "sonner";
import { Check, Save } from "lucide-react";
import { Check, Save, Square, Stethoscope, Trash2 } from "lucide-react";
import { Card } from "~/client/components/ui/card";
import { Button } from "~/client/components/ui/button";
import { Input } from "~/client/components/ui/input";
@ -20,8 +20,16 @@ import {
import type { Repository } from "~/client/lib/types";
import { REPOSITORY_BASE } from "~/client/lib/constants";
import { formatDateTime, formatTimeAgo } from "~/client/lib/datetime";
import { updateRepositoryMutation } from "~/client/api-client/@tanstack/react-query.gen";
import {
cancelDoctorMutation,
deleteRepositoryMutation,
startDoctorMutation,
updateRepositoryMutation,
} from "~/client/api-client/@tanstack/react-query.gen";
import type { CompressionMode, RepositoryConfig } from "~/schemas/restic";
import { DoctorReport } from "../components/doctor-report";
import { parseError } from "~/client/lib/errors";
import { useNavigate } from "react-router";
type Props = {
repository: Repository;
@ -41,10 +49,10 @@ const getEffectiveLocalPath = (repository: Repository): string | null => {
export const RepositoryInfoTabContent = ({ repository }: Props) => {
const [name, setName] = useState(repository.name);
const [compressionMode, setCompressionMode] = useState<CompressionMode>(
(repository.compressionMode as CompressionMode) || "off",
);
const [compressionMode, setCompressionMode] = useState<CompressionMode>(repository.compressionMode || "off");
const [showConfirmDialog, setShowConfirmDialog] = useState(false);
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
const navigate = useNavigate();
const effectiveLocalPath = getEffectiveLocalPath(repository);
@ -60,6 +68,40 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
},
});
const deleteRepo = useMutation({
...deleteRepositoryMutation(),
onSuccess: () => {
toast.success("Repository deleted successfully");
void navigate("/repositories");
},
onError: (error) => {
toast.error("Failed to delete repository", {
description: parseError(error)?.message,
});
},
});
const startDoctor = useMutation({
...startDoctorMutation(),
onError: (error) => {
toast.error("Failed to start doctor", {
description: parseError(error)?.message,
});
},
});
const cancelDoctor = useMutation({
...cancelDoctorMutation(),
onSuccess: () => {
toast.info("Doctor operation cancelled");
},
onError: (error) => {
toast.error("Failed to cancel doctor", {
description: parseError(error)?.message,
});
},
});
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
setShowConfirmDialog(true);
@ -72,6 +114,11 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
});
};
const handleConfirmDelete = () => {
setShowDeleteConfirm(false);
deleteRepo.mutate({ path: { id: repository.id } });
};
const hasChanges =
name !== repository.name || compressionMode !== ((repository.compressionMode as CompressionMode) || "off");
@ -81,35 +128,68 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
<>
<Card className="p-6">
<form onSubmit={handleSubmit} className="space-y-6">
<div>
<h3 className="text-lg font-semibold mb-4">Repository Settings</h3>
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div className="space-y-2">
<Label htmlFor="name">Name</Label>
<Input
id="name"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="Repository name"
maxLength={32}
minLength={2}
/>
<p className="text-sm text-muted-foreground">Unique identifier for the repository.</p>
</div>
<div className="space-y-2">
<Label htmlFor="compressionMode">Compression mode</Label>
<Select value={compressionMode} onValueChange={(val) => setCompressionMode(val as CompressionMode)}>
<SelectTrigger id="compressionMode">
<SelectValue placeholder="Select compression mode" />
</SelectTrigger>
<SelectContent>
<SelectItem value="off">Off</SelectItem>
<SelectItem value="auto">Auto</SelectItem>
<SelectItem value="max">Max</SelectItem>
</SelectContent>
</Select>
<p className="text-sm text-muted-foreground">Compression level for new data.</p>
</div>
<div className="flex flex-col sm:flex-row items-center justify-between gap-2">
<div>
<span className="text-lg font-semibold mb-4">Repository Settings</span>
</div>
<div className="flex gap-4">
{repository.status === "doctor" ? (
<Button
type="button"
variant="destructive"
loading={cancelDoctor.isPending}
onClick={() => cancelDoctor.mutate({ path: { id: repository.id } })}
>
<Square className="h-4 w-4 mr-2" />
<span>Cancel doctor</span>
</Button>
) : (
<Button
type="button"
onClick={() => startDoctor.mutate({ path: { id: repository.id } })}
disabled={startDoctor.isPending}
>
<Stethoscope className="h-4 w-4 mr-2" />
Run doctor
</Button>
)}
<Button
type="button"
variant="destructive"
onClick={() => setShowDeleteConfirm(true)}
disabled={deleteRepo.isPending}
>
<Trash2 className="h-4 w-4 mr-2" />
Delete
</Button>
</div>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
<div className="space-y-2">
<Label htmlFor="name">Name</Label>
<Input
id="name"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="Repository name"
maxLength={32}
minLength={2}
/>
<p className="text-sm text-muted-foreground">Unique identifier for the repository.</p>
</div>
<div className="space-y-2">
<Label htmlFor="compressionMode">Compression mode</Label>
<Select value={compressionMode} onValueChange={(val) => setCompressionMode(val as CompressionMode)}>
<SelectTrigger id="compressionMode">
<SelectValue placeholder="Select compression mode" />
</SelectTrigger>
<SelectContent>
<SelectItem value="off">Off</SelectItem>
<SelectItem value="auto">Auto</SelectItem>
<SelectItem value="max">Max</SelectItem>
</SelectContent>
</Select>
<p className="text-sm text-muted-foreground">Compression level for new data.</p>
</div>
</div>
@ -126,7 +206,7 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
</div>
{effectiveLocalPath && (
<div className="md:col-span-2">
<div className="text-sm font-medium text-muted-foreground">Effective Local Path</div>
<div className="text-sm font-medium text-muted-foreground">Local path</div>
<p className="mt-1 text-sm font-mono">{effectiveLocalPath}</p>
</div>
)}
@ -179,6 +259,8 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
</div>
</div>
<DoctorReport repositoryStatus={repository.status} result={repository.doctorResult} />
<div className="flex justify-end pt-4 border-t">
<Button type="submit" disabled={!hasChanges || updateMutation.isPending} loading={updateMutation.isPending}>
<Save className="h-4 w-4 mr-2" />
@ -203,6 +285,31 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
<AlertDialog open={showDeleteConfirm} onOpenChange={setShowDeleteConfirm}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Delete repository?</AlertDialogTitle>
<AlertDialogDescription>
Are you sure you want to delete the repository <strong>{repository.name}</strong>? This will not remove
the actual data from the backend storage, only the repository configuration will be deleted.
<br />
<br />
All backup schedules associated with this repository will also be removed.
</AlertDialogDescription>
</AlertDialogHeader>
<div className="flex gap-3 justify-end">
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleConfirmDelete}
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
>
<Trash2 className="h-4 w-4 mr-2" />
Delete repository
</AlertDialogAction>
</div>
</AlertDialogContent>
</AlertDialog>
</>
);
};

View file

@ -0,0 +1 @@
ALTER TABLE `repositories_table` ADD `doctor_result` text;

File diff suppressed because it is too large Load diff

View file

@ -302,6 +302,13 @@
"when": 1768687163221,
"tag": "0042_watery_liz_osborn",
"breakpoints": true
},
{
"idx": 43,
"version": "6",
"when": 1769085071955,
"tag": "0043_overjoyed_shen",
"breakpoints": true
}
]
}

View file

@ -127,10 +127,29 @@ export const REPOSITORY_STATUS = {
healthy: "healthy",
error: "error",
unknown: "unknown",
doctor: "doctor",
cancelled: "cancelled",
} as const;
export type RepositoryStatus = keyof typeof REPOSITORY_STATUS;
export const doctorStepSchema = type({
step: "string",
success: "boolean",
output: "string | null",
error: "string | null",
});
export type DoctorStep = typeof doctorStepSchema.infer;
export const doctorResultSchema = type({
success: "boolean",
steps: doctorStepSchema.array(),
completedAt: "number",
});
export type DoctorResult = typeof doctorResultSchema.infer;
export const OVERWRITE_MODES = {
always: "always",
ifChanged: "if-changed",

View file

@ -35,4 +35,63 @@ describe("RepositoryMutex", () => {
releaseShared2();
});
test("should remove aborted acquisitions from the wait queue", async () => {
const repoId = "abort-test";
const results: string[] = [];
const releaseShared1 = await repoMutex.acquireShared(repoId, "backup-1");
results.push("acquired-shared-1");
const controller = new AbortController();
const exclusivePromise = repoMutex.acquireExclusive(repoId, "unlock", controller.signal).catch((err) => {
results.push("aborted-exclusive");
throw err;
});
const shared2Promise = repoMutex.acquireShared(repoId, "backup-2").then((release) => {
results.push("acquired-shared-2");
return release;
});
await new Promise((resolve) => setTimeout(resolve, 50));
expect(results).toEqual(["acquired-shared-1"]);
controller.abort();
expect(exclusivePromise).rejects.toThrow();
expect(results).toEqual(["acquired-shared-1", "aborted-exclusive"]);
releaseShared1();
// After exclusive is aborted, shared-2 should be next
const releaseShared2 = await shared2Promise;
expect(results).toEqual(["acquired-shared-1", "aborted-exclusive", "acquired-shared-2"]);
releaseShared2();
});
test("should handle multiple aborts correctly", async () => {
const repoId = "multi-abort";
const releaseShared1 = await repoMutex.acquireShared(repoId, "backup-1");
const controller1 = new AbortController();
const controller2 = new AbortController();
const p1 = repoMutex.acquireExclusive(repoId, "ex-1", controller1.signal);
const p2 = repoMutex.acquireExclusive(repoId, "ex-2", controller2.signal);
const p3 = repoMutex.acquireExclusive(repoId, "ex-3");
controller2.abort();
expect(p2).rejects.toThrow();
controller1.abort();
expect(p1).rejects.toThrow();
releaseShared1();
const releaseEx3 = await p3;
expect(releaseEx3).toBeDefined();
releaseEx3();
});
});

View file

@ -1,5 +1,6 @@
import { EventEmitter } from "node:events";
import type { TypedEmitter } from "tiny-typed-emitter";
import type { DoctorResult } from "~/schemas/restic";
/**
* Event payloads for the SSE system
@ -36,6 +37,14 @@ interface ServerEvents {
"volume:unmounted": (data: { volumeName: string }) => void;
"volume:updated": (data: { volumeName: string }) => void;
"volume:status_changed": (data: { volumeName: string; status: string }) => void;
"doctor:started": (data: { repositoryId: string; repositoryName: string }) => void;
"doctor:completed": (
data: {
repositoryId: string;
repositoryName: string;
} & DoctorResult,
) => void;
"doctor:cancelled": (data: { repositoryId: string; repositoryName: string; error?: string }) => void;
}
/**

View file

@ -46,7 +46,11 @@ class RepositoryMutex {
}
}
async acquireShared(repositoryId: string, operation: string): Promise<() => void> {
async acquireShared(repositoryId: string, operation: string, signal?: AbortSignal): Promise<() => void> {
if (signal?.aborted) {
throw signal.reason || new Error("Operation aborted");
}
const state = this.getOrCreateState(repositoryId);
const hasExclusiveInQueue = state.waitQueue.some((item) => item.type === "exclusive");
@ -64,14 +68,40 @@ class RepositoryMutex {
logger.debug(
`[Mutex] Waiting for shared lock on repo ${repositoryId}: ${operation} (exclusive held by: ${state.exclusiveHolder?.operation ?? "none"}, queue: ${state.waitQueue.length})`,
);
const lockId = await new Promise<string>((resolve) => {
state.waitQueue.push({ type: "shared", operation, resolve });
let onAbort: () => void = () => {};
const lockId = await new Promise<string>((resolve, reject) => {
const waiter = { type: "shared" as const, operation, resolve };
state.waitQueue.push(waiter);
if (signal) {
onAbort = () => {
const index = state.waitQueue.indexOf(waiter);
if (index !== -1) {
state.waitQueue.splice(index, 1);
this.cleanupStateIfEmpty(repositoryId);
reject(signal.reason || new Error("Operation aborted"));
}
};
signal.addEventListener("abort", onAbort);
}
});
signal?.removeEventListener("abort", onAbort);
if (signal?.aborted) {
this.releaseShared(repositoryId, lockId);
throw signal.reason || new Error("Operation aborted");
}
return () => this.releaseShared(repositoryId, lockId);
}
async acquireExclusive(repositoryId: string, operation: string): Promise<() => void> {
async acquireExclusive(repositoryId: string, operation: string, signal?: AbortSignal): Promise<() => void> {
if (signal?.aborted) {
throw signal.reason || new Error("Operation aborted");
}
const state = this.getOrCreateState(repositoryId);
if (!state.exclusiveHolder && state.sharedHolders.size === 0 && state.waitQueue.length === 0) {
@ -87,10 +117,32 @@ class RepositoryMutex {
logger.debug(
`[Mutex] Waiting for exclusive lock on repo ${repositoryId}: ${operation} (shared: ${state.sharedHolders.size}, exclusive: ${state.exclusiveHolder ? "yes" : "no"}, queue: ${state.waitQueue.length})`,
);
const lockId = await new Promise<string>((resolve) => {
state.waitQueue.push({ type: "exclusive", operation, resolve });
let onAbort: () => void = () => {};
const lockId = await new Promise<string>((resolve, reject) => {
const waiter = { type: "exclusive" as const, operation, resolve };
state.waitQueue.push(waiter);
if (signal) {
onAbort = () => {
const index = state.waitQueue.indexOf(waiter);
if (index !== -1) {
state.waitQueue.splice(index, 1);
this.cleanupStateIfEmpty(repositoryId);
reject(signal.reason || new Error("Operation aborted"));
}
};
signal.addEventListener("abort", onAbort);
}
});
signal?.removeEventListener("abort", onAbort);
if (signal?.aborted) {
this.releaseExclusive(repositoryId, lockId);
throw signal.reason || new Error("Operation aborted");
}
logger.debug(`[Mutex] Acquired exclusive lock for repo ${repositoryId}: ${operation} (${lockId})`);
return () => this.releaseExclusive(repositoryId, lockId);
}

View file

@ -6,6 +6,7 @@ import type {
repositoryConfigSchema,
RepositoryStatus,
BandwidthUnit,
DoctorResult,
} from "~/schemas/restic";
import type { BackendStatus, BackendType, volumeConfigSchema } from "~/schemas/volumes";
import type { NotificationType, notificationConfigSchema } from "~/schemas/notifications";
@ -213,6 +214,7 @@ export const repositoriesTable = sqliteTable("repositories_table", {
status: text().$type<RepositoryStatus>().default("unknown"),
lastChecked: int("last_checked", { mode: "number" }),
lastError: text("last_error"),
doctorResult: text("doctor_result", { mode: "json" }).$type<DoctorResult>(),
uploadLimitEnabled: int("upload_limit_enabled", { mode: "boolean" }).notNull().default(false),
uploadLimitValue: real("upload_limit_value").notNull().default(1),
uploadLimitUnit: text("upload_limit_unit").$type<BandwidthUnit>().notNull().default("Mbps"),

View file

@ -309,7 +309,7 @@ const executeBackup = async (scheduleId: number, manual = false) => {
backupOptions.include = schedule.includePatterns.map((p) => processPattern(p, volumePath));
}
const releaseBackupLock = await repoMutex.acquireShared(repository.id, `backup:${volume.name}`);
const releaseBackupLock = await repoMutex.acquireShared(repository.id, `backup:${volume.name}`, abortController.signal);
let exitCode: number;
try {
const result = await restic.backup(repository.config, volumePath, {

View file

@ -3,6 +3,7 @@ import { streamSSE } from "hono/streaming";
import { logger } from "../../utils/logger";
import { serverEvents } from "../../core/events";
import { requireAuth } from "../auth/auth.middleware";
import type { DoctorResult } from "~/schemas/restic";
export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
logger.info("Client connected to SSE endpoint");
@ -91,6 +92,32 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
});
};
const onDoctorStarted = async (data: { repositoryId: string; repositoryName: string }) => {
await stream.writeSSE({
data: JSON.stringify(data),
event: "doctor:started",
});
};
const onDoctorCompleted = async (
data: {
repositoryId: string;
repositoryName: string;
} & DoctorResult,
) => {
await stream.writeSSE({
data: JSON.stringify(data),
event: "doctor:completed",
});
};
const onDoctorCancelled = async (data: { repositoryId: string; repositoryName: string; error?: string }) => {
await stream.writeSSE({
data: JSON.stringify(data),
event: "doctor:cancelled",
});
};
serverEvents.on("backup:started", onBackupStarted);
serverEvents.on("backup:progress", onBackupProgress);
serverEvents.on("backup:completed", onBackupCompleted);
@ -99,6 +126,9 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
serverEvents.on("volume:updated", onVolumeUpdated);
serverEvents.on("mirror:started", onMirrorStarted);
serverEvents.on("mirror:completed", onMirrorCompleted);
serverEvents.on("doctor:started", onDoctorStarted);
serverEvents.on("doctor:completed", onDoctorCompleted);
serverEvents.on("doctor:cancelled", onDoctorCancelled);
let keepAlive = true;
@ -113,6 +143,9 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
serverEvents.off("volume:updated", onVolumeUpdated);
serverEvents.off("mirror:started", onMirrorStarted);
serverEvents.off("mirror:completed", onMirrorCompleted);
serverEvents.off("doctor:started", onDoctorStarted);
serverEvents.off("doctor:completed", onDoctorCompleted);
serverEvents.off("doctor:cancelled", onDoctorCancelled);
});
while (keepAlive) {

View file

@ -0,0 +1,199 @@
import { eq } from "drizzle-orm";
import { db } from "../../db/db";
import { repositoriesTable } from "../../db/schema";
import { toMessage } from "../../utils/errors";
import { restic } from "../../utils/restic";
import { repoMutex } from "../../core/repository-mutex";
import { type DoctorStep, type DoctorResult, type RepositoryConfig } from "~/schemas/restic";
import { type } from "arktype";
import { serverEvents } from "../../core/events";
import { logger } from "../../utils/logger";
import { safeJsonParse } from "../../utils/json";
import { getOrganizationId } from "~/server/core/request-context";
class AbortError extends Error {
name = "AbortError";
}
const runUnlockStep = async (config: RepositoryConfig, signal?: AbortSignal) => {
const orgId = getOrganizationId();
const result = await restic.unlock(config, { signal, organizationId: orgId }).then(
(result) => ({ success: true, message: result.message, error: null }),
(error) => ({ success: false, message: null, error: toMessage(error) }),
);
return {
step: "unlock",
success: result.success,
output: result.message,
error: result.error,
};
};
const runCheckStep = async (config: RepositoryConfig, signal: AbortSignal) => {
const orgId = getOrganizationId();
const result = await restic.check(config, { readData: true, signal, organizationId: orgId }).then(
(result) => result,
(error) => ({ success: false, output: null, error: toMessage(error), hasErrors: true }),
);
return {
step: "check",
success: result.success,
output: result.output,
error: result.error,
};
};
const runRepairIndexStep = async (config: RepositoryConfig, signal: AbortSignal) => {
const orgId = getOrganizationId();
const result = await restic.repairIndex(config, { signal, organizationId: orgId }).then(
(result) => ({ success: true, output: result.output, error: null }),
(error) => ({ success: false, output: null, error: toMessage(error) }),
);
return {
step: "repair_index",
success: result.success,
output: result.output,
error: result.error,
};
};
const parseCheckOutput = (checkOutput: string | null) => {
const schema = type({ suggest_repair_index: "boolean", suggest_prune: "boolean" });
const parsedJson = safeJsonParse(checkOutput);
if (parsedJson === null) {
return null;
}
const parsed = schema(parsedJson);
if (parsed instanceof type.errors) {
logger.error(`Invalid check output format: ${parsed.summary}`);
return null;
}
return parsed;
};
const checkAbortSignal = (signal: AbortSignal | undefined): void => {
if (signal?.aborted) {
throw new AbortError("Doctor operation cancelled");
}
};
const determineRepositoryStatus = (steps: DoctorStep[]): "healthy" | "error" => {
const repairStep = steps.find((s) => s.step === "repair_index");
if (repairStep) {
return repairStep.success ? "healthy" : "error";
}
const checkStep = steps.find((s) => s.step === "check");
return checkStep?.success ? "healthy" : "error";
};
const saveDoctorResults = async (repositoryId: string, steps: DoctorStep[], finalStatus: "healthy" | "error") => {
const doctorResult: DoctorResult = {
success: steps.every((s) => s.success),
steps,
completedAt: Date.now(),
};
const finalError = steps.find((s) => s.error)?.error ?? null;
await db
.update(repositoriesTable)
.set({
status: finalStatus,
lastChecked: Date.now(),
lastError: finalError,
doctorResult,
})
.where(eq(repositoriesTable.id, repositoryId));
};
export const executeDoctor = async (
repositoryId: string,
repositoryConfig: RepositoryConfig,
repositoryName: string,
signal: AbortSignal,
) => {
const steps: DoctorStep[] = [];
try {
const releaseLock = await repoMutex.acquireExclusive(repositoryId, "doctor", signal);
try {
// Step 1: Unlock repository
const unlockStep = await runUnlockStep(repositoryConfig, signal);
steps.push(unlockStep);
checkAbortSignal(signal);
// Step 2: Check repository
const checkStep = await runCheckStep(repositoryConfig, signal);
steps.push(checkStep);
checkAbortSignal(signal);
// Step 3: Repair index if suggested
const checkOutput = parseCheckOutput(checkStep.output);
if (checkOutput?.suggest_repair_index) {
const repairStep = await runRepairIndexStep(repositoryConfig, signal);
steps.push(repairStep);
checkAbortSignal(signal);
}
} finally {
releaseLock();
}
const finalStatus = determineRepositoryStatus(steps);
await saveDoctorResults(repositoryId, steps, finalStatus);
serverEvents.emit("doctor:completed", {
repositoryId,
repositoryName,
success: steps.every((s) => s.success),
steps,
completedAt: Date.now(),
});
} catch (error) {
if (error instanceof AbortError) {
const doctorResult: DoctorResult = {
success: false,
steps,
completedAt: Date.now(),
};
await db
.update(repositoriesTable)
.set({
status: "cancelled",
lastChecked: Date.now(),
lastError: toMessage(error),
doctorResult,
})
.where(eq(repositoriesTable.id, repositoryId));
serverEvents.emit("doctor:cancelled", {
repositoryId,
repositoryName,
error: toMessage(error),
});
} else {
await db
.update(repositoriesTable)
.set({ status: "error", lastError: toMessage(error) })
.where(eq(repositoriesTable.id, repositoryId));
steps.push({ step: "doctor", success: false, output: null, error: toMessage(error) });
serverEvents.emit("doctor:completed", {
repositoryId,
repositoryName,
success: false,
steps,
completedAt: Date.now(),
});
}
}
};

View file

@ -7,7 +7,8 @@ import {
deleteSnapshotDto,
deleteSnapshotsBody,
deleteSnapshotsDto,
doctorRepositoryDto,
startDoctorDto,
cancelDoctorDto,
getRepositoryDto,
getSnapshotDetailsDto,
listRcloneRemotesDto,
@ -25,7 +26,8 @@ import {
type DeleteRepositoryDto,
type DeleteSnapshotDto,
type DeleteSnapshotsResponseDto,
type DoctorRepositoryDto,
type StartDoctorDto,
type CancelDoctorDto,
type GetRepositoryDto,
type GetSnapshotDetailsDto,
type ListRepositoriesDto,
@ -150,11 +152,19 @@ export const repositoriesController = new Hono()
return c.json<RestoreSnapshotDto>(result, 200);
})
.post("/:id/doctor", doctorRepositoryDto, async (c) => {
.post("/:id/doctor", startDoctorDto, async (c) => {
const { id } = c.req.param();
const result = await repositoriesService.doctorRepository(id);
return c.json<DoctorRepositoryDto>(result, 200);
const result = await repositoriesService.startDoctor(id);
return c.json<StartDoctorDto>(result, 202);
})
.delete("/:id/doctor", cancelDoctorDto, async (c) => {
const { id } = c.req.param();
const result = await repositoriesService.cancelDoctor(id);
return c.json<CancelDoctorDto>(result, 200);
})
.delete("/:id/snapshots/:snapshotId", deleteSnapshotDto, async (c) => {
const { id, snapshotId } = c.req.param();

View file

@ -6,6 +6,7 @@ import {
REPOSITORY_BACKENDS,
REPOSITORY_STATUS,
repositoryConfigSchema,
doctorResultSchema,
} from "~/schemas/restic";
export const repositorySchema = type({
@ -18,6 +19,7 @@ export const repositorySchema = type({
status: type.valueOf(REPOSITORY_STATUS).or("null"),
lastChecked: "number | null",
lastError: "string | null",
doctorResult: doctorResultSchema.or("null"),
createdAt: "number",
updatedAt: "number",
});
@ -317,36 +319,60 @@ export const restoreSnapshotDto = describeRoute({
});
/**
* Doctor a repository (unlock, check, repair)
* Start doctor operation
*/
export const doctorStepSchema = type({
step: "string",
success: "boolean",
output: "string | null",
error: "string | null",
export const startDoctorResponse = type({
message: "string",
repositoryId: "string",
});
export const doctorRepositoryResponse = type({
success: "boolean",
steps: doctorStepSchema.array(),
});
export type StartDoctorDto = typeof startDoctorResponse.infer;
export type DoctorRepositoryDto = typeof doctorRepositoryResponse.infer;
export const doctorRepositoryDto = describeRoute({
export const startDoctorDto = describeRoute({
description:
"Run doctor operations on a repository to fix common issues (unlock, check, repair index). Use this when the repository is locked or has errors.",
"Start an asynchronous doctor operation on a repository to fix common issues (unlock, check, repair index). The operation runs in the background and sends results via SSE events.",
tags: ["Repositories"],
operationId: "doctorRepository",
operationId: "startDoctor",
responses: {
200: {
description: "Doctor operation completed",
202: {
description: "Doctor operation started",
content: {
"application/json": {
schema: resolver(doctorRepositoryResponse),
schema: resolver(startDoctorResponse),
},
},
},
409: {
description: "Doctor operation already in progress",
},
},
});
/**
* Cancel running doctor operation
*/
export const cancelDoctorResponse = type({
message: "string",
});
export type CancelDoctorDto = typeof cancelDoctorResponse.infer;
export const cancelDoctorDto = describeRoute({
description: "Cancel a running doctor operation on a repository",
tags: ["Repositories"],
operationId: "cancelDoctor",
responses: {
200: {
description: "Doctor operation cancelled",
content: {
"application/json": {
schema: resolver(cancelDoctorResponse),
},
},
},
409: {
description: "No doctor operation is currently running",
},
},
});

View file

@ -1,6 +1,6 @@
import crypto from "node:crypto";
import { and, eq, or } from "drizzle-orm";
import { InternalServerError, NotFoundError } from "http-errors-enhanced";
import { ConflictError, InternalServerError, NotFoundError } from "http-errors-enhanced";
import { db } from "../../db/db";
import { repositoriesTable } from "../../db/schema";
import { toMessage } from "../../utils/errors";
@ -17,6 +17,11 @@ import {
type RepositoryConfig,
} from "~/schemas/restic";
import { getOrganizationId } from "~/server/core/request-context";
import { serverEvents } from "~/server/core/events";
import { executeDoctor } from "./doctor";
import { logger } from "~/server/utils/logger";
const runningDoctors = new Map<string, AbortController>();
const findRepository = async (idOrShortId: string) => {
const organizationId = getOrganizationId();
@ -344,96 +349,67 @@ const checkHealth = async (repositoryId: string) => {
}
};
const doctorRepository = async (id: string) => {
const organizationId = getOrganizationId();
const startDoctor = async (id: string) => {
const repository = await findRepository(id);
if (!repository) {
throw new NotFoundError("Repository not found");
}
const steps: Array<{ step: string; success: boolean; output: string | null; error: string | null }> = [];
const unlockResult = await restic.unlock(repository.config, organizationId).then(
(result) => ({ success: true, message: result.message, error: null }),
(error) => ({ success: false, message: null, error: toMessage(error) }),
);
steps.push({
step: "unlock",
success: unlockResult.success,
output: unlockResult.message,
error: unlockResult.error,
});
const releaseLock = await repoMutex.acquireExclusive(repository.id, "doctor");
try {
const checkResult = await restic.check(repository.config, { readData: false, organizationId }).then(
(result) => result,
(error) => ({ success: false, output: null, error: toMessage(error), hasErrors: true }),
);
steps.push({
step: "check",
success: checkResult.success,
output: checkResult.output,
error: checkResult.error,
});
if (checkResult.hasErrors) {
const repairResult = await restic.repairIndex(repository.config, organizationId).then(
(result) => ({ success: true, output: result.output, error: null }),
(error) => ({ success: false, output: null, error: toMessage(error) }),
);
steps.push({
step: "repair_index",
success: repairResult.success,
output: repairResult.output,
error: repairResult.error,
});
const recheckResult = await restic.check(repository.config, { readData: false, organizationId }).then(
(result) => result,
(error) => ({ success: false, output: null, error: toMessage(error), hasErrors: true }),
);
steps.push({
step: "recheck",
success: recheckResult.success,
output: recheckResult.output,
error: recheckResult.error,
});
}
} catch (error) {
steps.push({
step: "unexpected_error",
success: false,
output: null,
error: toMessage(error),
});
} finally {
releaseLock();
if (runningDoctors.has(repository.id)) {
throw new ConflictError("Doctor operation already in progress");
}
const doctorSucceeded = steps.every((step) => step.success);
const doctorError = steps.find((step) => step.error)?.error ?? null;
const abortController = new AbortController();
await db
.update(repositoriesTable)
.set({
status: doctorSucceeded ? "healthy" : "error",
lastChecked: Date.now(),
lastError: doctorError,
try {
await db.update(repositoriesTable).set({ status: "doctor" }).where(eq(repositoriesTable.id, repository.id));
serverEvents.emit("doctor:started", {
repositoryId: repository.id,
repositoryName: repository.name,
});
runningDoctors.set(repository.id, abortController);
} catch (error) {
runningDoctors.delete(repository.id);
throw error;
}
executeDoctor(repository.id, repository.config, repository.name, abortController.signal)
.catch((error) => {
logger.error(`Doctor background task failed: ${toMessage(error)}`);
})
.where(
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)),
);
.finally(() => {
runningDoctors.delete(repository.id);
});
return {
success: doctorSucceeded,
steps,
};
return { message: "Doctor operation started", repositoryId: repository.id };
};
const cancelDoctor = async (id: string) => {
const repository = await findRepository(id);
if (!repository) {
throw new NotFoundError("Repository not found");
}
const abortController = runningDoctors.get(repository.id);
if (!abortController) {
throw new ConflictError("No doctor operation is currently running");
}
abortController.abort();
runningDoctors.delete(repository.id);
await db.update(repositoriesTable).set({ status: "unknown" }).where(eq(repositoriesTable.id, repository.id));
serverEvents.emit("doctor:cancelled", {
repositoryId: repository.id,
repositoryName: repository.name,
});
return { message: "Doctor operation cancelled" };
};
const deleteSnapshot = async (id: string, snapshotId: string) => {
@ -546,7 +522,8 @@ export const repositoriesService = {
restoreSnapshot,
getSnapshotDetails,
checkHealth,
doctorRepository,
startDoctor,
cancelDoctor,
deleteSnapshot,
deleteSnapshots,
tagSnapshots,

11
app/server/utils/json.ts Normal file
View file

@ -0,0 +1,11 @@
export function safeJsonParse<T>(input: string | null | undefined): T | null {
if (!input) {
return null;
}
try {
return JSON.parse(input) as T;
} catch {
return null;
}
}

View file

@ -736,16 +736,21 @@ const ls = async (config: RepositoryConfig, snapshotId: string, organizationId:
return { snapshot, nodes };
};
const unlock = async (config: RepositoryConfig, organizationId: string) => {
const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, organizationId);
const env = await buildEnv(config, options.organizationId);
const args = ["unlock", "--repo", repoUrl, "--remove-all"];
addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env });
const res = await exec({ command: "restic", args, env, signal: options?.signal });
await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) {
logger.warn("Restic unlock was aborted by signal.");
return { success: false, message: "Operation aborted" };
}
if (res.exitCode !== 0) {
logger.error(`Restic unlock failed: ${res.stderr}`);
throw new ResticError(res.exitCode, res.stderr);
@ -755,7 +760,10 @@ const unlock = async (config: RepositoryConfig, organizationId: string) => {
return { success: true, message: "Repository unlocked successfully" };
};
const check = async (config: RepositoryConfig, options: { readData?: boolean; organizationId: string }) => {
const check = async (
config: RepositoryConfig,
options: { readData?: boolean; signal?: AbortSignal; organizationId: string },
) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, options.organizationId);
@ -767,9 +775,14 @@ const check = async (config: RepositoryConfig, options: { readData?: boolean; or
addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env });
const res = await exec({ command: "restic", args, env, signal: options?.signal });
await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) {
logger.warn("Restic check was aborted by signal.");
return { success: false, hasErrors: true, output: "", error: "Operation aborted" };
}
const { stdout, stderr } = res;
if (res.exitCode !== 0) {
@ -793,16 +806,21 @@ const check = async (config: RepositoryConfig, options: { readData?: boolean; or
};
};
const repairIndex = async (config: RepositoryConfig, organizationId: string) => {
const repairIndex = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, organizationId);
const env = await buildEnv(config, options.organizationId);
const args = ["repair", "index", "--repo", repoUrl];
addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env });
const res = await exec({ command: "restic", args, env, signal: options?.signal });
await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) {
logger.warn("Restic repair index was aborted by signal.");
return { success: false, message: "Operation aborted", output: "" };
}
const { stdout, stderr } = res;
if (res.exitCode !== 0) {