feat: background doctor operation
This commit is contained in:
Nicolas Meienberger
parent
36b17d73eb
commit
beb5792b45
20 changed files with 2672 additions and 232 deletions
|
|
@ -5,6 +5,7 @@ import { type DefaultError, queryOptions, type UseMutationOptions } from "@tanst
|
|||
import { client } from "../client.gen";
|
||||
import {
|
||||
browseFilesystem,
|
||||
cancelDoctor,
|
||||
createBackupSchedule,
|
||||
createNotificationDestination,
|
||||
createRepository,
|
||||
|
|
@ -15,7 +16,6 @@ import {
|
|||
deleteSnapshot,
|
||||
deleteSnapshots,
|
||||
deleteVolume,
|
||||
doctorRepository,
|
||||
downloadResticPassword,
|
||||
getBackupSchedule,
|
||||
getBackupScheduleForVolume,
|
||||
|
|
@ -44,6 +44,7 @@ import {
|
|||
restoreSnapshot,
|
||||
runBackupNow,
|
||||
runForget,
|
||||
startDoctor,
|
||||
stopBackup,
|
||||
tagSnapshots,
|
||||
testConnection,
|
||||
|
|
@ -59,6 +60,8 @@ import {
|
|||
import type {
|
||||
BrowseFilesystemData,
|
||||
BrowseFilesystemResponse,
|
||||
CancelDoctorData,
|
||||
CancelDoctorResponse,
|
||||
CreateBackupScheduleData,
|
||||
CreateBackupScheduleResponse,
|
||||
CreateNotificationDestinationData,
|
||||
|
|
@ -79,8 +82,6 @@ import type {
|
|||
DeleteSnapshotsResponse,
|
||||
DeleteVolumeData,
|
||||
DeleteVolumeResponse,
|
||||
DoctorRepositoryData,
|
||||
DoctorRepositoryResponse,
|
||||
DownloadResticPasswordData,
|
||||
DownloadResticPasswordResponse,
|
||||
GetBackupScheduleData,
|
||||
|
|
@ -135,6 +136,8 @@ import type {
|
|||
RunBackupNowResponse,
|
||||
RunForgetData,
|
||||
RunForgetResponse,
|
||||
StartDoctorData,
|
||||
StartDoctorResponse,
|
||||
StopBackupData,
|
||||
StopBackupResponse,
|
||||
TagSnapshotsData,
|
||||
|
|
@ -685,14 +688,33 @@ export const restoreSnapshotMutation = (
|
|||
};
|
||||
|
||||
/**
|
||||
* Run doctor operations on a repository to fix common issues (unlock, check, repair index). Use this when the repository is locked or has errors.
|
||||
* Cancel a running doctor operation on a repository
|
||||
*/
|
||||
export const doctorRepositoryMutation = (
|
||||
options?: Partial<Options<DoctorRepositoryData>>,
|
||||
): UseMutationOptions<DoctorRepositoryResponse, DefaultError, Options<DoctorRepositoryData>> => {
|
||||
const mutationOptions: UseMutationOptions<DoctorRepositoryResponse, DefaultError, Options<DoctorRepositoryData>> = {
|
||||
export const cancelDoctorMutation = (
|
||||
options?: Partial<Options<CancelDoctorData>>,
|
||||
): UseMutationOptions<CancelDoctorResponse, DefaultError, Options<CancelDoctorData>> => {
|
||||
const mutationOptions: UseMutationOptions<CancelDoctorResponse, DefaultError, Options<CancelDoctorData>> = {
|
||||
mutationFn: async (fnOptions) => {
|
||||
const { data } = await doctorRepository({
|
||||
const { data } = await cancelDoctor({
|
||||
...options,
|
||||
...fnOptions,
|
||||
throwOnError: true,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
};
|
||||
return mutationOptions;
|
||||
};
|
||||
|
||||
/**
|
||||
* Start an asynchronous doctor operation on a repository to fix common issues (unlock, check, repair index). The operation runs in the background and sends results via SSE events.
|
||||
*/
|
||||
export const startDoctorMutation = (
|
||||
options?: Partial<Options<StartDoctorData>>,
|
||||
): UseMutationOptions<StartDoctorResponse, DefaultError, Options<StartDoctorData>> => {
|
||||
const mutationOptions: UseMutationOptions<StartDoctorResponse, DefaultError, Options<StartDoctorData>> = {
|
||||
mutationFn: async (fnOptions) => {
|
||||
const { data } = await startDoctor({
|
||||
...options,
|
||||
...fnOptions,
|
||||
throwOnError: true,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
export {
|
||||
browseFilesystem,
|
||||
cancelDoctor,
|
||||
createBackupSchedule,
|
||||
createNotificationDestination,
|
||||
createRepository,
|
||||
|
|
@ -12,7 +13,6 @@ export {
|
|||
deleteSnapshot,
|
||||
deleteSnapshots,
|
||||
deleteVolume,
|
||||
doctorRepository,
|
||||
downloadResticPassword,
|
||||
getBackupSchedule,
|
||||
getBackupScheduleForVolume,
|
||||
|
|
@ -41,6 +41,7 @@ export {
|
|||
restoreSnapshot,
|
||||
runBackupNow,
|
||||
runForget,
|
||||
startDoctor,
|
||||
stopBackup,
|
||||
tagSnapshots,
|
||||
testConnection,
|
||||
|
|
@ -57,6 +58,10 @@ export type {
|
|||
BrowseFilesystemData,
|
||||
BrowseFilesystemResponse,
|
||||
BrowseFilesystemResponses,
|
||||
CancelDoctorData,
|
||||
CancelDoctorErrors,
|
||||
CancelDoctorResponse,
|
||||
CancelDoctorResponses,
|
||||
ClientOptions,
|
||||
CreateBackupScheduleData,
|
||||
CreateBackupScheduleResponse,
|
||||
|
|
@ -89,9 +94,6 @@ export type {
|
|||
DeleteVolumeData,
|
||||
DeleteVolumeResponse,
|
||||
DeleteVolumeResponses,
|
||||
DoctorRepositoryData,
|
||||
DoctorRepositoryResponse,
|
||||
DoctorRepositoryResponses,
|
||||
DownloadResticPasswordData,
|
||||
DownloadResticPasswordResponse,
|
||||
DownloadResticPasswordResponses,
|
||||
|
|
@ -176,6 +178,10 @@ export type {
|
|||
RunForgetData,
|
||||
RunForgetResponse,
|
||||
RunForgetResponses,
|
||||
StartDoctorData,
|
||||
StartDoctorErrors,
|
||||
StartDoctorResponse,
|
||||
StartDoctorResponses,
|
||||
StopBackupData,
|
||||
StopBackupErrors,
|
||||
StopBackupResponse,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ import { client } from "./client.gen";
|
|||
import type {
|
||||
BrowseFilesystemData,
|
||||
BrowseFilesystemResponses,
|
||||
CancelDoctorData,
|
||||
CancelDoctorErrors,
|
||||
CancelDoctorResponses,
|
||||
CreateBackupScheduleData,
|
||||
CreateBackupScheduleResponses,
|
||||
CreateNotificationDestinationData,
|
||||
|
|
@ -26,8 +29,6 @@ import type {
|
|||
DeleteSnapshotsResponses,
|
||||
DeleteVolumeData,
|
||||
DeleteVolumeResponses,
|
||||
DoctorRepositoryData,
|
||||
DoctorRepositoryResponses,
|
||||
DownloadResticPasswordData,
|
||||
DownloadResticPasswordResponses,
|
||||
GetBackupScheduleData,
|
||||
|
|
@ -85,6 +86,9 @@ import type {
|
|||
RunBackupNowResponses,
|
||||
RunForgetData,
|
||||
RunForgetResponses,
|
||||
StartDoctorData,
|
||||
StartDoctorErrors,
|
||||
StartDoctorResponses,
|
||||
StopBackupData,
|
||||
StopBackupErrors,
|
||||
StopBackupResponses,
|
||||
|
|
@ -405,12 +409,19 @@ export const restoreSnapshot = <ThrowOnError extends boolean = false>(
|
|||
});
|
||||
|
||||
/**
|
||||
* Run doctor operations on a repository to fix common issues (unlock, check, repair index). Use this when the repository is locked or has errors.
|
||||
* Cancel a running doctor operation on a repository
|
||||
*/
|
||||
export const doctorRepository = <ThrowOnError extends boolean = false>(
|
||||
options: Options<DoctorRepositoryData, ThrowOnError>,
|
||||
) =>
|
||||
(options.client ?? client).post<DoctorRepositoryResponses, unknown, ThrowOnError>({
|
||||
export const cancelDoctor = <ThrowOnError extends boolean = false>(options: Options<CancelDoctorData, ThrowOnError>) =>
|
||||
(options.client ?? client).delete<CancelDoctorResponses, CancelDoctorErrors, ThrowOnError>({
|
||||
url: "/api/v1/repositories/{id}/doctor",
|
||||
...options,
|
||||
});
|
||||
|
||||
/**
|
||||
* Start an asynchronous doctor operation on a repository to fix common issues (unlock, check, repair index). The operation runs in the background and sends results via SSE events.
|
||||
*/
|
||||
export const startDoctor = <ThrowOnError extends boolean = false>(options: Options<StartDoctorData, ThrowOnError>) =>
|
||||
(options.client ?? client).post<StartDoctorResponses, StartDoctorErrors, ThrowOnError>({
|
||||
url: "/api/v1/repositories/{id}/doctor",
|
||||
...options,
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,7 +11,10 @@ type ServerEventType =
|
|||
| "volume:unmounted"
|
||||
| "volume:updated"
|
||||
| "mirror:started"
|
||||
| "mirror:completed";
|
||||
| "mirror:completed"
|
||||
| "doctor:started"
|
||||
| "doctor:completed"
|
||||
| "doctor:cancelled";
|
||||
|
||||
export interface BackupEvent {
|
||||
scheduleId: number;
|
||||
|
|
@ -45,6 +48,21 @@ export interface MirrorEvent {
|
|||
error?: string;
|
||||
}
|
||||
|
||||
export interface DoctorEvent {
|
||||
repositoryId: string;
|
||||
repositoryName: string;
|
||||
}
|
||||
|
||||
export interface DoctorCompletedEvent extends DoctorEvent {
|
||||
success: boolean;
|
||||
steps: Array<{
|
||||
step: string;
|
||||
success: boolean;
|
||||
output: string | null;
|
||||
error: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
type EventHandler = (data: unknown) => void;
|
||||
|
||||
/**
|
||||
|
|
@ -156,6 +174,39 @@ export function useServerEvents() {
|
|||
});
|
||||
});
|
||||
|
||||
eventSource.addEventListener("doctor:started", (e) => {
|
||||
const data = JSON.parse(e.data) as DoctorEvent;
|
||||
console.info("[SSE] Doctor started:", data);
|
||||
|
||||
void queryClient.invalidateQueries();
|
||||
|
||||
handlersRef.current.get("doctor:started")?.forEach((handler) => {
|
||||
handler(data);
|
||||
});
|
||||
});
|
||||
|
||||
eventSource.addEventListener("doctor:completed", (e) => {
|
||||
const data = JSON.parse(e.data) as DoctorCompletedEvent;
|
||||
console.info("[SSE] Doctor completed:", data);
|
||||
|
||||
void queryClient.invalidateQueries();
|
||||
|
||||
handlersRef.current.get("doctor:completed")?.forEach((handler) => {
|
||||
handler(data);
|
||||
});
|
||||
});
|
||||
|
||||
eventSource.addEventListener("doctor:cancelled", (e) => {
|
||||
const data = JSON.parse(e.data) as DoctorEvent;
|
||||
console.info("[SSE] Doctor cancelled:", data);
|
||||
|
||||
void queryClient.invalidateQueries();
|
||||
|
||||
handlersRef.current.get("doctor:cancelled")?.forEach((handler) => {
|
||||
handler(data);
|
||||
});
|
||||
});
|
||||
|
||||
eventSource.onerror = (error) => {
|
||||
console.error("[SSE] Connection error:", error);
|
||||
};
|
||||
|
|
|
|||
88
app/client/modules/repositories/components/doctor-report.tsx
Normal file
88
app/client/modules/repositories/components/doctor-report.tsx
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import { AlertCircle, CheckCircle2 } from "lucide-react";
|
||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "~/client/components/ui/collapsible";
|
||||
import { formatDateTime } from "~/client/lib/datetime";
|
||||
import { cn } from "~/client/lib/utils";
|
||||
|
||||
type DoctorStep = {
|
||||
step: string;
|
||||
success: boolean;
|
||||
output: string | null;
|
||||
error: string | null;
|
||||
};
|
||||
|
||||
type DoctorResult = {
|
||||
success: boolean;
|
||||
steps: DoctorStep[];
|
||||
completedAt: number;
|
||||
};
|
||||
export const DoctorReport = ({
|
||||
result,
|
||||
repositoryStatus,
|
||||
}: {
|
||||
result?: DoctorResult | null;
|
||||
repositoryStatus: string | null;
|
||||
}) => {
|
||||
return (
|
||||
<div>
|
||||
<h3 className="text-lg font-semibold">Doctor Report</h3>
|
||||
{result && (
|
||||
<div className="space-y-2">
|
||||
<span className="text-xs text-muted-foreground">Completed {formatDateTime(result.completedAt)}</span>
|
||||
<div className="space-y-2 mt-2">
|
||||
{result.steps.map((step) => (
|
||||
<Collapsible key={step.step} className="border rounde overflow-hidden bg-muted/30 group">
|
||||
<CollapsibleTrigger className="w-full flex items-center justify-start p-3 hover:bg-muted/50 transition-colors">
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-sm font-medium">{step.step.replace("_", " ")}</span>
|
||||
{step.success ? (
|
||||
<CheckCircle2 className="h-4 w-4 text-green-500" />
|
||||
) : (
|
||||
<AlertCircle className="h-4 w-4 text-red-500" />
|
||||
)}
|
||||
</div>
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="border-t bg-muted/50">
|
||||
<div className="p-2 space-y-3">
|
||||
{step.output && (
|
||||
<pre className="text-xs font-mono bg-background/50 p-3 rounded border overflow-auto max-h-50 whitespace-pre-wrap">
|
||||
{step.output.startsWith("{") ? JSON.stringify(JSON.parse(step.output), null, 2) : step.output}
|
||||
</pre>
|
||||
)}
|
||||
{step.error && (
|
||||
<div className="space-y-1.5">
|
||||
<div className="text-[10px] uppercase font-bold text-red-500/70 px-1">Error</div>
|
||||
<pre className="text-xs font-mono bg-red-500/5 text-red-500 p-3 rounded border border-red-500/20 overflow-auto whitespace-pre-wrap">
|
||||
{step.error}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
{!step.output && !step.error && (
|
||||
<div className="text-xs text-muted-foreground italic px-1">No output recorded</div>
|
||||
)}
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div
|
||||
className={cn("mt-2 bg-muted/30 border rounded-lg p-6 text-center", {
|
||||
hidden: result !== null || repositoryStatus === "doctor",
|
||||
})}
|
||||
>
|
||||
<p className="text-sm text-muted-foreground">No doctor report available.</p>
|
||||
</div>
|
||||
<div
|
||||
className={cn("mt-2 border rounded-lg p-6 text-center", {
|
||||
hidden: repositoryStatus !== "doctor",
|
||||
})}
|
||||
>
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
<div className="h-4 w-4 animate-spin rounded-full border-2 border-primary border-t-transparent" />
|
||||
<p className="text-sm ">Doctor operation is currently running...</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -29,7 +29,7 @@ export const AdvancedForm = ({ form }: Props) => {
|
|||
|
||||
return (
|
||||
<Collapsible>
|
||||
<CollapsibleTrigger className="">Advanced Settings</CollapsibleTrigger>
|
||||
<CollapsibleTrigger>Advanced Settings</CollapsibleTrigger>
|
||||
<CollapsibleContent className="pb-4 space-y-4">
|
||||
<div className="space-y-4 mt-4">
|
||||
<div className="grid gap-6">
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@ import { redirect, useNavigate, useSearchParams } from "react-router";
|
|||
import { toast } from "sonner";
|
||||
import { useState, useEffect } from "react";
|
||||
import {
|
||||
cancelDoctorMutation,
|
||||
deleteRepositoryMutation,
|
||||
doctorRepositoryMutation,
|
||||
getRepositoryOptions,
|
||||
listSnapshotsOptions,
|
||||
startDoctorMutation,
|
||||
} from "~/client/api-client/@tanstack/react-query.gen";
|
||||
import { Button } from "~/client/components/ui/button";
|
||||
import {
|
||||
|
|
@ -25,7 +26,7 @@ import { cn } from "~/client/lib/utils";
|
|||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "~/client/components/ui/tabs";
|
||||
import { RepositoryInfoTabContent } from "../tabs/info";
|
||||
import { RepositorySnapshotsTabContent } from "../tabs/snapshots";
|
||||
import { Loader2, Stethoscope, Trash2 } from "lucide-react";
|
||||
import { Square, Stethoscope, Trash2 } from "lucide-react";
|
||||
|
||||
export const handle = {
|
||||
breadcrumb: (match: Route.MetaArgs) => [
|
||||
|
|
@ -52,8 +53,6 @@ export const clientLoader = async ({ params }: Route.ClientLoaderArgs) => {
|
|||
};
|
||||
|
||||
export default function RepositoryDetailsPage({ loaderData }: Route.ComponentProps) {
|
||||
const [showDoctorResults, setShowDoctorResults] = useState(false);
|
||||
|
||||
const navigate = useNavigate();
|
||||
const queryClient = useQueryClient();
|
||||
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
|
||||
|
|
@ -83,24 +82,22 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
|
|||
},
|
||||
});
|
||||
|
||||
const doctorMutation = useMutation({
|
||||
...doctorRepositoryMutation(),
|
||||
onSuccess: (data) => {
|
||||
if (data) {
|
||||
setShowDoctorResults(true);
|
||||
const startDoctor = useMutation({
|
||||
...startDoctorMutation(),
|
||||
onError: (error) => {
|
||||
toast.error("Failed to start doctor", {
|
||||
description: parseError(error)?.message,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
if (data.success) {
|
||||
toast.success("Repository doctor completed successfully");
|
||||
} else {
|
||||
toast.warning("Doctor completed with some issues", {
|
||||
description: "Check the details for more information",
|
||||
richColors: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
const cancelDoctor = useMutation({
|
||||
...cancelDoctorMutation(),
|
||||
onSuccess: () => {
|
||||
toast.info("Doctor operation cancelled");
|
||||
},
|
||||
onError: (error) => {
|
||||
toast.error("Failed to run doctor", {
|
||||
toast.error("Failed to cancel doctor", {
|
||||
description: parseError(error)?.message,
|
||||
});
|
||||
},
|
||||
|
|
@ -111,21 +108,6 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
|
|||
deleteRepo.mutate({ path: { id: data.id } });
|
||||
};
|
||||
|
||||
const getStepLabel = (step: string) => {
|
||||
switch (step) {
|
||||
case "unlock":
|
||||
return "Unlock Repository";
|
||||
case "check":
|
||||
return "Check Repository";
|
||||
case "repair_index":
|
||||
return "Repair Index";
|
||||
case "recheck":
|
||||
return "Re-check Repository";
|
||||
default:
|
||||
return step;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
|
|
@ -134,6 +116,7 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
|
|||
className={cn("inline-flex items-center gap-2 px-2 py-1 rounded-md text-xs bg-gray-500/10 text-gray-500", {
|
||||
"bg-green-500/10 text-green-500": data.status === "healthy",
|
||||
"bg-red-500/10 text-red-500": data.status === "error",
|
||||
"bg-blue-500/10 text-blue-500": data.status === "doctor",
|
||||
})}
|
||||
>
|
||||
{data.status || "unknown"}
|
||||
|
|
@ -141,23 +124,17 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
|
|||
<span className="text-xs bg-primary/10 rounded-md px-2 py-1">{data.type}</span>
|
||||
</div>
|
||||
<div className="flex gap-4">
|
||||
<Button
|
||||
onClick={() => doctorMutation.mutate({ path: { id: data.id } })}
|
||||
disabled={doctorMutation.isPending}
|
||||
variant={"outline"}
|
||||
>
|
||||
{doctorMutation.isPending ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Running doctor...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Stethoscope className="h-4 w-4 mr-2" />
|
||||
Run doctor
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
{data.status === "doctor" ? (
|
||||
<Button variant="destructive" onClick={() => cancelDoctor.mutate({ path: { id: data.id } })}>
|
||||
<Square className="h-4 w-4 mr-2" />
|
||||
<span>Cancel doctor</span>
|
||||
</Button>
|
||||
) : (
|
||||
<Button onClick={() => startDoctor.mutate({ path: { id: data.id } })} disabled={startDoctor.isPending}>
|
||||
<Stethoscope className="h-4 w-4 mr-2" />
|
||||
Run doctor
|
||||
</Button>
|
||||
)}
|
||||
<Button variant="destructive" onClick={() => setShowDeleteConfirm(true)} disabled={deleteRepo.isPending}>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
Delete
|
||||
|
|
@ -202,46 +179,6 @@ export default function RepositoryDetailsPage({ loaderData }: Route.ComponentPro
|
|||
</div>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
<AlertDialog open={showDoctorResults} onOpenChange={setShowDoctorResults}>
|
||||
<AlertDialogContent className="max-w-2xl">
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Doctor results</AlertDialogTitle>
|
||||
<AlertDialogDescription>Repository doctor operation completed</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
|
||||
{doctorMutation.data && (
|
||||
<div className="space-y-3 max-h-96 overflow-y-auto">
|
||||
{doctorMutation.data.steps.map((step) => (
|
||||
<div
|
||||
key={step.step}
|
||||
className={cn("border rounded-md p-3", {
|
||||
"bg-green-500/10 border-green-500/20": step.success,
|
||||
"bg-yellow-500/10 border-yellow-500/20": !step.success,
|
||||
})}
|
||||
>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<span className="font-medium text-sm">{getStepLabel(step.step)}</span>
|
||||
<span
|
||||
className={cn("text-xs px-2 py-1 rounded", {
|
||||
"bg-green-500/20 text-green-500": step.success,
|
||||
"bg-yellow-500/20 text-yellow-500": !step.success,
|
||||
})}
|
||||
>
|
||||
{step.success ? "Success" : "Warning"}
|
||||
</span>
|
||||
</div>
|
||||
{step.error && <p className="text-xs text-red-500 mt-1">{step.error}</p>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex justify-end">
|
||||
<Button onClick={() => setShowDoctorResults(false)}>Close</Button>
|
||||
</div>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import { REPOSITORY_BASE } from "~/client/lib/constants";
|
|||
import { formatDateTime, formatTimeAgo } from "~/client/lib/datetime";
|
||||
import { updateRepositoryMutation } from "~/client/api-client/@tanstack/react-query.gen";
|
||||
import type { CompressionMode, RepositoryConfig } from "~/schemas/restic";
|
||||
import { DoctorReport } from "../components/doctor-report";
|
||||
|
||||
type Props = {
|
||||
repository: Repository;
|
||||
|
|
@ -126,7 +127,7 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
|
|||
</div>
|
||||
{effectiveLocalPath && (
|
||||
<div className="md:col-span-2">
|
||||
<div className="text-sm font-medium text-muted-foreground">Effective Local Path</div>
|
||||
<div className="text-sm font-medium text-muted-foreground">Local path</div>
|
||||
<p className="mt-1 text-sm font-mono">{effectiveLocalPath}</p>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -179,6 +180,8 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<DoctorReport repositoryStatus={repository.status} result={repository.doctorResult} />
|
||||
|
||||
<div className="flex justify-end pt-4 border-t">
|
||||
<Button type="submit" disabled={!hasChanges || updateMutation.isPending} loading={updateMutation.isPending}>
|
||||
<Save className="h-4 w-4 mr-2" />
|
||||
|
|
|
|||
1
app/drizzle/0034_acoustic_sentry.sql
Normal file
1
app/drizzle/0034_acoustic_sentry.sql
Normal file
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE `repositories_table` ADD `doctor_result` text;
|
||||
1250
app/drizzle/meta/0034_snapshot.json
Normal file
1250
app/drizzle/meta/0034_snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -239,6 +239,13 @@
|
|||
"when": 1768497767122,
|
||||
"tag": "0033_chunky_tyrannus",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 34,
|
||||
"version": "6",
|
||||
"when": 1768571707915,
|
||||
"tag": "0034_acoustic_sentry",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -127,10 +127,28 @@ export const REPOSITORY_STATUS = {
|
|||
healthy: "healthy",
|
||||
error: "error",
|
||||
unknown: "unknown",
|
||||
doctor: "doctor",
|
||||
} as const;
|
||||
|
||||
export type RepositoryStatus = keyof typeof REPOSITORY_STATUS;
|
||||
|
||||
export const doctorStepSchema = type({
|
||||
step: "string",
|
||||
success: "boolean",
|
||||
output: "string | null",
|
||||
error: "string | null",
|
||||
});
|
||||
|
||||
export type DoctorStep = typeof doctorStepSchema.infer;
|
||||
|
||||
export const doctorResultSchema = type({
|
||||
success: "boolean",
|
||||
steps: doctorStepSchema.array(),
|
||||
completedAt: "number",
|
||||
});
|
||||
|
||||
export type DoctorResult = typeof doctorResultSchema.infer;
|
||||
|
||||
export const OVERWRITE_MODES = {
|
||||
always: "always",
|
||||
ifChanged: "if-changed",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { EventEmitter } from "node:events";
|
||||
import type { TypedEmitter } from "tiny-typed-emitter";
|
||||
import type { DoctorResult } from "~/schemas/restic";
|
||||
|
||||
/**
|
||||
* Event payloads for the SSE system
|
||||
|
|
@ -36,6 +37,14 @@ interface ServerEvents {
|
|||
"volume:unmounted": (data: { volumeName: string }) => void;
|
||||
"volume:updated": (data: { volumeName: string }) => void;
|
||||
"volume:status_changed": (data: { volumeName: string; status: string }) => void;
|
||||
"doctor:started": (data: { repositoryId: string; repositoryName: string }) => void;
|
||||
"doctor:completed": (
|
||||
data: {
|
||||
repositoryId: string;
|
||||
repositoryName: string;
|
||||
} & DoctorResult,
|
||||
) => void;
|
||||
"doctor:cancelled": (data: { repositoryId: string; repositoryName: string; error?: string }) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import type {
|
|||
repositoryConfigSchema,
|
||||
RepositoryStatus,
|
||||
BandwidthUnit,
|
||||
DoctorResult,
|
||||
} from "~/schemas/restic";
|
||||
import type { BackendStatus, BackendType, volumeConfigSchema } from "~/schemas/volumes";
|
||||
import type { NotificationType, notificationConfigSchema } from "~/schemas/notifications";
|
||||
|
|
@ -165,6 +166,7 @@ export const repositoriesTable = sqliteTable("repositories_table", {
|
|||
status: text().$type<RepositoryStatus>().default("unknown"),
|
||||
lastChecked: int("last_checked", { mode: "number" }),
|
||||
lastError: text("last_error"),
|
||||
doctorResult: text("doctor_result", { mode: "json" }).$type<DoctorResult>(),
|
||||
uploadLimitEnabled: int("upload_limit_enabled", { mode: "boolean" }).notNull().default(false),
|
||||
uploadLimitValue: real("upload_limit_value").notNull().default(1),
|
||||
uploadLimitUnit: text("upload_limit_unit").$type<BandwidthUnit>().notNull().default("Mbps"),
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { streamSSE } from "hono/streaming";
|
|||
import { logger } from "../../utils/logger";
|
||||
import { serverEvents } from "../../core/events";
|
||||
import { requireAuth } from "../auth/auth.middleware";
|
||||
import type { DoctorResult } from "~/schemas/restic";
|
||||
|
||||
export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
|
||||
logger.info("Client connected to SSE endpoint");
|
||||
|
|
@ -91,6 +92,32 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
|
|||
});
|
||||
};
|
||||
|
||||
const onDoctorStarted = async (data: { repositoryId: string; repositoryName: string }) => {
|
||||
await stream.writeSSE({
|
||||
data: JSON.stringify(data),
|
||||
event: "doctor:started",
|
||||
});
|
||||
};
|
||||
|
||||
const onDoctorCompleted = async (
|
||||
data: {
|
||||
repositoryId: string;
|
||||
repositoryName: string;
|
||||
} & DoctorResult,
|
||||
) => {
|
||||
await stream.writeSSE({
|
||||
data: JSON.stringify(data),
|
||||
event: "doctor:completed",
|
||||
});
|
||||
};
|
||||
|
||||
const onDoctorCancelled = async (data: { repositoryId: string; repositoryName: string; error?: string }) => {
|
||||
await stream.writeSSE({
|
||||
data: JSON.stringify(data),
|
||||
event: "doctor:cancelled",
|
||||
});
|
||||
};
|
||||
|
||||
serverEvents.on("backup:started", onBackupStarted);
|
||||
serverEvents.on("backup:progress", onBackupProgress);
|
||||
serverEvents.on("backup:completed", onBackupCompleted);
|
||||
|
|
@ -99,6 +126,9 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
|
|||
serverEvents.on("volume:updated", onVolumeUpdated);
|
||||
serverEvents.on("mirror:started", onMirrorStarted);
|
||||
serverEvents.on("mirror:completed", onMirrorCompleted);
|
||||
serverEvents.on("doctor:started", onDoctorStarted);
|
||||
serverEvents.on("doctor:completed", onDoctorCompleted);
|
||||
serverEvents.on("doctor:cancelled", onDoctorCancelled);
|
||||
|
||||
let keepAlive = true;
|
||||
|
||||
|
|
@ -113,6 +143,9 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
|
|||
serverEvents.off("volume:updated", onVolumeUpdated);
|
||||
serverEvents.off("mirror:started", onMirrorStarted);
|
||||
serverEvents.off("mirror:completed", onMirrorCompleted);
|
||||
serverEvents.off("doctor:started", onDoctorStarted);
|
||||
serverEvents.off("doctor:completed", onDoctorCompleted);
|
||||
serverEvents.off("doctor:cancelled", onDoctorCancelled);
|
||||
});
|
||||
|
||||
while (keepAlive) {
|
||||
|
|
|
|||
160
app/server/modules/repositories/doctor.ts
Normal file
160
app/server/modules/repositories/doctor.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../../db/db";
|
||||
import { repositoriesTable } from "../../db/schema";
|
||||
import { toMessage } from "../../utils/errors";
|
||||
import { restic } from "../../utils/restic";
|
||||
import { repoMutex } from "../../core/repository-mutex";
|
||||
import { type DoctorStep, type DoctorResult, type RepositoryConfig } from "~/schemas/restic";
|
||||
import { type } from "arktype";
|
||||
import { serverEvents } from "../../core/events";
|
||||
import { logger } from "../../utils/logger";
|
||||
|
||||
const runUnlockStep = async (config: RepositoryConfig): Promise<DoctorStep> => {
|
||||
const result = await restic.unlock(config).then(
|
||||
(result) => ({ success: true, message: result.message, error: null }),
|
||||
(error) => ({ success: false, message: null, error: toMessage(error) }),
|
||||
);
|
||||
|
||||
return {
|
||||
step: "unlock",
|
||||
success: result.success,
|
||||
output: result.message,
|
||||
error: result.error,
|
||||
};
|
||||
};
|
||||
|
||||
const runCheckStep = async (config: RepositoryConfig): Promise<DoctorStep> => {
|
||||
const result = await restic.check(config, { readData: true }).then(
|
||||
(result) => result,
|
||||
(error) => ({ success: false, output: null, error: toMessage(error), hasErrors: true }),
|
||||
);
|
||||
|
||||
return {
|
||||
step: "check",
|
||||
success: result.success,
|
||||
output: result.output,
|
||||
error: result.error,
|
||||
};
|
||||
};
|
||||
|
||||
const runRepairIndexStep = async (config: RepositoryConfig) => {
|
||||
const result = await restic.repairIndex(config).then(
|
||||
(result) => ({ success: true, output: result.output, error: null }),
|
||||
(error) => ({ success: false, output: null, error: toMessage(error) }),
|
||||
);
|
||||
|
||||
return {
|
||||
step: "repair_index",
|
||||
success: result.success,
|
||||
output: result.output,
|
||||
error: result.error,
|
||||
};
|
||||
};
|
||||
|
||||
const parseCheckOutput = (checkOutput: string | null) => {
|
||||
const schema = type({ suggest_repair_index: "boolean", suggest_prune: "boolean" });
|
||||
const parsed = schema(JSON.parse(checkOutput ?? "{}"));
|
||||
|
||||
if (parsed instanceof type.errors) {
|
||||
logger.error(`Invalid check output format: ${parsed.summary}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const checkAbortSignal = (signal: AbortSignal | undefined): void => {
|
||||
if (signal?.aborted) {
|
||||
throw new Error("Doctor operation cancelled");
|
||||
}
|
||||
};
|
||||
|
||||
const determineRepositoryStatus = (steps: DoctorStep[]): "healthy" | "error" => {
|
||||
const repairStep = steps.find((s) => s.step === "repair_index");
|
||||
if (repairStep) {
|
||||
return repairStep.success ? "healthy" : "error";
|
||||
}
|
||||
|
||||
const checkStep = steps.find((s) => s.step === "check");
|
||||
return checkStep?.success ? "healthy" : "error";
|
||||
};
|
||||
|
||||
const saveDoctorResults = async (repositoryId: string, steps: DoctorStep[], finalStatus: "healthy" | "error") => {
|
||||
const doctorResult: DoctorResult = {
|
||||
success: steps.every((s) => s.success),
|
||||
steps,
|
||||
completedAt: Date.now(),
|
||||
};
|
||||
|
||||
const finalError = steps.find((s) => s.error)?.error ?? null;
|
||||
|
||||
await db
|
||||
.update(repositoriesTable)
|
||||
.set({
|
||||
status: finalStatus,
|
||||
lastChecked: Date.now(),
|
||||
lastError: finalError,
|
||||
doctorResult,
|
||||
})
|
||||
.where(eq(repositoriesTable.id, repositoryId));
|
||||
};
|
||||
|
||||
export const executeDoctor = async (
|
||||
repositoryId: string,
|
||||
repositoryConfig: RepositoryConfig,
|
||||
repositoryName: string,
|
||||
signal: AbortSignal | undefined,
|
||||
) => {
|
||||
const steps: DoctorStep[] = [];
|
||||
|
||||
try {
|
||||
const releaseLock = await repoMutex.acquireExclusive(repositoryId, "doctor");
|
||||
|
||||
// Step 1: Unlock repository
|
||||
const unlockStep = await runUnlockStep(repositoryConfig);
|
||||
steps.push(unlockStep);
|
||||
checkAbortSignal(signal);
|
||||
|
||||
// Step 2: Check repository with exclusive lock
|
||||
try {
|
||||
const checkStep = await runCheckStep(repositoryConfig);
|
||||
steps.push(checkStep);
|
||||
checkAbortSignal(signal);
|
||||
|
||||
// Step 3: Repair index if suggested
|
||||
const checkOutput = parseCheckOutput(checkStep.output);
|
||||
if (checkOutput?.suggest_repair_index) {
|
||||
const repairStep = await runRepairIndexStep(repositoryConfig);
|
||||
steps.push(repairStep);
|
||||
checkAbortSignal(signal);
|
||||
}
|
||||
} finally {
|
||||
releaseLock();
|
||||
}
|
||||
|
||||
const finalStatus = determineRepositoryStatus(steps);
|
||||
await saveDoctorResults(repositoryId, steps, finalStatus);
|
||||
|
||||
serverEvents.emit("doctor:completed", {
|
||||
repositoryId,
|
||||
repositoryName,
|
||||
success: steps.every((s) => s.success),
|
||||
steps,
|
||||
completedAt: Date.now(),
|
||||
});
|
||||
} catch (error) {
|
||||
await db
|
||||
.update(repositoriesTable)
|
||||
.set({ status: "error", lastError: toMessage(error) })
|
||||
.where(eq(repositoriesTable.id, repositoryId));
|
||||
|
||||
steps.push({ step: "doctor", success: false, output: null, error: toMessage(error) });
|
||||
serverEvents.emit("doctor:completed", {
|
||||
repositoryId,
|
||||
repositoryName,
|
||||
success: false,
|
||||
steps,
|
||||
completedAt: Date.now(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -7,7 +7,8 @@ import {
|
|||
deleteSnapshotDto,
|
||||
deleteSnapshotsBody,
|
||||
deleteSnapshotsDto,
|
||||
doctorRepositoryDto,
|
||||
startDoctorDto,
|
||||
cancelDoctorDto,
|
||||
getRepositoryDto,
|
||||
getSnapshotDetailsDto,
|
||||
listRcloneRemotesDto,
|
||||
|
|
@ -25,7 +26,8 @@ import {
|
|||
type DeleteRepositoryDto,
|
||||
type DeleteSnapshotDto,
|
||||
type DeleteSnapshotsResponseDto,
|
||||
type DoctorRepositoryDto,
|
||||
type StartDoctorDto,
|
||||
type CancelDoctorDto,
|
||||
type GetRepositoryDto,
|
||||
type GetSnapshotDetailsDto,
|
||||
type ListRepositoriesDto,
|
||||
|
|
@ -152,12 +154,19 @@ export const repositoriesController = new Hono()
|
|||
|
||||
return c.json<RestoreSnapshotDto>(result, 200);
|
||||
})
|
||||
.post("/:id/doctor", doctorRepositoryDto, async (c) => {
|
||||
.post("/:id/doctor", startDoctorDto, async (c) => {
|
||||
const { id } = c.req.param();
|
||||
|
||||
const result = await repositoriesService.doctorRepository(id);
|
||||
const result = await repositoriesService.startDoctor(id);
|
||||
|
||||
return c.json<DoctorRepositoryDto>(result, 200);
|
||||
return c.json<StartDoctorDto>(result, 202);
|
||||
})
|
||||
.delete("/:id/doctor", cancelDoctorDto, async (c) => {
|
||||
const { id } = c.req.param();
|
||||
|
||||
const result = await repositoriesService.cancelDoctor(id);
|
||||
|
||||
return c.json<CancelDoctorDto>(result, 200);
|
||||
})
|
||||
.delete("/:id/snapshots/:snapshotId", deleteSnapshotDto, async (c) => {
|
||||
const { id, snapshotId } = c.req.param();
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import {
|
|||
REPOSITORY_BACKENDS,
|
||||
REPOSITORY_STATUS,
|
||||
repositoryConfigSchema,
|
||||
doctorStepSchema,
|
||||
doctorResultSchema,
|
||||
} from "~/schemas/restic";
|
||||
|
||||
export const repositorySchema = type({
|
||||
|
|
@ -18,6 +20,7 @@ export const repositorySchema = type({
|
|||
status: type.valueOf(REPOSITORY_STATUS).or("null"),
|
||||
lastChecked: "number | null",
|
||||
lastError: "string | null",
|
||||
doctorResult: doctorResultSchema.or("null"),
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
});
|
||||
|
|
@ -317,36 +320,60 @@ export const restoreSnapshotDto = describeRoute({
|
|||
});
|
||||
|
||||
/**
|
||||
* Doctor a repository (unlock, check, repair)
|
||||
* Start doctor operation
|
||||
*/
|
||||
export const doctorStepSchema = type({
|
||||
step: "string",
|
||||
success: "boolean",
|
||||
output: "string | null",
|
||||
error: "string | null",
|
||||
export const startDoctorResponse = type({
|
||||
message: "string",
|
||||
repositoryId: "string",
|
||||
});
|
||||
|
||||
export const doctorRepositoryResponse = type({
|
||||
success: "boolean",
|
||||
steps: doctorStepSchema.array(),
|
||||
});
|
||||
export type StartDoctorDto = typeof startDoctorResponse.infer;
|
||||
|
||||
export type DoctorRepositoryDto = typeof doctorRepositoryResponse.infer;
|
||||
|
||||
export const doctorRepositoryDto = describeRoute({
|
||||
export const startDoctorDto = describeRoute({
|
||||
description:
|
||||
"Run doctor operations on a repository to fix common issues (unlock, check, repair index). Use this when the repository is locked or has errors.",
|
||||
"Start an asynchronous doctor operation on a repository to fix common issues (unlock, check, repair index). The operation runs in the background and sends results via SSE events.",
|
||||
tags: ["Repositories"],
|
||||
operationId: "doctorRepository",
|
||||
operationId: "startDoctor",
|
||||
responses: {
|
||||
200: {
|
||||
description: "Doctor operation completed",
|
||||
202: {
|
||||
description: "Doctor operation started",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: resolver(doctorRepositoryResponse),
|
||||
schema: resolver(startDoctorResponse),
|
||||
},
|
||||
},
|
||||
},
|
||||
409: {
|
||||
description: "Doctor operation already in progress",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Cancel running doctor operation
|
||||
*/
|
||||
export const cancelDoctorResponse = type({
|
||||
message: "string",
|
||||
});
|
||||
|
||||
export type CancelDoctorDto = typeof cancelDoctorResponse.infer;
|
||||
|
||||
export const cancelDoctorDto = describeRoute({
|
||||
description: "Cancel a running doctor operation on a repository",
|
||||
tags: ["Repositories"],
|
||||
operationId: "cancelDoctor",
|
||||
responses: {
|
||||
200: {
|
||||
description: "Doctor operation cancelled",
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: resolver(cancelDoctorResponse),
|
||||
},
|
||||
},
|
||||
},
|
||||
409: {
|
||||
description: "No doctor operation is currently running",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import crypto from "node:crypto";
|
||||
import { eq, or } from "drizzle-orm";
|
||||
import { InternalServerError, NotFoundError } from "http-errors-enhanced";
|
||||
import { InternalServerError, NotFoundError, ConflictError } from "http-errors-enhanced";
|
||||
import { db } from "../../db/db";
|
||||
import { repositoriesTable } from "../../db/schema";
|
||||
import { toMessage } from "../../utils/errors";
|
||||
|
|
@ -16,6 +16,11 @@ import {
|
|||
type RepositoryConfig,
|
||||
} from "~/schemas/restic";
|
||||
import { type } from "arktype";
|
||||
import { serverEvents } from "../../core/events";
|
||||
import { logger } from "../../utils/logger";
|
||||
import { executeDoctor } from "./doctor";
|
||||
|
||||
const runningDoctors = new Map<string, AbortController>();
|
||||
|
||||
const findRepository = async (idOrShortId: string) => {
|
||||
return await db.query.repositoriesTable.findFirst({
|
||||
|
|
@ -321,93 +326,64 @@ const checkHealth = async (repositoryId: string) => {
|
|||
}
|
||||
};
|
||||
|
||||
const doctorRepository = async (id: string) => {
|
||||
const startDoctor = async (id: string) => {
|
||||
const repository = await findRepository(id);
|
||||
|
||||
if (!repository) {
|
||||
throw new NotFoundError("Repository not found");
|
||||
}
|
||||
|
||||
const steps: Array<{ step: string; success: boolean; output: string | null; error: string | null }> = [];
|
||||
|
||||
const unlockResult = await restic.unlock(repository.config).then(
|
||||
(result) => ({ success: true, message: result.message, error: null }),
|
||||
(error) => ({ success: false, message: null, error: toMessage(error) }),
|
||||
);
|
||||
|
||||
steps.push({
|
||||
step: "unlock",
|
||||
success: unlockResult.success,
|
||||
output: unlockResult.message,
|
||||
error: unlockResult.error,
|
||||
});
|
||||
|
||||
const releaseLock = await repoMutex.acquireExclusive(repository.id, "doctor");
|
||||
try {
|
||||
const checkResult = await restic.check(repository.config, { readData: false }).then(
|
||||
(result) => result,
|
||||
(error) => ({ success: false, output: null, error: toMessage(error), hasErrors: true }),
|
||||
);
|
||||
|
||||
steps.push({
|
||||
step: "check",
|
||||
success: checkResult.success,
|
||||
output: checkResult.output,
|
||||
error: checkResult.error,
|
||||
});
|
||||
|
||||
if (checkResult.hasErrors) {
|
||||
const repairResult = await restic.repairIndex(repository.config).then(
|
||||
(result) => ({ success: true, output: result.output, error: null }),
|
||||
(error) => ({ success: false, output: null, error: toMessage(error) }),
|
||||
);
|
||||
|
||||
steps.push({
|
||||
step: "repair_index",
|
||||
success: repairResult.success,
|
||||
output: repairResult.output,
|
||||
error: repairResult.error,
|
||||
});
|
||||
|
||||
const recheckResult = await restic.check(repository.config, { readData: false }).then(
|
||||
(result) => result,
|
||||
(error) => ({ success: false, output: null, error: toMessage(error), hasErrors: true }),
|
||||
);
|
||||
|
||||
steps.push({
|
||||
step: "recheck",
|
||||
success: recheckResult.success,
|
||||
output: recheckResult.output,
|
||||
error: recheckResult.error,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
steps.push({
|
||||
step: "unexpected_error",
|
||||
success: false,
|
||||
output: null,
|
||||
error: toMessage(error),
|
||||
});
|
||||
} finally {
|
||||
releaseLock();
|
||||
if (runningDoctors.has(repository.id)) {
|
||||
throw new ConflictError("Doctor operation already in progress");
|
||||
}
|
||||
|
||||
const doctorSucceeded = steps.every((step) => step.success);
|
||||
const doctorError = steps.find((step) => step.error)?.error ?? null;
|
||||
const abortController = new AbortController();
|
||||
runningDoctors.set(repository.id, abortController);
|
||||
|
||||
await db
|
||||
.update(repositoriesTable)
|
||||
.set({
|
||||
status: doctorSucceeded ? "healthy" : "error",
|
||||
lastChecked: Date.now(),
|
||||
lastError: doctorError,
|
||||
})
|
||||
.set({ status: "doctor", doctorResult: null })
|
||||
.where(eq(repositoriesTable.id, repository.id));
|
||||
|
||||
return {
|
||||
success: doctorSucceeded,
|
||||
steps,
|
||||
};
|
||||
serverEvents.emit("doctor:started", {
|
||||
repositoryId: repository.id,
|
||||
repositoryName: repository.name,
|
||||
});
|
||||
|
||||
executeDoctor(repository.id, repository.config, repository.name, abortController.signal)
|
||||
.catch((error) => {
|
||||
logger.error(`Doctor background task failed: ${toMessage(error)}`);
|
||||
})
|
||||
.finally(() => {
|
||||
runningDoctors.delete(repository.id);
|
||||
});
|
||||
|
||||
return { message: "Doctor operation started", repositoryId: repository.id };
|
||||
};
|
||||
|
||||
const cancelDoctor = async (id: string) => {
|
||||
const repository = await findRepository(id);
|
||||
|
||||
if (!repository) {
|
||||
throw new NotFoundError("Repository not found");
|
||||
}
|
||||
|
||||
await db.update(repositoriesTable).set({ status: "unknown" }).where(eq(repositoriesTable.id, repository.id));
|
||||
|
||||
const abortController = runningDoctors.get(repository.id);
|
||||
if (!abortController) {
|
||||
throw new ConflictError("No doctor operation is currently running");
|
||||
}
|
||||
|
||||
abortController.abort();
|
||||
runningDoctors.delete(repository.id);
|
||||
|
||||
serverEvents.emit("doctor:cancelled", {
|
||||
repositoryId: repository.id,
|
||||
repositoryName: repository.name,
|
||||
});
|
||||
|
||||
return { message: "Doctor operation cancelled" };
|
||||
};
|
||||
|
||||
const deleteSnapshot = async (id: string, snapshotId: string) => {
|
||||
|
|
@ -517,7 +493,8 @@ export const repositoriesService = {
|
|||
restoreSnapshot,
|
||||
getSnapshotDetails,
|
||||
checkHealth,
|
||||
doctorRepository,
|
||||
startDoctor,
|
||||
cancelDoctor,
|
||||
deleteSnapshot,
|
||||
deleteSnapshots,
|
||||
tagSnapshots,
|
||||
|
|
|
|||
Loading…
Reference in a new issue