parent
e39220c024
commit
1d6e7dcf65
4 changed files with 78 additions and 20 deletions
|
|
@ -20,6 +20,7 @@ import { toast } from "sonner";
|
||||||
import { handleRepositoryError } from "~/client/lib/errors";
|
import { handleRepositoryError } from "~/client/lib/errors";
|
||||||
import { formatShortDateTime, formatTimeAgo } from "~/client/lib/datetime";
|
import { formatShortDateTime, formatTimeAgo } from "~/client/lib/datetime";
|
||||||
import { Link } from "@tanstack/react-router";
|
import { Link } from "@tanstack/react-router";
|
||||||
|
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "~/client/components/ui/collapsible";
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
schedule: BackupSchedule;
|
schedule: BackupSchedule;
|
||||||
|
|
@ -89,6 +90,11 @@ export const ScheduleSummary = (props: Props) => {
|
||||||
handleStopBackup();
|
handleStopBackup();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const backupDetails =
|
||||||
|
schedule.lastBackupStatus === "warning"
|
||||||
|
? (schedule.lastBackupError ?? "Last backup completed with warnings. Check your container logs for more details.")
|
||||||
|
: schedule.lastBackupError;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
<Card className="@container">
|
<Card className="@container">
|
||||||
|
|
@ -198,22 +204,24 @@ export const ScheduleSummary = (props: Props) => {
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{schedule.lastBackupStatus === "warning" && (
|
{backupDetails && (schedule.lastBackupStatus === "warning" || schedule.lastBackupStatus === "error") && (
|
||||||
<div className="@medium:col-span-2 @wide:col-span-4">
|
<div className="@medium:col-span-2 @wide:col-span-4">
|
||||||
<p className="text-xs uppercase text-muted-foreground">Warning Details</p>
|
<Collapsible className="border border-border/50 rounded-lg overflow-hidden">
|
||||||
<p className="font-mono text-sm text-yellow-600 whitespace-pre-wrap wrap-break-word">
|
<CollapsibleTrigger className="w-full p-3 hover:bg-muted/50 transition-colors">
|
||||||
{schedule.lastBackupError ??
|
<span>{schedule.lastBackupStatus === "warning" ? "Warning details" : "Error details"}</span>
|
||||||
"Last backup completed with warnings. Check your container logs for more details."}
|
</CollapsibleTrigger>
|
||||||
</p>
|
<CollapsibleContent className="border-t border-border/50 bg-muted/30">
|
||||||
</div>
|
<div className="p-3">
|
||||||
)}
|
<p
|
||||||
|
className={`font-mono text-sm whitespace-pre-wrap wrap-break-word ${
|
||||||
{schedule.lastBackupError && schedule.lastBackupStatus === "error" && (
|
schedule.lastBackupStatus === "warning" ? "text-yellow-600" : "text-red-600"
|
||||||
<div className="@medium:col-span-2 @wide:col-span-4">
|
}`}
|
||||||
<p className="text-xs uppercase text-muted-foreground">Error details</p>
|
>
|
||||||
<p className="font-mono text-sm text-red-600 whitespace-pre-wrap wrap-break-word">
|
{backupDetails}
|
||||||
{schedule.lastBackupError}
|
</p>
|
||||||
</p>
|
</div>
|
||||||
|
</CollapsibleContent>
|
||||||
|
</Collapsible>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</CardContent>
|
</CardContent>
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import { generateBackupOutput } from "~/test/helpers/restic";
|
||||||
import { TEST_ORG_ID } from "~/test/helpers/organization";
|
import { TEST_ORG_ID } from "~/test/helpers/organization";
|
||||||
import * as context from "~/server/core/request-context";
|
import * as context from "~/server/core/request-context";
|
||||||
import * as spawnModule from "@zerobyte/core/node";
|
import * as spawnModule from "@zerobyte/core/node";
|
||||||
|
import type { SafeSpawnParams } from "@zerobyte/core/node";
|
||||||
import { restic } from "~/server/core/restic";
|
import { restic } from "~/server/core/restic";
|
||||||
import { NotFoundError, BadRequestError } from "http-errors-enhanced";
|
import { NotFoundError, BadRequestError } from "http-errors-enhanced";
|
||||||
import { scheduleQueries } from "../backups.queries";
|
import { scheduleQueries } from "../backups.queries";
|
||||||
|
|
@ -148,6 +149,32 @@ describe("backup execution - validation failures", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("stop backup", () => {
|
describe("stop backup", () => {
|
||||||
|
test("should keep restic warning details when backup completes with read errors", async () => {
|
||||||
|
const { resticBackupMock } = setup();
|
||||||
|
const volume = await createTestVolume();
|
||||||
|
const repository = await createTestRepository();
|
||||||
|
const schedule = await createTestBackupSchedule({
|
||||||
|
volumeId: volume.id,
|
||||||
|
repositoryId: repository.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
resticBackupMock.mockImplementationOnce((params: SafeSpawnParams) => {
|
||||||
|
params.onStderr?.("error: open /mnt/data/private.db: permission denied");
|
||||||
|
|
||||||
|
return Promise.resolve({
|
||||||
|
exitCode: 3,
|
||||||
|
summary: generateBackupOutput(),
|
||||||
|
error: "Warning: at least one source file could not be read",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await backupsExecutionService.executeBackup(schedule.id);
|
||||||
|
|
||||||
|
const updatedSchedule = await backupsService.getScheduleById(schedule.id);
|
||||||
|
expect(updatedSchedule.lastBackupStatus).toBe("warning");
|
||||||
|
expect(updatedSchedule.lastBackupError).toBe("error: open /mnt/data/private.db: permission denied");
|
||||||
|
});
|
||||||
|
|
||||||
test("should stop a running backup", async () => {
|
test("should stop a running backup", async () => {
|
||||||
// arrange
|
// arrange
|
||||||
const { resticBackupMock } = setup();
|
const { resticBackupMock } = setup();
|
||||||
|
|
|
||||||
|
|
@ -149,6 +149,7 @@ const finalizeSuccessfulBackup = async (
|
||||||
scheduleId: number,
|
scheduleId: number,
|
||||||
exitCode: number,
|
exitCode: number,
|
||||||
result: ResticBackupOutputDto | null,
|
result: ResticBackupOutputDto | null,
|
||||||
|
warningDetails: string | null,
|
||||||
) => {
|
) => {
|
||||||
const finalStatus = exitCode === 0 ? "success" : "warning";
|
const finalStatus = exitCode === 0 ? "success" : "warning";
|
||||||
|
|
||||||
|
|
@ -174,7 +175,7 @@ const finalizeSuccessfulBackup = async (
|
||||||
await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
|
await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
|
||||||
lastBackupAt: Date.now(),
|
lastBackupAt: Date.now(),
|
||||||
lastBackupStatus: finalStatus,
|
lastBackupStatus: finalStatus,
|
||||||
lastBackupError: null,
|
lastBackupError: finalStatus === "warning" ? warningDetails : null,
|
||||||
nextBackupAt,
|
nextBackupAt,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -285,7 +286,13 @@ const executeBackup = async (scheduleId: number, manual = false): Promise<void>
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const backupResult = await runBackupOperation(ctx, abortController.signal);
|
const backupResult = await runBackupOperation(ctx, abortController.signal);
|
||||||
await finalizeSuccessfulBackup(ctx, scheduleId, backupResult.exitCode, backupResult.result);
|
await finalizeSuccessfulBackup(
|
||||||
|
ctx,
|
||||||
|
scheduleId,
|
||||||
|
backupResult.exitCode,
|
||||||
|
backupResult.result,
|
||||||
|
backupResult.warningDetails,
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await handleBackupFailure(scheduleId, ctx.organizationId, error, ctx);
|
await handleBackupFailure(scheduleId, ctx.organizationId, error, ctx);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
||||||
|
|
@ -101,6 +101,7 @@ export const backup = async (
|
||||||
const logData = throttle((data: string) => {
|
const logData = throttle((data: string) => {
|
||||||
logger.info(data.trim());
|
logger.info(data.trim());
|
||||||
}, 5000);
|
}, 5000);
|
||||||
|
const stderrLines: string[] = [];
|
||||||
|
|
||||||
const streamProgress = throttle((data: string) => {
|
const streamProgress = throttle((data: string) => {
|
||||||
if (options.onProgress) {
|
if (options.onProgress) {
|
||||||
|
|
@ -134,6 +135,13 @@ export const backup = async (
|
||||||
streamProgress(data);
|
streamProgress(data);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
onStderr: (error) => {
|
||||||
|
const line = error.trim();
|
||||||
|
if (line.length > 0) {
|
||||||
|
stderrLines.push(line);
|
||||||
|
logger.error(`restic stderr: ${line}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (includeFile) {
|
if (includeFile) {
|
||||||
|
|
@ -146,7 +154,7 @@ export const backup = async (
|
||||||
|
|
||||||
if (options.signal?.aborted) {
|
if (options.signal?.aborted) {
|
||||||
logger.warn("Restic backup was aborted by signal.");
|
logger.warn("Restic backup was aborted by signal.");
|
||||||
return { result: null, exitCode: res.exitCode };
|
return { result: null, exitCode: res.exitCode, warningDetails: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (res.exitCode === 3) {
|
if (res.exitCode === 3) {
|
||||||
|
|
@ -174,8 +182,16 @@ export const backup = async (
|
||||||
|
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
logger.error(`Restic backup output validation failed: ${result.error.message}`);
|
logger.error(`Restic backup output validation failed: ${result.error.message}`);
|
||||||
return { result: null, exitCode: res.exitCode };
|
return {
|
||||||
|
result: null,
|
||||||
|
exitCode: res.exitCode,
|
||||||
|
warningDetails: stderrLines.length > 0 ? stderrLines.join("\n") : null,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return { result: result.data, exitCode: res.exitCode };
|
return {
|
||||||
|
result: result.data,
|
||||||
|
exitCode: res.exitCode,
|
||||||
|
warningDetails: stderrLines.length > 0 ? stderrLines.join("\n") : null,
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue