Compare commits
5 commits
main
...
graphite-b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc820f17b0 | ||
|
|
c2c5a1d235 | ||
|
|
7fba271107 | ||
|
|
c54ee10f73 | ||
|
|
1d6e7dcf65 |
8 changed files with 123 additions and 35 deletions
|
|
@ -40,7 +40,7 @@ export const StatusDot = ({ variant, label, animated }: StatusDotProps) => {
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger>
|
<TooltipTrigger aria-label={label}>
|
||||||
<span className="relative flex size-3 mx-auto">
|
<span className="relative flex size-3 mx-auto">
|
||||||
{statusMapping?.animated && (
|
{statusMapping?.animated && (
|
||||||
<span
|
<span
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,8 @@ export const BackupCard = ({ schedule }: { schedule: BackupSchedule }) => {
|
||||||
</div>
|
</div>
|
||||||
<BackupStatusDot
|
<BackupStatusDot
|
||||||
enabled={schedule.enabled}
|
enabled={schedule.enabled}
|
||||||
hasError={!!schedule.lastBackupError}
|
hasError={schedule.lastBackupStatus === "error"}
|
||||||
|
hasWarning={schedule.lastBackupStatus === "warning"}
|
||||||
isInProgress={schedule.lastBackupStatus === "in_progress"}
|
isInProgress={schedule.lastBackupStatus === "in_progress"}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -3,13 +3,15 @@ import { StatusDot } from "~/client/components/status-dot";
|
||||||
export const BackupStatusDot = ({
|
export const BackupStatusDot = ({
|
||||||
enabled,
|
enabled,
|
||||||
hasError,
|
hasError,
|
||||||
|
hasWarning,
|
||||||
isInProgress,
|
isInProgress,
|
||||||
}: {
|
}: {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
hasError?: boolean;
|
hasError?: boolean;
|
||||||
|
hasWarning?: boolean;
|
||||||
isInProgress?: boolean;
|
isInProgress?: boolean;
|
||||||
}) => {
|
}) => {
|
||||||
let variant: "success" | "neutral" | "error" | "info";
|
let variant: "success" | "neutral" | "error" | "warning" | "info";
|
||||||
let label: string;
|
let label: string;
|
||||||
|
|
||||||
if (isInProgress) {
|
if (isInProgress) {
|
||||||
|
|
@ -18,6 +20,9 @@ export const BackupStatusDot = ({
|
||||||
} else if (hasError) {
|
} else if (hasError) {
|
||||||
variant = "error";
|
variant = "error";
|
||||||
label = "Error";
|
label = "Error";
|
||||||
|
} else if (hasWarning) {
|
||||||
|
variant = "warning";
|
||||||
|
label = "Warning";
|
||||||
} else if (enabled) {
|
} else if (enabled) {
|
||||||
variant = "success";
|
variant = "success";
|
||||||
label = "Active";
|
label = "Active";
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,8 @@ import { toast } from "sonner";
|
||||||
import { handleRepositoryError } from "~/client/lib/errors";
|
import { handleRepositoryError } from "~/client/lib/errors";
|
||||||
import { formatShortDateTime, formatTimeAgo } from "~/client/lib/datetime";
|
import { formatShortDateTime, formatTimeAgo } from "~/client/lib/datetime";
|
||||||
import { Link } from "@tanstack/react-router";
|
import { Link } from "@tanstack/react-router";
|
||||||
|
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "~/client/components/ui/collapsible";
|
||||||
|
import { cn } from "~/client/lib/utils";
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
schedule: BackupSchedule;
|
schedule: BackupSchedule;
|
||||||
|
|
@ -198,22 +200,41 @@ export const ScheduleSummary = (props: Props) => {
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{schedule.lastBackupStatus === "warning" && (
|
{(schedule.lastBackupStatus === "warning" || schedule.lastBackupStatus === "error") && (
|
||||||
<div className="@medium:col-span-2 @wide:col-span-4">
|
<div className="@medium:col-span-2 @wide:col-span-4">
|
||||||
<p className="text-xs uppercase text-muted-foreground">Warning Details</p>
|
<Collapsible
|
||||||
<p className="font-mono text-sm text-yellow-600 whitespace-pre-wrap wrap-break-word">
|
className={cn("border border-border/50 rounded-lg overflow-hidden", {
|
||||||
|
"border-yellow-500/20 bg-yellow-500/5": schedule.lastBackupStatus === "warning",
|
||||||
|
"border-red-500/20 bg-red-500/5": schedule.lastBackupStatus === "error",
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<CollapsibleTrigger
|
||||||
|
className={cn("w-full justify-start p-3 hover:bg-muted/50 transition-colors", {
|
||||||
|
"hover:bg-yellow-500/10": schedule.lastBackupStatus === "warning",
|
||||||
|
"hover:bg-red-500/10": schedule.lastBackupStatus === "error",
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<span>{schedule.lastBackupStatus === "warning" ? "Warning details" : "Error details"}</span>
|
||||||
|
</CollapsibleTrigger>
|
||||||
|
<CollapsibleContent
|
||||||
|
className={cn("border-t border-border/50 bg-muted/30", {
|
||||||
|
"border-yellow-500/20 bg-yellow-500/8": schedule.lastBackupStatus === "warning",
|
||||||
|
"border-red-500/20 bg-red-500/8": schedule.lastBackupStatus === "error",
|
||||||
|
})}
|
||||||
|
>
|
||||||
|
<div className="p-3">
|
||||||
|
<p
|
||||||
|
className={cn("font-mono text-sm whitespace-pre-wrap wrap-break-word", {
|
||||||
|
"text-yellow-600": schedule.lastBackupStatus === "warning",
|
||||||
|
"text-red-600": schedule.lastBackupStatus === "error",
|
||||||
|
})}
|
||||||
|
>
|
||||||
{schedule.lastBackupError ??
|
{schedule.lastBackupError ??
|
||||||
"Last backup completed with warnings. Check your container logs for more details."}
|
"No additional details available. check your container logs for more information."}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
</CollapsibleContent>
|
||||||
|
</Collapsible>
|
||||||
{schedule.lastBackupError && schedule.lastBackupStatus === "error" && (
|
|
||||||
<div className="@medium:col-span-2 @wide:col-span-4">
|
|
||||||
<p className="text-xs uppercase text-muted-foreground">Error details</p>
|
|
||||||
<p className="font-mono text-sm text-red-600 whitespace-pre-wrap wrap-break-word">
|
|
||||||
{schedule.lastBackupError}
|
|
||||||
</p>
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</CardContent>
|
</CardContent>
|
||||||
|
|
|
||||||
|
|
@ -13,26 +13,35 @@ type NodeRuntimeRequest = Request & {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export const prepareApiRequest = (request: Request, timeoutMs: number) => {
|
type RequestInitWithDuplex = RequestInit & {
|
||||||
const nodeRequest = request as NodeRuntimeRequest;
|
duplex?: "half";
|
||||||
nodeRequest.runtime?.node?.res?.setTimeout(timeoutMs);
|
};
|
||||||
|
|
||||||
|
export const prepareApiRequest = (request: NodeRuntimeRequest, timeoutMs: number) => {
|
||||||
|
request.runtime?.node?.res?.setTimeout(timeoutMs);
|
||||||
|
|
||||||
if (config.trustProxy && request.headers.has("x-forwarded-for")) {
|
if (config.trustProxy && request.headers.has("x-forwarded-for")) {
|
||||||
return request.clone();
|
return request.clone();
|
||||||
}
|
}
|
||||||
|
|
||||||
const remoteAddress = nodeRequest.ip;
|
const remoteAddress = request.ip;
|
||||||
if (remoteAddress) {
|
|
||||||
const headers = new Headers(request.headers);
|
const headers = new Headers(request.headers);
|
||||||
headers.set("x-forwarded-for", remoteAddress);
|
|
||||||
|
|
||||||
return new Request(request, { headers });
|
if (remoteAddress) {
|
||||||
|
headers.set("x-forwarded-for", remoteAddress);
|
||||||
|
} else {
|
||||||
|
headers.delete("x-forwarded-for");
|
||||||
}
|
}
|
||||||
|
|
||||||
const headers = new Headers(request.headers);
|
const init: RequestInitWithDuplex = {
|
||||||
headers.delete("x-forwarded-for");
|
method: request.method,
|
||||||
|
headers,
|
||||||
|
body: request.body,
|
||||||
|
signal: request.signal,
|
||||||
|
duplex: request.body ? "half" : undefined,
|
||||||
|
};
|
||||||
|
|
||||||
return new Request(request, { headers });
|
return new Request(request.url, init);
|
||||||
};
|
};
|
||||||
|
|
||||||
const handle = ({ request }: { request: Request }) =>
|
const handle = ({ request }: { request: Request }) =>
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import { generateBackupOutput } from "~/test/helpers/restic";
|
||||||
import { TEST_ORG_ID } from "~/test/helpers/organization";
|
import { TEST_ORG_ID } from "~/test/helpers/organization";
|
||||||
import * as context from "~/server/core/request-context";
|
import * as context from "~/server/core/request-context";
|
||||||
import * as spawnModule from "@zerobyte/core/node";
|
import * as spawnModule from "@zerobyte/core/node";
|
||||||
|
import type { SafeSpawnParams } from "@zerobyte/core/node";
|
||||||
import { restic } from "~/server/core/restic";
|
import { restic } from "~/server/core/restic";
|
||||||
import { NotFoundError, BadRequestError } from "http-errors-enhanced";
|
import { NotFoundError, BadRequestError } from "http-errors-enhanced";
|
||||||
import { scheduleQueries } from "../backups.queries";
|
import { scheduleQueries } from "../backups.queries";
|
||||||
|
|
@ -17,7 +18,9 @@ import { fromAny } from "@total-typescript/shoehorn";
|
||||||
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
||||||
|
|
||||||
const setup = () => {
|
const setup = () => {
|
||||||
const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }));
|
const resticBackupMock = mock((_: SafeSpawnParams) =>
|
||||||
|
Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }),
|
||||||
|
);
|
||||||
const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null }));
|
const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null }));
|
||||||
const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" }));
|
const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" }));
|
||||||
const refreshStatsMock = mock(() =>
|
const refreshStatsMock = mock(() =>
|
||||||
|
|
@ -148,6 +151,32 @@ describe("backup execution - validation failures", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("stop backup", () => {
|
describe("stop backup", () => {
|
||||||
|
test("should keep restic warning details when backup completes with read errors", async () => {
|
||||||
|
const { resticBackupMock } = setup();
|
||||||
|
const volume = await createTestVolume();
|
||||||
|
const repository = await createTestRepository();
|
||||||
|
const schedule = await createTestBackupSchedule({
|
||||||
|
volumeId: volume.id,
|
||||||
|
repositoryId: repository.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
resticBackupMock.mockImplementationOnce((params: SafeSpawnParams) => {
|
||||||
|
params.onStderr?.("error: open /mnt/data/private.db: permission denied");
|
||||||
|
|
||||||
|
return Promise.resolve({
|
||||||
|
exitCode: 3,
|
||||||
|
summary: generateBackupOutput(),
|
||||||
|
error: "Warning: at least one source file could not be read",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await backupsExecutionService.executeBackup(schedule.id);
|
||||||
|
|
||||||
|
const updatedSchedule = await backupsService.getScheduleById(schedule.id);
|
||||||
|
expect(updatedSchedule.lastBackupStatus).toBe("warning");
|
||||||
|
expect(updatedSchedule.lastBackupError).toBe("error: open /mnt/data/private.db: permission denied");
|
||||||
|
});
|
||||||
|
|
||||||
test("should stop a running backup", async () => {
|
test("should stop a running backup", async () => {
|
||||||
// arrange
|
// arrange
|
||||||
const { resticBackupMock } = setup();
|
const { resticBackupMock } = setup();
|
||||||
|
|
|
||||||
|
|
@ -149,6 +149,7 @@ const finalizeSuccessfulBackup = async (
|
||||||
scheduleId: number,
|
scheduleId: number,
|
||||||
exitCode: number,
|
exitCode: number,
|
||||||
result: ResticBackupOutputDto | null,
|
result: ResticBackupOutputDto | null,
|
||||||
|
warningDetails: string | null,
|
||||||
) => {
|
) => {
|
||||||
const finalStatus = exitCode === 0 ? "success" : "warning";
|
const finalStatus = exitCode === 0 ? "success" : "warning";
|
||||||
|
|
||||||
|
|
@ -174,7 +175,7 @@ const finalizeSuccessfulBackup = async (
|
||||||
await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
|
await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
|
||||||
lastBackupAt: Date.now(),
|
lastBackupAt: Date.now(),
|
||||||
lastBackupStatus: finalStatus,
|
lastBackupStatus: finalStatus,
|
||||||
lastBackupError: null,
|
lastBackupError: finalStatus === "warning" ? warningDetails : null,
|
||||||
nextBackupAt,
|
nextBackupAt,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -285,7 +286,13 @@ const executeBackup = async (scheduleId: number, manual = false): Promise<void>
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const backupResult = await runBackupOperation(ctx, abortController.signal);
|
const backupResult = await runBackupOperation(ctx, abortController.signal);
|
||||||
await finalizeSuccessfulBackup(ctx, scheduleId, backupResult.exitCode, backupResult.result);
|
await finalizeSuccessfulBackup(
|
||||||
|
ctx,
|
||||||
|
scheduleId,
|
||||||
|
backupResult.exitCode,
|
||||||
|
backupResult.result,
|
||||||
|
backupResult.warningDetails,
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await handleBackupFailure(scheduleId, ctx.organizationId, error, ctx);
|
await handleBackupFailure(scheduleId, ctx.organizationId, error, ctx);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
||||||
|
|
@ -101,6 +101,7 @@ export const backup = async (
|
||||||
const logData = throttle((data: string) => {
|
const logData = throttle((data: string) => {
|
||||||
logger.info(data.trim());
|
logger.info(data.trim());
|
||||||
}, 5000);
|
}, 5000);
|
||||||
|
const stderrLines: string[] = [];
|
||||||
|
|
||||||
const streamProgress = throttle((data: string) => {
|
const streamProgress = throttle((data: string) => {
|
||||||
if (options.onProgress) {
|
if (options.onProgress) {
|
||||||
|
|
@ -134,6 +135,13 @@ export const backup = async (
|
||||||
streamProgress(data);
|
streamProgress(data);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
onStderr: (error) => {
|
||||||
|
const line = error.trim();
|
||||||
|
if (line.length > 0) {
|
||||||
|
stderrLines.push(line);
|
||||||
|
logger.error(`restic stderr: ${line}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (includeFile) {
|
if (includeFile) {
|
||||||
|
|
@ -146,7 +154,7 @@ export const backup = async (
|
||||||
|
|
||||||
if (options.signal?.aborted) {
|
if (options.signal?.aborted) {
|
||||||
logger.warn("Restic backup was aborted by signal.");
|
logger.warn("Restic backup was aborted by signal.");
|
||||||
return { result: null, exitCode: res.exitCode };
|
return { result: null, exitCode: res.exitCode, warningDetails: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (res.exitCode === 3) {
|
if (res.exitCode === 3) {
|
||||||
|
|
@ -174,8 +182,16 @@ export const backup = async (
|
||||||
|
|
||||||
if (!result.success) {
|
if (!result.success) {
|
||||||
logger.error(`Restic backup output validation failed: ${result.error.message}`);
|
logger.error(`Restic backup output validation failed: ${result.error.message}`);
|
||||||
return { result: null, exitCode: res.exitCode };
|
return {
|
||||||
|
result: null,
|
||||||
|
exitCode: res.exitCode,
|
||||||
|
warningDetails: stderrLines.length > 0 ? stderrLines.join("\n") : null,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
return { result: result.data, exitCode: res.exitCode };
|
return {
|
||||||
|
result: result.data,
|
||||||
|
exitCode: res.exitCode,
|
||||||
|
warningDetails: stderrLines.length > 0 ? stderrLines.join("\n") : null,
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue