diff --git a/app/client/components/status-dot.tsx b/app/client/components/status-dot.tsx index 3f550bc4..8a2097b5 100644 --- a/app/client/components/status-dot.tsx +++ b/app/client/components/status-dot.tsx @@ -40,7 +40,7 @@ export const StatusDot = ({ variant, label, animated }: StatusDotProps) => { return ( - + {statusMapping?.animated && ( { diff --git a/app/client/modules/backups/components/backup-status-dot.tsx b/app/client/modules/backups/components/backup-status-dot.tsx index 6d30224d..f099a68e 100644 --- a/app/client/modules/backups/components/backup-status-dot.tsx +++ b/app/client/modules/backups/components/backup-status-dot.tsx @@ -3,13 +3,15 @@ import { StatusDot } from "~/client/components/status-dot"; export const BackupStatusDot = ({ enabled, hasError, + hasWarning, isInProgress, }: { enabled: boolean; hasError?: boolean; + hasWarning?: boolean; isInProgress?: boolean; }) => { - let variant: "success" | "neutral" | "error" | "info"; + let variant: "success" | "neutral" | "error" | "warning" | "info"; let label: string; if (isInProgress) { @@ -18,6 +20,9 @@ export const BackupStatusDot = ({ } else if (hasError) { variant = "error"; label = "Error"; + } else if (hasWarning) { + variant = "warning"; + label = "Warning"; } else if (enabled) { variant = "success"; label = "Active"; diff --git a/app/client/modules/backups/components/schedule-summary.tsx b/app/client/modules/backups/components/schedule-summary.tsx index f978d3c3..089b6c5f 100644 --- a/app/client/modules/backups/components/schedule-summary.tsx +++ b/app/client/modules/backups/components/schedule-summary.tsx @@ -20,6 +20,8 @@ import { toast } from "sonner"; import { handleRepositoryError } from "~/client/lib/errors"; import { formatShortDateTime, formatTimeAgo } from "~/client/lib/datetime"; import { Link } from "@tanstack/react-router"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "~/client/components/ui/collapsible"; +import { cn } from "~/client/lib/utils"; type Props = { schedule: BackupSchedule; @@ -198,22 +200,41 @@ export const ScheduleSummary = (props: Props) => {

- {schedule.lastBackupStatus === "warning" && ( + {(schedule.lastBackupStatus === "warning" || schedule.lastBackupStatus === "error") && (
-

Warning Details

-

- {schedule.lastBackupError ?? - "Last backup completed with warnings. Check your container logs for more details."} -

-
- )} - - {schedule.lastBackupError && schedule.lastBackupStatus === "error" && ( -
-

Error details

-

- {schedule.lastBackupError} -

+ + + {schedule.lastBackupStatus === "warning" ? "Warning details" : "Error details"} + + +
+

+ {schedule.lastBackupError ?? + "No additional details available. check your container logs for more information."} +

+
+ +
)} diff --git a/app/routes/api.$.ts b/app/routes/api.$.ts index 80bdbb0e..f36dfa84 100644 --- a/app/routes/api.$.ts +++ b/app/routes/api.$.ts @@ -13,26 +13,35 @@ type NodeRuntimeRequest = Request & { }; }; -export const prepareApiRequest = (request: Request, timeoutMs: number) => { - const nodeRequest = request as NodeRuntimeRequest; - nodeRequest.runtime?.node?.res?.setTimeout(timeoutMs); +type RequestInitWithDuplex = RequestInit & { + duplex?: "half"; +}; + +export const prepareApiRequest = (request: NodeRuntimeRequest, timeoutMs: number) => { + request.runtime?.node?.res?.setTimeout(timeoutMs); if (config.trustProxy && request.headers.has("x-forwarded-for")) { return request.clone(); } - const remoteAddress = nodeRequest.ip; - if (remoteAddress) { - const headers = new Headers(request.headers); - headers.set("x-forwarded-for", remoteAddress); + const remoteAddress = request.ip; + const headers = new Headers(request.headers); - return new Request(request, { headers }); + if (remoteAddress) { + headers.set("x-forwarded-for", remoteAddress); + } else { + headers.delete("x-forwarded-for"); } - const headers = new Headers(request.headers); - headers.delete("x-forwarded-for"); + const init: RequestInitWithDuplex = { + method: request.method, + headers, + body: request.body, + signal: request.signal, + duplex: request.body ? "half" : undefined, + }; - return new Request(request, { headers }); + return new Request(request.url, init); }; const handle = ({ request }: { request: Request }) => diff --git a/app/server/modules/backups/__tests__/backups.execution.test.ts b/app/server/modules/backups/__tests__/backups.execution.test.ts index fb563469..bc984c58 100644 --- a/app/server/modules/backups/__tests__/backups.execution.test.ts +++ b/app/server/modules/backups/__tests__/backups.execution.test.ts @@ -10,6 +10,7 @@ import { generateBackupOutput } from "~/test/helpers/restic"; import { TEST_ORG_ID } from "~/test/helpers/organization"; import * as context from "~/server/core/request-context"; import * as spawnModule from "@zerobyte/core/node"; +import type { SafeSpawnParams } from "@zerobyte/core/node"; import { restic } from "~/server/core/restic"; import { NotFoundError, BadRequestError } from "http-errors-enhanced"; import { scheduleQueries } from "../backups.queries"; @@ -17,7 +18,9 @@ import { fromAny } from "@total-typescript/shoehorn"; import { repositoriesService } from "~/server/modules/repositories/repositories.service"; const setup = () => { - const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" })); + const resticBackupMock = mock((_: SafeSpawnParams) => + Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }), + ); const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null })); const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" })); const refreshStatsMock = mock(() => @@ -148,6 +151,32 @@ describe("backup execution - validation failures", () => { }); describe("stop backup", () => { + test("should keep restic warning details when backup completes with read errors", async () => { + const { resticBackupMock } = setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + + resticBackupMock.mockImplementationOnce((params: SafeSpawnParams) => { + params.onStderr?.("error: open /mnt/data/private.db: permission denied"); + + return Promise.resolve({ + exitCode: 3, + summary: generateBackupOutput(), + error: "Warning: at least one source file could not be read", + }); + }); + + await backupsExecutionService.executeBackup(schedule.id); + + const updatedSchedule = await backupsService.getScheduleById(schedule.id); + expect(updatedSchedule.lastBackupStatus).toBe("warning"); + expect(updatedSchedule.lastBackupError).toBe("error: open /mnt/data/private.db: permission denied"); + }); + test("should stop a running backup", async () => { // arrange const { resticBackupMock } = setup(); diff --git a/app/server/modules/backups/backups.execution.ts b/app/server/modules/backups/backups.execution.ts index e9389334..f446514a 100644 --- a/app/server/modules/backups/backups.execution.ts +++ b/app/server/modules/backups/backups.execution.ts @@ -149,6 +149,7 @@ const finalizeSuccessfulBackup = async ( scheduleId: number, exitCode: number, result: ResticBackupOutputDto | null, + warningDetails: string | null, ) => { const finalStatus = exitCode === 0 ? "success" : "warning"; @@ -174,7 +175,7 @@ const finalizeSuccessfulBackup = async ( await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, { lastBackupAt: Date.now(), lastBackupStatus: finalStatus, - lastBackupError: null, + lastBackupError: finalStatus === "warning" ? warningDetails : null, nextBackupAt, }); @@ -285,7 +286,13 @@ const executeBackup = async (scheduleId: number, manual = false): Promise try { const backupResult = await runBackupOperation(ctx, abortController.signal); - await finalizeSuccessfulBackup(ctx, scheduleId, backupResult.exitCode, backupResult.result); + await finalizeSuccessfulBackup( + ctx, + scheduleId, + backupResult.exitCode, + backupResult.result, + backupResult.warningDetails, + ); } catch (error) { await handleBackupFailure(scheduleId, ctx.organizationId, error, ctx); } finally { diff --git a/packages/core/src/restic/commands/backup.ts b/packages/core/src/restic/commands/backup.ts index aa27aa98..b0022db8 100644 --- a/packages/core/src/restic/commands/backup.ts +++ b/packages/core/src/restic/commands/backup.ts @@ -101,6 +101,7 @@ export const backup = async ( const logData = throttle((data: string) => { logger.info(data.trim()); }, 5000); + const stderrLines: string[] = []; const streamProgress = throttle((data: string) => { if (options.onProgress) { @@ -134,6 +135,13 @@ export const backup = async ( streamProgress(data); } }, + onStderr: (error) => { + const line = error.trim(); + if (line.length > 0) { + stderrLines.push(line); + logger.error(`restic stderr: ${line}`); + } + }, }); if (includeFile) { @@ -146,7 +154,7 @@ export const backup = async ( if (options.signal?.aborted) { logger.warn("Restic backup was aborted by signal."); - return { result: null, exitCode: res.exitCode }; + return { result: null, exitCode: res.exitCode, warningDetails: null }; } if (res.exitCode === 3) { @@ -174,8 +182,16 @@ export const backup = async ( if (!result.success) { logger.error(`Restic backup output validation failed: ${result.error.message}`); - return { result: null, exitCode: res.exitCode }; + return { + result: null, + exitCode: res.exitCode, + warningDetails: stderrLines.length > 0 ? stderrLines.join("\n") : null, + }; } - return { result: result.data, exitCode: res.exitCode }; + return { + result: result.data, + exitCode: res.exitCode, + warningDetails: stderrLines.length > 0 ? stderrLines.join("\n") : null, + }; };