From 3fc3039d9e7381dfc964f06d5749c8776b6cf5fb Mon Sep 17 00:00:00 2001 From: Nicolas Meienberger Date: Wed, 26 Nov 2025 18:43:22 +0100 Subject: [PATCH] feat: report partial backups with warnings --- app/client/api-client/types.gen.ts | 10 ++--- app/client/components/volume-icon.tsx | 4 +- .../backups/components/schedule-summary.tsx | 10 +++++ app/schemas/notifications.ts | 1 + app/server/core/events.ts | 2 +- app/server/db/schema.ts | 2 +- app/server/modules/backups/backups.dto.ts | 2 +- app/server/modules/backups/backups.service.ts | 14 +++--- .../modules/events/events.controller.ts | 2 +- .../notifications/notifications.service.ts | 22 +++++++++- app/server/utils/errors.ts | 22 ++++++++++ app/server/utils/restic.ts | 43 +++++++++++-------- app/server/utils/spawn.ts | 3 +- docker-compose.yml | 5 +-- 14 files changed, 102 insertions(+), 40 deletions(-) diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index 8a983f9c..fb477402 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -1181,7 +1181,7 @@ export type ListBackupSchedulesResponses = { includePatterns: Array | null; lastBackupAt: number | null; lastBackupError: string | null; - lastBackupStatus: 'error' | 'in_progress' | 'success' | null; + lastBackupStatus: 'error' | 'in_progress' | 'success' | 'warning' | null; nextBackupAt: number | null; repository: { compressionMode: 'auto' | 'better' | 'fastest' | 'max' | 'off' | null; @@ -1351,7 +1351,7 @@ export type CreateBackupScheduleResponses = { includePatterns: Array | null; lastBackupAt: number | null; lastBackupError: string | null; - lastBackupStatus: 'error' | 'in_progress' | 'success' | null; + lastBackupStatus: 'error' | 'in_progress' | 'success' | 'warning' | null; nextBackupAt: number | null; repositoryId: string; retentionPolicy: { @@ -1412,7 +1412,7 @@ export type GetBackupScheduleResponses = { includePatterns: Array | null; lastBackupAt: number | null; lastBackupError: string | null; - lastBackupStatus: 'error' | 'in_progress' | 'success' | null; + lastBackupStatus: 'error' | 'in_progress' | 'success' | 'warning' | null; nextBackupAt: number | null; repository: { compressionMode: 'auto' | 'better' | 'fastest' | 'max' | 'off' | null; @@ -1583,7 +1583,7 @@ export type UpdateBackupScheduleResponses = { includePatterns: Array | null; lastBackupAt: number | null; lastBackupError: string | null; - lastBackupStatus: 'error' | 'in_progress' | 'success' | null; + lastBackupStatus: 'error' | 'in_progress' | 'success' | 'warning' | null; nextBackupAt: number | null; repositoryId: string; retentionPolicy: { @@ -1624,7 +1624,7 @@ export type GetBackupScheduleForVolumeResponses = { includePatterns: Array | null; lastBackupAt: number | null; lastBackupError: string | null; - lastBackupStatus: 'error' | 'in_progress' | 'success' | null; + lastBackupStatus: 'error' | 'in_progress' | 'success' | 'warning' | null; nextBackupAt: number | null; repository: { compressionMode: 'auto' | 'better' | 'fastest' | 'max' | 'off' | null; diff --git a/app/client/components/volume-icon.tsx b/app/client/components/volume-icon.tsx index d2936c3e..4935215a 100644 --- a/app/client/components/volume-icon.tsx +++ b/app/client/components/volume-icon.tsx @@ -41,12 +41,12 @@ const getIconAndColor = (backend: BackendType) => { } }; -export const VolumeIcon = ({ backend, size = 10 }: VolumeIconProps) => { +export const VolumeIcon = ({ backend }: VolumeIconProps) => { const { icon: Icon, label } = getIconAndColor(backend); return ( - + {label} ); diff --git a/app/client/modules/backups/components/schedule-summary.tsx b/app/client/modules/backups/components/schedule-summary.tsx index ba18d803..4422c775 100644 --- a/app/client/modules/backups/components/schedule-summary.tsx +++ b/app/client/modules/backups/components/schedule-summary.tsx @@ -164,10 +164,20 @@ export const ScheduleSummary = (props: Props) => { {schedule.lastBackupStatus === "success" && "✓ Success"} {schedule.lastBackupStatus === "error" && "✗ Error"} {schedule.lastBackupStatus === "in_progress" && "⟳ in progress..."} + {schedule.lastBackupStatus === "warning" && "! Warning"} {!schedule.lastBackupStatus && "—"}

+ {schedule.lastBackupStatus === "warning" && ( +
+

Warning Details

+

+ Last backup completed with warnings. Check your container logs for more details. +

+
+ )} + {schedule.lastBackupError && (

Error Details

diff --git a/app/schemas/notifications.ts b/app/schemas/notifications.ts index 09a4d988..33bafccf 100644 --- a/app/schemas/notifications.ts +++ b/app/schemas/notifications.ts @@ -83,6 +83,7 @@ export const NOTIFICATION_EVENTS = { start: "start", success: "success", failure: "failure", + warning: "warning", } as const; export type NotificationEvent = keyof typeof NOTIFICATION_EVENTS; diff --git a/app/server/core/events.ts b/app/server/core/events.ts index 8e55c0e7..33fbb42a 100644 --- a/app/server/core/events.ts +++ b/app/server/core/events.ts @@ -22,7 +22,7 @@ interface ServerEvents { scheduleId: number; volumeName: string; repositoryName: string; - status: "success" | "error" | "stopped"; + status: "success" | "error" | "stopped" | "warning"; }) => void; "volume:mounted": (data: { volumeName: string }) => void; "volume:unmounted": (data: { volumeName: string }) => void; diff --git a/app/server/db/schema.ts b/app/server/db/schema.ts index bdcb37a8..f4842d6b 100644 --- a/app/server/db/schema.ts +++ b/app/server/db/schema.ts @@ -85,7 +85,7 @@ export const backupSchedulesTable = sqliteTable("backup_schedules_table", { excludePatterns: text("exclude_patterns", { mode: "json" }).$type().default([]), includePatterns: text("include_patterns", { mode: "json" }).$type().default([]), lastBackupAt: int("last_backup_at", { mode: "number" }), - lastBackupStatus: text("last_backup_status").$type<"success" | "error" | "in_progress">(), + lastBackupStatus: text("last_backup_status").$type<"success" | "error" | "in_progress" | "warning">(), lastBackupError: text("last_backup_error"), nextBackupAt: int("next_backup_at", { mode: "number" }), createdAt: int("created_at", { mode: "number" }).notNull().default(sql`(unixepoch())`), diff --git a/app/server/modules/backups/backups.dto.ts b/app/server/modules/backups/backups.dto.ts index 8e11e77a..95c4d49c 100644 --- a/app/server/modules/backups/backups.dto.ts +++ b/app/server/modules/backups/backups.dto.ts @@ -25,7 +25,7 @@ const backupScheduleSchema = type({ excludePatterns: "string[] | null", includePatterns: "string[] | null", lastBackupAt: "number | null", - lastBackupStatus: "'success' | 'error' | 'in_progress' | null", + lastBackupStatus: "'success' | 'error' | 'in_progress' | 'warning' | null", lastBackupError: "string | null", nextBackupAt: "number | null", createdAt: "number", diff --git a/app/server/modules/backups/backups.service.ts b/app/server/modules/backups/backups.service.ts index 669fbfba..e36bcf1a 100644 --- a/app/server/modules/backups/backups.service.ts +++ b/app/server/modules/backups/backups.service.ts @@ -236,7 +236,7 @@ const executeBackup = async (scheduleId: number, manual = false) => { backupOptions.include = schedule.includePatterns; } - await restic.backup(repository.config, volumePath, { + const { exitCode } = await restic.backup(repository.config, volumePath, { ...backupOptions, compressionMode: repository.compressionMode ?? "auto", onProgress: (progress) => { @@ -258,24 +258,28 @@ const executeBackup = async (scheduleId: number, manual = false) => { .update(backupSchedulesTable) .set({ lastBackupAt: Date.now(), - lastBackupStatus: "success", + lastBackupStatus: exitCode === 0 ? "success" : "warning", lastBackupError: null, nextBackupAt: nextBackupAt, updatedAt: Date.now(), }) .where(eq(backupSchedulesTable.id, scheduleId)); - logger.info(`Backup completed successfully for volume ${volume.name} to repository ${repository.name}`); + if (exitCode !== 0) { + logger.warn(`Backup completed with warnings for volume ${volume.name} to repository ${repository.name}`); + } else { + logger.info(`Backup completed successfully for volume ${volume.name} to repository ${repository.name}`); + } serverEvents.emit("backup:completed", { scheduleId, volumeName: volume.name, repositoryName: repository.name, - status: "success", + status: exitCode === 0 ? "success" : "warning", }); notificationsService - .sendBackupNotification(scheduleId, "success", { + .sendBackupNotification(scheduleId, exitCode === 0 ? "success" : "warning", { volumeName: volume.name, repositoryName: repository.name, }) diff --git a/app/server/modules/events/events.controller.ts b/app/server/modules/events/events.controller.ts index 971f0b92..a93dc27e 100644 --- a/app/server/modules/events/events.controller.ts +++ b/app/server/modules/events/events.controller.ts @@ -41,7 +41,7 @@ export const eventsController = new Hono().get("/", (c) => { scheduleId: number; volumeName: string; repositoryName: string; - status: "success" | "error" | "stopped"; + status: "success" | "error" | "stopped" | "warning"; }) => { stream.writeSSE({ data: JSON.stringify(data), diff --git a/app/server/modules/notifications/notifications.service.ts b/app/server/modules/notifications/notifications.service.ts index 7e7a8a45..a3a83c58 100644 --- a/app/server/modules/notifications/notifications.service.ts +++ b/app/server/modules/notifications/notifications.service.ts @@ -291,6 +291,7 @@ const sendBackupNotification = async ( case "success": return assignment.notifyOnSuccess; case "failure": + case "warning": return assignment.notifyOnFailure; default: return false; @@ -367,7 +368,7 @@ function buildNotificationMessage( case "success": return { - title: "✅ Backup Completed Successfully", + title: "✅ Backup Completed successfully", body: [ `Volume: ${context.volumeName}`, `Repository: ${context.repositoryName}`, @@ -381,9 +382,26 @@ function buildNotificationMessage( .join("\n"), }; + case "warning": + return { + title: "! Backup completed with warnings", + body: [ + `Volume: ${context.volumeName}`, + `Repository: ${context.repositoryName}`, + context.duration ? `Duration: ${Math.round(context.duration / 1000)}s` : null, + context.filesProcessed !== undefined ? `Files: ${context.filesProcessed}` : null, + context.bytesProcessed ? `Size: ${context.bytesProcessed}` : null, + context.snapshotId ? `Snapshot: ${context.snapshotId}` : null, + context.error ? `Warning: ${context.error}` : null, + `Time: ${date} - ${time}`, + ] + .filter(Boolean) + .join("\n"), + }; + case "failure": return { - title: "❌ Backup Failed", + title: "❌ Backup failed", body: [ `Volume: ${context.volumeName}`, `Repository: ${context.repositoryName}`, diff --git a/app/server/utils/errors.ts b/app/server/utils/errors.ts index 11660517..22379a36 100644 --- a/app/server/utils/errors.ts +++ b/app/server/utils/errors.ts @@ -17,3 +17,25 @@ export const toMessage = (err: unknown): string => { const message = err instanceof Error ? err.message : String(err); return sanitizeSensitiveData(message); }; + +const resticErrorCodes: Record = { + 1: "Command failed: An error occurred while executing the command.", + 2: "Go runtime error: A runtime error occurred in the Go program.", + 3: "Backup could not read all files: Some files could not be read during backup.", + 10: "Repository not found: The specified repository could not be found.", + 11: "Failed to lock repository: Unable to acquire a lock on the repository. Try to run doctor on the repository.", + 12: "Wrong repository password: The provided password for the repository is incorrect.", + 130: "Backup interrupted: The backup process was interrupted.", +}; + +export class ResticError extends Error { + code: number; + + constructor(code: number, stderr: string) { + const message = resticErrorCodes[code] || `Unknown restic error with code ${code}`; + super(`${message}\n${stderr}`); + + this.code = code; + this.name = "ResticError"; + } +} diff --git a/app/server/utils/restic.ts b/app/server/utils/restic.ts index 1ef957fd..3e4b0378 100644 --- a/app/server/utils/restic.ts +++ b/app/server/utils/restic.ts @@ -10,6 +10,8 @@ import { cryptoUtils } from "./crypto"; import type { RetentionPolicy } from "../modules/backups/backups.dto"; import { safeSpawn } from "./spawn"; import type { CompressionMode, RepositoryConfig } from "~/schemas/restic"; +import type { RepositoryConfig } from "~/schemas/restic"; +import { ResticError } from "./errors"; const backupOutputSchema = type({ message_type: "'summary'", @@ -313,34 +315,41 @@ const backup = async ( streamProgress(data); } }, - onStderr: (error) => { - logger.error(error.trim()); - }, finally: async () => { includeFile && (await fs.unlink(includeFile).catch(() => {})); await cleanupTemporaryKeys(config, env); }, }); - if (res.exitCode !== 0) { - logger.error(`Restic backup failed: ${res.stderr}`); + if (res.exitCode === 3) { + logger.error(`Restic backup encountered read errors: ${res.stderr.toString()}`); + } + + if (res.exitCode !== 0 && res.exitCode !== 3) { + logger.error(`Restic backup failed: ${res.stderr.toString()}`); logger.error(`Command executed: restic ${args.join(" ")}`); - throw new Error(`Restic backup failed: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr.toString()); } const lastLine = stdout.trim(); - const resSummary = JSON.parse(lastLine ?? "{}"); + let summaryLine = ""; + try { + const resSummary = JSON.parse(lastLine ?? "{}"); + summaryLine = resSummary; + } catch (_) { + logger.warn("Failed to parse restic backup output JSON summary.", lastLine); + summaryLine = "{}"; + } - const result = backupOutputSchema(resSummary); + const result = backupOutputSchema(summaryLine); if (result instanceof type.errors) { logger.error(`Restic backup output validation failed: ${result}`); - - throw new Error(`Restic backup output validation failed: ${result}`); + return { result: null, exitCode: res.exitCode }; } - return result; + return { result, exitCode: res.exitCode }; }; const restoreOutputSchema = type({ @@ -404,7 +413,7 @@ const restore = async ( if (res.exitCode !== 0) { logger.error(`Restic restore failed: ${res.stderr}`); - throw new Error(`Restic restore failed: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr.toString()); } const stdout = res.text(); @@ -517,7 +526,7 @@ const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra: if (res.exitCode !== 0) { logger.error(`Restic forget failed: ${res.stderr}`); - throw new Error(`Restic forget failed: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr.toString()); } return { success: true }; @@ -535,7 +544,7 @@ const deleteSnapshot = async (config: RepositoryConfig, snapshotId: string) => { if (res.exitCode !== 0) { logger.error(`Restic snapshot deletion failed: ${res.stderr}`); - throw new Error(`Failed to delete snapshot: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr.toString()); } return { success: true }; @@ -585,7 +594,7 @@ const ls = async (config: RepositoryConfig, snapshotId: string, path?: string) = if (res.exitCode !== 0) { logger.error(`Restic ls failed: ${res.stderr}`); - throw new Error(`Restic ls failed: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr.toString()); } // The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes @@ -636,7 +645,7 @@ const unlock = async (config: RepositoryConfig) => { if (res.exitCode !== 0) { logger.error(`Restic unlock failed: ${res.stderr}`); - throw new Error(`Restic unlock failed: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr.toString()); } logger.info(`Restic unlock succeeded for repository: ${repoUrl}`); @@ -697,7 +706,7 @@ const repairIndex = async (config: RepositoryConfig) => { if (res.exitCode !== 0) { logger.error(`Restic repair index failed: ${stderr}`); - throw new Error(`Restic repair index failed: ${stderr}`); + throw new ResticError(res.exitCode, stderr); } logger.info(`Restic repair index completed for repository: ${repoUrl}`); diff --git a/app/server/utils/spawn.ts b/app/server/utils/spawn.ts index fd465aa1..6ef86cba 100644 --- a/app/server/utils/spawn.ts +++ b/app/server/utils/spawn.ts @@ -41,9 +41,8 @@ export const safeSpawn = (params: Params) => { child.stderr.on("data", (data) => { if (callbacks.onStderr) { callbacks.onStderr(data.toString()); - } else { - stderrData += data.toString(); } + stderrData += data.toString(); }); child.on("error", async (error) => { diff --git a/docker-compose.yml b/docker-compose.yml index 2695c23f..2478d6bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,9 +20,8 @@ services: - ./app:/app/app - ~/.config/rclone:/root/.config/rclone - - /var/lib/zerobyte:/var/lib/zerobyte:rshared - - /run/docker/plugins:/run/docker/plugins - - /var/run/docker.sock:/var/run/docker.sock + # - /run/docker/plugins:/run/docker/plugins + # - /var/run/docker.sock:/var/run/docker.sock zerobyte-prod: build: