feat(backup): retry backup on failure

app/drizzle/20260407172925_fresh_warlock/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `backup_schedules_table` ADD `failure_retry_count` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE `backup_schedules_table` ADD `max_retries` integer DEFAULT 3 NOT NULL;--> statement-breakpoint
+ALTER TABLE `backup_schedules_table` ADD `retry_delay` integer DEFAULT 3600000 NOT NULL;

app/server/db/schema.ts

@@ -311,6 +311,9 @@ export const backupSchedulesTable = sqliteTable("backup_schedules_table", {
 	oneFileSystem: int("one_file_system", { mode: "boolean" }).notNull().default(false),
 	customResticParams: text("custom_restic_params", { mode: "json" }).$type<string[]>().default([]),
 	sortOrder: int("sort_order", { mode: "number" }).notNull().default(0),
+	failureRetryCount: int("failure_retry_count").notNull().default(0),
+	maxRetries: int("max_retries").notNull().default(3),
+	retryDelay: int("retry_delay").notNull().default(3600000),
 	createdAt: int("created_at", { mode: "number" })
 		.notNull()
 		.default(sql`(unixepoch() * 1000)`),

app/server/modules/backups/backups.queries.ts

@@ -42,6 +42,7 @@ export const scheduleQueries = {
 			lastBackupAt?: number;
 			lastBackupError?: string | null;
 			nextBackupAt?: number | null;
+			failureRetryCount?: number;
 		},
 	) => {
 		return db

app/server/modules/backups/helpers/backup-lifecycle.ts

@@ -167,6 +167,7 @@ export async function finalizeSuccessfulBackup(
 		lastBackupStatus: finalStatus,
 		lastBackupError: finalStatus === "warning" ? warningDetails : null,
 		nextBackupAt: ctx.schedule.cronExpression ? calculateNextRun(ctx.schedule.cronExpression) : null,
+		failureRetryCount: 0,
 	});
 
 	if (finalStatus === "warning") {
@@ -219,10 +220,55 @@ export async function handleBackupFailure(
 		return;
 	}
 
-	const { schedule, volume, repository } = partialContext;
+	// Determine if the backup should be retried
+	const schedule = partialContext.schedule;
+	const currentRetryCount = schedule.failureRetryCount;
+	const maxRetries = schedule.maxRetries;
+	const shouldRetry = currentRetryCount < maxRetries;
+
+	if (shouldRetry) {
+		const nextBackupAt = Date.now() + schedule.retryDelay;
+
+		await scheduleQueries.updateStatus(scheduleId, organizationId, {
+			lastBackupAt: Date.now(),
+			lastBackupStatus: "error",
+			lastBackupError: errorDetails,
+			nextBackupAt,
+			failureRetryCount: currentRetryCount + 1,
+		});
+
+		logger.warn(
+			`Backup ${schedule?.name} failed. Scheduling retry ${currentRetryCount + 1}/${maxRetries} for 1 hour from now: ${errorMessage}`,
+		);
+
+		if (partialContext?.volume && partialContext?.repository) {
+			serverEvents.emit("backup:completed", {
+				organizationId,
+				scheduleId: schedule!.shortId,
+				volumeName: partialContext.volume.name,
+				repositoryName: partialContext.repository.name,
+				status: "error",
+			});
+
+			notificationsService
+				.sendBackupNotification(scheduleId, "failure", {
+					volumeName: partialContext.volume.name,
+					repositoryName: partialContext.repository.name,
+					scheduleName: schedule!.name,
+					error: `${errorDetails}\n\nRetrying in 1 hour (attempt ${currentRetryCount + 1}/${maxRetries})`,
+				})
+				.catch((notifError) => {
+					logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);
+				});
+		}
+
+		return;
+	}
+
+	const { volume, repository } = partialContext;
 
 	logger.error(
-		`Backup ${schedule.name} failed for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
+		`Backup ${schedule?.name} failed after ${maxRetries} retries for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
 	);
 
 	serverEvents.emit("backup:completed", {
@@ -238,7 +284,7 @@ export async function handleBackupFailure(
 			volumeName: volume.name,
 			repositoryName: repository.name,
 			scheduleName: schedule.name,
-			error: errorDetails,
+			error: `${errorDetails}\n\nFailed after ${maxRetries} retry attempts.`,
 		})
 		.catch((notifError) => {
 			logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);