danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(backup): retry backup on failure

Browse source
This commit is contained in:
DerPenz 2026-04-07 19:47:49 +02:00 committed by Nico
parent 4520335ebc
commit feb69ef7a7
5 changed files with 2387 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
app/drizzle/20260407172925_fresh_warlock/migration.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE `backup_schedules_table` ADD `failure_retry_count` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE `backup_schedules_table` ADD `max_retries` integer DEFAULT 3 NOT NULL;--> statement-breakpoint
ALTER TABLE `backup_schedules_table` ADD `retry_delay` integer DEFAULT 3600000 NOT NULL;

2331
app/drizzle/20260407172925_fresh_warlock/snapshot.json Normal file
View file

File diff suppressed because it is too large Load diff

3
app/server/db/schema.ts
View file

@ -311,6 +311,9 @@ export const backupSchedulesTable = sqliteTable("backup_schedules_table", {
oneFileSystem: int("one_file_system", { mode: "boolean" }).notNull().default(false),
customResticParams: text("custom_restic_params", { mode: "json" }).$type<string[]>().default([]),
sortOrder: int("sort_order", { mode: "number" }).notNull().default(0),
failureRetryCount: int("failure_retry_count").notNull().default(0),
maxRetries: int("max_retries").notNull().default(3),
retryDelay: int("retry_delay").notNull().default(3600000),
createdAt: int("created_at", { mode: "number" })
.notNull()
.default(sql`(unixepoch() * 1000)`),

1
app/server/modules/backups/backups.queries.ts
View file

@ -42,6 +42,7 @@ export const scheduleQueries = {
lastBackupAt?: number;
lastBackupError?: string | null;
nextBackupAt?: number | null;
failureRetryCount?: number;
},
) => {
return db

52
app/server/modules/backups/helpers/backup-lifecycle.ts
View file

@ -167,6 +167,7 @@ export async function finalizeSuccessfulBackup(
lastBackupStatus: finalStatus,
lastBackupError: finalStatus === "warning" ? warningDetails : null,
nextBackupAt: ctx.schedule.cronExpression ? calculateNextRun(ctx.schedule.cronExpression) : null,
failureRetryCount: 0,
});
if (finalStatus === "warning") {
@ -219,10 +220,55 @@ export async function handleBackupFailure(
return;
}
const { schedule, volume, repository } = partialContext;
// Determine if the backup should be retried
const schedule = partialContext.schedule;
const currentRetryCount = schedule.failureRetryCount;
const maxRetries = schedule.maxRetries;
const shouldRetry = currentRetryCount < maxRetries;
if (shouldRetry) {
const nextBackupAt = Date.now() + schedule.retryDelay;
await scheduleQueries.updateStatus(scheduleId, organizationId, {
lastBackupAt: Date.now(),
lastBackupStatus: "error",
lastBackupError: errorDetails,
nextBackupAt,
failureRetryCount: currentRetryCount + 1,
});
logger.warn(
`Backup ${schedule?.name} failed. Scheduling retry ${currentRetryCount + 1}/${maxRetries} for 1 hour from now: ${errorMessage}`,
);
if (partialContext?.volume && partialContext?.repository) {
serverEvents.emit("backup:completed", {
organizationId,
scheduleId: schedule!.shortId,
volumeName: partialContext.volume.name,
repositoryName: partialContext.repository.name,
status: "error",
});
notificationsService
.sendBackupNotification(scheduleId, "failure", {
volumeName: partialContext.volume.name,
repositoryName: partialContext.repository.name,
scheduleName: schedule!.name,
error: `${errorDetails}\n\nRetrying in 1 hour (attempt ${currentRetryCount + 1}/${maxRetries})`,
})
.catch((notifError) => {
logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);
});
}
return;
}
const { volume, repository } = partialContext;
logger.error(
`Backup ${schedule.name} failed for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
`Backup ${schedule?.name} failed after ${maxRetries} retries for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
);
serverEvents.emit("backup:completed", {
@ -238,7 +284,7 @@ export async function handleBackupFailure(
volumeName: volume.name,
repositoryName: repository.name,
scheduleName: schedule.name,
error: errorDetails,
error: `${errorDetails}\n\nFailed after ${maxRetries} retry attempts.`,
})
.catch((notifError) => {
logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);