danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'Der-Penz-retry-backup'

Browse source
This commit is contained in:
Nicolas Meienberger 2026-04-12 10:50:02 +02:00
commit 7ea9899385
No known key found for this signature in database
14 changed files with 2573 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
app/client/api-client/types.gen.ts
View file

@ -2434,6 +2434,8 @@ export type ListBackupSchedulesResponses = {
includePatterns: Array<string> | null;
oneFileSystem: boolean;
customResticParams: Array<string> | null;
maxRetries: number;
retryDelay: number;
lastBackupAt: number | null;
lastBackupStatus: 'success' | 'error' | 'in_progress' | 'warning' | null;
lastBackupError: string | null;
@ -2710,6 +2712,8 @@ export type CreateBackupScheduleData = {
oneFileSystem?: boolean;
tags?: Array<string>;
customResticParams?: Array<string>;
maxRetries?: number;
retryDelay?: number;
};
path?: never;
query?: never;
@ -2743,6 +2747,8 @@ export type CreateBackupScheduleResponses = {
includePatterns: Array<string> | null;
oneFileSystem: boolean;
customResticParams: Array<string> | null;
maxRetries: number;
retryDelay: number;
lastBackupAt: number | null;
lastBackupStatus: 'success' | 'error' | 'in_progress' | 'warning' | null;
lastBackupError: string | null;
@ -2810,6 +2816,8 @@ export type GetBackupScheduleResponses = {
includePatterns: Array<string> | null;
oneFileSystem: boolean;
customResticParams: Array<string> | null;
maxRetries: number;
retryDelay: number;
lastBackupAt: number | null;
lastBackupStatus: 'success' | 'error' | 'in_progress' | 'warning' | null;
lastBackupError: string | null;
@ -3085,6 +3093,8 @@ export type UpdateBackupScheduleData = {
oneFileSystem?: boolean;
tags?: Array<string>;
customResticParams?: Array<string>;
maxRetries?: number;
retryDelay?: number;
};
path: {
shortId: string;
@ -3120,6 +3130,8 @@ export type UpdateBackupScheduleResponses = {
includePatterns: Array<string> | null;
oneFileSystem: boolean;
customResticParams: Array<string> | null;
maxRetries: number;
retryDelay: number;
lastBackupAt: number | null;
lastBackupStatus: 'success' | 'error' | 'in_progress' | 'warning' | null;
lastBackupError: string | null;
@ -3167,6 +3179,8 @@ export type GetBackupScheduleForVolumeResponses = {
includePatterns: Array<string> | null;
oneFileSystem: boolean;
customResticParams: Array<string> | null;
maxRetries: number;
retryDelay: number;
lastBackupAt: number | null;
lastBackupStatus: 'success' | 'error' | 'in_progress' | 'warning' | null;
lastBackupError: string | null;

92
app/client/modules/backups/components/create-schedule-form/advanced-section.tsx
View file

@ -2,6 +2,7 @@ import { FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessa
import { Textarea } from "~/client/components/ui/textarea";
import type { UseFormReturn } from "react-hook-form";
import type { InternalFormValues } from "./types";
import { Input } from "~/client/components/ui/input";
type AdvancedSectionProps = {
form: UseFormReturn<InternalFormValues>;
@ -9,27 +10,74 @@ type AdvancedSectionProps = {
export const AdvancedSection = ({ form }: AdvancedSectionProps) => {
return (
<FormField
control={form.control}
name="customResticParamsText"
render={({ field }) => (
<FormItem>
<FormLabel>Custom restic parameters</FormLabel>
<FormControl>
<Textarea
{...field}
placeholder="--exclude-larger-than 500M&#10;--no-scan&#10;--read-concurrency 8"
className="font-mono text-sm min-h-24"
/>
</FormControl>
<FormDescription>
Advanced: enter one restic flag per line (e.g.{" "}
<code className="bg-muted px-1 rounded">--exclude-larger-than 500M</code>). Only the supported flag list is
accepted.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<>
<FormField
control={form.control}
name="maxRetries"
render={({ field }) => (
<FormItem>
<FormLabel>Maximum retries</FormLabel>
<FormControl>
<Input
{...field}
type="number"
min={0}
max={32}
value={field.value ?? ""}
placeholder="e.g., 2"
onChange={(e) => field.onChange(e.target.value ? Number(e.target.value) : undefined)}
/>
</FormControl>
<FormDescription>Maximum number of retry attempts if a backup fails (default: 2).</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="retryDelay"
render={({ field }) => (
<FormItem>
<FormLabel>Retry delay</FormLabel>
<FormControl>
<Input
{...field}
type="number"
min={1}
max={1440}
step={1}
placeholder="e.g., 15"
value={field.value ?? ""}
onChange={(e) => field.onChange(e.target.value ? Number(e.target.value) : undefined)}
/>
</FormControl>
<FormDescription>Delay in minutes before retrying a failed backup (default: 15 minutes).</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="customResticParamsText"
render={({ field }) => (
<FormItem>
<FormLabel>Custom restic parameters</FormLabel>
<FormControl>
<Textarea
{...field}
placeholder="--exclude-larger-than 500M&#10;--no-scan&#10;--read-concurrency 8"
className="font-mono text-sm min-h-24"
/>
</FormControl>
<FormDescription>
Advanced: enter one restic flag per line (e.g.{" "}
<code className="bg-muted px-1 rounded">--exclude-larger-than 500M</code>). Only the supported flag list
is accepted.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
</>
);
};

4
app/client/modules/backups/components/create-schedule-form/index.tsx
View file

@ -44,6 +44,8 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:
customResticParamsText,
includePaths,
cronExpression,
maxRetries,
retryDelay,
...rest
} = data;
const excludePatterns = parseMultilineEntries(excludePatternsText);
@ -59,6 +61,8 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:
excludePatterns,
excludeIfPresent,
customResticParams,
maxRetries,
retryDelay,
});
},
[onSubmit],

2
app/client/modules/backups/components/create-schedule-form/types.ts
View file

@ -20,6 +20,8 @@ export const internalFormSchema = z.object({
keepYearly: z.number().optional(),
oneFileSystem: z.boolean().optional(),
customResticParamsText: z.string().optional(),
maxRetries: z.number().min(0).max(32).optional(),
retryDelay: z.number().min(1).max(1440).optional(),
});
export const weeklyDays = [

2
app/client/modules/backups/components/create-schedule-form/utils.ts
View file

@ -26,6 +26,8 @@ export const backupScheduleToFormValues = (schedule?: BackupSchedule): InternalF
excludeIfPresentText: schedule.excludeIfPresent?.join("\n") || undefined,
oneFileSystem: schedule.oneFileSystem ?? false,
customResticParamsText: schedule.customResticParams?.join("\n") ?? "",
maxRetries: schedule.maxRetries,
retryDelay: schedule.retryDelay ? schedule.retryDelay / (60 * 1000) : undefined, // Convert ms to minutes
...cronValues,
...schedule.retentionPolicy,
};

2
app/client/modules/backups/routes/create-backup.tsx
View file

@ -75,6 +75,8 @@ export function CreateBackupPage() {
excludeIfPresent: formValues.excludeIfPresent,
oneFileSystem: formValues.oneFileSystem,
customResticParams: formValues.customResticParams,
maxRetries: formValues.maxRetries,
retryDelay: formValues.retryDelay,
},
});
};

3
app/drizzle/20260407172925_fresh_warlock/migration.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE `backup_schedules_table` ADD `failure_retry_count` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE `backup_schedules_table` ADD `max_retries` integer DEFAULT 2 NOT NULL;--> statement-breakpoint
ALTER TABLE `backup_schedules_table` ADD `retry_delay` integer DEFAULT 900000 NOT NULL;

2331
app/drizzle/20260407172925_fresh_warlock/snapshot.json Normal file
View file

File diff suppressed because it is too large Load diff

3
app/server/db/schema.ts
View file

@ -311,6 +311,9 @@ export const backupSchedulesTable = sqliteTable("backup_schedules_table", {
oneFileSystem: int("one_file_system", { mode: "boolean" }).notNull().default(false),
customResticParams: text("custom_restic_params", { mode: "json" }).$type<string[]>().default([]),
sortOrder: int("sort_order", { mode: "number" }).notNull().default(0),
failureRetryCount: int("failure_retry_count").notNull().default(0),
maxRetries: int("max_retries").notNull().default(2),
retryDelay: int("retry_delay").notNull().default(900000),
createdAt: int("created_at", { mode: "number" })
.notNull()
.default(sql`(unixepoch() * 1000)`),

53
app/server/modules/backups/__tests__/backups.execution.test.ts
View file

@ -233,6 +233,59 @@ describe("stop backup", () => {
expect(resticBackupMock).not.toHaveBeenCalled();
});
test("should clear failureRetryCount when a scheduled retry is cancelled", async () => {
const { resticBackupMock } = setup();
const volume = await createTestVolume();
const repository = await createTestRepository();
const schedule = await createTestBackupSchedule({
volumeId: volume.id,
repositoryId: repository.id,
cronExpression: "0 0 1 1 *",
maxRetries: 3,
retryDelay: 60 * 1000,
});
resticBackupMock.mockImplementationOnce(() =>
Promise.resolve({ exitCode: 1, summary: generateBackupOutput(), error: "retry me" }),
);
await backupsExecutionService.executeBackup(schedule.id);
const failedSchedule = await backupsService.getScheduleById(schedule.id);
expect(failedSchedule.failureRetryCount).toBe(1);
resticBackupMock.mockImplementationOnce(({ signal }: SafeSpawnParams) => {
return new Promise((resolve) => {
if (signal?.aborted) {
resolve({ exitCode: 1, summary: "", error: "" });
return;
}
signal?.addEventListener(
"abort",
() => {
resolve({ exitCode: 1, summary: "", error: "" });
},
{ once: true },
);
});
});
const executePromise = backupsExecutionService.executeBackup(schedule.id);
await waitForExpect(async () => {
const retryingSchedule = await backupsService.getScheduleById(schedule.id);
expect(retryingSchedule.lastBackupStatus).toBe("in_progress");
});
await backupsExecutionService.stopBackup(schedule.id);
await executePromise;
const cancelledSchedule = await backupsService.getScheduleById(schedule.id);
expect(cancelledSchedule.lastBackupStatus).toBe("warning");
expect(cancelledSchedule.failureRetryCount).toBe(0);
});
test("should throw ConflictError when trying to stop non-running backup", async () => {
// arrange
setup();

6
app/server/modules/backups/backups.dto.ts
View file

@ -31,6 +31,8 @@ const backupScheduleSchema = z.object({
includePatterns: z.array(z.string()).nullable(),
oneFileSystem: z.boolean(),
customResticParams: z.array(z.string()).nullable(),
maxRetries: z.number(),
retryDelay: z.number(),
lastBackupAt: z.number().nullable(),
lastBackupStatus: z.enum(["success", "error", "in_progress", "warning"]).nullable(),
lastBackupError: z.string().nullable(),
@ -128,6 +130,8 @@ export const createBackupScheduleBody = z.object({
oneFileSystem: z.boolean().optional(),
tags: z.array(z.string()).optional(),
customResticParams: z.array(z.string()).optional(),
maxRetries: z.number().min(0).max(32).optional(),
retryDelay: z.number().min(1).max(1440).optional(),
});
export type CreateBackupScheduleBody = z.infer<typeof createBackupScheduleBody>;
@ -165,6 +169,8 @@ export const updateBackupScheduleBody = z.object({
oneFileSystem: z.boolean().optional(),
tags: z.array(z.string()).optional(),
customResticParams: z.array(z.string()).optional(),
maxRetries: z.number().min(0).max(32).optional(),
retryDelay: z.number().min(1).max(1440).optional(),
});
export type UpdateBackupScheduleBody = z.infer<typeof updateBackupScheduleBody>;

1
app/server/modules/backups/backups.queries.ts
View file

@ -42,6 +42,7 @@ export const scheduleQueries = {
lastBackupAt?: number;
lastBackupError?: string | null;
nextBackupAt?: number | null;
failureRetryCount?: number;
},
) => {
return db

13
app/server/modules/backups/backups.service.ts
View file

@ -111,6 +111,8 @@ const createSchedule = async (data: CreateBackupScheduleBody) => {
customResticParams: data.customResticParams ?? [],
nextBackupAt: nextBackupAt,
shortId: generateShortId(),
maxRetries: data.maxRetries,
retryDelay: data.retryDelay ? data.retryDelay * 60 * 1000 : undefined,
organizationId,
})
.returning();
@ -163,10 +165,11 @@ const updateSchedule = async (scheduleIdOrShortId: number | string, data: Update
const cronExpression = data.cronExpression ?? schedule.cronExpression;
const nextBackupAt =
data.cronExpression === "" ? null : data.cronExpression ? calculateNextRun(cronExpression) : schedule.nextBackupAt;
const retryDelay = data.retryDelay ? data.retryDelay * 60 * 1000 : schedule.retryDelay;
const [updated] = await db
.update(backupSchedulesTable)
.set({ ...data, repositoryId: repository.id, nextBackupAt, updatedAt: Date.now() })
.set({ ...data, repositoryId: repository.id, nextBackupAt, updatedAt: Date.now(), retryDelay })
.where(and(eq(backupSchedulesTable.id, schedule.id), eq(backupSchedulesTable.organizationId, organizationId)))
.returning();
@ -386,7 +389,7 @@ const executeBackup = async (scheduleId: number, manual = false) => {
const result = await validateBackupExecution(scheduleId, manual);
if (result.type !== "success") {
return handleValidationResult(scheduleId, result);
return handleValidationResult(scheduleId, result, manual);
}
const { context: ctx } = result;
@ -423,7 +426,7 @@ const executeBackup = async (scheduleId: number, manual = false) => {
switch (executionResult.status) {
case "unavailable":
return handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, ctx);
return handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, manual, ctx);
case "completed":
return finalizeSuccessfulBackup(
ctx,
@ -432,7 +435,7 @@ const executeBackup = async (scheduleId: number, manual = false) => {
executionResult.warningDetails,
);
case "failed":
return handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, ctx);
return handleBackupFailure(scheduleId, ctx.organizationId, executionResult.error, manual, ctx);
case "cancelled":
return handleBackupCancellation(scheduleId, ctx.organizationId, executionResult.message);
}
@ -444,7 +447,7 @@ const executeBackup = async (scheduleId: number, manual = false) => {
return;
}
return handleBackupFailure(scheduleId, ctx.organizationId, error, ctx);
return handleBackupFailure(scheduleId, ctx.organizationId, error, manual, ctx);
} finally {
backupExecutor.untrack(scheduleId, abortController);
cache.del(cacheKeys.backup.progress(scheduleId));

83
app/server/modules/backups/helpers/backup-lifecycle.ts
View file

@ -85,7 +85,11 @@ export async function validateBackupExecution(scheduleId: number, manual = false
};
}
export async function handleValidationResult(scheduleId: number, result: ValidationFailure | ValidationSkipped) {
export async function handleValidationResult(
scheduleId: number,
result: ValidationFailure | ValidationSkipped,
manual: boolean,
) {
const organizationId = getOrganizationId();
if (result.type === "skipped") {
@ -93,7 +97,7 @@ export async function handleValidationResult(scheduleId: number, result: Validat
return;
}
await handleBackupFailure(scheduleId, organizationId, result.error, result.partialContext);
await handleBackupFailure(scheduleId, organizationId, result.error, manual, result.partialContext);
}
export function emitBackupStarted(ctx: BackupContext, scheduleId: number) {
@ -167,6 +171,7 @@ export async function finalizeSuccessfulBackup(
lastBackupStatus: finalStatus,
lastBackupError: finalStatus === "warning" ? warningDetails : null,
nextBackupAt: ctx.schedule.cronExpression ? calculateNextRun(ctx.schedule.cronExpression) : null,
failureRetryCount: 0,
});
if (finalStatus === "warning") {
@ -204,6 +209,7 @@ export async function handleBackupFailure(
scheduleId: number,
organizationId: string,
error: unknown,
manual: boolean,
partialContext?: Partial<BackupContext>,
) {
const errorMessage = toMessage(error);
@ -219,11 +225,65 @@ export async function handleBackupFailure(
return;
}
const { schedule, volume, repository } = partialContext;
// Determine if the backup should be retried
const schedule = partialContext.schedule;
const currentRetryCount = schedule.failureRetryCount;
const maxRetries = schedule.maxRetries;
const shouldRetry = currentRetryCount < maxRetries;
const nextRetryBackupAt = Date.now() + schedule.retryDelay;
const nextScheduledBackupAt = calculateNextRun(schedule.cronExpression);
logger.error(
`Backup ${schedule.name} failed for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
);
if (!manual && shouldRetry && nextRetryBackupAt < nextScheduledBackupAt) {
await scheduleQueries.updateStatus(scheduleId, organizationId, {
nextBackupAt: nextRetryBackupAt,
failureRetryCount: currentRetryCount + 1,
});
const delayMinutes = Math.round((schedule.retryDelay / (60 * 1000)) * 10) / 10;
logger.error(
`Backup ${schedule.name} failed. Scheduling retry ${currentRetryCount + 1}/${maxRetries} for ${delayMinutes} minutes from now: ${errorMessage}`,
);
if (partialContext?.volume && partialContext?.repository) {
serverEvents.emit("backup:completed", {
organizationId,
scheduleId: schedule.shortId,
volumeName: partialContext.volume.name,
repositoryName: partialContext.repository.name,
status: "error",
});
notificationsService
.sendBackupNotification(scheduleId, "failure", {
volumeName: partialContext.volume.name,
repositoryName: partialContext.repository.name,
scheduleName: schedule.name,
error: `${errorDetails}\n\nRetrying in ${delayMinutes} minutes (attempt ${currentRetryCount + 1}/${maxRetries})`,
})
.catch((notifError) => {
logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);
});
}
return;
}
await scheduleQueries.updateStatus(scheduleId, organizationId, {
failureRetryCount: 0,
});
const { volume, repository } = partialContext;
if (manual) {
logger.error(
`Manual backup ${schedule.name} failed for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
);
} else {
logger.error(
`Backup ${schedule.name} failed after ${maxRetries} retries for volume ${volume.name} to repository ${repository.name}: ${errorMessage}`,
);
}
serverEvents.emit("backup:completed", {
organizationId,
@ -233,15 +293,19 @@ export async function handleBackupFailure(
status: "error",
});
const errorNotificationMessage = manual
? `${errorDetails}`
: `${errorDetails}\n\nFailed after ${maxRetries} retry attempts.`;
notificationsService
.sendBackupNotification(scheduleId, "failure", {
volumeName: volume.name,
repositoryName: repository.name,
scheduleName: schedule.name,
error: errorDetails,
error: errorNotificationMessage,
})
.catch((notifError) => {
logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);
.catch((notifyError) => {
logger.error(`Failed to send backup failure notification: ${toMessage(notifyError)}`);
});
}
@ -255,5 +319,6 @@ export async function handleBackupCancellation(
lastBackupAt: shouldSetLastBackupAt ? Date.now() : undefined,
lastBackupStatus: "warning",
lastBackupError: message ?? "Backup was stopped by the user",
failureRetryCount: 0,
});
}