feat(backup): added retry policy into job settings
This commit is contained in:
parent
feb69ef7a7
commit
7e9b1fafa3
5 changed files with 82 additions and 24 deletions
|
|
@ -2,6 +2,7 @@ import { FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessa
|
|||
import { Textarea } from "~/client/components/ui/textarea";
|
||||
import type { UseFormReturn } from "react-hook-form";
|
||||
import type { InternalFormValues } from "./types";
|
||||
import { Input } from "~/client/components/ui/input";
|
||||
|
||||
type AdvancedSectionProps = {
|
||||
form: UseFormReturn<InternalFormValues>;
|
||||
|
|
@ -9,27 +10,72 @@ type AdvancedSectionProps = {
|
|||
|
||||
export const AdvancedSection = ({ form }: AdvancedSectionProps) => {
|
||||
return (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="customResticParamsText"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Custom restic parameters</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
{...field}
|
||||
placeholder="--exclude-larger-than 500M --no-scan --read-concurrency 8"
|
||||
className="font-mono text-sm min-h-24"
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Advanced: enter one restic flag per line (e.g.{" "}
|
||||
<code className="bg-muted px-1 rounded">--exclude-larger-than 500M</code>). Only the supported flag list is
|
||||
accepted.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="maxRetries"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Maximum retries</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
type="number"
|
||||
min={0}
|
||||
max={32}
|
||||
value={field.value ?? ""}
|
||||
onChange={(e) => field.onChange(e.target.value ? Number(e.target.value) : undefined)}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>Maximum number of retry attempts if a backup fails (default: 0).</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="retryDelay"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Retry delay</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
type="number"
|
||||
min={0}
|
||||
max={1440}
|
||||
step={1}
|
||||
value={field.value ?? ""}
|
||||
onChange={(e) => field.onChange(e.target.value ? Number(e.target.value) : undefined)}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>Delay in minutes before retrying a failed backup (default: 60 minutes).</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="customResticParamsText"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Custom restic parameters</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
{...field}
|
||||
placeholder="--exclude-larger-than 500M --no-scan --read-concurrency 8"
|
||||
className="font-mono text-sm min-h-24"
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Advanced: enter one restic flag per line (e.g.{" "}
|
||||
<code className="bg-muted px-1 rounded">--exclude-larger-than 500M</code>). Only the supported flag list
|
||||
is accepted.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:
|
|||
customResticParamsText,
|
||||
includePaths,
|
||||
cronExpression,
|
||||
maxRetries,
|
||||
retryDelay,
|
||||
...rest
|
||||
} = data;
|
||||
const excludePatterns = parseMultilineEntries(excludePatternsText);
|
||||
|
|
@ -59,6 +61,8 @@ export const CreateScheduleForm = ({ initialValues, formId, onSubmit, volume }:
|
|||
excludePatterns,
|
||||
excludeIfPresent,
|
||||
customResticParams,
|
||||
maxRetries,
|
||||
retryDelay: retryDelay ? retryDelay * 60 * 1000 : 3600000,
|
||||
});
|
||||
},
|
||||
[onSubmit],
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ export const internalFormSchema = z.object({
|
|||
keepYearly: z.number().optional(),
|
||||
oneFileSystem: z.boolean().optional(),
|
||||
customResticParamsText: z.string().optional(),
|
||||
maxRetries: z.number().min(0).max(32).optional(),
|
||||
retryDelay: z.number().min(1).max(1440).optional(),
|
||||
});
|
||||
|
||||
export const weeklyDays = [
|
||||
|
|
@ -42,4 +44,6 @@ export type BackupScheduleFormValues = Omit<
|
|||
excludeIfPresent?: string[];
|
||||
includePatterns?: string[];
|
||||
customResticParams?: string[];
|
||||
maxRetries?: number;
|
||||
retryDelay?: number;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ export const backupScheduleToFormValues = (schedule?: BackupSchedule): InternalF
|
|||
excludeIfPresentText: schedule.excludeIfPresent?.join("\n") || undefined,
|
||||
oneFileSystem: schedule.oneFileSystem ?? false,
|
||||
customResticParamsText: schedule.customResticParams?.join("\n") ?? "",
|
||||
maxRetries: schedule.maxRetries ?? 5,
|
||||
retryDelay: schedule.retryDelay ? schedule.retryDelay / (60 * 60 * 1000) : 1, // Convert ms to hours
|
||||
...cronValues,
|
||||
...schedule.retentionPolicy,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -237,8 +237,10 @@ export async function handleBackupFailure(
|
|||
failureRetryCount: currentRetryCount + 1,
|
||||
});
|
||||
|
||||
const delayMinutes = Math.round((schedule.retryDelay / (60 * 1000)) * 10) / 10;
|
||||
|
||||
logger.warn(
|
||||
`Backup ${schedule?.name} failed. Scheduling retry ${currentRetryCount + 1}/${maxRetries} for 1 hour from now: ${errorMessage}`,
|
||||
`Backup ${schedule?.name} failed. Scheduling retry ${currentRetryCount + 1}/${maxRetries} for ${delayMinutes} minutes from now: ${errorMessage}`,
|
||||
);
|
||||
|
||||
if (partialContext?.volume && partialContext?.repository) {
|
||||
|
|
@ -255,7 +257,7 @@ export async function handleBackupFailure(
|
|||
volumeName: partialContext.volume.name,
|
||||
repositoryName: partialContext.repository.name,
|
||||
scheduleName: schedule!.name,
|
||||
error: `${errorDetails}\n\nRetrying in 1 hour (attempt ${currentRetryCount + 1}/${maxRetries})`,
|
||||
error: `${errorDetails}\n\nRetrying in ${delayMinutes} minutes (attempt ${currentRetryCount + 1}/${maxRetries})`,
|
||||
})
|
||||
.catch((notifError) => {
|
||||
logger.error(`Failed to send backup failure notification: ${toMessage(notifError)}`);
|
||||
|
|
|
|||
Loading…
Reference in a new issue