danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Adds bandwidth limiting to repositories

Browse source 
Implements bandwidth limits for repository uploads and downloads.

Adds UI components to configure bandwidth limits.
Adds database fields to store bandwidth limit configuration.
Integrates bandwidth limiting into restic commands.
This commit is contained in:
Raj Dave 2026-01-10 14:03:45 +03:00
parent 8ba855f4ca
commit db127afe5a
9 changed files with 1561 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -20,3 +20,5 @@ data/
.env*
!.env.example
!.env.test
.idea/

185
app/client/modules/repositories/components/repository-forms/advanced-tls-form.tsx
View file

@ -7,12 +7,15 @@ import {
FormLabel,
FormMessage,
} from "../../../../components/ui/form";
import { Input } from "../../../../components/ui/input";
import { Textarea } from "../../../../components/ui/textarea";
import { Checkbox } from "../../../../components/ui/checkbox";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "../../../../components/ui/select";
import { Tooltip, TooltipContent, TooltipTrigger } from "../../../../components/ui/tooltip";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "../../../../components/ui/collapsible";
import type { RepositoryFormValues } from "../create-repository-form";
import { cn } from "~/client/lib/utils";
import { BANDWIDTH_UNITS } from "~/schemas/restic";
type Props = {
form: UseFormReturn<RepositoryFormValues>;
@ -21,11 +24,193 @@ type Props = {
export const AdvancedForm = ({ form }: Props) => {
const insecureTls = form.watch("insecureTls");
const cacert = form.watch("cacert");
const uploadEnabled = form.watch("uploadLimit.enabled");
const downloadEnabled = form.watch("downloadLimit.enabled");
return (
<Collapsible>
<CollapsibleTrigger className="">Advanced Settings</CollapsibleTrigger>
<CollapsibleContent className="pb-4 space-y-4">
{/* Bandwidth Limit Controls */}
<div className="space-y-6 rounded-lg border bg-card p-6">
<div className="flex items-center gap-3">
<div>
<h3 className="text-base font-semibold">Bandwidth Limits</h3>
<p className="text-sm text-muted-foreground">
Control upload and download speeds to prevent saturating network bandwidth
</p>
</div>
</div>
<div className="grid gap-6 sm:grid-cols-2">
{/* Upload Limit */}
<div className="space-y-4 rounded-lg border bg-background/50 p-4">
<div className="flex items-center gap-2">
<h4 className="text-sm font-medium">Upload Limit</h4>
</div>
<FormField
control={form.control}
name="uploadLimit.enabled"
render={({ field }) => (
<FormItem className="flex flex-row items-start space-x-3 space-y-0">
<FormControl>
<Checkbox
checked={field.value ?? false}
onCheckedChange={field.onChange}
/>
</FormControl>
<div className="space-y-1 leading-none">
<FormLabel>Enable upload speed limit</FormLabel>
<FormDescription className="text-xs">
Limit upload speed to the repository
</FormDescription>
</div>
</FormItem>
)}
/>
{uploadEnabled && (
<div className="space-y-3 pt-2">
<div className="flex items-center gap-2">
<FormField
control={form.control}
name="uploadLimit.value"
render={({ field }) => (
<FormItem className="flex-1">
<FormControl>
<div className="relative">
<Input
type="number"
placeholder="10"
min="0"
step="0.1"
className="pr-12"
{...field}
onChange={(e) => field.onChange(parseFloat(e.target.value) || 0)}
/>
<div className="absolute inset-y-0 right-0 flex items-center pr-3">
<div className="h-4 w-px bg-border" />
</div>
</div>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="uploadLimit.unit"
render={({ field }) => (
<FormItem className="w-24">
<Select onValueChange={field.onChange} defaultValue={field.value || "MB/s"} value={field.value || "MB/s"}>
<FormControl>
<SelectTrigger className="text-xs">
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
{Object.keys(BANDWIDTH_UNITS).map((unit) => (
<SelectItem key={unit} value={unit} className="text-xs">
{unit}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
</div>
</div>
)}
</div>
{/* Download Limit */}
<div className="space-y-4 rounded-lg border bg-background/50 p-4">
<div className="flex items-center gap-2">
<h4 className="text-sm font-medium">Download Limit</h4>
</div>
<FormField
control={form.control}
name="downloadLimit.enabled"
render={({ field }) => (
<FormItem className="flex flex-row items-start space-x-3 space-y-0">
<FormControl>
<Checkbox
checked={field.value ?? false}
onCheckedChange={field.onChange}
/>
</FormControl>
<div className="space-y-1 leading-none">
<FormLabel>Enable download speed limit</FormLabel>
<FormDescription className="text-xs">
Limit download speed from the repository
</FormDescription>
</div>
</FormItem>
)}
/>
{downloadEnabled && (
<div className="space-y-3 pt-2">
<div className="flex items-center gap-2">
<FormField
control={form.control}
name="downloadLimit.value"
render={({ field }) => (
<FormItem className="flex-1">
<FormControl>
<div className="relative">
<Input
type="number"
placeholder="10"
min="0"
step="0.1"
className="pr-12"
{...field}
onChange={(e) => field.onChange(parseFloat(e.target.value) || 0)}
/>
<div className="absolute inset-y-0 right-0 flex items-center pr-3">
<div className="h-4 w-px bg-border" />
</div>
</div>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="downloadLimit.unit"
render={({ field }) => (
<FormItem className="w-24">
<Select onValueChange={field.onChange} defaultValue={field.value || "MB/s"} value={field.value || "MB/s"}>
<FormControl>
<SelectTrigger className="text-xs">
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
{Object.keys(BANDWIDTH_UNITS).map((unit) => (
<SelectItem key={unit} value={unit} className="text-xs">
{unit}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
</div>
</div>
)}
</div>
</div>
</div>
<FormField
control={form.control}
name="insecureTls"

6
app/drizzle/0032_outstanding_ultron.sql Normal file
View file

@ -0,0 +1,6 @@
ALTER TABLE `repositories_table` ADD `upload_limit_enabled` integer DEFAULT false NOT NULL;--> statement-breakpoint
ALTER TABLE `repositories_table` ADD `upload_limit_value` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE `repositories_table` ADD `upload_limit_unit` text DEFAULT 'MB/s' NOT NULL;--> statement-breakpoint
ALTER TABLE `repositories_table` ADD `download_limit_enabled` integer DEFAULT false NOT NULL;--> statement-breakpoint
ALTER TABLE `repositories_table` ADD `download_limit_value` integer DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE `repositories_table` ADD `download_limit_unit` text DEFAULT 'MB/s' NOT NULL;

1243
app/drizzle/meta/0032_snapshot.json Normal file
View file

File diff suppressed because it is too large Load diff

7
app/drizzle/meta/_journal.json
View file

@ -225,6 +225,13 @@
"when": 1767863951955,
"tag": "0031_graceful_squadron_supreme",
"breakpoints": true
},
{
"idx": 32,
"version": "6",
"when": 1768040838612,
"tag": "0032_outstanding_ultron",
"breakpoints": true
}
]
}

20
app/schemas/restic.ts
View file

@ -13,12 +13,32 @@ export const REPOSITORY_BACKENDS = {
export type RepositoryBackend = keyof typeof REPOSITORY_BACKENDS;
export const BANDWIDTH_UNITS = {
"KB/s": "KB/s",
"MB/s": "MB/s",
"GB/s": "GB/s",
} as const;
export type BandwidthUnit = keyof typeof BANDWIDTH_UNITS;
// Bandwidth limit configuration
export const bandwidthLimitSchema = type({
enabled: "boolean = false",
value: "number >= 0",
unit: type.valueOf(BANDWIDTH_UNITS).default("MB/s"),
});
export type BandwidthLimit = typeof bandwidthLimitSchema.infer;
// Common fields for all repository configs
const baseRepositoryConfigSchema = type({
isExistingRepository: "boolean?",
customPassword: "string?",
cacert: "string?",
insecureTls: "boolean?",
// Bandwidth controls
uploadLimit: bandwidthLimitSchema.optional(),
downloadLimit: bandwidthLimitSchema.optional(),
});
export const s3RepositoryConfigSchema = type({

7
app/server/db/schema.ts
View file

@ -159,6 +159,13 @@ export const repositoriesTable = sqliteTable("repositories_table", {
status: text().$type<RepositoryStatus>().default("unknown"),
lastChecked: int("last_checked", { mode: "number" }),
lastError: text("last_error"),
// Bandwidth limit fields
uploadLimitEnabled: int("upload_limit_enabled", { mode: "boolean" }).notNull().default(false),
uploadLimitValue: int("upload_limit_value").notNull().default(0),
uploadLimitUnit: text("upload_limit_unit").notNull().default("MB/s"),
downloadLimitEnabled: int("download_limit_enabled", { mode: "boolean" }).notNull().default(false),
downloadLimitValue: int("download_limit_value").notNull().default(0),
downloadLimitUnit: text("download_limit_unit").notNull().default("MB/s"),
createdAt: int("created_at", { mode: "number" })
.notNull()
.default(sql`(unixepoch() * 1000)`),

0
app/server/utils/rclone-dummy.ts Normal file
View file

121
app/server/utils/restic.ts
View file

@ -10,7 +10,7 @@ import { logger } from "./logger";
import { cryptoUtils } from "./crypto";
import type { RetentionPolicy } from "../modules/backups/backups.dto";
import { safeSpawn } from "./spawn";
import type { CompressionMode, RepositoryConfig, OverwriteMode } from "~/schemas/restic";
import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic";
import { ResticError } from "./errors";
const backupOutputSchema = type({
@ -229,7 +229,7 @@ const init = async (config: RepositoryConfig) => {
const env = await buildEnv(config);
const args = ["init", "--repo", repoUrl];
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -321,7 +321,7 @@ const backup = async (
}
}
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const logData = throttle((data: string) => {
logger.info(data.trim());
@ -453,7 +453,7 @@ const restore = async (
}
}
addCommonArgs(args, env);
addCommonArgs(args, env, config);
logger.debug(`Executing: restic ${args.join(" ")}`);
const res = await safeSpawn({ command: "restic", args, env });
@ -516,7 +516,7 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[] }
}
}
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -565,7 +565,7 @@ const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra:
}
args.push("--prune");
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -587,7 +587,7 @@ const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[])
}
const args: string[] = ["--repo", repoUrl, "forget", ...snapshotIds, "--prune"];
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -636,7 +636,7 @@ const tagSnapshots = async (
}
}
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -686,7 +686,7 @@ const ls = async (config: RepositoryConfig, snapshotId: string, path?: string) =
args.push(path);
}
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -737,7 +737,7 @@ const unlock = async (config: RepositoryConfig) => {
const env = await buildEnv(config);
const args = ["unlock", "--repo", repoUrl, "--remove-all"];
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -761,7 +761,7 @@ const check = async (config: RepositoryConfig, options?: { readData?: boolean })
args.push("--read-data");
}
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -794,7 +794,7 @@ const repairIndex = async (config: RepositoryConfig) => {
const env = await buildEnv(config);
const args = ["repair", "index", "--repo", repoUrl];
addCommonArgs(args, env);
addCommonArgs(args, env, config);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
@ -846,7 +846,7 @@ const copy = async (
args.push("latest");
}
addCommonArgs(args, env);
addCommonArgs(args, env, destConfig);
if (sourceConfig.backend === "sftp" && sourceEnv._SFTP_SSH_ARGS) {
args.push("-o", `sftp.args=${sourceEnv._SFTP_SSH_ARGS}`);
@ -874,29 +874,32 @@ const copy = async (
};
};
export const cleanupTemporaryKeys = async (env: Record<string, string>) => {
if (env._SFTP_KEY_PATH) {
await fs.unlink(env._SFTP_KEY_PATH).catch(() => {});
// Helper function to convert bandwidth limit to restic format
const formatBandwidthLimit = (limit: BandwidthLimit): string => {
if (!limit.enabled || limit.value <= 0) {
return "";
}
if (env._SFTP_KNOWN_HOSTS_PATH) {
await fs.unlink(env._SFTP_KNOWN_HOSTS_PATH).catch(() => {});
// Convert to bytes per second for restic
let bytesPerSecond: number;
switch (limit.unit) {
case "KB/s":
bytesPerSecond = limit.value * 1024;
break;
case "MB/s":
bytesPerSecond = limit.value * 1024 * 1024;
break;
case "GB/s":
bytesPerSecond = limit.value * 1024 * 1024 * 1024;
break;
default:
return "";
}
if (env.RESTIC_PASSWORD_FILE && env.RESTIC_PASSWORD_FILE !== RESTIC_PASS_FILE) {
await fs.unlink(env.RESTIC_PASSWORD_FILE).catch(() => {});
}
if (env.GOOGLE_APPLICATION_CREDENTIALS) {
await fs.unlink(env.GOOGLE_APPLICATION_CREDENTIALS).catch(() => {});
}
if (env.RESTIC_CACERT) {
await fs.unlink(env.RESTIC_CACERT).catch(() => {});
}
return `${Math.floor(bytesPerSecond)}`;
};
export const addCommonArgs = (args: string[], env: Record<string, string>) => {
export const addCommonArgs = (args: string[], env: Record<string, string>, config?: RepositoryConfig) => {
args.push("--json");
if (env._SFTP_SSH_ARGS) {
@ -910,6 +913,40 @@ export const addCommonArgs = (args: string[], env: Record<string, string>) => {
if (env.RESTIC_CACERT) {
args.push("--cacert", env.RESTIC_CACERT);
}
// Add bandwidth limits if configuration is provided
if (config) {
// Handle upload limit
if (config.uploadLimit?.enabled) {
const uploadLimit = formatBandwidthLimit(config.uploadLimit);
if (uploadLimit) {
if (config.backend === "rclone") {
// For rclone backends, use --bwlimit
args.push("-o", `rclone.bwlimit=${uploadLimit}`);
} else {
// For restic backends, use --limit-upload
args.push("--limit-upload", uploadLimit);
}
}
}
// Handle download limit
if (config.downloadLimit?.enabled) {
const downloadLimit = formatBandwidthLimit(config.downloadLimit);
if (downloadLimit) {
if (config.backend === "rclone") {
// For rclone, bwlimit affects both upload and download
// If both limits are set, use the more restrictive one
const uploadLimit = config.uploadLimit?.enabled ? formatBandwidthLimit(config.uploadLimit) : "";
const effectiveLimit = uploadLimit && parseInt(uploadLimit) < parseInt(downloadLimit) ? uploadLimit : downloadLimit;
args.push("-o", `rclone.bwlimit=${effectiveLimit}`);
} else {
// For restic backends, use --limit-download
args.push("--limit-download", downloadLimit);
}
}
}
}
};
export const restic = {
@ -928,3 +965,27 @@ export const restic = {
repairIndex,
copy,
};
// Helper function to clean up temporary files
const cleanupTemporaryKeys = async (env: Record<string, string>) => {
const keysToClean = ['_SFTP_KEY_PATH', '_SFTP_KNOWN_HOSTS_PATH', 'RESTIC_CACERT', 'GOOGLE_APPLICATION_CREDENTIALS'];
for (const key of keysToClean) {
if (env[key]) {
try {
await fs.unlink(env[key]);
} catch (error) {
// Ignore errors when cleaning up temporary files
}
}
}
// Clean up custom password files
if (env.RESTIC_PASSWORD_FILE && env.RESTIC_PASSWORD_FILE !== RESTIC_PASS_FILE) {
try {
await fs.unlink(env.RESTIC_PASSWORD_FILE);
} catch (error) {
// Ignore errors when cleaning up temporary files
}
}
};