danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor(retention badges): use restic forget dry run result instead of manual calc (#494)

Browse source 
* refactor(retention badges): use restic forget dry run result instead of manual calc

* chore: PR feedbacks
This commit is contained in:
Nico 2026-02-10 20:14:35 +01:00 committed by GitHub
parent 12d0eda6ef
commit cb7988b8ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 150 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/client/components/retention-category-badges.tsx
View file

@ -8,7 +8,7 @@ interface RetentionCategoryBadgesProps {
}
const categoryColors: Record<string, string> = {
latest: "bg-blue-500/20 text-blue-700 dark:text-blue-300 border-blue-500/30",
last: "bg-blue-500/20 text-blue-700 dark:text-blue-300 border-blue-500/30",
hourly: "bg-cyan-500/20 text-cyan-700 dark:text-cyan-300 border-cyan-500/30",
daily: "bg-green-500/20 text-green-700 dark:text-green-300 border-green-500/30",
weekly: "bg-orange-500/20 text-orange-700 dark:text-orange-300 border-orange-500/30",
@ -17,7 +17,7 @@ const categoryColors: Record<string, string> = {
};
const categoryLabels: Record<string, string> = {
latest: "Latest",
last: "Last",
hourly: "Hourly",
daily: "Daily",
weekly: "Weekly",
@ -45,7 +45,7 @@ export function RetentionCategoryBadges({ categories, className }: RetentionCate
return null;
}
const order = ["latest", "hourly", "daily", "weekly", "monthly", "yearly"];
const order = ["last", "hourly", "daily", "weekly", "monthly", "yearly"];
const sortedCategories = [...categories].sort((a, b) => {
const indexA = order.indexOf(a);
const indexB = order.indexOf(b);

2
app/server/modules/backups/__tests__/backups.execution.test.ts
View file

@ -14,7 +14,7 @@ import { restic } from "~/server/utils/restic";
import { NotFoundError, BadRequestError } from "http-errors-enhanced";
const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }));
const resticForgetMock = mock(() => Promise.resolve({ success: true }));
const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null }));
const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" }));
beforeEach(() => {

2
app/server/modules/backups/__tests__/backups.patterns.test.ts
View file

@ -15,7 +15,7 @@ const backupMock = mock(() => Promise.resolve({ exitCode: 0, result: JSON.parse(
beforeEach(() => {
backupMock.mockClear();
spyOn(restic, "backup").mockImplementation(backupMock);
spyOn(restic, "forget").mockImplementation(mock(() => Promise.resolve({ success: true })));
spyOn(restic, "forget").mockImplementation(mock(() => Promise.resolve({ success: true, data: null })));
spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID);
});

2
app/server/modules/backups/backups.execution.ts
View file

@ -145,6 +145,7 @@ const finalizeSuccessfulBackup = async (ctx: BackupContext, scheduleId: number,
});
cache.delByPrefix(`snapshots:${ctx.repository.id}:`);
cache.del(`retention:${ctx.schedule.shortId}`);
const nextBackupAt = calculateNextRun(ctx.schedule.cronExpression);
await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
@ -320,6 +321,7 @@ const runForget = async (scheduleId: number, repositoryId?: string) => {
try {
await restic.forget(repository.config, schedule.retentionPolicy, { tag: schedule.shortId, organizationId });
cache.delByPrefix(`snapshots:${repository.id}:`);
cache.del(`retention:${schedule.shortId}`);
} finally {
releaseLock();
}

21
app/server/modules/repositories/repositories.controller.ts
View file

@ -43,11 +43,8 @@ import {
type UpdateRepositoryDto,
} from "./repositories.dto";
import { repositoriesService } from "./repositories.service";
import { backupsService } from "../backups/backups.service";
import { getRcloneRemoteInfo, listRcloneRemotes } from "../../utils/rclone";
import { requireAuth, requireOrgAdmin } from "../auth/auth.middleware";
import { computeRetentionCategories } from "../../utils/retention-categories";
import { logger } from "~/server/utils/logger";
import { toMessage } from "~/server/utils/errors";
import { requireDevPanel } from "../auth/dev-panel.middleware";
@ -94,21 +91,11 @@ export const repositoriesController = new Hono()
.get("/:id/snapshots", listSnapshotsDto, validator("query", listSnapshotsFilters), async (c) => {
const { id } = c.req.param();
const { backupId } = c.req.valid("query");
const res = await repositoriesService.listSnapshots(id, backupId);
let retentionCategories: Map<string, string[]> = new Map();
if (backupId) {
try {
const schedule = await backupsService.getScheduleByShortId(backupId);
const snapshotsForCategories = res.map((snapshot) => ({
short_id: snapshot.short_id,
time: new Date(snapshot.time).getTime(),
}));
retentionCategories = computeRetentionCategories(snapshotsForCategories, schedule.retentionPolicy);
} catch (error) {
logger.warn(`Failed to fetch retention policy for backup ID ${backupId}`, toMessage(error));
}
}
const [res, retentionCategories] = await Promise.all([
repositoriesService.listSnapshots(id, backupId),
repositoriesService.getRetentionCategories(id, backupId),
]);
const snapshots = res.map((snapshot) => {
const { summary } = snapshot;

47
app/server/modules/repositories/repositories.service.ts
View file

@ -21,6 +21,8 @@ import { getOrganizationId } from "~/server/core/request-context";
import { serverEvents } from "~/server/core/events";
import { executeDoctor } from "./doctor";
import { logger } from "~/server/utils/logger";
import { parseRetentionCategories, type RetentionCategory } from "~/server/utils/retention-categories";
import { backupsService } from "../backups/backups.service";
const runningDoctors = new Map<string, AbortController>();
@ -592,6 +594,50 @@ const execResticCommand = async (
return { exitCode: result.exitCode };
};
const getRetentionCategories = async (repositoryId: string, scheduleId?: string) => {
if (!scheduleId) {
return new Map<string, RetentionCategory[]>();
}
const cacheKey = `retention:${scheduleId}`;
const cached = cache.get<Record<string, RetentionCategory[]>>(cacheKey);
if (cached && Object.keys(cached).length > 0) {
return new Map(Object.entries(cached));
}
try {
const [schedule, repositoryResult] = await Promise.all([
backupsService.getScheduleByShortId(scheduleId),
repositoriesService.getRepository(repositoryId),
]);
if (!schedule?.retentionPolicy) {
return new Map<string, RetentionCategory[]>();
}
const { repository } = repositoryResult;
const dryRunResults = await restic.forget(repository.config, schedule.retentionPolicy, {
tag: scheduleId,
organizationId: getOrganizationId(),
dryRun: true,
});
if (!dryRunResults.data) {
return new Map<string, RetentionCategory[]>();
}
const categories = parseRetentionCategories(dryRunResults.data);
cache.set(cacheKey, Object.fromEntries(categories));
return categories;
} catch (error) {
logger.error(`Failed to get retention categories: ${toMessage(error)}`);
return new Map<string, RetentionCategory[]>();
}
};
export const repositoriesService = {
listRepositories,
createRepository,
@ -610,4 +656,5 @@ export const repositoriesService = {
tagSnapshots,
refreshSnapshots,
execResticCommand,
getRetentionCategories,
};

67
app/server/utils/restic.ts
View file

@ -13,6 +13,7 @@ import { safeSpawn, exec } from "./spawn";
import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic";
import { ResticError } from "./errors";
import { db } from "../db/db";
import { safeJsonParse } from "./json";
const backupOutputSchema = type({
message_type: "'summary'",
@ -552,16 +553,70 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; o
return result;
};
export type ResticForgetResponse = ForgetGroup[];
export interface ForgetGroup {
tags: string[] | null;
host: string;
paths: string[] | null;
keep: Snapshot[];
remove: Snapshot[] | null;
reasons: ForgetReason[];
}
export interface Snapshot {
time: string;
parent?: string;
tree: string;
paths: string[];
hostname: string;
username?: string;
uid?: number;
gid?: number;
excludes?: string[] | null;
tags?: string[] | null;
program_version?: string;
summary?: SnapshotSummary;
id: string;
short_id: string;
}
export interface SnapshotSummary {
backup_start: string;
backup_end: string;
files_new: number;
files_changed: number;
files_unmodified: number;
dirs_new: number;
dirs_changed: number;
dirs_unmodified: number;
data_blobs: number;
tree_blobs: number;
data_added: number;
data_added_packed: number;
total_files_processed: number;
total_bytes_processed: number;
}
export interface ForgetReason {
snapshot: Snapshot;
matches: string[];
}
const forget = async (
config: RepositoryConfig,
options: RetentionPolicy,
extra: { tag: string; organizationId: string },
extra: { tag: string; organizationId: string; dryRun?: boolean },
) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, extra.organizationId);
const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag];
if (extra.dryRun) {
args.push("--dry-run", "--no-lock");
}
if (options.keepLast) {
args.push("--keep-last", String(options.keepLast));
}
@ -584,7 +639,10 @@ const forget = async (
args.push("--keep-within-duration", options.keepWithinDuration);
}
args.push("--prune");
if (!extra.dryRun) {
args.push("--prune");
}
addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env });
@ -595,7 +653,10 @@ const forget = async (
throw new ResticError(res.exitCode, res.stderr);
}
return { success: true };
const lines = res.stdout.split("\n").filter((line) => line.trim());
const result = extra.dryRun ? safeJsonParse<ResticForgetResponse>(lines.at(-1) ?? "[]") : null;
return { success: true, data: result };
};
const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[], organizationId: string) => {

69
app/server/utils/retention-categories.ts
View file

@ -1,51 +1,38 @@
import { format } from "date-fns";
import type { RetentionPolicy } from "../modules/backups/backups.dto";
import type { ResticForgetResponse } from "./restic";
export type RetentionCategory = "latest" | "hourly" | "daily" | "weekly" | "monthly" | "yearly";
export type RetentionCategory = "last" | "hourly" | "daily" | "weekly" | "monthly" | "yearly";
interface SnapshotInfo {
short_id: string;
time: number;
}
const MATCH_TO_CATEGORY: Record<string, RetentionCategory> = {
"last snapshot": "last",
"hourly snapshot": "hourly",
"daily snapshot": "daily",
"weekly snapshot": "weekly",
"monthly snapshot": "monthly",
"yearly snapshot": "yearly",
"oldest hourly snapshot": "hourly",
"oldest daily snapshot": "daily",
"oldest weekly snapshot": "weekly",
"oldest monthly snapshot": "monthly",
"oldest yearly snapshot": "yearly",
};
const RETENTION_RULES = [
{ prop: "keepHourly", tag: "hourly", fmt: "yyyy-MM-dd-HH" },
{ prop: "keepDaily", tag: "daily", fmt: "yyyy-MM-dd" },
{ prop: "keepWeekly", tag: "weekly", fmt: "RRRR-'W'II" },
{ prop: "keepMonthly", tag: "monthly", fmt: "yyyy-MM" },
{ prop: "keepYearly", tag: "yearly", fmt: "yyyy" },
] as const;
export const computeRetentionCategories = (snapshots: SnapshotInfo[], policy: RetentionPolicy | null) => {
export const parseRetentionCategories = (dryRunResults: ResticForgetResponse) => {
const categories = new Map<string, RetentionCategory[]>();
if (snapshots.length === 0) return categories;
for (const group of dryRunResults) {
for (const reason of group.reasons) {
const { short_id } = reason.snapshot;
const categoryList: RetentionCategory[] = [];
const sorted = [...snapshots].sort((a, b) => b.time - a.time);
for (const match of reason.matches) {
const category = MATCH_TO_CATEGORY[match];
if (category && !categoryList.includes(category)) {
categoryList.push(category);
}
}
const addTag = (id: string, tag: RetentionCategory) => {
const tags = categories.get(id) ?? [];
if (!tags.includes(tag)) categories.set(id, [...tags, tag]);
};
addTag(sorted[0].short_id, "latest");
if (!policy) return categories;
for (const { prop, tag, fmt } of RETENTION_RULES) {
const limit = policy[prop];
if (!limit || limit <= 0) continue;
const seenBuckets = new Set<string>();
let count = 0;
for (const snapshot of sorted) {
if (count >= limit) break;
const bucketKey = format(snapshot.time, fmt);
if (!seenBuckets.has(bucketKey)) {
seenBuckets.add(bucketKey);
addTag(snapshot.short_id, tag);
count++;
if (categoryList.length > 0) {
categories.set(short_id, categoryList);
}
}
}