refactor(retention badges): use restic forget dry run result instead of manual calc (#494)

* refactor(retention badges): use restic forget dry run result instead of manual calc

* chore: PR feedbacks

app/client/components/retention-category-badges.tsx

@@ -8,7 +8,7 @@ interface RetentionCategoryBadgesProps {
 }
 
 const categoryColors: Record<string, string> = {
-	latest: "bg-blue-500/20 text-blue-700 dark:text-blue-300 border-blue-500/30",
+	last: "bg-blue-500/20 text-blue-700 dark:text-blue-300 border-blue-500/30",
 	hourly: "bg-cyan-500/20 text-cyan-700 dark:text-cyan-300 border-cyan-500/30",
 	daily: "bg-green-500/20 text-green-700 dark:text-green-300 border-green-500/30",
 	weekly: "bg-orange-500/20 text-orange-700 dark:text-orange-300 border-orange-500/30",
@@ -17,7 +17,7 @@ const categoryColors: Record<string, string> = {
 };
 
 const categoryLabels: Record<string, string> = {
-	latest: "Latest",
+	last: "Last",
 	hourly: "Hourly",
 	daily: "Daily",
 	weekly: "Weekly",
@@ -45,7 +45,7 @@ export function RetentionCategoryBadges({ categories, className }: RetentionCate
 		return null;
 	}
 
-	const order = ["latest", "hourly", "daily", "weekly", "monthly", "yearly"];
+	const order = ["last", "hourly", "daily", "weekly", "monthly", "yearly"];
 	const sortedCategories = [...categories].sort((a, b) => {
 		const indexA = order.indexOf(a);
 		const indexB = order.indexOf(b);

app/server/modules/backups/__tests__/backups.execution.test.ts

@@ -14,7 +14,7 @@ import { restic } from "~/server/utils/restic";
 import { NotFoundError, BadRequestError } from "http-errors-enhanced";
 
 const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }));
-const resticForgetMock = mock(() => Promise.resolve({ success: true }));
+const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null }));
 const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" }));
 
 beforeEach(() => {

app/server/modules/backups/__tests__/backups.patterns.test.ts

@@ -15,7 +15,7 @@ const backupMock = mock(() => Promise.resolve({ exitCode: 0, result: JSON.parse(
 beforeEach(() => {
 	backupMock.mockClear();
 	spyOn(restic, "backup").mockImplementation(backupMock);
-	spyOn(restic, "forget").mockImplementation(mock(() => Promise.resolve({ success: true })));
+	spyOn(restic, "forget").mockImplementation(mock(() => Promise.resolve({ success: true, data: null })));
 	spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID);
 });
 

app/server/modules/backups/backups.execution.ts

@@ -145,6 +145,7 @@ const finalizeSuccessfulBackup = async (ctx: BackupContext, scheduleId: number,
 	});
 
 	cache.delByPrefix(`snapshots:${ctx.repository.id}:`);
+	cache.del(`retention:${ctx.schedule.shortId}`);
 
 	const nextBackupAt = calculateNextRun(ctx.schedule.cronExpression);
 	await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
@@ -320,6 +321,7 @@ const runForget = async (scheduleId: number, repositoryId?: string) => {
 	try {
 		await restic.forget(repository.config, schedule.retentionPolicy, { tag: schedule.shortId, organizationId });
 		cache.delByPrefix(`snapshots:${repository.id}:`);
+		cache.del(`retention:${schedule.shortId}`);
 	} finally {
 		releaseLock();
 	}

app/server/modules/repositories/repositories.controller.ts

@@ -43,11 +43,8 @@ import {
 	type UpdateRepositoryDto,
 } from "./repositories.dto";
 import { repositoriesService } from "./repositories.service";
-import { backupsService } from "../backups/backups.service";
 import { getRcloneRemoteInfo, listRcloneRemotes } from "../../utils/rclone";
 import { requireAuth, requireOrgAdmin } from "../auth/auth.middleware";
-import { computeRetentionCategories } from "../../utils/retention-categories";
-import { logger } from "~/server/utils/logger";
 import { toMessage } from "~/server/utils/errors";
 import { requireDevPanel } from "../auth/dev-panel.middleware";
 
@@ -94,21 +91,11 @@ export const repositoriesController = new Hono()
 	.get("/:id/snapshots", listSnapshotsDto, validator("query", listSnapshotsFilters), async (c) => {
 		const { id } = c.req.param();
 		const { backupId } = c.req.valid("query");
-		const res = await repositoriesService.listSnapshots(id, backupId);
 
-		let retentionCategories: Map<string, string[]> = new Map();
+		const [res, retentionCategories] = await Promise.all([
-		if (backupId) {
+			repositoriesService.listSnapshots(id, backupId),
-			try {
+			repositoriesService.getRetentionCategories(id, backupId),
-				const schedule = await backupsService.getScheduleByShortId(backupId);
+		]);
-				const snapshotsForCategories = res.map((snapshot) => ({
-					short_id: snapshot.short_id,
-					time: new Date(snapshot.time).getTime(),
-				}));
-				retentionCategories = computeRetentionCategories(snapshotsForCategories, schedule.retentionPolicy);
-			} catch (error) {
-				logger.warn(`Failed to fetch retention policy for backup ID ${backupId}`, toMessage(error));
-			}
-		}
 
 		const snapshots = res.map((snapshot) => {
 			const { summary } = snapshot;

app/server/modules/repositories/repositories.service.ts

@@ -21,6 +21,8 @@ import { getOrganizationId } from "~/server/core/request-context";
 import { serverEvents } from "~/server/core/events";
 import { executeDoctor } from "./doctor";
 import { logger } from "~/server/utils/logger";
+import { parseRetentionCategories, type RetentionCategory } from "~/server/utils/retention-categories";
+import { backupsService } from "../backups/backups.service";
 
 const runningDoctors = new Map<string, AbortController>();
 
@@ -592,6 +594,50 @@ const execResticCommand = async (
 	return { exitCode: result.exitCode };
 };
 
+const getRetentionCategories = async (repositoryId: string, scheduleId?: string) => {
+	if (!scheduleId) {
+		return new Map<string, RetentionCategory[]>();
+	}
+
+	const cacheKey = `retention:${scheduleId}`;
+	const cached = cache.get<Record<string, RetentionCategory[]>>(cacheKey);
+
+	if (cached && Object.keys(cached).length > 0) {
+		return new Map(Object.entries(cached));
+	}
+
+	try {
+		const [schedule, repositoryResult] = await Promise.all([
+			backupsService.getScheduleByShortId(scheduleId),
+			repositoriesService.getRepository(repositoryId),
+		]);
+
+		if (!schedule?.retentionPolicy) {
+			return new Map<string, RetentionCategory[]>();
+		}
+
+		const { repository } = repositoryResult;
+
+		const dryRunResults = await restic.forget(repository.config, schedule.retentionPolicy, {
+			tag: scheduleId,
+			organizationId: getOrganizationId(),
+			dryRun: true,
+		});
+
+		if (!dryRunResults.data) {
+			return new Map<string, RetentionCategory[]>();
+		}
+
+		const categories = parseRetentionCategories(dryRunResults.data);
+		cache.set(cacheKey, Object.fromEntries(categories));
+
+		return categories;
+	} catch (error) {
+		logger.error(`Failed to get retention categories: ${toMessage(error)}`);
+		return new Map<string, RetentionCategory[]>();
+	}
+};
+
 export const repositoriesService = {
 	listRepositories,
 	createRepository,
@@ -610,4 +656,5 @@ export const repositoriesService = {
 	tagSnapshots,
 	refreshSnapshots,
 	execResticCommand,
+	getRetentionCategories,
 };

app/server/utils/restic.ts

@@ -13,6 +13,7 @@ import { safeSpawn, exec } from "./spawn";
 import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic";
 import { ResticError } from "./errors";
 import { db } from "../db/db";
+import { safeJsonParse } from "./json";
 
 const backupOutputSchema = type({
 	message_type: "'summary'",
@@ -552,16 +553,70 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; o
 	return result;
 };
 
+export type ResticForgetResponse = ForgetGroup[];
+
+export interface ForgetGroup {
+	tags: string[] | null;
+	host: string;
+	paths: string[] | null;
+	keep: Snapshot[];
+	remove: Snapshot[] | null;
+	reasons: ForgetReason[];
+}
+
+export interface Snapshot {
+	time: string;
+	parent?: string;
+	tree: string;
+	paths: string[];
+	hostname: string;
+	username?: string;
+	uid?: number;
+	gid?: number;
+	excludes?: string[] | null;
+	tags?: string[] | null;
+	program_version?: string;
+	summary?: SnapshotSummary;
+	id: string;
+	short_id: string;
+}
+
+export interface SnapshotSummary {
+	backup_start: string;
+	backup_end: string;
+	files_new: number;
+	files_changed: number;
+	files_unmodified: number;
+	dirs_new: number;
+	dirs_changed: number;
+	dirs_unmodified: number;
+	data_blobs: number;
+	tree_blobs: number;
+	data_added: number;
+	data_added_packed: number;
+	total_files_processed: number;
+	total_bytes_processed: number;
+}
+
+export interface ForgetReason {
+	snapshot: Snapshot;
+	matches: string[];
+}
+
 const forget = async (
 	config: RepositoryConfig,
 	options: RetentionPolicy,
-	extra: { tag: string; organizationId: string },
+	extra: { tag: string; organizationId: string; dryRun?: boolean },
 ) => {
 	const repoUrl = buildRepoUrl(config);
 	const env = await buildEnv(config, extra.organizationId);
 
 	const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag];
 
+	if (extra.dryRun) {
+		args.push("--dry-run", "--no-lock");
+	}
+
 	if (options.keepLast) {
 		args.push("--keep-last", String(options.keepLast));
 	}
@@ -584,7 +639,10 @@ const forget = async (
 		args.push("--keep-within-duration", options.keepWithinDuration);
 	}
 
-	args.push("--prune");
+	if (!extra.dryRun) {
+		args.push("--prune");
+	}
+
 	addCommonArgs(args, env, config);
 
 	const res = await exec({ command: "restic", args, env });
@@ -595,7 +653,10 @@ const forget = async (
 		throw new ResticError(res.exitCode, res.stderr);
 	}
 
-	return { success: true };
+	const lines = res.stdout.split("\n").filter((line) => line.trim());
+	const result = extra.dryRun ? safeJsonParse<ResticForgetResponse>(lines.at(-1) ?? "[]") : null;
+
+	return { success: true, data: result };
 };
 
 const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[], organizationId: string) => {

app/server/utils/retention-categories.ts

@@ -1,51 +1,38 @@
-import { format } from "date-fns";
+import type { ResticForgetResponse } from "./restic";
-import type { RetentionPolicy } from "../modules/backups/backups.dto";
 
-export type RetentionCategory = "latest" | "hourly" | "daily" | "weekly" | "monthly" | "yearly";
+export type RetentionCategory = "last" | "hourly" | "daily" | "weekly" | "monthly" | "yearly";
 
-interface SnapshotInfo {
+const MATCH_TO_CATEGORY: Record<string, RetentionCategory> = {
-	short_id: string;
+	"last snapshot": "last",
-	time: number;
+	"hourly snapshot": "hourly",
-}
+	"daily snapshot": "daily",
+	"weekly snapshot": "weekly",
+	"monthly snapshot": "monthly",
+	"yearly snapshot": "yearly",
+	"oldest hourly snapshot": "hourly",
+	"oldest daily snapshot": "daily",
+	"oldest weekly snapshot": "weekly",
+	"oldest monthly snapshot": "monthly",
+	"oldest yearly snapshot": "yearly",
+};
 
-const RETENTION_RULES = [
+export const parseRetentionCategories = (dryRunResults: ResticForgetResponse) => {
-	{ prop: "keepHourly", tag: "hourly", fmt: "yyyy-MM-dd-HH" },
-	{ prop: "keepDaily", tag: "daily", fmt: "yyyy-MM-dd" },
-	{ prop: "keepWeekly", tag: "weekly", fmt: "RRRR-'W'II" },
-	{ prop: "keepMonthly", tag: "monthly", fmt: "yyyy-MM" },
-	{ prop: "keepYearly", tag: "yearly", fmt: "yyyy" },
-] as const;
-
-export const computeRetentionCategories = (snapshots: SnapshotInfo[], policy: RetentionPolicy | null) => {
 	const categories = new Map<string, RetentionCategory[]>();
 
-	if (snapshots.length === 0) return categories;
+	for (const group of dryRunResults) {
+		for (const reason of group.reasons) {
+			const { short_id } = reason.snapshot;
+			const categoryList: RetentionCategory[] = [];
 
-	const sorted = [...snapshots].sort((a, b) => b.time - a.time);
+			for (const match of reason.matches) {
+				const category = MATCH_TO_CATEGORY[match];
+				if (category && !categoryList.includes(category)) {
+					categoryList.push(category);
+				}
+			}
 
-	const addTag = (id: string, tag: RetentionCategory) => {
+			if (categoryList.length > 0) {
-		const tags = categories.get(id) ?? [];
+				categories.set(short_id, categoryList);
-		if (!tags.includes(tag)) categories.set(id, [...tags, tag]);
-	};
-	addTag(sorted[0].short_id, "latest");
-
-	if (!policy) return categories;
-
-	for (const { prop, tag, fmt } of RETENTION_RULES) {
-		const limit = policy[prop];
-		if (!limit || limit <= 0) continue;
-
-		const seenBuckets = new Set<string>();
-		let count = 0;
-
-		for (const snapshot of sorted) {
-			if (count >= limit) break;
-
-			const bucketKey = format(snapshot.time, fmt);
-			if (!seenBuckets.has(bucketKey)) {
-				seenBuckets.add(bucketKey);
-				addTag(snapshot.short_id, tag);
-				count++;
 			}
 		}
 	}