diff --git a/app/client/components/retention-category-badges.tsx b/app/client/components/retention-category-badges.tsx index 97a16f15..d3acfec1 100644 --- a/app/client/components/retention-category-badges.tsx +++ b/app/client/components/retention-category-badges.tsx @@ -8,7 +8,7 @@ interface RetentionCategoryBadgesProps { } const categoryColors: Record = { - latest: "bg-blue-500/20 text-blue-700 dark:text-blue-300 border-blue-500/30", + last: "bg-blue-500/20 text-blue-700 dark:text-blue-300 border-blue-500/30", hourly: "bg-cyan-500/20 text-cyan-700 dark:text-cyan-300 border-cyan-500/30", daily: "bg-green-500/20 text-green-700 dark:text-green-300 border-green-500/30", weekly: "bg-orange-500/20 text-orange-700 dark:text-orange-300 border-orange-500/30", @@ -17,7 +17,7 @@ const categoryColors: Record = { }; const categoryLabels: Record = { - latest: "Latest", + last: "Last", hourly: "Hourly", daily: "Daily", weekly: "Weekly", @@ -45,7 +45,7 @@ export function RetentionCategoryBadges({ categories, className }: RetentionCate return null; } - const order = ["latest", "hourly", "daily", "weekly", "monthly", "yearly"]; + const order = ["last", "hourly", "daily", "weekly", "monthly", "yearly"]; const sortedCategories = [...categories].sort((a, b) => { const indexA = order.indexOf(a); const indexB = order.indexOf(b); diff --git a/app/server/modules/backups/__tests__/backups.execution.test.ts b/app/server/modules/backups/__tests__/backups.execution.test.ts index a0e03207..e24397bc 100644 --- a/app/server/modules/backups/__tests__/backups.execution.test.ts +++ b/app/server/modules/backups/__tests__/backups.execution.test.ts @@ -14,7 +14,7 @@ import { restic } from "~/server/utils/restic"; import { NotFoundError, BadRequestError } from "http-errors-enhanced"; const resticBackupMock = mock(() => Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" })); -const resticForgetMock = mock(() => Promise.resolve({ success: true })); +const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null })); const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" })); beforeEach(() => { diff --git a/app/server/modules/backups/__tests__/backups.patterns.test.ts b/app/server/modules/backups/__tests__/backups.patterns.test.ts index 841a71f6..2e26a990 100644 --- a/app/server/modules/backups/__tests__/backups.patterns.test.ts +++ b/app/server/modules/backups/__tests__/backups.patterns.test.ts @@ -15,7 +15,7 @@ const backupMock = mock(() => Promise.resolve({ exitCode: 0, result: JSON.parse( beforeEach(() => { backupMock.mockClear(); spyOn(restic, "backup").mockImplementation(backupMock); - spyOn(restic, "forget").mockImplementation(mock(() => Promise.resolve({ success: true }))); + spyOn(restic, "forget").mockImplementation(mock(() => Promise.resolve({ success: true, data: null }))); spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID); }); diff --git a/app/server/modules/backups/backups.execution.ts b/app/server/modules/backups/backups.execution.ts index 93947fab..da639ea7 100644 --- a/app/server/modules/backups/backups.execution.ts +++ b/app/server/modules/backups/backups.execution.ts @@ -145,6 +145,7 @@ const finalizeSuccessfulBackup = async (ctx: BackupContext, scheduleId: number, }); cache.delByPrefix(`snapshots:${ctx.repository.id}:`); + cache.del(`retention:${ctx.schedule.shortId}`); const nextBackupAt = calculateNextRun(ctx.schedule.cronExpression); await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, { @@ -320,6 +321,7 @@ const runForget = async (scheduleId: number, repositoryId?: string) => { try { await restic.forget(repository.config, schedule.retentionPolicy, { tag: schedule.shortId, organizationId }); cache.delByPrefix(`snapshots:${repository.id}:`); + cache.del(`retention:${schedule.shortId}`); } finally { releaseLock(); } diff --git a/app/server/modules/repositories/repositories.controller.ts b/app/server/modules/repositories/repositories.controller.ts index c0ed0a93..6448bb4a 100644 --- a/app/server/modules/repositories/repositories.controller.ts +++ b/app/server/modules/repositories/repositories.controller.ts @@ -43,11 +43,8 @@ import { type UpdateRepositoryDto, } from "./repositories.dto"; import { repositoriesService } from "./repositories.service"; -import { backupsService } from "../backups/backups.service"; import { getRcloneRemoteInfo, listRcloneRemotes } from "../../utils/rclone"; import { requireAuth, requireOrgAdmin } from "../auth/auth.middleware"; -import { computeRetentionCategories } from "../../utils/retention-categories"; -import { logger } from "~/server/utils/logger"; import { toMessage } from "~/server/utils/errors"; import { requireDevPanel } from "../auth/dev-panel.middleware"; @@ -94,21 +91,11 @@ export const repositoriesController = new Hono() .get("/:id/snapshots", listSnapshotsDto, validator("query", listSnapshotsFilters), async (c) => { const { id } = c.req.param(); const { backupId } = c.req.valid("query"); - const res = await repositoriesService.listSnapshots(id, backupId); - let retentionCategories: Map = new Map(); - if (backupId) { - try { - const schedule = await backupsService.getScheduleByShortId(backupId); - const snapshotsForCategories = res.map((snapshot) => ({ - short_id: snapshot.short_id, - time: new Date(snapshot.time).getTime(), - })); - retentionCategories = computeRetentionCategories(snapshotsForCategories, schedule.retentionPolicy); - } catch (error) { - logger.warn(`Failed to fetch retention policy for backup ID ${backupId}`, toMessage(error)); - } - } + const [res, retentionCategories] = await Promise.all([ + repositoriesService.listSnapshots(id, backupId), + repositoriesService.getRetentionCategories(id, backupId), + ]); const snapshots = res.map((snapshot) => { const { summary } = snapshot; diff --git a/app/server/modules/repositories/repositories.service.ts b/app/server/modules/repositories/repositories.service.ts index 855bba2a..62a0c6ef 100644 --- a/app/server/modules/repositories/repositories.service.ts +++ b/app/server/modules/repositories/repositories.service.ts @@ -21,6 +21,8 @@ import { getOrganizationId } from "~/server/core/request-context"; import { serverEvents } from "~/server/core/events"; import { executeDoctor } from "./doctor"; import { logger } from "~/server/utils/logger"; +import { parseRetentionCategories, type RetentionCategory } from "~/server/utils/retention-categories"; +import { backupsService } from "../backups/backups.service"; const runningDoctors = new Map(); @@ -592,6 +594,50 @@ const execResticCommand = async ( return { exitCode: result.exitCode }; }; +const getRetentionCategories = async (repositoryId: string, scheduleId?: string) => { + if (!scheduleId) { + return new Map(); + } + + const cacheKey = `retention:${scheduleId}`; + const cached = cache.get>(cacheKey); + + if (cached && Object.keys(cached).length > 0) { + return new Map(Object.entries(cached)); + } + + try { + const [schedule, repositoryResult] = await Promise.all([ + backupsService.getScheduleByShortId(scheduleId), + repositoriesService.getRepository(repositoryId), + ]); + + if (!schedule?.retentionPolicy) { + return new Map(); + } + + const { repository } = repositoryResult; + + const dryRunResults = await restic.forget(repository.config, schedule.retentionPolicy, { + tag: scheduleId, + organizationId: getOrganizationId(), + dryRun: true, + }); + + if (!dryRunResults.data) { + return new Map(); + } + + const categories = parseRetentionCategories(dryRunResults.data); + cache.set(cacheKey, Object.fromEntries(categories)); + + return categories; + } catch (error) { + logger.error(`Failed to get retention categories: ${toMessage(error)}`); + return new Map(); + } +}; + export const repositoriesService = { listRepositories, createRepository, @@ -610,4 +656,5 @@ export const repositoriesService = { tagSnapshots, refreshSnapshots, execResticCommand, + getRetentionCategories, }; diff --git a/app/server/utils/restic.ts b/app/server/utils/restic.ts index 0d91d2ae..f0c1b76e 100644 --- a/app/server/utils/restic.ts +++ b/app/server/utils/restic.ts @@ -13,6 +13,7 @@ import { safeSpawn, exec } from "./spawn"; import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic"; import { ResticError } from "./errors"; import { db } from "../db/db"; +import { safeJsonParse } from "./json"; const backupOutputSchema = type({ message_type: "'summary'", @@ -552,16 +553,70 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; o return result; }; +export type ResticForgetResponse = ForgetGroup[]; + +export interface ForgetGroup { + tags: string[] | null; + host: string; + paths: string[] | null; + keep: Snapshot[]; + remove: Snapshot[] | null; + reasons: ForgetReason[]; +} + +export interface Snapshot { + time: string; + parent?: string; + tree: string; + paths: string[]; + hostname: string; + username?: string; + uid?: number; + gid?: number; + excludes?: string[] | null; + tags?: string[] | null; + program_version?: string; + summary?: SnapshotSummary; + id: string; + short_id: string; +} + +export interface SnapshotSummary { + backup_start: string; + backup_end: string; + files_new: number; + files_changed: number; + files_unmodified: number; + dirs_new: number; + dirs_changed: number; + dirs_unmodified: number; + data_blobs: number; + tree_blobs: number; + data_added: number; + data_added_packed: number; + total_files_processed: number; + total_bytes_processed: number; +} + +export interface ForgetReason { + snapshot: Snapshot; + matches: string[]; +} + const forget = async ( config: RepositoryConfig, options: RetentionPolicy, - extra: { tag: string; organizationId: string }, + extra: { tag: string; organizationId: string; dryRun?: boolean }, ) => { const repoUrl = buildRepoUrl(config); const env = await buildEnv(config, extra.organizationId); const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag]; + if (extra.dryRun) { + args.push("--dry-run", "--no-lock"); + } + if (options.keepLast) { args.push("--keep-last", String(options.keepLast)); } @@ -584,7 +639,10 @@ const forget = async ( args.push("--keep-within-duration", options.keepWithinDuration); } - args.push("--prune"); + if (!extra.dryRun) { + args.push("--prune"); + } + addCommonArgs(args, env, config); const res = await exec({ command: "restic", args, env }); @@ -595,7 +653,10 @@ const forget = async ( throw new ResticError(res.exitCode, res.stderr); } - return { success: true }; + const lines = res.stdout.split("\n").filter((line) => line.trim()); + const result = extra.dryRun ? safeJsonParse(lines.at(-1) ?? "[]") : null; + + return { success: true, data: result }; }; const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[], organizationId: string) => { diff --git a/app/server/utils/retention-categories.ts b/app/server/utils/retention-categories.ts index c27dd2f3..755d3fd6 100644 --- a/app/server/utils/retention-categories.ts +++ b/app/server/utils/retention-categories.ts @@ -1,51 +1,38 @@ -import { format } from "date-fns"; -import type { RetentionPolicy } from "../modules/backups/backups.dto"; +import type { ResticForgetResponse } from "./restic"; -export type RetentionCategory = "latest" | "hourly" | "daily" | "weekly" | "monthly" | "yearly"; +export type RetentionCategory = "last" | "hourly" | "daily" | "weekly" | "monthly" | "yearly"; -interface SnapshotInfo { - short_id: string; - time: number; -} +const MATCH_TO_CATEGORY: Record = { + "last snapshot": "last", + "hourly snapshot": "hourly", + "daily snapshot": "daily", + "weekly snapshot": "weekly", + "monthly snapshot": "monthly", + "yearly snapshot": "yearly", + "oldest hourly snapshot": "hourly", + "oldest daily snapshot": "daily", + "oldest weekly snapshot": "weekly", + "oldest monthly snapshot": "monthly", + "oldest yearly snapshot": "yearly", +}; -const RETENTION_RULES = [ - { prop: "keepHourly", tag: "hourly", fmt: "yyyy-MM-dd-HH" }, - { prop: "keepDaily", tag: "daily", fmt: "yyyy-MM-dd" }, - { prop: "keepWeekly", tag: "weekly", fmt: "RRRR-'W'II" }, - { prop: "keepMonthly", tag: "monthly", fmt: "yyyy-MM" }, - { prop: "keepYearly", tag: "yearly", fmt: "yyyy" }, -] as const; - -export const computeRetentionCategories = (snapshots: SnapshotInfo[], policy: RetentionPolicy | null) => { +export const parseRetentionCategories = (dryRunResults: ResticForgetResponse) => { const categories = new Map(); - if (snapshots.length === 0) return categories; + for (const group of dryRunResults) { + for (const reason of group.reasons) { + const { short_id } = reason.snapshot; + const categoryList: RetentionCategory[] = []; - const sorted = [...snapshots].sort((a, b) => b.time - a.time); + for (const match of reason.matches) { + const category = MATCH_TO_CATEGORY[match]; + if (category && !categoryList.includes(category)) { + categoryList.push(category); + } + } - const addTag = (id: string, tag: RetentionCategory) => { - const tags = categories.get(id) ?? []; - if (!tags.includes(tag)) categories.set(id, [...tags, tag]); - }; - addTag(sorted[0].short_id, "latest"); - - if (!policy) return categories; - - for (const { prop, tag, fmt } of RETENTION_RULES) { - const limit = policy[prop]; - if (!limit || limit <= 0) continue; - - const seenBuckets = new Set(); - let count = 0; - - for (const snapshot of sorted) { - if (count >= limit) break; - - const bucketKey = format(snapshot.time, fmt); - if (!seenBuckets.has(bucketKey)) { - seenBuckets.add(bucketKey); - addTag(snapshot.short_id, tag); - count++; + if (categoryList.length > 0) { + categories.set(short_id, categoryList); } } }