From 45fd9f9de17bc5b44ff750aaa492ba0371a015e6 Mon Sep 17 00:00:00 2001 From: Nico <47644445+nicotsx@users.noreply.github.com> Date: Sat, 21 Feb 2026 10:52:55 +0100 Subject: [PATCH] feat: repository used space (#551) * feat: repository used space refactor: use a smarter cache key logic for easier invalidations * chore: pr feedbacks --- .../api-client/@tanstack/react-query.gen.ts | 28 ++ app/client/api-client/index.ts | 4 + app/client/api-client/sdk.gen.ts | 13 + app/client/api-client/types.gen.ts | 25 ++ .../components/compression-stats-chart.tsx | 134 ++++++++++ app/client/modules/repositories/tabs/info.tsx | 252 +++++++++--------- app/schemas/restic-dto.ts | 10 + .../modules/backups/backups.execution.ts | 10 +- .../__tests__/repositories.controller.test.ts | 12 + .../__tests__/repositories.service.test.ts | 2 +- .../repositories/repositories.controller.ts | 8 + .../modules/repositories/repositories.dto.ts | 25 +- .../repositories/repositories.service.ts | 59 ++-- app/server/modules/system/system.service.ts | 4 +- app/server/utils/cache.ts | 28 +- app/server/utils/restic.ts | 28 ++ 16 files changed, 483 insertions(+), 159 deletions(-) create mode 100644 app/client/modules/repositories/components/compression-stats-chart.tsx diff --git a/app/client/api-client/@tanstack/react-query.gen.ts b/app/client/api-client/@tanstack/react-query.gen.ts index bbad7cc8..95287be8 100644 --- a/app/client/api-client/@tanstack/react-query.gen.ts +++ b/app/client/api-client/@tanstack/react-query.gen.ts @@ -33,6 +33,7 @@ import { getNotificationDestination, getRegistrationStatus, getRepository, + getRepositoryStats, getScheduleMirrors, getScheduleNotifications, getSnapshotDetails, @@ -117,6 +118,8 @@ import type { GetRegistrationStatusResponse, GetRepositoryData, GetRepositoryResponse, + GetRepositoryStatsData, + GetRepositoryStatsResponse, GetScheduleMirrorsData, GetScheduleMirrorsResponse, GetScheduleNotificationsData, @@ -688,6 +691,31 @@ export const updateRepositoryMutation = ( return mutationOptions; }; +export const getRepositoryStatsQueryKey = (options: Options) => + createQueryKey("getRepositoryStats", options); + +/** + * Get repository storage and compression statistics + */ +export const getRepositoryStatsOptions = (options: Options) => + queryOptions< + GetRepositoryStatsResponse, + DefaultError, + GetRepositoryStatsResponse, + ReturnType + >({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await getRepositoryStats({ + ...options, + ...queryKey[0], + signal, + throwOnError: true, + }); + return data; + }, + queryKey: getRepositoryStatsQueryKey(options), + }); + /** * Delete multiple snapshots from a repository */ diff --git a/app/client/api-client/index.ts b/app/client/api-client/index.ts index a2851cf5..dadecaf4 100644 --- a/app/client/api-client/index.ts +++ b/app/client/api-client/index.ts @@ -24,6 +24,7 @@ export { getNotificationDestination, getRegistrationStatus, getRepository, + getRepositoryStats, getScheduleMirrors, getScheduleNotifications, getSnapshotDetails, @@ -135,6 +136,9 @@ export type { GetRepositoryData, GetRepositoryResponse, GetRepositoryResponses, + GetRepositoryStatsData, + GetRepositoryStatsResponse, + GetRepositoryStatsResponses, GetScheduleMirrorsData, GetScheduleMirrorsResponse, GetScheduleMirrorsResponses, diff --git a/app/client/api-client/sdk.gen.ts b/app/client/api-client/sdk.gen.ts index 87103d93..14aa4fc2 100644 --- a/app/client/api-client/sdk.gen.ts +++ b/app/client/api-client/sdk.gen.ts @@ -52,6 +52,8 @@ import type { GetRegistrationStatusResponses, GetRepositoryData, GetRepositoryResponses, + GetRepositoryStatsData, + GetRepositoryStatsResponses, GetScheduleMirrorsData, GetScheduleMirrorsResponses, GetScheduleNotificationsData, @@ -363,6 +365,17 @@ export const updateRepository = ( }, }); +/** + * Get repository storage and compression statistics + */ +export const getRepositoryStats = ( + options: Options, +) => + (options.client ?? client).get({ + url: "/api/v1/repositories/{shortId}/stats", + ...options, + }); + /** * Delete multiple snapshots from a repository */ diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index 71c57dba..eac7e763 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -1770,6 +1770,31 @@ export type UpdateRepositoryResponses = { export type UpdateRepositoryResponse = UpdateRepositoryResponses[keyof UpdateRepositoryResponses]; +export type GetRepositoryStatsData = { + body?: never; + path: { + shortId: string; + }; + query?: never; + url: "/api/v1/repositories/{shortId}/stats"; +}; + +export type GetRepositoryStatsResponses = { + /** + * Repository statistics + */ + 200: { + compression_progress?: number; + compression_ratio?: number; + compression_space_saving?: number; + snapshots_count?: number; + total_size?: number; + total_uncompressed_size?: number; + }; +}; + +export type GetRepositoryStatsResponse = GetRepositoryStatsResponses[keyof GetRepositoryStatsResponses]; + export type DeleteSnapshotsData = { body?: { snapshotIds: Array; diff --git a/app/client/modules/repositories/components/compression-stats-chart.tsx b/app/client/modules/repositories/components/compression-stats-chart.tsx new file mode 100644 index 00000000..74996c78 --- /dev/null +++ b/app/client/modules/repositories/components/compression-stats-chart.tsx @@ -0,0 +1,134 @@ +import { useQuery } from "@tanstack/react-query"; +import { Archive } from "lucide-react"; +import { getRepositoryStatsOptions } from "~/client/api-client/@tanstack/react-query.gen"; +import { ByteSize } from "~/client/components/bytes-size"; +import { Card, CardContent, CardHeader, CardTitle } from "~/client/components/ui/card"; + +type Props = { + repositoryShortId: string; +}; + +const toSafeNumber = (value: number | undefined) => { + if (typeof value !== "number" || !Number.isFinite(value)) { + return 0; + } + + return Math.max(0, value); +}; + +export function CompressionStatsChart({ repositoryShortId }: Props) { + const { + data: stats, + isLoading, + error, + } = useQuery({ + ...getRepositoryStatsOptions({ path: { shortId: repositoryShortId } }), + retry: false, + }); + + const storedSize = toSafeNumber(stats?.total_size); + const uncompressedSize = toSafeNumber(stats?.total_uncompressed_size); + const savedSize = uncompressedSize > storedSize ? uncompressedSize - storedSize : 0; + + const compressionRatio = toSafeNumber(stats?.compression_ratio); + + const rawCompressionProgress = toSafeNumber(stats?.compression_progress); + const compressionProgressPercent = Math.min(100, Math.max(0, rawCompressionProgress)); + + const spaceSavingPercent = toSafeNumber(stats?.compression_space_saving); + const snapshotsCount = Math.round(toSafeNumber(stats?.snapshots_count)); + + const hasStats = !!stats && (storedSize > 0 || uncompressedSize > 0 || snapshotsCount > 0); + + if (isLoading) { + return ( + +

Loading compression statistics...

+ + ); + } + + if (error) { + return ( + +

Failed to load compression statistics

+

{error.message}

+ + ); + } + + if (!hasStats) { + return ( + +

+ No compression statistics available yet. Run a backup to populate repository stats. +

+ + ); + } + + return ( + + + + + Compression Statistics + + + +
+
+
+ Stored Size +
+
+ +
+
+ +
+
+ Uncompressed +
+
+ +
+
+ +
+
+ Space Saved +
+
+ {spaceSavingPercent.toFixed(1)}% + +
+
+ +
+
+ Ratio +
+
+ + {compressionRatio > 0 ? `${compressionRatio.toFixed(2)}x` : "—"} + +
+
+ +
+
+ Snapshots +
+
+ {snapshotsCount.toLocaleString()} + + {compressionProgressPercent.toFixed(1)}% compressed + +
+
+
+ + + ); +} diff --git a/app/client/modules/repositories/tabs/info.tsx b/app/client/modules/repositories/tabs/info.tsx index 9f973db9..2d11ac23 100644 --- a/app/client/modules/repositories/tabs/info.tsx +++ b/app/client/modules/repositories/tabs/info.tsx @@ -25,6 +25,7 @@ import type { RepositoryConfig } from "~/schemas/restic"; import { DoctorReport } from "../components/doctor-report"; import { parseError } from "~/client/lib/errors"; import { useNavigate } from "@tanstack/react-router"; +import { CompressionStatsChart } from "../components/compression-stats-chart"; type Props = { repository: Repository; @@ -96,143 +97,148 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => { return ( <> - -
-
- Repository Settings -
-
- - {repository.status === "doctor" ? ( +
+ +
+
+ Repository Settings +
+
+ + {repository.status === "doctor" ? ( + + ) : ( + + )} + - ) : ( - - )} - - -
-
- -
-

Current Configuration

-
-
-
Name
-

{repository.name}

-
-
-
Compression mode
-

{repository.compressionMode || "off"}

-
-
-

Repository Information

-
-
-
Backend
-

{repository.type}

-
-
-
Status
-

{repository.status || "unknown"}

-
- {effectiveLocalPath && ( -
-
Local path
-

{effectiveLocalPath}

-
- )} -
-
Created at
-

{formatDateTime(repository.createdAt)}

-
-
-
Last checked
-

{formatTimeAgo(repository.lastChecked)}

-
- {config.cacert && ( -
-
CA Certificate
-

- configured -

-
- )} - {"insecureTls" in config && ( -
-
TLS Certificate Validation
-

- {config.insecureTls ? ( - disabled - ) : ( - enabled - )} -

-
- )} -
-
- - {repository.lastError && (
-
-

Last Error

-
-
-

{repository.lastError}

+

Current Configuration

+
+
+
Name
+

{repository.name}

+
+
+
Compression mode
+

{repository.compressionMode || "off"}

+
- )} -
-

Configuration

-
- 
{JSON.stringify(repository.config, null, 2)}
+
+

Repository Information

+
+
+
Backend
+

{repository.type}

+
+
+
Status
+

{repository.status || "unknown"}

+
+ {effectiveLocalPath && ( +
+
Local path
+

{effectiveLocalPath}

+
+ )} +
+
Created at
+

{formatDateTime(repository.createdAt)}

+
+
+
Last checked
+

{formatTimeAgo(repository.lastChecked)}

+
+ {config.cacert && ( +
+
CA Certificate
+

+ configured +

+
+ )} + {"insecureTls" in config && ( +
+
TLS Certificate Validation
+

+ {config.insecureTls ? ( + disabled + ) : ( + enabled + )} +

+
+ )} +
-
- - +
+ {repository.lastError && ( +
+
+

Last Error

+
+
+

{repository.lastError}

+
+
+ )} + +
+

Configuration

+
+ 
{JSON.stringify(repository.config, null, 2)}
+
+
+
+ + + + +
diff --git a/app/schemas/restic-dto.ts b/app/schemas/restic-dto.ts index 5abd2733..2683834b 100644 --- a/app/schemas/restic-dto.ts +++ b/app/schemas/restic-dto.ts @@ -61,6 +61,15 @@ export const resticRestoreOutputSchema = type({ bytes_skipped: "number", }); +export const resticStatsSchema = type({ + total_size: "number = 0", + total_uncompressed_size: "number = 0", + compression_ratio: "number = 0", + compression_progress: "number = 0", + compression_space_saving: "number = 0", + snapshots_count: "number = 0", +}); + export type ResticSnapshotSummaryDto = typeof resticSnapshotSummarySchema.infer; export type ResticBackupRunSummaryDto = typeof resticBackupRunSummarySchema.infer; export type ResticBackupOutputDto = typeof resticBackupOutputSchema.infer; @@ -68,3 +77,4 @@ export type ResticBackupProgressMetricsDto = typeof resticBackupProgressMetricsS export type ResticBackupProgressDto = typeof resticBackupProgressSchema.infer; export type ResticRestoreOutputDto = typeof resticRestoreOutputSchema.infer; +export type ResticStatsDto = typeof resticStatsSchema.infer; diff --git a/app/server/modules/backups/backups.execution.ts b/app/server/modules/backups/backups.execution.ts index ab954ece..6a0b044c 100644 --- a/app/server/modules/backups/backups.execution.ts +++ b/app/server/modules/backups/backups.execution.ts @@ -2,7 +2,7 @@ import { NotFoundError, BadRequestError, ConflictError } from "http-errors-enhan import type { BackupSchedule, Volume, Repository } from "../../db/schema"; import { restic } from "../../utils/restic"; import { logger } from "../../utils/logger"; -import { cache } from "../../utils/cache"; +import { cache, cacheKeys } from "../../utils/cache"; import { getVolumePath } from "../volumes/helpers"; import { toMessage } from "../../utils/errors"; import { serverEvents } from "../../core/events"; @@ -150,8 +150,7 @@ const finalizeSuccessfulBackup = async ( logger.error(`Background mirror copy failed for schedule ${scheduleId}: ${toMessage(error)}`); }); - cache.delByPrefix(`snapshots:${ctx.repository.id}:`); - cache.del(`retention:${ctx.schedule.shortId}`); + cache.delByPrefix(cacheKeys.repository.all(ctx.repository.id)); const nextBackupAt = calculateNextRun(ctx.schedule.cronExpression); await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, { @@ -328,8 +327,7 @@ const runForget = async (scheduleId: number, repositoryId?: string) => { try { await restic.forget(repository.config, schedule.retentionPolicy, { tag: schedule.shortId, organizationId }); - cache.delByPrefix(`snapshots:${repository.id}:`); - cache.del(`retention:${schedule.shortId}`); + cache.delByPrefix(cacheKeys.repository.all(repository.id)); } finally { releaseLock(); } @@ -393,7 +391,7 @@ const copyToSingleMirror = async ( try { await restic.copy(sourceRepository.config, mirror.repository.config, { tag: schedule.shortId, organizationId }); - cache.delByPrefix(`snapshots:${mirror.repository.id}:`); + cache.delByPrefix(cacheKeys.repository.all(mirror.repository.id)); } finally { releaseSource(); releaseMirror(); diff --git a/app/server/modules/repositories/__tests__/repositories.controller.test.ts b/app/server/modules/repositories/__tests__/repositories.controller.test.ts index 3f887b0e..40c6c706 100644 --- a/app/server/modules/repositories/__tests__/repositories.controller.test.ts +++ b/app/server/modules/repositories/__tests__/repositories.controller.test.ts @@ -72,6 +72,7 @@ describe("repositories security", () => { { method: "POST", path: "/api/v1/repositories" }, { method: "GET", path: "/api/v1/repositories/rclone-remotes" }, { method: "GET", path: "/api/v1/repositories/test-repo" }, + { method: "GET", path: "/api/v1/repositories/test-repo/stats" }, { method: "DELETE", path: "/api/v1/repositories/test-repo" }, { method: "GET", path: "/api/v1/repositories/test-repo/snapshots" }, { method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot" }, @@ -115,6 +116,17 @@ describe("repositories security", () => { expect(body.message).toBe("Repository not found"); }); + test("should return 404 for stats of non-existent repository", async () => { + const { token } = await createTestSession(); + const res = await app.request("/api/v1/repositories/non-existent-repo/stats", { + headers: getAuthHeaders(token), + }); + + expect(res.status).toBe(404); + const body = await res.json(); + expect(body.message).toBe("Repository not found"); + }); + test("should return 400 for invalid payload on create", async () => { const { token } = await createTestSession(); const res = await app.request("/api/v1/repositories", { diff --git a/app/server/modules/repositories/__tests__/repositories.service.test.ts b/app/server/modules/repositories/__tests__/repositories.service.test.ts index 079884ba..456414b3 100644 --- a/app/server/modules/repositories/__tests__/repositories.service.test.ts +++ b/app/server/modules/repositories/__tests__/repositories.service.test.ts @@ -244,7 +244,7 @@ describe("repositoriesService.dumpSnapshot", () => { ); spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots); - await expect( + expect( withContext({ organizationId, userId: user.id }, () => repositoriesService.dumpSnapshot(shortId, "snapshot-no-kind", `${basePath}/documents/report.txt`), ), diff --git a/app/server/modules/repositories/repositories.controller.ts b/app/server/modules/repositories/repositories.controller.ts index b6dc31bc..c8911676 100644 --- a/app/server/modules/repositories/repositories.controller.ts +++ b/app/server/modules/repositories/repositories.controller.ts @@ -13,6 +13,7 @@ import { startDoctorDto, cancelDoctorDto, getRepositoryDto, + getRepositoryStatsDto, getSnapshotDetailsDto, refreshSnapshotsDto, listRcloneRemotesDto, @@ -38,6 +39,7 @@ import { type StartDoctorDto, type CancelDoctorDto, type GetRepositoryDto, + type GetRepositoryStatsDto, type GetSnapshotDetailsDto, type RefreshSnapshotsDto, type ListRepositoriesDto, @@ -89,6 +91,12 @@ export const repositoriesController = new Hono() return c.json(res.repository, 200); }) + .get("/:shortId/stats", getRepositoryStatsDto, async (c) => { + const { shortId } = c.req.param(); + const stats = await repositoriesService.getRepositoryStats(shortId); + + return c.json(stats, 200); + }) .delete("/:shortId", deleteRepositoryDto, async (c) => { const { shortId } = c.req.param(); await repositoriesService.deleteRepository(shortId); diff --git a/app/server/modules/repositories/repositories.dto.ts b/app/server/modules/repositories/repositories.dto.ts index 0226ddce..8e932bd4 100644 --- a/app/server/modules/repositories/repositories.dto.ts +++ b/app/server/modules/repositories/repositories.dto.ts @@ -8,7 +8,7 @@ import { repositoryConfigSchema, doctorResultSchema, } from "~/schemas/restic"; -import { resticSnapshotSummarySchema } from "~/schemas/restic-dto"; +import { resticSnapshotSummarySchema, resticStatsSchema } from "~/schemas/restic-dto"; export const repositorySchema = type({ id: "string", @@ -109,6 +109,29 @@ export const getRepositoryDto = describeRoute({ }, }); +/** + * Get repository stats + */ +export const repositoryStatsSchema = resticStatsSchema; +export const getRepositoryStatsResponse = repositoryStatsSchema; +export type GetRepositoryStatsDto = typeof getRepositoryStatsResponse.infer; + +export const getRepositoryStatsDto = describeRoute({ + description: "Get repository storage and compression statistics", + tags: ["Repositories"], + operationId: "getRepositoryStats", + responses: { + 200: { + description: "Repository statistics", + content: { + "application/json": { + schema: resolver(getRepositoryStatsResponse), + }, + }, + }, + }, +}); + /** * Delete a repository */ diff --git a/app/server/modules/repositories/repositories.service.ts b/app/server/modules/repositories/repositories.service.ts index 5342f91e..b32ab0b5 100644 --- a/app/server/modules/repositories/repositories.service.ts +++ b/app/server/modules/repositories/repositories.service.ts @@ -16,7 +16,7 @@ import { parseRetentionCategories, type RetentionCategory } from "~/server/utils import { repoMutex } from "../../core/repository-mutex"; import { db } from "../../db/db"; import { repositoriesTable } from "../../db/schema"; -import { cache } from "../../utils/cache"; +import { cache, cacheKeys } from "../../utils/cache"; import { cryptoUtils } from "../../utils/crypto"; import { toMessage } from "../../utils/errors"; import { generateShortId } from "../../utils/id"; @@ -194,6 +194,30 @@ const getRepository = async (shortId: string) => { return { repository }; }; +const getRepositoryStats = async (shortId: string) => { + const organizationId = getOrganizationId(); + const repository = await findRepository(shortId); + + if (!repository) { + throw new NotFoundError("Repository not found"); + } + + const cacheKey = cacheKeys.repository.stats(repository.id); + const cached = cache.get>>(cacheKey); + if (cached) { + return cached; + } + + const releaseLock = await repoMutex.acquireShared(repository.id, "stats"); + try { + const stats = await restic.stats(repository.config, { organizationId }); + cache.set(cacheKey, stats); + return stats; + } finally { + releaseLock(); + } +}; + const deleteRepository = async (shortId: string) => { const repository = await findRepository(shortId); @@ -209,8 +233,7 @@ const deleteRepository = async (shortId: string) => { and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)), ); - cache.delByPrefix(`snapshots:${repository.id}:`); - cache.delByPrefix(`ls:${repository.id}:`); + cache.delByPrefix(cacheKeys.repository.all(repository.id)); }; /** @@ -229,7 +252,7 @@ const listSnapshots = async (shortId: string, backupId?: string) => { throw new NotFoundError("Repository not found"); } - const cacheKey = `snapshots:${repository.id}:${backupId || "all"}`; + const cacheKey = cacheKeys.repository.snapshots(repository.id, backupId); const cached = cache.get>>(cacheKey); if (cached) { return cached; @@ -269,7 +292,7 @@ const listSnapshotFiles = async ( const offset = options?.offset ?? 0; const limit = options?.limit ?? 500; - const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`; + const cacheKey = cacheKeys.repository.ls(repository.id, snapshotId, path, offset, limit); type LsResult = { snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null; nodes: { name: string; type: string; path: string; size?: number; mode?: number }[]; @@ -459,7 +482,7 @@ const getSnapshotDetails = async (shortId: string, snapshotId: string) => { throw new NotFoundError("Repository not found"); } - const cacheKey = `snapshots:${repository.id}:all`; + const cacheKey = cacheKeys.repository.snapshots(repository.id); let snapshots = cache.get>>(cacheKey); if (!snapshots) { @@ -589,8 +612,7 @@ const deleteSnapshot = async (shortId: string, snapshotId: string) => { const releaseLock = await repoMutex.acquireExclusive(repository.id, `delete:${snapshotId}`); try { await restic.deleteSnapshot(repository.config, snapshotId, organizationId); - cache.delByPrefix(`snapshots:${repository.id}:`); - cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`); + cache.delByPrefix(cacheKeys.repository.all(repository.id)); } finally { releaseLock(); } @@ -607,10 +629,7 @@ const deleteSnapshots = async (shortId: string, snapshotIds: string[]) => { const releaseLock = await repoMutex.acquireExclusive(repository.id, `delete:bulk`); try { await restic.deleteSnapshots(repository.config, snapshotIds, organizationId); - cache.delByPrefix(`snapshots:${repository.id}:`); - for (const snapshotId of snapshotIds) { - cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`); - } + cache.delByPrefix(cacheKeys.repository.all(repository.id)); } finally { releaseLock(); } @@ -631,10 +650,7 @@ const tagSnapshots = async ( const releaseLock = await repoMutex.acquireExclusive(repository.id, `tag:bulk`); try { await restic.tagSnapshots(repository.config, snapshotIds, tags, organizationId); - cache.delByPrefix(`snapshots:${repository.id}:`); - for (const snapshotId of snapshotIds) { - cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`); - } + cache.delByPrefix(cacheKeys.repository.all(repository.id)); } finally { releaseLock(); } @@ -648,13 +664,12 @@ const refreshSnapshots = async (shortId: string) => { throw new NotFoundError("Repository not found"); } - cache.delByPrefix(`snapshots:${repository.id}:`); - cache.delByPrefix(`ls:${repository.id}:`); + cache.delByPrefix(cacheKeys.repository.all(repository.id)); const releaseLock = await repoMutex.acquireShared(repository.id, "refresh"); try { const snapshots = await restic.snapshots(repository.config, { organizationId }); - const cacheKey = `snapshots:${repository.id}:all`; + const cacheKey = cacheKeys.repository.snapshots(repository.id); cache.set(cacheKey, snapshots); return { @@ -731,8 +746,7 @@ const updateRepository = async (shortId: string, updates: UpdateRepositoryBody) } if (configChanged) { - cache.delByPrefix(`snapshots:${existing.id}:`); - cache.delByPrefix(`ls:${existing.id}:`); + cache.delByPrefix(cacheKeys.repository.all(existing.id)); } return { repository: updated }; @@ -791,7 +805,7 @@ const getRetentionCategories = async (repositoryId: string, scheduleId?: string) return new Map(); } - const cacheKey = `retention:${scheduleId}`; + const cacheKey = cacheKeys.repository.retention(repositoryId, scheduleId); const cached = cache.get>(cacheKey); if (cached && Object.keys(cached).length > 0) { @@ -834,6 +848,7 @@ export const repositoriesService = { listRepositories, createRepository, getRepository, + getRepositoryStats, deleteRepository, updateRepository, listSnapshots, diff --git a/app/server/modules/system/system.service.ts b/app/server/modules/system/system.service.ts index 17260f00..1933ba90 100644 --- a/app/server/modules/system/system.service.ts +++ b/app/server/modules/system/system.service.ts @@ -2,7 +2,7 @@ import { getCapabilities } from "../../core/capabilities"; import { config } from "../../core/config"; import type { UpdateInfoDto } from "./system.dto"; import semver from "semver"; -import { cache } from "../../utils/cache"; +import { cache, cacheKeys } from "../../utils/cache"; import { logger } from "~/server/utils/logger"; import { db } from "../../db/db"; import { appMetadataTable } from "../../db/schema"; @@ -24,7 +24,7 @@ interface GitHubRelease { } const getUpdates = async (): Promise => { - const CACHE_KEY = `system:updates:${config.appVersion}`; + const CACHE_KEY = cacheKeys.system.githubReleases(config.appVersion); const cached = cache.get(CACHE_KEY); if (cached) { diff --git a/app/server/utils/cache.ts b/app/server/utils/cache.ts index f402c20f..7af4e0e1 100644 --- a/app/server/utils/cache.ts +++ b/app/server/utils/cache.ts @@ -54,14 +54,20 @@ export const createCache = (options: CacheOptions = {}) => { stmt.run(key); }; + const escapeLikePattern = (pattern: string): string => { + return pattern.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_"); + }; + const delByPrefix = (prefix: string) => { - const stmt = db.prepare("DELETE FROM cache WHERE key LIKE ?"); - stmt.run(`${prefix}%`); + const escapedPrefix = escapeLikePattern(prefix); + const stmt = db.prepare("DELETE FROM cache WHERE key LIKE ? ESCAPE '\\'"); + stmt.run(`${escapedPrefix}%`); }; const getByPrefix = (prefix: string): { key: string; value: T }[] => { - const stmt = db.prepare("SELECT key, value, expiration FROM cache WHERE key LIKE ?"); - const rows = stmt.all(`${prefix}%`) as { key: string; value: string; expiration: number }[]; + const escapedPrefix = escapeLikePattern(prefix); + const stmt = db.prepare("SELECT key, value, expiration FROM cache WHERE key LIKE ? ESCAPE '\\'"); + const rows = stmt.all(`${escapedPrefix}%`) as { key: string; value: string; expiration: number }[]; const now = Date.now(); const results: { key: string; value: T }[] = []; @@ -99,4 +105,18 @@ export const createCache = (options: CacheOptions = {}) => { }; }; +export const cacheKeys = { + repository: { + all: (repositoryId: string) => `repo:${repositoryId}:`, + stats: (repositoryId: string) => `repo:${repositoryId}:stats`, + snapshots: (repositoryId: string, backupId = "all") => `repo:${repositoryId}:snapshots:${backupId}`, + ls: (repositoryId: string, snapshotId: string, path = "root", offset: number, limit: number) => + `repo:${repositoryId}:ls:${snapshotId}:${path}:${offset}:${limit}`, + retention: (repositoryId: string, scheduleId: string) => `repo:${repositoryId}:retention:${scheduleId}`, + }, + system: { + githubReleases: (version: string) => `system:updates:${version}`, + }, +}; + export const cache = createCache(); diff --git a/app/server/utils/restic.ts b/app/server/utils/restic.ts index 6c710977..ada00097 100644 --- a/app/server/utils/restic.ts +++ b/app/server/utils/restic.ts @@ -15,6 +15,7 @@ import { resticBackupProgressSchema, resticRestoreOutputSchema, resticSnapshotSummarySchema, + resticStatsSchema, } from "~/schemas/restic-dto"; import { config as appConfig } from "../core/config"; import { DEFAULT_EXCLUDES, RESTIC_CACHE_DIR, RESTIC_PASS_FILE } from "../core/constants"; @@ -691,6 +692,32 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; o return result; }; +const stats = async (config: RepositoryConfig, options: { organizationId: string }) => { + const repoUrl = buildRepoUrl(config); + const env = await buildEnv(config, options.organizationId); + + const args = ["--repo", repoUrl, "stats", "--mode", "raw-data"]; + addCommonArgs(args, env, config); + + const res = await exec({ command: "restic", args, env }); + await cleanupTemporaryKeys(env); + + if (res.exitCode !== 0) { + logger.error(`Restic stats retrieval failed: ${res.stderr}`); + throw new ResticError(res.exitCode, res.stderr); + } + + const parsedJson = safeJsonParse(res.stdout); + const result = resticStatsSchema(parsedJson); + + if (result instanceof type.errors) { + logger.error(`Restic stats output validation failed: ${result.summary}`); + throw new Error(`Restic stats output validation failed: ${result.summary}`); + } + + return result; +}; + export type ResticForgetResponse = ForgetGroup[]; export interface ForgetGroup { @@ -1230,6 +1257,7 @@ export const restic = { restore, dump, snapshots, + stats, forget, deleteSnapshot, deleteSnapshots,