feat: repository used space (#551)

* feat: repository used space

refactor: use a smarter cache key logic for easier invalidations

* chore: pr feedbacks
This commit is contained in:
Nico 2026-02-21 10:52:55 +01:00 committed by GitHub
parent 182d39a887
commit 45fd9f9de1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 483 additions and 159 deletions

View file

@ -33,6 +33,7 @@ import {
getNotificationDestination,
getRegistrationStatus,
getRepository,
getRepositoryStats,
getScheduleMirrors,
getScheduleNotifications,
getSnapshotDetails,
@ -117,6 +118,8 @@ import type {
GetRegistrationStatusResponse,
GetRepositoryData,
GetRepositoryResponse,
GetRepositoryStatsData,
GetRepositoryStatsResponse,
GetScheduleMirrorsData,
GetScheduleMirrorsResponse,
GetScheduleNotificationsData,
@ -688,6 +691,31 @@ export const updateRepositoryMutation = (
return mutationOptions;
};
export const getRepositoryStatsQueryKey = (options: Options<GetRepositoryStatsData>) =>
createQueryKey("getRepositoryStats", options);
/**
* Get repository storage and compression statistics
*/
export const getRepositoryStatsOptions = (options: Options<GetRepositoryStatsData>) =>
queryOptions<
GetRepositoryStatsResponse,
DefaultError,
GetRepositoryStatsResponse,
ReturnType<typeof getRepositoryStatsQueryKey>
>({
queryFn: async ({ queryKey, signal }) => {
const { data } = await getRepositoryStats({
...options,
...queryKey[0],
signal,
throwOnError: true,
});
return data;
},
queryKey: getRepositoryStatsQueryKey(options),
});
/**
* Delete multiple snapshots from a repository
*/

View file

@ -24,6 +24,7 @@ export {
getNotificationDestination,
getRegistrationStatus,
getRepository,
getRepositoryStats,
getScheduleMirrors,
getScheduleNotifications,
getSnapshotDetails,
@ -135,6 +136,9 @@ export type {
GetRepositoryData,
GetRepositoryResponse,
GetRepositoryResponses,
GetRepositoryStatsData,
GetRepositoryStatsResponse,
GetRepositoryStatsResponses,
GetScheduleMirrorsData,
GetScheduleMirrorsResponse,
GetScheduleMirrorsResponses,

View file

@ -52,6 +52,8 @@ import type {
GetRegistrationStatusResponses,
GetRepositoryData,
GetRepositoryResponses,
GetRepositoryStatsData,
GetRepositoryStatsResponses,
GetScheduleMirrorsData,
GetScheduleMirrorsResponses,
GetScheduleNotificationsData,
@ -363,6 +365,17 @@ export const updateRepository = <ThrowOnError extends boolean = false>(
},
});
/**
* Get repository storage and compression statistics
*/
export const getRepositoryStats = <ThrowOnError extends boolean = false>(
options: Options<GetRepositoryStatsData, ThrowOnError>,
) =>
(options.client ?? client).get<GetRepositoryStatsResponses, unknown, ThrowOnError>({
url: "/api/v1/repositories/{shortId}/stats",
...options,
});
/**
* Delete multiple snapshots from a repository
*/

View file

@ -1770,6 +1770,31 @@ export type UpdateRepositoryResponses = {
export type UpdateRepositoryResponse = UpdateRepositoryResponses[keyof UpdateRepositoryResponses];
export type GetRepositoryStatsData = {
body?: never;
path: {
shortId: string;
};
query?: never;
url: "/api/v1/repositories/{shortId}/stats";
};
export type GetRepositoryStatsResponses = {
/**
* Repository statistics
*/
200: {
compression_progress?: number;
compression_ratio?: number;
compression_space_saving?: number;
snapshots_count?: number;
total_size?: number;
total_uncompressed_size?: number;
};
};
export type GetRepositoryStatsResponse = GetRepositoryStatsResponses[keyof GetRepositoryStatsResponses];
export type DeleteSnapshotsData = {
body?: {
snapshotIds: Array<string>;

View file

@ -0,0 +1,134 @@
import { useQuery } from "@tanstack/react-query";
import { Archive } from "lucide-react";
import { getRepositoryStatsOptions } from "~/client/api-client/@tanstack/react-query.gen";
import { ByteSize } from "~/client/components/bytes-size";
import { Card, CardContent, CardHeader, CardTitle } from "~/client/components/ui/card";
type Props = {
repositoryShortId: string;
};
const toSafeNumber = (value: number | undefined) => {
if (typeof value !== "number" || !Number.isFinite(value)) {
return 0;
}
return Math.max(0, value);
};
export function CompressionStatsChart({ repositoryShortId }: Props) {
const {
data: stats,
isLoading,
error,
} = useQuery({
...getRepositoryStatsOptions({ path: { shortId: repositoryShortId } }),
retry: false,
});
const storedSize = toSafeNumber(stats?.total_size);
const uncompressedSize = toSafeNumber(stats?.total_uncompressed_size);
const savedSize = uncompressedSize > storedSize ? uncompressedSize - storedSize : 0;
const compressionRatio = toSafeNumber(stats?.compression_ratio);
const rawCompressionProgress = toSafeNumber(stats?.compression_progress);
const compressionProgressPercent = Math.min(100, Math.max(0, rawCompressionProgress));
const spaceSavingPercent = toSafeNumber(stats?.compression_space_saving);
const snapshotsCount = Math.round(toSafeNumber(stats?.snapshots_count));
const hasStats = !!stats && (storedSize > 0 || uncompressedSize > 0 || snapshotsCount > 0);
if (isLoading) {
return (
<Card className="p-6">
<p className="text-sm text-muted-foreground">Loading compression statistics...</p>
</Card>
);
}
if (error) {
return (
<Card className="p-6">
<p className="text-sm font-medium text-destructive">Failed to load compression statistics</p>
<p className="mt-2 text-sm text-muted-foreground wrap-break-word">{error.message}</p>
</Card>
);
}
if (!hasStats) {
return (
<Card className="p-6">
<p className="text-sm text-muted-foreground">
No compression statistics available yet. Run a backup to populate repository stats.
</p>
</Card>
);
}
return (
<Card className="flex flex-col h-full">
<CardHeader className="pb-4">
<CardTitle className="flex items-center gap-2 text-base font-semibold">
<Archive className="h-4 w-4" />
Compression Statistics
</CardTitle>
</CardHeader>
<CardContent className="flex-1">
<div className="grid grid-cols-2 md:grid-cols-3 gap-y-6 gap-x-6">
<div className="flex flex-col gap-1.5">
<div className="flex items-center gap-2 text-muted-foreground">
<span className="text-xs font-medium uppercase tracking-wider">Stored Size</span>
</div>
<div className="flex items-baseline gap-2">
<ByteSize base={1024} bytes={storedSize} className="text-2xl font-bold font-mono text-foreground" />
</div>
</div>
<div className="flex flex-col gap-1.5">
<div className="flex items-center gap-2 text-muted-foreground">
<span className="text-xs font-medium uppercase tracking-wider">Uncompressed</span>
</div>
<div className="flex items-baseline gap-2">
<ByteSize base={1024} bytes={uncompressedSize} className="text-2xl font-bold font-mono text-foreground" />
</div>
</div>
<div className="flex flex-col gap-1.5">
<div className="flex items-center gap-2 text-muted-foreground">
<span className="text-xs font-medium uppercase tracking-wider">Space Saved</span>
</div>
<div className="flex items-baseline gap-2">
<span className="text-2xl font-bold font-mono text-foreground">{spaceSavingPercent.toFixed(1)}%</span>
<ByteSize base={1024} bytes={savedSize} className="text-sm text-muted-foreground font-mono" />
</div>
</div>
<div className="flex flex-col gap-1.5">
<div className="flex items-center gap-2 text-muted-foreground">
<span className="text-xs font-medium uppercase tracking-wider">Ratio</span>
</div>
<div className="flex items-baseline gap-2">
<span className="text-2xl font-bold font-mono text-foreground">
{compressionRatio > 0 ? `${compressionRatio.toFixed(2)}x` : "—"}
</span>
</div>
</div>
<div className="flex flex-col gap-1.5">
<div className="flex items-center gap-2 text-muted-foreground">
<span className="text-xs font-medium uppercase tracking-wider">Snapshots</span>
</div>
<div className="flex items-baseline gap-2">
<span className="text-2xl font-bold font-mono text-foreground">{snapshotsCount.toLocaleString()}</span>
<span className="text-sm text-muted-foreground font-mono">
{compressionProgressPercent.toFixed(1)}% compressed
</span>
</div>
</div>
</div>
</CardContent>
</Card>
);
}

View file

@ -25,6 +25,7 @@ import type { RepositoryConfig } from "~/schemas/restic";
import { DoctorReport } from "../components/doctor-report";
import { parseError } from "~/client/lib/errors";
import { useNavigate } from "@tanstack/react-router";
import { CompressionStatsChart } from "../components/compression-stats-chart";
type Props = {
repository: Repository;
@ -96,143 +97,148 @@ export const RepositoryInfoTabContent = ({ repository }: Props) => {
return (
<>
<Card className="p-6 @container">
<div className="flex flex-col @xl:flex-row items-start @xl:items-center justify-between gap-4">
<div>
<span className="text-lg font-semibold">Repository Settings</span>
</div>
<div className="flex flex-col @xl:flex-row w-full @xl:w-auto gap-2">
<Button
type="button"
variant="outline"
onClick={() => navigate({ to: `/repositories/${repository.shortId}/edit` })}
>
<Pencil className="h-4 w-4 mr-2" />
Edit
</Button>
{repository.status === "doctor" ? (
<div className="grid gap-4">
<Card className="p-6 @container">
<div className="flex flex-col @xl:flex-row items-start @xl:items-center justify-between gap-4">
<div>
<span className="text-lg font-semibold">Repository Settings</span>
</div>
<div className="flex flex-col @xl:flex-row w-full @xl:w-auto gap-2">
<Button
type="button"
variant="outline"
onClick={() => navigate({ to: `/repositories/${repository.shortId}/edit` })}
>
<Pencil className="h-4 w-4 mr-2" />
Edit
</Button>
{repository.status === "doctor" ? (
<Button
type="button"
variant="destructive"
loading={cancelDoctor.isPending}
onClick={() => cancelDoctor.mutate({ path: { shortId: repository.shortId } })}
>
<Square className="h-4 w-4 mr-2" />
<span>Cancel doctor</span>
</Button>
) : (
<Button
type="button"
onClick={() => startDoctor.mutate({ path: { shortId: repository.shortId } })}
disabled={startDoctor.isPending}
>
<Stethoscope className="h-4 w-4 mr-2" />
Run doctor
</Button>
)}
<Button
type="button"
variant="outline"
onClick={() => unlockRepo.mutate({ path: { shortId: repository.shortId } })}
loading={unlockRepo.isPending}
>
<Unlock className="h-4 w-4 mr-2" />
Unlock
</Button>
<Button
type="button"
variant="destructive"
loading={cancelDoctor.isPending}
onClick={() => cancelDoctor.mutate({ path: { shortId: repository.shortId } })}
onClick={() => setShowDeleteConfirm(true)}
disabled={deleteRepo.isPending}
>
<Square className="h-4 w-4 mr-2" />
<span>Cancel doctor</span>
<Trash2 className="h-4 w-4 mr-2" />
Delete
</Button>
) : (
<Button
type="button"
onClick={() => startDoctor.mutate({ path: { shortId: repository.shortId } })}
disabled={startDoctor.isPending}
>
<Stethoscope className="h-4 w-4 mr-2" />
Run doctor
</Button>
)}
<Button
type="button"
variant="outline"
onClick={() => unlockRepo.mutate({ path: { shortId: repository.shortId } })}
loading={unlockRepo.isPending}
>
<Unlock className="h-4 w-4 mr-2" />
Unlock
</Button>
<Button
type="button"
variant="destructive"
onClick={() => setShowDeleteConfirm(true)}
disabled={deleteRepo.isPending}
>
<Trash2 className="h-4 w-4 mr-2" />
Delete
</Button>
</div>
</div>
<div>
<h3 className="text-lg font-semibold mb-4">Current Configuration</h3>
<div className="grid grid-cols-1 @xl:grid-cols-2 gap-4">
<div>
<div className="text-sm font-medium text-muted-foreground">Name</div>
<p className="mt-1 text-sm">{repository.name}</p>
</div>
<div>
<div className="text-sm font-medium text-muted-foreground">Compression mode</div>
<p className="mt-1 text-sm">{repository.compressionMode || "off"}</p>
</div>
</div>
</div>
<div>
<h3 className="text-lg font-semibold mb-4">Repository Information</h3>
<div className="grid grid-cols-1 @xl:grid-cols-2 gap-4">
<div>
<div className="text-sm font-medium text-muted-foreground">Backend</div>
<p className="mt-1 text-sm">{repository.type}</p>
</div>
<div>
<div className="text-sm font-medium text-muted-foreground">Status</div>
<p className="mt-1 text-sm">{repository.status || "unknown"}</p>
</div>
{effectiveLocalPath && (
<div>
<div className="text-sm font-medium text-muted-foreground">Local path</div>
<p className="mt-1 text-sm font-mono">{effectiveLocalPath}</p>
</div>
)}
<div>
<div className="text-sm font-medium text-muted-foreground">Created at</div>
<p className="mt-1 text-sm">{formatDateTime(repository.createdAt)}</p>
</div>
<div>
<div className="text-sm font-medium text-muted-foreground">Last checked</div>
<p className="mt-1 text-sm">{formatTimeAgo(repository.lastChecked)}</p>
</div>
{config.cacert && (
<div>
<div className="text-sm font-medium text-muted-foreground">CA Certificate</div>
<p className="mt-1 text-sm">
<span className="text-green-500">configured</span>
</p>
</div>
)}
{"insecureTls" in config && (
<div>
<div className="text-sm font-medium text-muted-foreground">TLS Certificate Validation</div>
<p className="mt-1 text-sm">
{config.insecureTls ? (
<span className="text-red-500">disabled</span>
) : (
<span className="text-green-500">enabled</span>
)}
</p>
</div>
)}
</div>
</div>
{repository.lastError && (
<div>
<div className="flex items-center justify-between mb-4">
<h3 className="text-lg font-semibold text-red-500">Last Error</h3>
</div>
<div className="bg-red-500/10 border border-red-500/20 rounded-md p-4">
<p className="text-sm text-red-500 wrap-break-word">{repository.lastError}</p>
<h3 className="text-lg font-semibold mb-4">Current Configuration</h3>
<div className="grid grid-cols-1 @xl:grid-cols-2 gap-4">
<div>
<div className="text-sm font-medium text-muted-foreground">Name</div>
<p className="mt-1 text-sm">{repository.name}</p>
</div>
<div>
<div className="text-sm font-medium text-muted-foreground">Compression mode</div>
<p className="mt-1 text-sm">{repository.compressionMode || "off"}</p>
</div>
</div>
</div>
)}
<div>
<h3 className="text-lg font-semibold mb-4">Configuration</h3>
<div className="bg-muted/50 rounded-md p-4">
<pre className="text-sm overflow-auto">{JSON.stringify(repository.config, null, 2)}</pre>
<div>
<h3 className="text-lg font-semibold mb-4">Repository Information</h3>
<div className="grid grid-cols-1 @xl:grid-cols-2 gap-4">
<div>
<div className="text-sm font-medium text-muted-foreground">Backend</div>
<p className="mt-1 text-sm">{repository.type}</p>
</div>
<div>
<div className="text-sm font-medium text-muted-foreground">Status</div>
<p className="mt-1 text-sm">{repository.status || "unknown"}</p>
</div>
{effectiveLocalPath && (
<div>
<div className="text-sm font-medium text-muted-foreground">Local path</div>
<p className="mt-1 text-sm font-mono">{effectiveLocalPath}</p>
</div>
)}
<div>
<div className="text-sm font-medium text-muted-foreground">Created at</div>
<p className="mt-1 text-sm">{formatDateTime(repository.createdAt)}</p>
</div>
<div>
<div className="text-sm font-medium text-muted-foreground">Last checked</div>
<p className="mt-1 text-sm">{formatTimeAgo(repository.lastChecked)}</p>
</div>
{config.cacert && (
<div>
<div className="text-sm font-medium text-muted-foreground">CA Certificate</div>
<p className="mt-1 text-sm">
<span className="text-green-500">configured</span>
</p>
</div>
)}
{"insecureTls" in config && (
<div>
<div className="text-sm font-medium text-muted-foreground">TLS Certificate Validation</div>
<p className="mt-1 text-sm">
{config.insecureTls ? (
<span className="text-red-500">disabled</span>
) : (
<span className="text-green-500">enabled</span>
)}
</p>
</div>
)}
</div>
</div>
</div>
<DoctorReport repositoryStatus={repository.status} result={repository.doctorResult} />
</Card>
<div>
{repository.lastError && (
<div>
<div className="flex items-center justify-between mb-4">
<h3 className="text-lg font-semibold text-red-500">Last Error</h3>
</div>
<div className="bg-red-500/10 border border-red-500/20 rounded-md p-4">
<p className="text-sm text-red-500 wrap-break-word">{repository.lastError}</p>
</div>
</div>
)}
<div>
<h3 className="text-lg font-semibold mb-4">Configuration</h3>
<div className="bg-muted/50 rounded-md p-4">
<pre className="text-sm overflow-auto">{JSON.stringify(repository.config, null, 2)}</pre>
</div>
</div>
</div>
<DoctorReport repositoryStatus={repository.status} result={repository.doctorResult} />
</Card>
<CompressionStatsChart repositoryShortId={repository.shortId} />
</div>
<AlertDialog open={showDeleteConfirm} onOpenChange={setShowDeleteConfirm}>
<AlertDialogContent>

View file

@ -61,6 +61,15 @@ export const resticRestoreOutputSchema = type({
bytes_skipped: "number",
});
export const resticStatsSchema = type({
total_size: "number = 0",
total_uncompressed_size: "number = 0",
compression_ratio: "number = 0",
compression_progress: "number = 0",
compression_space_saving: "number = 0",
snapshots_count: "number = 0",
});
export type ResticSnapshotSummaryDto = typeof resticSnapshotSummarySchema.infer;
export type ResticBackupRunSummaryDto = typeof resticBackupRunSummarySchema.infer;
export type ResticBackupOutputDto = typeof resticBackupOutputSchema.infer;
@ -68,3 +77,4 @@ export type ResticBackupProgressMetricsDto = typeof resticBackupProgressMetricsS
export type ResticBackupProgressDto = typeof resticBackupProgressSchema.infer;
export type ResticRestoreOutputDto = typeof resticRestoreOutputSchema.infer;
export type ResticStatsDto = typeof resticStatsSchema.infer;

View file

@ -2,7 +2,7 @@ import { NotFoundError, BadRequestError, ConflictError } from "http-errors-enhan
import type { BackupSchedule, Volume, Repository } from "../../db/schema";
import { restic } from "../../utils/restic";
import { logger } from "../../utils/logger";
import { cache } from "../../utils/cache";
import { cache, cacheKeys } from "../../utils/cache";
import { getVolumePath } from "../volumes/helpers";
import { toMessage } from "../../utils/errors";
import { serverEvents } from "../../core/events";
@ -150,8 +150,7 @@ const finalizeSuccessfulBackup = async (
logger.error(`Background mirror copy failed for schedule ${scheduleId}: ${toMessage(error)}`);
});
cache.delByPrefix(`snapshots:${ctx.repository.id}:`);
cache.del(`retention:${ctx.schedule.shortId}`);
cache.delByPrefix(cacheKeys.repository.all(ctx.repository.id));
const nextBackupAt = calculateNextRun(ctx.schedule.cronExpression);
await scheduleQueries.updateStatus(scheduleId, ctx.organizationId, {
@ -328,8 +327,7 @@ const runForget = async (scheduleId: number, repositoryId?: string) => {
try {
await restic.forget(repository.config, schedule.retentionPolicy, { tag: schedule.shortId, organizationId });
cache.delByPrefix(`snapshots:${repository.id}:`);
cache.del(`retention:${schedule.shortId}`);
cache.delByPrefix(cacheKeys.repository.all(repository.id));
} finally {
releaseLock();
}
@ -393,7 +391,7 @@ const copyToSingleMirror = async (
try {
await restic.copy(sourceRepository.config, mirror.repository.config, { tag: schedule.shortId, organizationId });
cache.delByPrefix(`snapshots:${mirror.repository.id}:`);
cache.delByPrefix(cacheKeys.repository.all(mirror.repository.id));
} finally {
releaseSource();
releaseMirror();

View file

@ -72,6 +72,7 @@ describe("repositories security", () => {
{ method: "POST", path: "/api/v1/repositories" },
{ method: "GET", path: "/api/v1/repositories/rclone-remotes" },
{ method: "GET", path: "/api/v1/repositories/test-repo" },
{ method: "GET", path: "/api/v1/repositories/test-repo/stats" },
{ method: "DELETE", path: "/api/v1/repositories/test-repo" },
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots" },
{ method: "GET", path: "/api/v1/repositories/test-repo/snapshots/test-snapshot" },
@ -115,6 +116,17 @@ describe("repositories security", () => {
expect(body.message).toBe("Repository not found");
});
test("should return 404 for stats of non-existent repository", async () => {
const { token } = await createTestSession();
const res = await app.request("/api/v1/repositories/non-existent-repo/stats", {
headers: getAuthHeaders(token),
});
expect(res.status).toBe(404);
const body = await res.json();
expect(body.message).toBe("Repository not found");
});
test("should return 400 for invalid payload on create", async () => {
const { token } = await createTestSession();
const res = await app.request("/api/v1/repositories", {

View file

@ -244,7 +244,7 @@ describe("repositoriesService.dumpSnapshot", () => {
);
spyOn(restic, "snapshots").mockImplementation(snapshotsMock as typeof restic.snapshots);
await expect(
expect(
withContext({ organizationId, userId: user.id }, () =>
repositoriesService.dumpSnapshot(shortId, "snapshot-no-kind", `${basePath}/documents/report.txt`),
),

View file

@ -13,6 +13,7 @@ import {
startDoctorDto,
cancelDoctorDto,
getRepositoryDto,
getRepositoryStatsDto,
getSnapshotDetailsDto,
refreshSnapshotsDto,
listRcloneRemotesDto,
@ -38,6 +39,7 @@ import {
type StartDoctorDto,
type CancelDoctorDto,
type GetRepositoryDto,
type GetRepositoryStatsDto,
type GetSnapshotDetailsDto,
type RefreshSnapshotsDto,
type ListRepositoriesDto,
@ -89,6 +91,12 @@ export const repositoriesController = new Hono()
return c.json<GetRepositoryDto>(res.repository, 200);
})
.get("/:shortId/stats", getRepositoryStatsDto, async (c) => {
const { shortId } = c.req.param();
const stats = await repositoriesService.getRepositoryStats(shortId);
return c.json<GetRepositoryStatsDto>(stats, 200);
})
.delete("/:shortId", deleteRepositoryDto, async (c) => {
const { shortId } = c.req.param();
await repositoriesService.deleteRepository(shortId);

View file

@ -8,7 +8,7 @@ import {
repositoryConfigSchema,
doctorResultSchema,
} from "~/schemas/restic";
import { resticSnapshotSummarySchema } from "~/schemas/restic-dto";
import { resticSnapshotSummarySchema, resticStatsSchema } from "~/schemas/restic-dto";
export const repositorySchema = type({
id: "string",
@ -109,6 +109,29 @@ export const getRepositoryDto = describeRoute({
},
});
/**
* Get repository stats
*/
export const repositoryStatsSchema = resticStatsSchema;
export const getRepositoryStatsResponse = repositoryStatsSchema;
export type GetRepositoryStatsDto = typeof getRepositoryStatsResponse.infer;
export const getRepositoryStatsDto = describeRoute({
description: "Get repository storage and compression statistics",
tags: ["Repositories"],
operationId: "getRepositoryStats",
responses: {
200: {
description: "Repository statistics",
content: {
"application/json": {
schema: resolver(getRepositoryStatsResponse),
},
},
},
},
});
/**
* Delete a repository
*/

View file

@ -16,7 +16,7 @@ import { parseRetentionCategories, type RetentionCategory } from "~/server/utils
import { repoMutex } from "../../core/repository-mutex";
import { db } from "../../db/db";
import { repositoriesTable } from "../../db/schema";
import { cache } from "../../utils/cache";
import { cache, cacheKeys } from "../../utils/cache";
import { cryptoUtils } from "../../utils/crypto";
import { toMessage } from "../../utils/errors";
import { generateShortId } from "../../utils/id";
@ -194,6 +194,30 @@ const getRepository = async (shortId: string) => {
return { repository };
};
const getRepositoryStats = async (shortId: string) => {
const organizationId = getOrganizationId();
const repository = await findRepository(shortId);
if (!repository) {
throw new NotFoundError("Repository not found");
}
const cacheKey = cacheKeys.repository.stats(repository.id);
const cached = cache.get<Awaited<ReturnType<typeof restic.stats>>>(cacheKey);
if (cached) {
return cached;
}
const releaseLock = await repoMutex.acquireShared(repository.id, "stats");
try {
const stats = await restic.stats(repository.config, { organizationId });
cache.set(cacheKey, stats);
return stats;
} finally {
releaseLock();
}
};
const deleteRepository = async (shortId: string) => {
const repository = await findRepository(shortId);
@ -209,8 +233,7 @@ const deleteRepository = async (shortId: string) => {
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)),
);
cache.delByPrefix(`snapshots:${repository.id}:`);
cache.delByPrefix(`ls:${repository.id}:`);
cache.delByPrefix(cacheKeys.repository.all(repository.id));
};
/**
@ -229,7 +252,7 @@ const listSnapshots = async (shortId: string, backupId?: string) => {
throw new NotFoundError("Repository not found");
}
const cacheKey = `snapshots:${repository.id}:${backupId || "all"}`;
const cacheKey = cacheKeys.repository.snapshots(repository.id, backupId);
const cached = cache.get<Awaited<ReturnType<typeof restic.snapshots>>>(cacheKey);
if (cached) {
return cached;
@ -269,7 +292,7 @@ const listSnapshotFiles = async (
const offset = options?.offset ?? 0;
const limit = options?.limit ?? 500;
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`;
const cacheKey = cacheKeys.repository.ls(repository.id, snapshotId, path, offset, limit);
type LsResult = {
snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null;
nodes: { name: string; type: string; path: string; size?: number; mode?: number }[];
@ -459,7 +482,7 @@ const getSnapshotDetails = async (shortId: string, snapshotId: string) => {
throw new NotFoundError("Repository not found");
}
const cacheKey = `snapshots:${repository.id}:all`;
const cacheKey = cacheKeys.repository.snapshots(repository.id);
let snapshots = cache.get<Awaited<ReturnType<typeof restic.snapshots>>>(cacheKey);
if (!snapshots) {
@ -589,8 +612,7 @@ const deleteSnapshot = async (shortId: string, snapshotId: string) => {
const releaseLock = await repoMutex.acquireExclusive(repository.id, `delete:${snapshotId}`);
try {
await restic.deleteSnapshot(repository.config, snapshotId, organizationId);
cache.delByPrefix(`snapshots:${repository.id}:`);
cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`);
cache.delByPrefix(cacheKeys.repository.all(repository.id));
} finally {
releaseLock();
}
@ -607,10 +629,7 @@ const deleteSnapshots = async (shortId: string, snapshotIds: string[]) => {
const releaseLock = await repoMutex.acquireExclusive(repository.id, `delete:bulk`);
try {
await restic.deleteSnapshots(repository.config, snapshotIds, organizationId);
cache.delByPrefix(`snapshots:${repository.id}:`);
for (const snapshotId of snapshotIds) {
cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`);
}
cache.delByPrefix(cacheKeys.repository.all(repository.id));
} finally {
releaseLock();
}
@ -631,10 +650,7 @@ const tagSnapshots = async (
const releaseLock = await repoMutex.acquireExclusive(repository.id, `tag:bulk`);
try {
await restic.tagSnapshots(repository.config, snapshotIds, tags, organizationId);
cache.delByPrefix(`snapshots:${repository.id}:`);
for (const snapshotId of snapshotIds) {
cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`);
}
cache.delByPrefix(cacheKeys.repository.all(repository.id));
} finally {
releaseLock();
}
@ -648,13 +664,12 @@ const refreshSnapshots = async (shortId: string) => {
throw new NotFoundError("Repository not found");
}
cache.delByPrefix(`snapshots:${repository.id}:`);
cache.delByPrefix(`ls:${repository.id}:`);
cache.delByPrefix(cacheKeys.repository.all(repository.id));
const releaseLock = await repoMutex.acquireShared(repository.id, "refresh");
try {
const snapshots = await restic.snapshots(repository.config, { organizationId });
const cacheKey = `snapshots:${repository.id}:all`;
const cacheKey = cacheKeys.repository.snapshots(repository.id);
cache.set(cacheKey, snapshots);
return {
@ -731,8 +746,7 @@ const updateRepository = async (shortId: string, updates: UpdateRepositoryBody)
}
if (configChanged) {
cache.delByPrefix(`snapshots:${existing.id}:`);
cache.delByPrefix(`ls:${existing.id}:`);
cache.delByPrefix(cacheKeys.repository.all(existing.id));
}
return { repository: updated };
@ -791,7 +805,7 @@ const getRetentionCategories = async (repositoryId: string, scheduleId?: string)
return new Map<string, RetentionCategory[]>();
}
const cacheKey = `retention:${scheduleId}`;
const cacheKey = cacheKeys.repository.retention(repositoryId, scheduleId);
const cached = cache.get<Record<string, RetentionCategory[]>>(cacheKey);
if (cached && Object.keys(cached).length > 0) {
@ -834,6 +848,7 @@ export const repositoriesService = {
listRepositories,
createRepository,
getRepository,
getRepositoryStats,
deleteRepository,
updateRepository,
listSnapshots,

View file

@ -2,7 +2,7 @@ import { getCapabilities } from "../../core/capabilities";
import { config } from "../../core/config";
import type { UpdateInfoDto } from "./system.dto";
import semver from "semver";
import { cache } from "../../utils/cache";
import { cache, cacheKeys } from "../../utils/cache";
import { logger } from "~/server/utils/logger";
import { db } from "../../db/db";
import { appMetadataTable } from "../../db/schema";
@ -24,7 +24,7 @@ interface GitHubRelease {
}
const getUpdates = async (): Promise<UpdateInfoDto> => {
const CACHE_KEY = `system:updates:${config.appVersion}`;
const CACHE_KEY = cacheKeys.system.githubReleases(config.appVersion);
const cached = cache.get<UpdateInfoDto>(CACHE_KEY);
if (cached) {

View file

@ -54,14 +54,20 @@ export const createCache = (options: CacheOptions = {}) => {
stmt.run(key);
};
const escapeLikePattern = (pattern: string): string => {
return pattern.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
};
const delByPrefix = (prefix: string) => {
const stmt = db.prepare("DELETE FROM cache WHERE key LIKE ?");
stmt.run(`${prefix}%`);
const escapedPrefix = escapeLikePattern(prefix);
const stmt = db.prepare("DELETE FROM cache WHERE key LIKE ? ESCAPE '\\'");
stmt.run(`${escapedPrefix}%`);
};
const getByPrefix = <T>(prefix: string): { key: string; value: T }[] => {
const stmt = db.prepare("SELECT key, value, expiration FROM cache WHERE key LIKE ?");
const rows = stmt.all(`${prefix}%`) as { key: string; value: string; expiration: number }[];
const escapedPrefix = escapeLikePattern(prefix);
const stmt = db.prepare("SELECT key, value, expiration FROM cache WHERE key LIKE ? ESCAPE '\\'");
const rows = stmt.all(`${escapedPrefix}%`) as { key: string; value: string; expiration: number }[];
const now = Date.now();
const results: { key: string; value: T }[] = [];
@ -99,4 +105,18 @@ export const createCache = (options: CacheOptions = {}) => {
};
};
export const cacheKeys = {
repository: {
all: (repositoryId: string) => `repo:${repositoryId}:`,
stats: (repositoryId: string) => `repo:${repositoryId}:stats`,
snapshots: (repositoryId: string, backupId = "all") => `repo:${repositoryId}:snapshots:${backupId}`,
ls: (repositoryId: string, snapshotId: string, path = "root", offset: number, limit: number) =>
`repo:${repositoryId}:ls:${snapshotId}:${path}:${offset}:${limit}`,
retention: (repositoryId: string, scheduleId: string) => `repo:${repositoryId}:retention:${scheduleId}`,
},
system: {
githubReleases: (version: string) => `system:updates:${version}`,
},
};
export const cache = createCache();

View file

@ -15,6 +15,7 @@ import {
resticBackupProgressSchema,
resticRestoreOutputSchema,
resticSnapshotSummarySchema,
resticStatsSchema,
} from "~/schemas/restic-dto";
import { config as appConfig } from "../core/config";
import { DEFAULT_EXCLUDES, RESTIC_CACHE_DIR, RESTIC_PASS_FILE } from "../core/constants";
@ -691,6 +692,32 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; o
return result;
};
const stats = async (config: RepositoryConfig, options: { organizationId: string }) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, options.organizationId);
const args = ["--repo", repoUrl, "stats", "--mode", "raw-data"];
addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env });
await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) {
logger.error(`Restic stats retrieval failed: ${res.stderr}`);
throw new ResticError(res.exitCode, res.stderr);
}
const parsedJson = safeJsonParse<unknown>(res.stdout);
const result = resticStatsSchema(parsedJson);
if (result instanceof type.errors) {
logger.error(`Restic stats output validation failed: ${result.summary}`);
throw new Error(`Restic stats output validation failed: ${result.summary}`);
}
return result;
};
export type ResticForgetResponse = ForgetGroup[];
export interface ForgetGroup {
@ -1230,6 +1257,7 @@ export const restic = {
restore,
dump,
snapshots,
stats,
forget,
deleteSnapshot,
deleteSnapshots,