From 7f1874a571287b2f64fda76df8d466a65af073e0 Mon Sep 17 00:00:00 2001 From: Nicolas Meienberger Date: Thu, 12 Feb 2026 17:53:31 +0100 Subject: [PATCH] refactor: centralize restic backup schemas --- app/client/components/backup-summary-card.tsx | 20 +--- app/client/hooks/use-server-events.ts | 47 ++------- .../components/backup-progress-card.tsx | 7 +- app/schemas/events-dto.ts | 39 ++++++++ app/schemas/restic-dto.ts | 70 +++++++++++++ app/server/core/events.ts | 45 ++------- .../modules/backups/backups.execution.ts | 7 +- .../modules/events/events.controller.ts | 50 ++-------- .../notifications/notifications.service.ts | 10 +- .../modules/repositories/repositories.dto.ts | 18 +--- app/server/utils/restic.ts | 99 ++++--------------- 11 files changed, 167 insertions(+), 245 deletions(-) create mode 100644 app/schemas/events-dto.ts create mode 100644 app/schemas/restic-dto.ts diff --git a/app/client/components/backup-summary-card.tsx b/app/client/components/backup-summary-card.tsx index 05027685..5ceb554a 100644 --- a/app/client/components/backup-summary-card.tsx +++ b/app/client/components/backup-summary-card.tsx @@ -1,26 +1,10 @@ import { Card, CardContent } from "~/client/components/ui/card"; import { ByteSize } from "~/client/components/bytes-size"; +import type { ResticSnapshotSummaryDto } from "~/schemas/restic-dto"; import { formatDuration } from "~/utils/utils"; -type BackupSummary = { - backup_start: string; - backup_end: string; - files_new: number; - files_changed: number; - files_unmodified: number; - dirs_new: number; - dirs_changed: number; - dirs_unmodified: number; - data_blobs: number; - tree_blobs: number; - data_added: number; - data_added_packed?: number | null; - total_files_processed: number; - total_bytes_processed: number; -}; - type Props = { - summary?: BackupSummary | null; + summary?: ResticSnapshotSummaryDto | null; }; const formatCount = (value: number) => value.toLocaleString(); diff --git a/app/client/hooks/use-server-events.ts b/app/client/hooks/use-server-events.ts index 688ec98c..1e1ae264 100644 --- a/app/client/hooks/use-server-events.ts +++ b/app/client/hooks/use-server-events.ts @@ -1,5 +1,10 @@ import { useEffect, useRef } from "react"; import { useQueryClient } from "@tanstack/react-query"; +import type { + BackupCompletedEventDto, + BackupProgressEventDto, + BackupStartedEventDto, +} from "~/schemas/events-dto"; type ServerEventType = | "connected" @@ -16,42 +21,6 @@ type ServerEventType = | "doctor:completed" | "doctor:cancelled"; -export interface BackupEvent { - scheduleId: number; - volumeName: string; - repositoryName: string; - status?: "success" | "error"; - summary?: { - files_new: number; - files_changed: number; - files_unmodified: number; - dirs_new: number; - dirs_changed: number; - dirs_unmodified: number; - data_blobs: number; - tree_blobs: number; - data_added: number; - data_added_packed?: number; - total_files_processed: number; - total_bytes_processed: number; - total_duration: number; - snapshot_id: string; - }; -} - -export interface BackupProgressEvent { - scheduleId: number; - volumeName: string; - repositoryName: string; - seconds_elapsed: number; - percent_done: number; - total_files: number; - files_done: number; - total_bytes: number; - bytes_done: number; - current_files: string[]; -} - export interface VolumeEvent { volumeName: string; } @@ -103,7 +72,7 @@ export function useServerEvents() { eventSource.addEventListener("heartbeat", () => {}); eventSource.addEventListener("backup:started", (e) => { - const data = JSON.parse(e.data) as BackupEvent; + const data = JSON.parse(e.data) as BackupStartedEventDto; console.info("[SSE] Backup started:", data); handlersRef.current.get("backup:started")?.forEach((handler) => { @@ -112,7 +81,7 @@ export function useServerEvents() { }); eventSource.addEventListener("backup:progress", (e) => { - const data = JSON.parse(e.data) as BackupProgressEvent; + const data = JSON.parse(e.data) as BackupProgressEventDto; handlersRef.current.get("backup:progress")?.forEach((handler) => { handler(data); @@ -120,7 +89,7 @@ export function useServerEvents() { }); eventSource.addEventListener("backup:completed", (e) => { - const data = JSON.parse(e.data) as BackupEvent; + const data = JSON.parse(e.data) as BackupCompletedEventDto; console.info("[SSE] Backup completed:", data); void queryClient.invalidateQueries(); diff --git a/app/client/modules/backups/components/backup-progress-card.tsx b/app/client/modules/backups/components/backup-progress-card.tsx index 50ea5f03..45ae948d 100644 --- a/app/client/modules/backups/components/backup-progress-card.tsx +++ b/app/client/modules/backups/components/backup-progress-card.tsx @@ -2,7 +2,8 @@ import { useEffect, useState } from "react"; import { ByteSize } from "~/client/components/bytes-size"; import { Card } from "~/client/components/ui/card"; import { Progress } from "~/client/components/ui/progress"; -import { type BackupProgressEvent, useServerEvents } from "~/client/hooks/use-server-events"; +import { useServerEvents } from "~/client/hooks/use-server-events"; +import type { BackupProgressEventDto } from "~/schemas/events-dto"; import { formatDuration } from "~/utils/utils"; import { formatBytes } from "../../../../utils/format-bytes"; @@ -12,11 +13,11 @@ type Props = { export const BackupProgressCard = ({ scheduleId }: Props) => { const { addEventListener } = useServerEvents(); - const [progress, setProgress] = useState(null); + const [progress, setProgress] = useState(null); useEffect(() => { const unsubscribe = addEventListener("backup:progress", (data) => { - const progressData = data as BackupProgressEvent; + const progressData = data as BackupProgressEventDto; if (progressData.scheduleId === scheduleId) { setProgress(progressData); } diff --git a/app/schemas/events-dto.ts b/app/schemas/events-dto.ts new file mode 100644 index 00000000..4ff9b02f --- /dev/null +++ b/app/schemas/events-dto.ts @@ -0,0 +1,39 @@ +import { type } from "arktype"; +import { resticBackupProgressMetricsSchema, resticBackupRunSummarySchema } from "~/schemas/restic-dto"; + +export const backupEventStatusSchema = type("'success' | 'error' | 'stopped' | 'warning'"); + +const backupEventBaseSchema = type({ + scheduleId: "number", + volumeName: "string", + repositoryName: "string", +}); + +const organizationScopedSchema = type({ + organizationId: "string", +}); + +export const backupStartedEventSchema = backupEventBaseSchema; + +export const backupProgressEventSchema = backupEventBaseSchema.and(resticBackupProgressMetricsSchema); + +export const backupCompletedEventSchema = backupEventBaseSchema.and( + type({ + status: backupEventStatusSchema, + summary: resticBackupRunSummarySchema.optional(), + }), +); + +export const serverBackupStartedEventSchema = organizationScopedSchema.and(backupStartedEventSchema); + +export const serverBackupProgressEventSchema = organizationScopedSchema.and(backupProgressEventSchema); + +export const serverBackupCompletedEventSchema = organizationScopedSchema.and(backupCompletedEventSchema); + +export type BackupEventStatusDto = typeof backupEventStatusSchema.infer; +export type BackupStartedEventDto = typeof backupStartedEventSchema.infer; +export type BackupProgressEventDto = typeof backupProgressEventSchema.infer; +export type BackupCompletedEventDto = typeof backupCompletedEventSchema.infer; +export type ServerBackupStartedEventDto = typeof serverBackupStartedEventSchema.infer; +export type ServerBackupProgressEventDto = typeof serverBackupProgressEventSchema.infer; +export type ServerBackupCompletedEventDto = typeof serverBackupCompletedEventSchema.infer; diff --git a/app/schemas/restic-dto.ts b/app/schemas/restic-dto.ts new file mode 100644 index 00000000..5abd2733 --- /dev/null +++ b/app/schemas/restic-dto.ts @@ -0,0 +1,70 @@ +import { type } from "arktype"; + +export const resticSummaryBaseSchema = type({ + files_new: "number", + files_changed: "number", + files_unmodified: "number", + dirs_new: "number", + dirs_changed: "number", + dirs_unmodified: "number", + data_blobs: "number", + tree_blobs: "number", + data_added: "number", + data_added_packed: "number?", + total_files_processed: "number", + total_bytes_processed: "number", +}); + +export const resticSnapshotSummarySchema = resticSummaryBaseSchema.and( + type({ + backup_start: "string", + backup_end: "string", + }), +); + +export const resticBackupRunSummarySchema = resticSummaryBaseSchema.and( + type({ + total_duration: "number", + snapshot_id: "string", + }), +); + +export const resticBackupOutputSchema = resticBackupRunSummarySchema.and( + type({ + message_type: "'summary'", + }), +); + +export const resticBackupProgressMetricsSchema = type({ + seconds_elapsed: "number", + percent_done: "number", + total_files: "number", + files_done: "number", + total_bytes: "number", + bytes_done: "number", + current_files: "string[]", +}); + +export const resticBackupProgressSchema = resticBackupProgressMetricsSchema.and( + type({ + message_type: "'status'", + }), +); + +export const resticRestoreOutputSchema = type({ + message_type: "'summary'", + total_files: "number?", + files_restored: "number", + files_skipped: "number", + total_bytes: "number?", + bytes_restored: "number?", + bytes_skipped: "number", +}); + +export type ResticSnapshotSummaryDto = typeof resticSnapshotSummarySchema.infer; +export type ResticBackupRunSummaryDto = typeof resticBackupRunSummarySchema.infer; +export type ResticBackupOutputDto = typeof resticBackupOutputSchema.infer; +export type ResticBackupProgressMetricsDto = typeof resticBackupProgressMetricsSchema.infer; +export type ResticBackupProgressDto = typeof resticBackupProgressSchema.infer; + +export type ResticRestoreOutputDto = typeof resticRestoreOutputSchema.infer; diff --git a/app/server/core/events.ts b/app/server/core/events.ts index cd60a17e..45da39f6 100644 --- a/app/server/core/events.ts +++ b/app/server/core/events.ts @@ -1,48 +1,19 @@ import { EventEmitter } from "node:events"; import type { TypedEmitter } from "tiny-typed-emitter"; import type { DoctorResult } from "~/schemas/restic"; +import type { + ServerBackupCompletedEventDto, + ServerBackupProgressEventDto, + ServerBackupStartedEventDto, +} from "~/schemas/events-dto"; /** * Event payloads for the SSE system */ interface ServerEvents { - "backup:started": (data: { organizationId: string; scheduleId: number; volumeName: string; repositoryName: string }) => void; - "backup:progress": (data: { - organizationId: string; - scheduleId: number; - volumeName: string; - repositoryName: string; - seconds_elapsed: number; - percent_done: number; - total_files: number; - files_done: number; - total_bytes: number; - bytes_done: number; - current_files: string[]; - }) => void; - "backup:completed": (data: { - organizationId: string; - scheduleId: number; - volumeName: string; - repositoryName: string; - status: "success" | "error" | "stopped" | "warning"; - summary?: { - files_new: number; - files_changed: number; - files_unmodified: number; - dirs_new: number; - dirs_changed: number; - dirs_unmodified: number; - data_blobs: number; - tree_blobs: number; - data_added: number; - data_added_packed?: number; - total_files_processed: number; - total_bytes_processed: number; - total_duration: number; - snapshot_id: string; - }; - }) => void; + "backup:started": (data: ServerBackupStartedEventDto) => void; + "backup:progress": (data: ServerBackupProgressEventDto) => void; + "backup:completed": (data: ServerBackupCompletedEventDto) => void; "mirror:started": (data: { organizationId: string; scheduleId: number; diff --git a/app/server/modules/backups/backups.execution.ts b/app/server/modules/backups/backups.execution.ts index 65f3bd0c..98f3c991 100644 --- a/app/server/modules/backups/backups.execution.ts +++ b/app/server/modules/backups/backups.execution.ts @@ -1,6 +1,6 @@ import { NotFoundError, BadRequestError, ConflictError } from "http-errors-enhanced"; import type { BackupSchedule, Volume, Repository } from "../../db/schema"; -import { restic, type BackupOutput } from "../../utils/restic"; +import { restic } from "../../utils/restic"; import { logger } from "../../utils/logger"; import { cache } from "../../utils/cache"; import { getVolumePath } from "../volumes/helpers"; @@ -11,6 +11,7 @@ import { repoMutex } from "../../core/repository-mutex"; import { getOrganizationId } from "~/server/core/request-context"; import { scheduleQueries, mirrorQueries, repositoryQueries } from "./backups.queries"; import { calculateNextRun, createBackupOptions } from "./backup.helpers"; +import type { ResticBackupOutputDto } from "~/schemas/restic-dto"; const runningBackups = new Map(); @@ -131,7 +132,7 @@ const runBackupOperation = async (ctx: BackupContext, signal: AbortSignal) => { } }; -const buildBackupSummary = (result: BackupOutput | null | undefined) => { +const buildBackupSummary = (result: ResticBackupOutputDto | null | undefined) => { if (!result) return undefined; return result; }; @@ -140,7 +141,7 @@ const finalizeSuccessfulBackup = async ( ctx: BackupContext, scheduleId: number, exitCode: number, - result: BackupOutput | null, + result: ResticBackupOutputDto | null, ) => { const finalStatus = exitCode === 0 ? "success" : "warning"; const summary = buildBackupSummary(result); diff --git a/app/server/modules/events/events.controller.ts b/app/server/modules/events/events.controller.ts index 709ee4c3..3d7a778b 100644 --- a/app/server/modules/events/events.controller.ts +++ b/app/server/modules/events/events.controller.ts @@ -4,6 +4,11 @@ import { logger } from "../../utils/logger"; import { serverEvents } from "../../core/events"; import { requireAuth } from "../auth/auth.middleware"; import type { DoctorResult } from "~/schemas/restic"; +import type { + ServerBackupCompletedEventDto, + ServerBackupProgressEventDto, + ServerBackupStartedEventDto, +} from "~/schemas/events-dto"; export const eventsController = new Hono().use(requireAuth).get("/", (c) => { logger.info("Client connected to SSE endpoint"); @@ -15,12 +20,7 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => { event: "connected", }); - const onBackupStarted = async (data: { - organizationId: string; - scheduleId: number; - volumeName: string; - repositoryName: string; - }) => { + const onBackupStarted = async (data: ServerBackupStartedEventDto) => { if (data.organizationId !== organizationId) return; await stream.writeSSE({ data: JSON.stringify(data), @@ -28,19 +28,7 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => { }); }; - const onBackupProgress = async (data: { - organizationId: string; - scheduleId: number; - volumeName: string; - repositoryName: string; - seconds_elapsed: number; - percent_done: number; - total_files: number; - files_done: number; - total_bytes: number; - bytes_done: number; - current_files: string[]; - }) => { + const onBackupProgress = async (data: ServerBackupProgressEventDto) => { if (data.organizationId !== organizationId) return; await stream.writeSSE({ data: JSON.stringify(data), @@ -48,29 +36,7 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => { }); }; - const onBackupCompleted = async (data: { - organizationId: string; - scheduleId: number; - volumeName: string; - repositoryName: string; - status: "success" | "error" | "stopped" | "warning"; - summary?: { - files_new: number; - files_changed: number; - files_unmodified: number; - dirs_new: number; - dirs_changed: number; - dirs_unmodified: number; - data_blobs: number; - tree_blobs: number; - data_added: number; - data_added_packed?: number; - total_files_processed: number; - total_bytes_processed: number; - total_duration: number; - snapshot_id: string; - }; - }) => { + const onBackupCompleted = async (data: ServerBackupCompletedEventDto) => { if (data.organizationId !== organizationId) return; await stream.writeSSE({ data: JSON.stringify(data), diff --git a/app/server/modules/notifications/notifications.service.ts b/app/server/modules/notifications/notifications.service.ts index a458d30e..4f548eed 100644 --- a/app/server/modules/notifications/notifications.service.ts +++ b/app/server/modules/notifications/notifications.service.ts @@ -9,10 +9,10 @@ import { import { cryptoUtils } from "../../utils/crypto"; import { logger } from "../../utils/logger"; import { sendNotification } from "../../utils/shoutrrr"; -import type { BackupOutput } from "../../utils/restic"; import { formatDuration } from "~/utils/utils"; import { buildShoutrrrUrl } from "./builders"; import { notificationConfigSchema, type NotificationConfig, type NotificationEvent } from "~/schemas/notifications"; +import type { ResticBackupRunSummaryDto } from "~/schemas/restic-dto"; import { toMessage } from "../../utils/errors"; import { type } from "arktype"; import { getOrganizationId } from "~/server/core/request-context"; @@ -318,8 +318,6 @@ const updateScheduleNotifications = async ( return getScheduleNotifications(scheduleId); }; -type BackupSummary = Omit; - const formatBytesText = (bytes: number) => { const { text, unit } = formatBytes(bytes, { base: 1024, @@ -330,7 +328,7 @@ const formatBytesText = (bytes: number) => { return unit ? `${text} ${unit}` : text; }; -const buildBackupSummaryLines = (summary?: BackupSummary) => { +const buildBackupSummaryLines = (summary?: ResticBackupRunSummaryDto) => { if (!summary) return []; const safeNumber = (value: number | undefined) => (typeof value === "number" && Number.isFinite(value) ? value : 0); @@ -374,7 +372,7 @@ const sendBackupNotification = async ( filesProcessed?: number; bytesProcessed?: string; snapshotId?: string; - summary?: BackupSummary; + summary?: ResticBackupRunSummaryDto; }, ) => { try { @@ -454,7 +452,7 @@ function buildNotificationMessage( filesProcessed?: number; bytesProcessed?: string; snapshotId?: string; - summary?: BackupSummary; + summary?: ResticBackupRunSummaryDto; }, ) { const backupName = context.scheduleName ?? "backup"; diff --git a/app/server/modules/repositories/repositories.dto.ts b/app/server/modules/repositories/repositories.dto.ts index d7e234cf..dda6dbc7 100644 --- a/app/server/modules/repositories/repositories.dto.ts +++ b/app/server/modules/repositories/repositories.dto.ts @@ -8,6 +8,7 @@ import { repositoryConfigSchema, doctorResultSchema, } from "~/schemas/restic"; +import { resticSnapshotSummarySchema } from "~/schemas/restic-dto"; export const repositorySchema = type({ id: "string", @@ -180,22 +181,7 @@ export const snapshotSchema = type({ tags: "string[]", retentionCategories: "string[]", hostname: "string?", - summary: type({ - backup_start: "string", - backup_end: "string", - files_new: "number", - files_changed: "number", - files_unmodified: "number", - dirs_new: "number", - dirs_changed: "number", - dirs_unmodified: "number", - data_blobs: "number", - tree_blobs: "number", - data_added: "number", - data_added_packed: "number?", - total_files_processed: "number", - total_bytes_processed: "number", - }).optional(), + summary: resticSnapshotSummarySchema.optional(), }); const listSnapshotsResponse = snapshotSchema.array(); diff --git a/app/server/utils/restic.ts b/app/server/utils/restic.ts index 43c10bb3..db4382d4 100644 --- a/app/server/utils/restic.ts +++ b/app/server/utils/restic.ts @@ -11,29 +11,19 @@ import { cryptoUtils } from "./crypto"; import type { RetentionPolicy } from "../modules/backups/backups.dto"; import { safeSpawn, exec } from "./spawn"; import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic"; +import { + resticBackupOutputSchema, + resticBackupProgressSchema, + resticRestoreOutputSchema, + resticSnapshotSummarySchema, + type ResticBackupProgressDto, + type ResticRestoreOutputDto, + type ResticSnapshotSummaryDto, +} from "~/schemas/restic-dto"; import { ResticError } from "./errors"; import { db } from "../db/db"; import { safeJsonParse } from "./json"; -const backupOutputSchema = type({ - message_type: "'summary'", - files_new: "number", - files_changed: "number", - files_unmodified: "number", - dirs_new: "number", - dirs_changed: "number", - dirs_unmodified: "number", - data_blobs: "number", - tree_blobs: "number", - data_added: "number", - data_added_packed: "number?", - total_files_processed: "number", - total_bytes_processed: "number", - total_duration: "number", - snapshot_id: "string", -}); -export type BackupOutput = typeof backupOutputSchema.infer; - const snapshotInfoSchema = type({ gid: "number?", hostname: "string", @@ -46,22 +36,7 @@ const snapshotInfoSchema = type({ uid: "number?", username: "string?", tags: "string[]?", - summary: type({ - backup_end: "string", - backup_start: "string", - data_added: "number", - data_added_packed: "number", - data_blobs: "number", - dirs_changed: "number", - dirs_new: "number", - dirs_unmodified: "number", - files_changed: "number", - files_new: "number", - files_unmodified: "number", - total_bytes_processed: "number", - total_files_processed: "number", - tree_blobs: "number", - }).optional(), + summary: resticSnapshotSummarySchema.optional(), }); export const buildRepoUrl = (config: RepositoryConfig): string => { @@ -268,19 +243,6 @@ const init = async (config: RepositoryConfig, organizationId: string, options?: return { success: true, error: null }; }; -const backupProgressSchema = type({ - message_type: "'status'", - seconds_elapsed: "number", - percent_done: "number", - total_files: "number", - files_done: "number", - total_bytes: "number", - bytes_done: "number", - current_files: "string[]", -}); - -export type BackupProgress = typeof backupProgressSchema.infer; - const backup = async ( config: RepositoryConfig, source: string, @@ -293,7 +255,7 @@ const backup = async ( oneFileSystem?: boolean; compressionMode?: CompressionMode; signal?: AbortSignal; - onProgress?: (progress: BackupProgress) => void; + onProgress?: (progress: ResticBackupProgressDto) => void; }, ) => { const repoUrl = buildRepoUrl(config); @@ -357,7 +319,7 @@ const backup = async ( if (options?.onProgress) { try { const jsonData = JSON.parse(data); - const progress = backupProgressSchema(jsonData); + const progress = resticBackupProgressSchema(jsonData); if (!(progress instanceof type.errors)) { options.onProgress(progress); } @@ -412,7 +374,7 @@ const backup = async ( } logger.debug(`Restic backup output last line: ${summaryLine}`); - const result = backupOutputSchema(summaryLine); + const result = resticBackupOutputSchema(summaryLine); if (result instanceof type.errors) { logger.error(`Restic backup output validation failed: ${result.summary}`); @@ -422,16 +384,6 @@ const backup = async ( return { result, exitCode: res.exitCode }; }; -const restoreOutputSchema = type({ - message_type: "'summary'", - total_files: "number?", - files_restored: "number", - files_skipped: "number", - total_bytes: "number?", - bytes_restored: "number?", - bytes_skipped: "number", -}); - const restore = async ( config: RepositoryConfig, snapshotId: string, @@ -499,18 +451,20 @@ const restore = async ( } logger.debug(`Restic restore output last line: ${summaryLine}`); - const result = restoreOutputSchema(summaryLine); + const result = resticRestoreOutputSchema(summaryLine); if (result instanceof type.errors) { logger.warn(`Restic restore output validation failed: ${result.summary}`); logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`); - return { + const fallback: ResticRestoreOutputDto = { message_type: "summary" as const, total_files: 0, files_restored: 0, files_skipped: 0, bytes_skipped: 0, }; + + return fallback; } logger.info( @@ -577,28 +531,11 @@ export interface Snapshot { excludes?: string[] | null; tags?: string[] | null; program_version?: string; - summary?: SnapshotSummary; + summary?: ResticSnapshotSummaryDto; id: string; short_id: string; } -export interface SnapshotSummary { - backup_start: string; - backup_end: string; - files_new: number; - files_changed: number; - files_unmodified: number; - dirs_new: number; - dirs_changed: number; - dirs_unmodified: number; - data_blobs: number; - tree_blobs: number; - data_added: number; - data_added_packed: number; - total_files_processed: number; - total_bytes_processed: number; -} - export interface ForgetReason { snapshot: Snapshot; matches: string[];