feat: restore progress

This commit is contained in:
Nicolas Meienberger 2026-01-02 14:13:45 +01:00
parent 1017f1a38b
commit c8bc611d82
9 changed files with 753 additions and 137 deletions

View file

@ -9,6 +9,7 @@ import { Label } from "~/client/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "~/client/components/ui/select"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "~/client/components/ui/select";
import { PathSelector } from "~/client/components/path-selector"; import { PathSelector } from "~/client/components/path-selector";
import { FileTree } from "~/client/components/file-tree"; import { FileTree } from "~/client/components/file-tree";
import { RestoreProgress } from "~/client/components/restore-progress";
import { listSnapshotFilesOptions, restoreSnapshotMutation } from "~/client/api-client/@tanstack/react-query.gen"; import { listSnapshotFilesOptions, restoreSnapshotMutation } from "~/client/api-client/@tanstack/react-query.gen";
import { useFileBrowser } from "~/client/hooks/use-file-browser"; import { useFileBrowser } from "~/client/hooks/use-file-browser";
import { OVERWRITE_MODES, type OverwriteMode } from "~/schemas/restic"; import { OVERWRITE_MODES, type OverwriteMode } from "~/schemas/restic";
@ -168,6 +169,8 @@ export function RestoreForm({ snapshot, repository, snapshotId, returnPath }: Re
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6"> <div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
<div className="space-y-6"> <div className="space-y-6">
{isRestoring && <RestoreProgress repositoryId={repository.id} snapshotId={snapshotId} />}
<Card> <Card>
<CardHeader> <CardHeader>
<CardTitle>Restore Location</CardTitle> <CardTitle>Restore Location</CardTitle>

View file

@ -0,0 +1,92 @@
import { useEffect, useState } from "react";
import { ByteSize } from "~/client/components/bytes-size";
import { Card } from "~/client/components/ui/card";
import { Progress } from "~/client/components/ui/progress";
import { useServerEvents } from "~/client/hooks/use-server-events";
import type { RestoreCompletedEventDto, RestoreProgressEventDto } from "~/schemas/events-dto";
import { formatBytes } from "~/utils/format-bytes";
import { formatDuration } from "~/utils/utils";
type Props = {
repositoryId: string;
snapshotId: string;
};
export const RestoreProgress = ({ repositoryId, snapshotId }: Props) => {
const { addEventListener } = useServerEvents();
const [progress, setProgress] = useState<RestoreProgressEventDto | null>(null);
useEffect(() => {
const unsubscribe = addEventListener("restore:progress", (data) => {
const progressData = data as RestoreProgressEventDto;
if (progressData.repositoryId === repositoryId && progressData.snapshotId === snapshotId) {
setProgress(progressData);
}
});
const unsubscribeComplete = addEventListener("restore:completed", (data) => {
const completedData = data as RestoreCompletedEventDto;
if (completedData.repositoryId === repositoryId && completedData.snapshotId === snapshotId) {
setProgress(null);
}
});
return () => {
unsubscribe();
unsubscribeComplete();
};
}, [addEventListener, repositoryId, snapshotId]);
if (!progress) {
return (
<Card className="p-4">
<div className="flex items-center gap-2">
<div className="h-4 w-4 animate-spin rounded-full border-2 border-primary border-t-transparent" />
<span className="font-medium">Restore in progress</span>
</div>
</Card>
);
}
const percentDone = Math.round(progress.percent_done * 100);
const speed = formatBytes(progress.bytes_done / progress.seconds_elapsed);
return (
<Card className="p-4">
<div className="flex items-center justify-between mb-4">
<div className="flex items-center gap-2">
<div className="h-4 w-4 animate-spin rounded-full border-2 border-primary border-t-transparent" />
<span className="font-medium">Restore in progress</span>
</div>
<span className="text-sm font-medium text-primary">{percentDone}%</span>
</div>
<Progress value={percentDone} className="h-2 mb-4" />
<div className="grid grid-cols-2 gap-4 text-sm">
<div>
<p className="text-xs uppercase text-muted-foreground">Files</p>
<p className="font-medium">
{progress.files_done.toLocaleString()} / {progress.total_files.toLocaleString()}
</p>
</div>
<div>
<p className="text-xs uppercase text-muted-foreground">Data</p>
<p className="font-medium">
<ByteSize bytes={progress.bytes_done} /> / <ByteSize bytes={progress.total_bytes} />
</p>
</div>
<div>
<p className="text-xs uppercase text-muted-foreground">Elapsed</p>
<p className="font-medium">{formatDuration(progress.seconds_elapsed)}</p>
</div>
<div>
<p className="text-xs uppercase text-muted-foreground">Speed</p>
<p className="font-medium">
{progress.seconds_elapsed > 0 ? `${speed.text} ${speed.unit}/s` : "Calculating..."}
</p>
</div>
</div>
</Card>
);
};

View file

@ -1,9 +1,12 @@
import { useCallback, useEffect, useRef } from "react";
import { useQueryClient } from "@tanstack/react-query"; import { useQueryClient } from "@tanstack/react-query";
import { useCallback, useEffect, useRef } from "react";
import type { import type {
BackupCompletedEventDto, BackupCompletedEventDto,
BackupProgressEventDto, BackupProgressEventDto,
BackupStartedEventDto, BackupStartedEventDto,
RestoreCompletedEventDto,
RestoreProgressEventDto,
RestoreStartedEventDto,
} from "~/schemas/events-dto"; } from "~/schemas/events-dto";
type ServerEventType = type ServerEventType =
@ -17,6 +20,9 @@ type ServerEventType =
| "volume:updated" | "volume:updated"
| "mirror:started" | "mirror:started"
| "mirror:completed" | "mirror:completed"
| "restore:started"
| "restore:progress"
| "restore:completed"
| "doctor:started" | "doctor:started"
| "doctor:completed" | "doctor:completed"
| "doctor:cancelled"; | "doctor:cancelled";
@ -161,6 +167,32 @@ export function useServerEvents() {
}); });
}); });
eventSource.addEventListener("restore:started", (e) => {
const data = JSON.parse(e.data) as RestoreStartedEventDto;
console.info("[SSE] Restore started:", data);
handlersRef.current.get("restore:started")?.forEach((handler) => {
handler(data);
});
});
eventSource.addEventListener("restore:progress", (e) => {
const data = JSON.parse(e.data) as RestoreProgressEventDto;
handlersRef.current.get("restore:progress")?.forEach((handler) => {
handler(data);
});
});
eventSource.addEventListener("restore:completed", (e) => {
const data = JSON.parse(e.data) as RestoreCompletedEventDto;
console.info("[SSE] Restore completed:", data);
handlersRef.current.get("restore:completed")?.forEach((handler) => {
handler(data);
});
});
eventSource.addEventListener("doctor:started", (e) => { eventSource.addEventListener("doctor:started", (e) => {
const data = JSON.parse(e.data) as DoctorEvent; const data = JSON.parse(e.data) as DoctorEvent;
console.info("[SSE] Doctor started:", data); console.info("[SSE] Doctor started:", data);

View file

@ -2,6 +2,7 @@ import { type } from "arktype";
import { resticBackupProgressMetricsSchema, resticBackupRunSummarySchema } from "~/schemas/restic-dto"; import { resticBackupProgressMetricsSchema, resticBackupRunSummarySchema } from "~/schemas/restic-dto";
export const backupEventStatusSchema = type("'success' | 'error' | 'stopped' | 'warning'"); export const backupEventStatusSchema = type("'success' | 'error' | 'stopped' | 'warning'");
export const restoreEventStatusSchema = type("'success' | 'error'");
const backupEventBaseSchema = type({ const backupEventBaseSchema = type({
scheduleId: "number", scheduleId: "number",
@ -13,6 +14,20 @@ const organizationScopedSchema = type({
organizationId: "string", organizationId: "string",
}); });
const restoreEventBaseSchema = type({
repositoryId: "string",
snapshotId: "string",
});
const restoreProgressMetricsSchema = type({
seconds_elapsed: "number",
percent_done: "number",
total_files: "number",
files_done: "number",
total_bytes: "number",
bytes_done: "number",
});
export const backupStartedEventSchema = backupEventBaseSchema; export const backupStartedEventSchema = backupEventBaseSchema;
export const backupProgressEventSchema = backupEventBaseSchema.and(resticBackupProgressMetricsSchema); export const backupProgressEventSchema = backupEventBaseSchema.and(resticBackupProgressMetricsSchema);
@ -24,16 +39,39 @@ export const backupCompletedEventSchema = backupEventBaseSchema.and(
}), }),
); );
export const restoreStartedEventSchema = restoreEventBaseSchema;
export const restoreProgressEventSchema = restoreEventBaseSchema.and(restoreProgressMetricsSchema);
export const restoreCompletedEventSchema = restoreEventBaseSchema.and(
type({
status: restoreEventStatusSchema,
error: "string?",
}),
);
export const serverBackupStartedEventSchema = organizationScopedSchema.and(backupStartedEventSchema); export const serverBackupStartedEventSchema = organizationScopedSchema.and(backupStartedEventSchema);
export const serverBackupProgressEventSchema = organizationScopedSchema.and(backupProgressEventSchema); export const serverBackupProgressEventSchema = organizationScopedSchema.and(backupProgressEventSchema);
export const serverBackupCompletedEventSchema = organizationScopedSchema.and(backupCompletedEventSchema); export const serverBackupCompletedEventSchema = organizationScopedSchema.and(backupCompletedEventSchema);
export const serverRestoreStartedEventSchema = organizationScopedSchema.and(restoreStartedEventSchema);
export const serverRestoreProgressEventSchema = organizationScopedSchema.and(restoreProgressEventSchema);
export const serverRestoreCompletedEventSchema = organizationScopedSchema.and(restoreCompletedEventSchema);
export type BackupEventStatusDto = typeof backupEventStatusSchema.infer; export type BackupEventStatusDto = typeof backupEventStatusSchema.infer;
export type BackupStartedEventDto = typeof backupStartedEventSchema.infer; export type BackupStartedEventDto = typeof backupStartedEventSchema.infer;
export type BackupProgressEventDto = typeof backupProgressEventSchema.infer; export type BackupProgressEventDto = typeof backupProgressEventSchema.infer;
export type BackupCompletedEventDto = typeof backupCompletedEventSchema.infer; export type BackupCompletedEventDto = typeof backupCompletedEventSchema.infer;
export type RestoreStartedEventDto = typeof restoreStartedEventSchema.infer;
export type RestoreProgressEventDto = typeof restoreProgressEventSchema.infer;
export type RestoreCompletedEventDto = typeof restoreCompletedEventSchema.infer;
export type ServerBackupStartedEventDto = typeof serverBackupStartedEventSchema.infer; export type ServerBackupStartedEventDto = typeof serverBackupStartedEventSchema.infer;
export type ServerBackupProgressEventDto = typeof serverBackupProgressEventSchema.infer; export type ServerBackupProgressEventDto = typeof serverBackupProgressEventSchema.infer;
export type ServerBackupCompletedEventDto = typeof serverBackupCompletedEventSchema.infer; export type ServerBackupCompletedEventDto = typeof serverBackupCompletedEventSchema.infer;
export type ServerRestoreStartedEventDto = typeof serverRestoreStartedEventSchema.infer;
export type ServerRestoreProgressEventDto = typeof serverRestoreProgressEventSchema.infer;
export type ServerRestoreCompletedEventDto = typeof serverRestoreCompletedEventSchema.infer;

View file

@ -1,11 +1,14 @@
import { EventEmitter } from "node:events"; import { EventEmitter } from "node:events";
import type { TypedEmitter } from "tiny-typed-emitter"; import type { TypedEmitter } from "tiny-typed-emitter";
import type { DoctorResult } from "~/schemas/restic";
import type { import type {
ServerBackupCompletedEventDto, ServerBackupCompletedEventDto,
ServerBackupProgressEventDto, ServerBackupProgressEventDto,
ServerBackupStartedEventDto, ServerBackupStartedEventDto,
ServerRestoreCompletedEventDto,
ServerRestoreProgressEventDto,
ServerRestoreStartedEventDto,
} from "~/schemas/events-dto"; } from "~/schemas/events-dto";
import type { DoctorResult } from "~/schemas/restic";
/** /**
* Event payloads for the SSE system * Event payloads for the SSE system
@ -14,6 +17,9 @@ interface ServerEvents {
"backup:started": (data: ServerBackupStartedEventDto) => void; "backup:started": (data: ServerBackupStartedEventDto) => void;
"backup:progress": (data: ServerBackupProgressEventDto) => void; "backup:progress": (data: ServerBackupProgressEventDto) => void;
"backup:completed": (data: ServerBackupCompletedEventDto) => void; "backup:completed": (data: ServerBackupCompletedEventDto) => void;
"restore:started": (data: ServerRestoreStartedEventDto) => void;
"restore:progress": (data: ServerRestoreProgressEventDto) => void;
"restore:completed": (data: ServerRestoreCompletedEventDto) => void;
"mirror:started": (data: { "mirror:started": (data: {
organizationId: string; organizationId: string;
scheduleId: number; scheduleId: number;

View file

@ -1,14 +1,17 @@
import { Hono } from "hono"; import { Hono } from "hono";
import { streamSSE } from "hono/streaming"; import { streamSSE } from "hono/streaming";
import { logger } from "../../utils/logger";
import { serverEvents } from "../../core/events";
import { requireAuth } from "../auth/auth.middleware";
import type { DoctorResult } from "~/schemas/restic";
import type { import type {
ServerBackupCompletedEventDto, ServerBackupCompletedEventDto,
ServerBackupProgressEventDto, ServerBackupProgressEventDto,
ServerBackupStartedEventDto, ServerBackupStartedEventDto,
ServerRestoreCompletedEventDto,
ServerRestoreProgressEventDto,
ServerRestoreStartedEventDto,
} from "~/schemas/events-dto"; } from "~/schemas/events-dto";
import type { DoctorResult } from "~/schemas/restic";
import { serverEvents } from "../../core/events";
import { logger } from "../../utils/logger";
import { requireAuth } from "../auth/auth.middleware";
export const eventsController = new Hono().use(requireAuth).get("/", (c) => { export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
logger.info("Client connected to SSE endpoint"); logger.info("Client connected to SSE endpoint");
@ -96,6 +99,30 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
}); });
}; };
const onRestoreStarted = async (data: ServerRestoreStartedEventDto) => {
if (data.organizationId !== organizationId) return;
await stream.writeSSE({
data: JSON.stringify(data),
event: "restore:started",
});
};
const onRestoreProgress = async (data: ServerRestoreProgressEventDto) => {
if (data.organizationId !== organizationId) return;
await stream.writeSSE({
data: JSON.stringify(data),
event: "restore:progress",
});
};
const onRestoreCompleted = async (data: ServerRestoreCompletedEventDto) => {
if (data.organizationId !== organizationId) return;
await stream.writeSSE({
data: JSON.stringify(data),
event: "restore:completed",
});
};
const onDoctorStarted = async (data: { organizationId: string; repositoryId: string; repositoryName: string }) => { const onDoctorStarted = async (data: { organizationId: string; repositoryId: string; repositoryName: string }) => {
if (data.organizationId !== organizationId) return; if (data.organizationId !== organizationId) return;
await stream.writeSSE({ await stream.writeSSE({
@ -139,6 +166,9 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
serverEvents.on("volume:updated", onVolumeUpdated); serverEvents.on("volume:updated", onVolumeUpdated);
serverEvents.on("mirror:started", onMirrorStarted); serverEvents.on("mirror:started", onMirrorStarted);
serverEvents.on("mirror:completed", onMirrorCompleted); serverEvents.on("mirror:completed", onMirrorCompleted);
serverEvents.on("restore:started", onRestoreStarted);
serverEvents.on("restore:progress", onRestoreProgress);
serverEvents.on("restore:completed", onRestoreCompleted);
serverEvents.on("doctor:started", onDoctorStarted); serverEvents.on("doctor:started", onDoctorStarted);
serverEvents.on("doctor:completed", onDoctorCompleted); serverEvents.on("doctor:completed", onDoctorCompleted);
serverEvents.on("doctor:cancelled", onDoctorCancelled); serverEvents.on("doctor:cancelled", onDoctorCancelled);
@ -159,6 +189,9 @@ export const eventsController = new Hono().use(requireAuth).get("/", (c) => {
serverEvents.off("volume:updated", onVolumeUpdated); serverEvents.off("volume:updated", onVolumeUpdated);
serverEvents.off("mirror:started", onMirrorStarted); serverEvents.off("mirror:started", onMirrorStarted);
serverEvents.off("mirror:completed", onMirrorCompleted); serverEvents.off("mirror:completed", onMirrorCompleted);
serverEvents.off("restore:started", onRestoreStarted);
serverEvents.off("restore:progress", onRestoreProgress);
serverEvents.off("restore:completed", onRestoreCompleted);
serverEvents.off("doctor:started", onDoctorStarted); serverEvents.off("doctor:started", onDoctorStarted);
serverEvents.off("doctor:completed", onDoctorCompleted); serverEvents.off("doctor:completed", onDoctorCompleted);
serverEvents.off("doctor:cancelled", onDoctorCancelled); serverEvents.off("doctor:cancelled", onDoctorCancelled);

View file

@ -143,6 +143,8 @@ export const repositoriesController = new Hono()
summary: snapshot.summary, summary: snapshot.summary,
}; };
c.header("Cache-Control", "max-age=300, stale-while-revalidate=600");
return c.json<GetSnapshotDetailsDto>(response, 200); return c.json<GetSnapshotDetailsDto>(response, 200);
}) })
.get( .get(

View file

@ -1,29 +1,43 @@
import crypto from "node:crypto"; import crypto from "node:crypto";
import { and, eq } from "drizzle-orm";
import { BadRequestError, ConflictError, InternalServerError, NotFoundError } from "http-errors-enhanced";
import { db } from "../../db/db";
import { repositoriesTable } from "../../db/schema";
import { toMessage } from "../../utils/errors";
import { generateShortId } from "../../utils/id";
import { restic, buildEnv, buildRepoUrl, addCommonArgs, cleanupTemporaryKeys } from "../../utils/restic";
import { safeSpawn } from "../../utils/spawn";
import { cryptoUtils } from "../../utils/crypto";
import { cache } from "../../utils/cache";
import { repoMutex } from "../../core/repository-mutex";
import { type } from "arktype"; import { type } from "arktype";
import { and, eq } from "drizzle-orm";
import {
BadRequestError,
ConflictError,
InternalServerError,
NotFoundError,
} from "http-errors-enhanced";
import { import {
repositoryConfigSchema,
type CompressionMode, type CompressionMode,
type OverwriteMode, type OverwriteMode,
type RepositoryConfig, type RepositoryConfig,
repositoryConfigSchema,
} from "~/schemas/restic"; } from "~/schemas/restic";
import { getOrganizationId } from "~/server/core/request-context";
import { serverEvents } from "~/server/core/events"; import { serverEvents } from "~/server/core/events";
import { executeDoctor } from "./doctor"; import { getOrganizationId } from "~/server/core/request-context";
import { logger } from "~/server/utils/logger"; import { logger } from "~/server/utils/logger";
import { parseRetentionCategories, type RetentionCategory } from "~/server/utils/retention-categories"; import {
parseRetentionCategories,
type RetentionCategory,
} from "~/server/utils/retention-categories";
import { repoMutex } from "../../core/repository-mutex";
import { db } from "../../db/db";
import { repositoriesTable } from "../../db/schema";
import { cache } from "../../utils/cache";
import { cryptoUtils } from "../../utils/crypto";
import { toMessage } from "../../utils/errors";
import { generateShortId } from "../../utils/id";
import {
addCommonArgs,
buildEnv,
buildRepoUrl,
cleanupTemporaryKeys,
restic,
} from "../../utils/restic";
import { safeSpawn } from "../../utils/spawn";
import { backupsService } from "../backups/backups.service"; import { backupsService } from "../backups/backups.service";
import type { UpdateRepositoryBody } from "./repositories.dto"; import type { UpdateRepositoryBody } from "./repositories.dto";
import { executeDoctor } from "./doctor";
const runningDoctors = new Map<string, AbortController>(); const runningDoctors = new Map<string, AbortController>();
@ -31,22 +45,31 @@ const findRepository = async (idOrShortId: string) => {
const organizationId = getOrganizationId(); const organizationId = getOrganizationId();
return await db.query.repositoriesTable.findFirst({ return await db.query.repositoriesTable.findFirst({
where: { where: {
AND: [{ OR: [{ id: idOrShortId }, { shortId: idOrShortId }] }, { organizationId }], AND: [
{ OR: [{ id: idOrShortId }, { shortId: idOrShortId }] },
{ organizationId },
],
}, },
}); });
}; };
const listRepositories = async () => { const listRepositories = async () => {
const organizationId = getOrganizationId(); const organizationId = getOrganizationId();
const repositories = await db.query.repositoriesTable.findMany({ where: { organizationId } }); const repositories = await db.query.repositoriesTable.findMany({
where: { organizationId },
});
return repositories; return repositories;
}; };
const encryptConfig = async (config: RepositoryConfig): Promise<RepositoryConfig> => { const encryptConfig = async (
config: RepositoryConfig,
): Promise<RepositoryConfig> => {
const encryptedConfig: Record<string, unknown> = { ...config }; const encryptedConfig: Record<string, unknown> = { ...config };
if (config.customPassword) { if (config.customPassword) {
encryptedConfig.customPassword = await cryptoUtils.sealSecret(config.customPassword); encryptedConfig.customPassword = await cryptoUtils.sealSecret(
config.customPassword,
);
} }
if (config.cacert) { if (config.cacert) {
@ -56,25 +79,39 @@ const encryptConfig = async (config: RepositoryConfig): Promise<RepositoryConfig
switch (config.backend) { switch (config.backend) {
case "s3": case "s3":
case "r2": case "r2":
encryptedConfig.accessKeyId = await cryptoUtils.sealSecret(config.accessKeyId); encryptedConfig.accessKeyId = await cryptoUtils.sealSecret(
encryptedConfig.secretAccessKey = await cryptoUtils.sealSecret(config.secretAccessKey); config.accessKeyId,
);
encryptedConfig.secretAccessKey = await cryptoUtils.sealSecret(
config.secretAccessKey,
);
break; break;
case "gcs": case "gcs":
encryptedConfig.credentialsJson = await cryptoUtils.sealSecret(config.credentialsJson); encryptedConfig.credentialsJson = await cryptoUtils.sealSecret(
config.credentialsJson,
);
break; break;
case "azure": case "azure":
encryptedConfig.accountKey = await cryptoUtils.sealSecret(config.accountKey); encryptedConfig.accountKey = await cryptoUtils.sealSecret(
config.accountKey,
);
break; break;
case "rest": case "rest":
if (config.username) { if (config.username) {
encryptedConfig.username = await cryptoUtils.sealSecret(config.username); encryptedConfig.username = await cryptoUtils.sealSecret(
config.username,
);
} }
if (config.password) { if (config.password) {
encryptedConfig.password = await cryptoUtils.sealSecret(config.password); encryptedConfig.password = await cryptoUtils.sealSecret(
config.password,
);
} }
break; break;
case "sftp": case "sftp":
encryptedConfig.privateKey = await cryptoUtils.sealSecret(config.privateKey); encryptedConfig.privateKey = await cryptoUtils.sealSecret(
config.privateKey,
);
break; break;
} }
@ -156,7 +193,9 @@ const createRepository = async (name: string, config: RepositoryConfig, compress
error = result.error; error = result.error;
} else { } else {
const initResult = await restic.init(encryptedConfig, organizationId, { timeoutMs: 20000 }); const initResult = await restic.init(encryptedConfig, organizationId, {
timeoutMs: 20000,
});
error = initResult.error; error = initResult.error;
} }
@ -164,7 +203,12 @@ const createRepository = async (name: string, config: RepositoryConfig, compress
await db await db
.update(repositoriesTable) .update(repositoriesTable)
.set({ status: "healthy", lastChecked: Date.now(), lastError: null }) .set({ status: "healthy", lastChecked: Date.now(), lastError: null })
.where(and(eq(repositoriesTable.id, id), eq(repositoriesTable.organizationId, organizationId))); .where(
and(
eq(repositoriesTable.id, id),
eq(repositoriesTable.organizationId, organizationId),
),
);
return { repository: created, status: 201 }; return { repository: created, status: 201 };
} }
@ -172,9 +216,16 @@ const createRepository = async (name: string, config: RepositoryConfig, compress
const errorMessage = toMessage(error); const errorMessage = toMessage(error);
await db await db
.delete(repositoriesTable) .delete(repositoriesTable)
.where(and(eq(repositoriesTable.id, id), eq(repositoriesTable.organizationId, organizationId))); .where(
and(
eq(repositoriesTable.id, id),
eq(repositoriesTable.organizationId, organizationId),
),
);
throw new InternalServerError(`Failed to initialize repository: ${errorMessage}`); throw new InternalServerError(
`Failed to initialize repository: ${errorMessage}`,
);
}; };
const getRepository = async (id: string) => { const getRepository = async (id: string) => {
@ -199,7 +250,10 @@ const deleteRepository = async (id: string) => {
await db await db
.delete(repositoriesTable) .delete(repositoriesTable)
.where( .where(
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)), and(
eq(repositoriesTable.id, repository.id),
eq(repositoriesTable.organizationId, repository.organizationId),
),
); );
cache.delByPrefix(`snapshots:${repository.id}:`); cache.delByPrefix(`snapshots:${repository.id}:`);
@ -223,7 +277,8 @@ const listSnapshots = async (id: string, backupId?: string) => {
} }
const cacheKey = `snapshots:${repository.id}:${backupId || "all"}`; const cacheKey = `snapshots:${repository.id}:${backupId || "all"}`;
const cached = cache.get<Awaited<ReturnType<typeof restic.snapshots>>>(cacheKey); const cached =
cache.get<Awaited<ReturnType<typeof restic.snapshots>>>(cacheKey);
if (cached) { if (cached) {
return cached; return cached;
} }
@ -233,7 +288,10 @@ const listSnapshots = async (id: string, backupId?: string) => {
let snapshots = []; let snapshots = [];
if (backupId) { if (backupId) {
snapshots = await restic.snapshots(repository.config, { tags: [backupId], organizationId }); snapshots = await restic.snapshots(repository.config, {
tags: [backupId],
organizationId,
});
} else { } else {
snapshots = await restic.snapshots(repository.config, { organizationId }); snapshots = await restic.snapshots(repository.config, { organizationId });
} }
@ -264,9 +322,26 @@ const listSnapshotFiles = async (
const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`; const cacheKey = `ls:${repository.id}:${snapshotId}:${path || "root"}:${offset}:${limit}`;
type LsResult = { type LsResult = {
snapshot: { id: string; short_id: string; time: string; hostname: string; paths: string[] } | null; snapshot: {
nodes: { name: string; type: string; path: string; size?: number; mode?: number }[]; id: string;
pagination: { offset: number; limit: number; total: number; hasMore: boolean }; short_id: string;
time: string;
hostname: string;
paths: string[];
} | null;
nodes: {
name: string;
type: string;
path: string;
size?: number;
mode?: number;
}[];
pagination: {
offset: number;
limit: number;
total: number;
hasMore: boolean;
};
}; };
const cached = cache.get<LsResult>(cacheKey); const cached = cache.get<LsResult>(cacheKey);
if (cached?.snapshot) { if (cached?.snapshot) {
@ -280,9 +355,18 @@ const listSnapshotFiles = async (
}; };
} }
const releaseLock = await repoMutex.acquireShared(repository.id, `ls:${snapshotId}`); const releaseLock = await repoMutex.acquireShared(
repository.id,
`ls:${snapshotId}`,
);
try { try {
const result = await restic.ls(repository.config, snapshotId, organizationId, path, { offset, limit }); const result = await restic.ls(
repository.config,
snapshotId,
organizationId,
path,
{ offset, limit },
);
if (!result.snapshot) { if (!result.snapshot) {
throw new NotFoundError("Snapshot not found or empty"); throw new NotFoundError("Snapshot not found or empty");
@ -332,9 +416,50 @@ const restoreSnapshot = async (
const target = options?.targetPath || "/"; const target = options?.targetPath || "/";
const releaseLock = await repoMutex.acquireShared(repository.id, `restore:${snapshotId}`); const releaseLock = await repoMutex.acquireShared(
repository.id,
`restore:${snapshotId}`,
);
try { try {
const result = await restic.restore(repository.config, snapshotId, target, { ...options, organizationId }); serverEvents.emit("restore:started", {
organizationId,
repositoryId: repository.id,
snapshotId,
});
const result = await restic.restore(repository.config, snapshotId, target, {
...options,
organizationId,
onProgress: (progress) => {
const {
seconds_elapsed,
percent_done,
total_files,
files_done,
total_bytes,
bytes_done,
} = progress;
serverEvents.emit("restore:progress", {
organizationId,
repositoryId: repository.id,
snapshotId,
seconds_elapsed,
percent_done,
total_files,
files_done,
total_bytes,
bytes_done,
});
},
});
serverEvents.emit("restore:completed", {
organizationId,
repositoryId: repository.id,
snapshotId,
status: "success",
});
return { return {
success: true, success: true,
@ -342,6 +467,15 @@ const restoreSnapshot = async (
filesRestored: result.files_restored, filesRestored: result.files_restored,
filesSkipped: result.files_skipped, filesSkipped: result.files_skipped,
}; };
} catch (error) {
serverEvents.emit("restore:completed", {
organizationId,
repositoryId: repository.id,
snapshotId,
status: "error",
error: toMessage(error),
});
throw error;
} finally { } finally {
releaseLock(); releaseLock();
} }
@ -356,10 +490,14 @@ const getSnapshotDetails = async (id: string, snapshotId: string) => {
} }
const cacheKey = `snapshots:${repository.id}:all`; const cacheKey = `snapshots:${repository.id}:all`;
let snapshots = cache.get<Awaited<ReturnType<typeof restic.snapshots>>>(cacheKey); let snapshots =
cache.get<Awaited<ReturnType<typeof restic.snapshots>>>(cacheKey);
if (!snapshots) { if (!snapshots) {
const releaseLock = await repoMutex.acquireShared(repository.id, `snapshot_details:${snapshotId}`); const releaseLock = await repoMutex.acquireShared(
repository.id,
`snapshot_details:${snapshotId}`,
);
try { try {
snapshots = await restic.snapshots(repository.config, { organizationId }); snapshots = await restic.snapshots(repository.config, { organizationId });
cache.set(cacheKey, snapshots); cache.set(cacheKey, snapshots);
@ -368,7 +506,9 @@ const getSnapshotDetails = async (id: string, snapshotId: string) => {
} }
} }
const snapshot = snapshots.find((snap) => snap.id === snapshotId || snap.short_id === snapshotId); const snapshot = snapshots.find(
(snap) => snap.id === snapshotId || snap.short_id === snapshotId,
);
if (!snapshot) { if (!snapshot) {
void refreshSnapshots(id).catch(() => {}); void refreshSnapshots(id).catch(() => {});
@ -389,7 +529,9 @@ const checkHealth = async (repositoryId: string) => {
const releaseLock = await repoMutex.acquireExclusive(repository.id, "check"); const releaseLock = await repoMutex.acquireExclusive(repository.id, "check");
try { try {
const { hasErrors, error } = await restic.check(repository.config, { organizationId }); const { hasErrors, error } = await restic.check(repository.config, {
organizationId,
});
await db await db
.update(repositoriesTable) .update(repositoriesTable)
@ -399,7 +541,10 @@ const checkHealth = async (repositoryId: string) => {
lastError: error, lastError: error,
}) })
.where( .where(
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)), and(
eq(repositoriesTable.id, repository.id),
eq(repositoriesTable.organizationId, repository.organizationId),
),
); );
return { lastError: error }; return { lastError: error };
@ -422,7 +567,10 @@ const startDoctor = async (id: string) => {
const abortController = new AbortController(); const abortController = new AbortController();
try { try {
await db.update(repositoriesTable).set({ status: "doctor" }).where(eq(repositoriesTable.id, repository.id)); await db
.update(repositoriesTable)
.set({ status: "doctor" })
.where(eq(repositoriesTable.id, repository.id));
serverEvents.emit("doctor:started", { serverEvents.emit("doctor:started", {
organizationId: repository.organizationId, organizationId: repository.organizationId,
@ -436,7 +584,12 @@ const startDoctor = async (id: string) => {
throw error; throw error;
} }
executeDoctor(repository.id, repository.config, repository.name, abortController.signal) executeDoctor(
repository.id,
repository.config,
repository.name,
abortController.signal,
)
.catch((error) => { .catch((error) => {
logger.error(`Doctor background task failed: ${toMessage(error)}`); logger.error(`Doctor background task failed: ${toMessage(error)}`);
}) })
@ -456,14 +609,20 @@ const cancelDoctor = async (id: string) => {
const abortController = runningDoctors.get(repository.id); const abortController = runningDoctors.get(repository.id);
if (!abortController) { if (!abortController) {
await db.update(repositoriesTable).set({ status: "unknown" }).where(eq(repositoriesTable.id, repository.id)); await db
.update(repositoriesTable)
.set({ status: "unknown" })
.where(eq(repositoriesTable.id, repository.id));
throw new ConflictError("No doctor operation is currently running"); throw new ConflictError("No doctor operation is currently running");
} }
abortController.abort(); abortController.abort();
runningDoctors.delete(repository.id); runningDoctors.delete(repository.id);
await db.update(repositoriesTable).set({ status: "unknown" }).where(eq(repositoriesTable.id, repository.id)); await db
.update(repositoriesTable)
.set({ status: "unknown" })
.where(eq(repositoriesTable.id, repository.id));
serverEvents.emit("doctor:cancelled", { serverEvents.emit("doctor:cancelled", {
organizationId: repository.organizationId, organizationId: repository.organizationId,
@ -482,7 +641,10 @@ const deleteSnapshot = async (id: string, snapshotId: string) => {
throw new NotFoundError("Repository not found"); throw new NotFoundError("Repository not found");
} }
const releaseLock = await repoMutex.acquireExclusive(repository.id, `delete:${snapshotId}`); const releaseLock = await repoMutex.acquireExclusive(
repository.id,
`delete:${snapshotId}`,
);
try { try {
await restic.deleteSnapshot(repository.config, snapshotId, organizationId); await restic.deleteSnapshot(repository.config, snapshotId, organizationId);
cache.delByPrefix(`snapshots:${repository.id}:`); cache.delByPrefix(`snapshots:${repository.id}:`);
@ -500,9 +662,16 @@ const deleteSnapshots = async (id: string, snapshotIds: string[]) => {
throw new NotFoundError("Repository not found"); throw new NotFoundError("Repository not found");
} }
const releaseLock = await repoMutex.acquireExclusive(repository.id, `delete:bulk`); const releaseLock = await repoMutex.acquireExclusive(
repository.id,
`delete:bulk`,
);
try { try {
await restic.deleteSnapshots(repository.config, snapshotIds, organizationId); await restic.deleteSnapshots(
repository.config,
snapshotIds,
organizationId,
);
cache.delByPrefix(`snapshots:${repository.id}:`); cache.delByPrefix(`snapshots:${repository.id}:`);
for (const snapshotId of snapshotIds) { for (const snapshotId of snapshotIds) {
cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`); cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`);
@ -524,9 +693,17 @@ const tagSnapshots = async (
throw new NotFoundError("Repository not found"); throw new NotFoundError("Repository not found");
} }
const releaseLock = await repoMutex.acquireExclusive(repository.id, `tag:bulk`); const releaseLock = await repoMutex.acquireExclusive(
repository.id,
`tag:bulk`,
);
try { try {
await restic.tagSnapshots(repository.config, snapshotIds, tags, organizationId); await restic.tagSnapshots(
repository.config,
snapshotIds,
tags,
organizationId,
);
cache.delByPrefix(`snapshots:${repository.id}:`); cache.delByPrefix(`snapshots:${repository.id}:`);
for (const snapshotId of snapshotIds) { for (const snapshotId of snapshotIds) {
cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`); cache.delByPrefix(`ls:${repository.id}:${snapshotId}:`);
@ -549,7 +726,9 @@ const refreshSnapshots = async (id: string) => {
const releaseLock = await repoMutex.acquireShared(repository.id, "refresh"); const releaseLock = await repoMutex.acquireShared(repository.id, "refresh");
try { try {
const snapshots = await restic.snapshots(repository.config, { organizationId }); const snapshots = await restic.snapshots(repository.config, {
organizationId,
});
const cacheKey = `snapshots:${repository.id}:all`; const cacheKey = `snapshots:${repository.id}:all`;
cache.set(cacheKey, snapshots); cache.set(cacheKey, snapshots);
@ -676,14 +855,24 @@ const execResticCommand = async (
} }
addCommonArgs(resticArgs, env, repository.config); addCommonArgs(resticArgs, env, repository.config);
const result = await safeSpawn({ command: "restic", args: resticArgs, env, signal, onStdout, onStderr }); const result = await safeSpawn({
command: "restic",
args: resticArgs,
env,
signal,
onStdout,
onStderr,
});
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
return { exitCode: result.exitCode }; return { exitCode: result.exitCode };
}; };
const getRetentionCategories = async (repositoryId: string, scheduleId?: string) => { const getRetentionCategories = async (
repositoryId: string,
scheduleId?: string,
) => {
if (!scheduleId) { if (!scheduleId) {
return new Map<string, RetentionCategory[]>(); return new Map<string, RetentionCategory[]>();
} }
@ -707,11 +896,15 @@ const getRetentionCategories = async (repositoryId: string, scheduleId?: string)
const { repository } = repositoryResult; const { repository } = repositoryResult;
const dryRunResults = await restic.forget(repository.config, schedule.retentionPolicy, { const dryRunResults = await restic.forget(
tag: scheduleId, repository.config,
organizationId: getOrganizationId(), schedule.retentionPolicy,
dryRun: true, {
}); tag: scheduleId,
organizationId: getOrganizationId(),
dryRun: true,
},
);
if (!dryRunResults.data) { if (!dryRunResults.data) {
return new Map<string, RetentionCategory[]>(); return new Map<string, RetentionCategory[]>();

View file

@ -1,28 +1,38 @@
import crypto from "node:crypto"; import crypto from "node:crypto";
import fs from "node:fs/promises"; import fs from "node:fs/promises";
import path from "node:path";
import os from "node:os"; import os from "node:os";
import { throttle } from "es-toolkit"; import path from "node:path";
import { type } from "arktype"; import { type } from "arktype";
import { RESTIC_PASS_FILE, DEFAULT_EXCLUDES, RESTIC_CACHE_DIR } from "../core/constants"; import { throttle } from "es-toolkit";
import { config as appConfig } from "../core/config"; import type {
import { logger } from "./logger"; BandwidthLimit,
import { cryptoUtils } from "./crypto"; CompressionMode,
import type { RetentionPolicy } from "../modules/backups/backups.dto"; OverwriteMode,
import { safeSpawn, exec } from "./spawn"; RepositoryConfig,
import type { CompressionMode, RepositoryConfig, OverwriteMode, BandwidthLimit } from "~/schemas/restic"; } from "~/schemas/restic";
import { import {
type ResticBackupProgressDto,
type ResticRestoreOutputDto,
type ResticSnapshotSummaryDto,
resticBackupOutputSchema, resticBackupOutputSchema,
resticBackupProgressSchema, resticBackupProgressSchema,
resticRestoreOutputSchema, resticRestoreOutputSchema,
resticSnapshotSummarySchema, resticSnapshotSummarySchema,
type ResticBackupProgressDto,
type ResticRestoreOutputDto,
type ResticSnapshotSummaryDto,
} from "~/schemas/restic-dto"; } from "~/schemas/restic-dto";
import { ResticError } from "./errors"; import { config as appConfig } from "../core/config";
import {
DEFAULT_EXCLUDES,
REPOSITORY_BASE,
RESTIC_CACHE_DIR,
RESTIC_PASS_FILE,
} from "../core/constants";
import { db } from "../db/db"; import { db } from "../db/db";
import type { RetentionPolicy } from "../modules/backups/backups.dto";
import { cryptoUtils } from "./crypto";
import { ResticError } from "./errors";
import { safeJsonParse } from "./json"; import { safeJsonParse } from "./json";
import { logger } from "./logger";
import { exec, safeSpawn } from "./spawn";
const snapshotInfoSchema = type({ const snapshotInfoSchema = type({
gid: "number?", gid: "number?",
@ -62,25 +72,37 @@ export const buildRepoUrl = (config: RepositoryConfig): string => {
case "sftp": case "sftp":
return `sftp:${config.user}@${config.host}:${config.path}`; return `sftp:${config.user}@${config.host}:${config.path}`;
default: { default: {
throw new Error(`Unsupported repository backend: ${JSON.stringify(config)}`); throw new Error(
`Unsupported repository backend: ${JSON.stringify(config)}`,
);
} }
} }
}; };
export const buildEnv = async (config: RepositoryConfig, organizationId: string) => { export const buildEnv = async (
config: RepositoryConfig,
organizationId: string,
) => {
const env: Record<string, string> = { const env: Record<string, string> = {
RESTIC_CACHE_DIR, RESTIC_CACHE_DIR,
PATH: process.env.PATH || "/usr/local/bin:/usr/bin:/bin", PATH: process.env.PATH || "/usr/local/bin:/usr/bin:/bin",
}; };
if (config.isExistingRepository && config.customPassword) { if (config.isExistingRepository && config.customPassword) {
const decryptedPassword = await cryptoUtils.resolveSecret(config.customPassword); const decryptedPassword = await cryptoUtils.resolveSecret(
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`); config.customPassword,
);
const passwordFilePath = path.join(
"/tmp",
`zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`,
);
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 }); await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
env.RESTIC_PASSWORD_FILE = passwordFilePath; env.RESTIC_PASSWORD_FILE = passwordFilePath;
} else { } else {
const org = await db.query.organization.findFirst({ where: { id: organizationId } }); const org = await db.query.organization.findFirst({
where: { id: organizationId },
});
if (!org) { if (!org) {
throw new Error(`Organization ${organizationId} not found`); throw new Error(`Organization ${organizationId} not found`);
@ -88,10 +110,17 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
const metadata = org.metadata; const metadata = org.metadata;
if (!metadata?.resticPassword) { if (!metadata?.resticPassword) {
throw new Error(`Restic password not configured for organization ${organizationId}`); throw new Error(
`Restic password not configured for organization ${organizationId}`,
);
} else { } else {
const decryptedPassword = await cryptoUtils.resolveSecret(metadata.resticPassword); const decryptedPassword = await cryptoUtils.resolveSecret(
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`); metadata.resticPassword,
);
const passwordFilePath = path.join(
"/tmp",
`zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`,
);
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 }); await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
env.RESTIC_PASSWORD_FILE = passwordFilePath; env.RESTIC_PASSWORD_FILE = passwordFilePath;
} }
@ -99,8 +128,12 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
switch (config.backend) { switch (config.backend) {
case "s3": case "s3":
env.AWS_ACCESS_KEY_ID = await cryptoUtils.resolveSecret(config.accessKeyId); env.AWS_ACCESS_KEY_ID = await cryptoUtils.resolveSecret(
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.resolveSecret(config.secretAccessKey); config.accessKeyId,
);
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.resolveSecret(
config.secretAccessKey,
);
// Huawei OBS requires virtual-hosted style access // Huawei OBS requires virtual-hosted style access
if (config.endpoint.includes("myhuaweicloud")) { if (config.endpoint.includes("myhuaweicloud")) {
@ -108,22 +141,35 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
} }
break; break;
case "r2": case "r2":
env.AWS_ACCESS_KEY_ID = await cryptoUtils.resolveSecret(config.accessKeyId); env.AWS_ACCESS_KEY_ID = await cryptoUtils.resolveSecret(
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.resolveSecret(config.secretAccessKey); config.accessKeyId,
);
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.resolveSecret(
config.secretAccessKey,
);
env.AWS_REGION = "auto"; env.AWS_REGION = "auto";
env.AWS_S3_FORCE_PATH_STYLE = "true"; env.AWS_S3_FORCE_PATH_STYLE = "true";
break; break;
case "gcs": { case "gcs": {
const decryptedCredentials = await cryptoUtils.resolveSecret(config.credentialsJson); const decryptedCredentials = await cryptoUtils.resolveSecret(
const credentialsPath = path.join("/tmp", `zerobyte-gcs-${crypto.randomBytes(8).toString("hex")}.json`); config.credentialsJson,
await fs.writeFile(credentialsPath, decryptedCredentials, { mode: 0o600 }); );
const credentialsPath = path.join(
"/tmp",
`zerobyte-gcs-${crypto.randomBytes(8).toString("hex")}.json`,
);
await fs.writeFile(credentialsPath, decryptedCredentials, {
mode: 0o600,
});
env.GOOGLE_PROJECT_ID = config.projectId; env.GOOGLE_PROJECT_ID = config.projectId;
env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath; env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;
break; break;
} }
case "azure": { case "azure": {
env.AZURE_ACCOUNT_NAME = config.accountName; env.AZURE_ACCOUNT_NAME = config.accountName;
env.AZURE_ACCOUNT_KEY = await cryptoUtils.resolveSecret(config.accountKey); env.AZURE_ACCOUNT_KEY = await cryptoUtils.resolveSecret(
config.accountKey,
);
if (config.endpointSuffix) { if (config.endpointSuffix) {
env.AZURE_ENDPOINT_SUFFIX = config.endpointSuffix; env.AZURE_ENDPOINT_SUFFIX = config.endpointSuffix;
} }
@ -131,16 +177,23 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
} }
case "rest": { case "rest": {
if (config.username) { if (config.username) {
env.RESTIC_REST_USERNAME = await cryptoUtils.resolveSecret(config.username); env.RESTIC_REST_USERNAME = await cryptoUtils.resolveSecret(
config.username,
);
} }
if (config.password) { if (config.password) {
env.RESTIC_REST_PASSWORD = await cryptoUtils.resolveSecret(config.password); env.RESTIC_REST_PASSWORD = await cryptoUtils.resolveSecret(
config.password,
);
} }
break; break;
} }
case "sftp": { case "sftp": {
const decryptedKey = await cryptoUtils.resolveSecret(config.privateKey); const decryptedKey = await cryptoUtils.resolveSecret(config.privateKey);
const keyPath = path.join("/tmp", `zerobyte-ssh-${crypto.randomBytes(8).toString("hex")}`); const keyPath = path.join(
"/tmp",
`zerobyte-ssh-${crypto.randomBytes(8).toString("hex")}`,
);
let normalizedKey = decryptedKey.replace(/\r\n/g, "\n"); let normalizedKey = decryptedKey.replace(/\r\n/g, "\n");
if (!normalizedKey.endsWith("\n")) { if (!normalizedKey.endsWith("\n")) {
@ -148,8 +201,12 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
} }
if (normalizedKey.includes("ENCRYPTED")) { if (normalizedKey.includes("ENCRYPTED")) {
logger.error("SFTP: Private key appears to be passphrase-protected. Please use an unencrypted key."); logger.error(
throw new Error("Passphrase-protected SSH keys are not supported. Please provide an unencrypted private key."); "SFTP: Private key appears to be passphrase-protected. Please use an unencrypted key.",
);
throw new Error(
"Passphrase-protected SSH keys are not supported. Please provide an unencrypted private key.",
);
} }
await fs.writeFile(keyPath, normalizedKey, { mode: 0o600 }); await fs.writeFile(keyPath, normalizedKey, { mode: 0o600 });
@ -184,12 +241,25 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
]; ];
if (config.skipHostKeyCheck || !config.knownHosts) { if (config.skipHostKeyCheck || !config.knownHosts) {
sshArgs.push("-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"); sshArgs.push(
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
);
} else if (config.knownHosts) { } else if (config.knownHosts) {
const knownHostsPath = path.join("/tmp", `zerobyte-known-hosts-${crypto.randomBytes(8).toString("hex")}`); const knownHostsPath = path.join(
"/tmp",
`zerobyte-known-hosts-${crypto.randomBytes(8).toString("hex")}`,
);
await fs.writeFile(knownHostsPath, config.knownHosts, { mode: 0o600 }); await fs.writeFile(knownHostsPath, config.knownHosts, { mode: 0o600 });
env._SFTP_KNOWN_HOSTS_PATH = knownHostsPath; env._SFTP_KNOWN_HOSTS_PATH = knownHostsPath;
sshArgs.push("-o", "StrictHostKeyChecking=yes", "-o", `UserKnownHostsFile=${knownHostsPath}`); sshArgs.push(
"-o",
"StrictHostKeyChecking=yes",
"-o",
`UserKnownHostsFile=${knownHostsPath}`,
);
} }
if (config.port && config.port !== 22) { if (config.port && config.port !== 22) {
@ -204,7 +274,10 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
if (config.cacert) { if (config.cacert) {
const decryptedCert = await cryptoUtils.resolveSecret(config.cacert); const decryptedCert = await cryptoUtils.resolveSecret(config.cacert);
const certPath = path.join("/tmp", `zerobyte-cacert-${crypto.randomBytes(8).toString("hex")}.pem`); const certPath = path.join(
"/tmp",
`zerobyte-cacert-${crypto.randomBytes(8).toString("hex")}.pem`,
);
await fs.writeFile(certPath, decryptedCert, { mode: 0o600 }); await fs.writeFile(certPath, decryptedCert, { mode: 0o600 });
env.RESTIC_CACERT = certPath; env.RESTIC_CACERT = certPath;
} }
@ -216,7 +289,11 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
return env; return env;
}; };
const init = async (config: RepositoryConfig, organizationId: string, options?: { timeoutMs?: number }) => { const init = async (
config: RepositoryConfig,
organizationId: string,
options?: { timeoutMs?: number },
) => {
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
logger.info(`Initializing restic repository at ${repoUrl}...`); logger.info(`Initializing restic repository at ${repoUrl}...`);
@ -226,7 +303,12 @@ const init = async (config: RepositoryConfig, organizationId: string, options?:
const args = ["init", "--repo", repoUrl]; const args = ["init", "--repo", repoUrl];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env, timeout: options?.timeoutMs ?? 20000 }); const res = await exec({
command: "restic",
args,
env,
timeout: options?.timeoutMs ?? 20000,
});
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (res.exitCode !== 0) { if (res.exitCode !== 0) {
@ -256,7 +338,13 @@ const backup = async (
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, options?.organizationId); const env = await buildEnv(config, options?.organizationId);
const args: string[] = ["--repo", repoUrl, "backup", "--compression", options?.compressionMode ?? "auto"]; const args: string[] = [
"--repo",
repoUrl,
"backup",
"--compression",
options?.compressionMode ?? "auto",
];
if (options?.oneFileSystem) { if (options?.oneFileSystem) {
args.push("--one-file-system"); args.push("--one-file-system");
@ -274,7 +362,9 @@ const backup = async (
let includeFile: string | null = null; let includeFile: string | null = null;
if (options?.include && options.include.length > 0) { if (options?.include && options.include.length > 0) {
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-restic-include-")); const tmp = await fs.mkdtemp(
path.join(os.tmpdir(), "zerobyte-restic-include-"),
);
includeFile = path.join(tmp, `include.txt`); includeFile = path.join(tmp, `include.txt`);
await fs.writeFile(includeFile, options.include.join("\n"), "utf-8"); await fs.writeFile(includeFile, options.include.join("\n"), "utf-8");
@ -290,7 +380,9 @@ const backup = async (
let excludeFile: string | null = null; let excludeFile: string | null = null;
if (options?.exclude && options.exclude.length > 0) { if (options?.exclude && options.exclude.length > 0) {
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-restic-exclude-")); const tmp = await fs.mkdtemp(
path.join(os.tmpdir(), "zerobyte-restic-exclude-"),
);
excludeFile = path.join(tmp, `exclude.txt`); excludeFile = path.join(tmp, `exclude.txt`);
await fs.writeFile(excludeFile, options.exclude.join("\n"), "utf-8"); await fs.writeFile(excludeFile, options.exclude.join("\n"), "utf-8");
@ -379,6 +471,17 @@ const backup = async (
return { result, exitCode: res.exitCode }; return { result, exitCode: res.exitCode };
}; };
const restoreProgressSchema = type({
message_type: "'status'",
seconds_elapsed: "number",
percent_done: "number",
total_files: "number",
files_done: "number",
total_bytes: "number",
bytes_done: "number",
});
export type RestoreProgress = typeof restoreProgressSchema.infer;
const restore = async ( const restore = async (
config: RepositoryConfig, config: RepositoryConfig,
snapshotId: string, snapshotId: string,
@ -390,17 +493,30 @@ const restore = async (
excludeXattr?: string[]; excludeXattr?: string[];
delete?: boolean; delete?: boolean;
overwrite?: OverwriteMode; overwrite?: OverwriteMode;
onProgress?: (progress: RestoreProgress) => void;
signal?: AbortSignal;
}, },
) => { ) => {
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, options.organizationId); const env = await buildEnv(config, options.organizationId);
const args: string[] = ["--repo", repoUrl, "restore", snapshotId, "--target", target]; const args: string[] = [
"--repo",
repoUrl,
"restore",
snapshotId,
"--target",
target,
];
if (options?.overwrite) { if (options?.overwrite) {
args.push("--overwrite", options.overwrite); args.push("--overwrite", options.overwrite);
} }
if (options?.delete) {
args.push("--delete");
}
if (options?.include?.length) { if (options?.include?.length) {
for (const pattern of options.include) { for (const pattern of options.include) {
args.push("--include", pattern); args.push("--include", pattern);
@ -421,8 +537,32 @@ const restore = async (
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const streamProgress = throttle((data: string) => {
if (options?.onProgress) {
try {
const jsonData = JSON.parse(data);
const progress = restoreProgressSchema(jsonData);
if (!(progress instanceof type.errors)) {
options.onProgress(progress);
}
} catch (_) {
// Ignore JSON parse errors for non-JSON lines
}
}
}, 1000);
logger.debug(`Executing: restic ${args.join(" ")}`); logger.debug(`Executing: restic ${args.join(" ")}`);
const res = await safeSpawn({ command: "restic", args, env }); const res = await safeSpawn({
command: "restic",
args,
env,
signal: options?.signal,
onStdout: (data) => {
if (options?.onProgress) {
streamProgress(data);
}
},
});
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
@ -432,21 +572,27 @@ const restore = async (
} }
const lastLine = res.summary.trim(); const lastLine = res.summary.trim();
let summaryLine = ""; let summaryLine: unknown = {};
try { try {
const resSummary = JSON.parse(lastLine ?? "{}"); summaryLine = JSON.parse(lastLine ?? "{}");
summaryLine = resSummary;
} catch (_) { } catch (_) {
logger.warn("Failed to parse restic backup output JSON summary.", lastLine); logger.warn(
summaryLine = "{}"; "Failed to parse restic restore output JSON summary.",
lastLine,
);
summaryLine = {};
} }
logger.debug(`Restic restore output last line: ${summaryLine}`); logger.debug(
`Restic restore output last line: ${JSON.stringify(summaryLine)}`,
);
const result = resticRestoreOutputSchema(summaryLine); const result = resticRestoreOutputSchema(summaryLine);
if (result instanceof type.errors) { if (result instanceof type.errors) {
logger.warn(`Restic restore output validation failed: ${result.summary}`); logger.warn(`Restic restore output validation failed: ${result.summary}`);
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`); logger.info(
`Restic restore completed for snapshot ${snapshotId} to target ${target}`,
);
const fallback: ResticRestoreOutputDto = { const fallback: ResticRestoreOutputDto = {
message_type: "summary" as const, message_type: "summary" as const,
total_files: 0, total_files: 0,
@ -465,7 +611,10 @@ const restore = async (
return result; return result;
}; };
const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; organizationId: string }) => { const snapshots = async (
config: RepositoryConfig,
options: { tags?: string[]; organizationId: string },
) => {
const { tags, organizationId } = options; const { tags, organizationId } = options;
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
@ -492,8 +641,12 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[]; o
const result = snapshotInfoSchema.array()(JSON.parse(res.stdout)); const result = snapshotInfoSchema.array()(JSON.parse(res.stdout));
if (result instanceof type.errors) { if (result instanceof type.errors) {
logger.error(`Restic snapshots output validation failed: ${result.summary}`); logger.error(
throw new Error(`Restic snapshots output validation failed: ${result.summary}`); `Restic snapshots output validation failed: ${result.summary}`,
);
throw new Error(
`Restic snapshots output validation failed: ${result.summary}`,
);
} }
return result; return result;
@ -540,7 +693,15 @@ const forget = async (
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, extra.organizationId); const env = await buildEnv(config, extra.organizationId);
const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag]; const args: string[] = [
"--repo",
repoUrl,
"forget",
"--group-by",
"tags",
"--tag",
extra.tag,
];
if (extra.dryRun) { if (extra.dryRun) {
args.push("--dry-run", "--no-lock"); args.push("--dry-run", "--no-lock");
@ -583,12 +744,18 @@ const forget = async (
} }
const lines = res.stdout.split("\n").filter((line) => line.trim()); const lines = res.stdout.split("\n").filter((line) => line.trim());
const result = extra.dryRun ? safeJsonParse<ResticForgetResponse>(lines.at(-1) ?? "[]") : null; const result = extra.dryRun
? safeJsonParse<ResticForgetResponse>(lines.at(-1) ?? "[]")
: null;
return { success: true, data: result }; return { success: true, data: result };
}; };
const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[], organizationId: string) => { const deleteSnapshots = async (
config: RepositoryConfig,
snapshotIds: string[],
organizationId: string,
) => {
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, organizationId); const env = await buildEnv(config, organizationId);
@ -596,7 +763,13 @@ const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[],
throw new Error("No snapshot IDs provided for deletion."); throw new Error("No snapshot IDs provided for deletion.");
} }
const args: string[] = ["--repo", repoUrl, "forget", ...snapshotIds, "--prune"]; const args: string[] = [
"--repo",
repoUrl,
"forget",
...snapshotIds,
"--prune",
];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env }); const res = await exec({ command: "restic", args, env });
@ -610,7 +783,11 @@ const deleteSnapshots = async (config: RepositoryConfig, snapshotIds: string[],
return { success: true }; return { success: true };
}; };
const deleteSnapshot = async (config: RepositoryConfig, snapshotId: string, organizationId: string) => { const deleteSnapshot = async (
config: RepositoryConfig,
snapshotId: string,
organizationId: string,
) => {
return deleteSnapshots(config, [snapshotId], organizationId); return deleteSnapshots(config, [snapshotId], organizationId);
}; };
@ -777,14 +954,22 @@ const ls = async (
}; };
}; };
const unlock = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => { const unlock = async (
config: RepositoryConfig,
options: { signal?: AbortSignal; organizationId: string },
) => {
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, options.organizationId); const env = await buildEnv(config, options.organizationId);
const args = ["unlock", "--repo", repoUrl, "--remove-all"]; const args = ["unlock", "--repo", repoUrl, "--remove-all"];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env, signal: options?.signal }); const res = await exec({
command: "restic",
args,
env,
signal: options?.signal,
});
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) { if (options?.signal?.aborted) {
@ -816,12 +1001,22 @@ const check = async (
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env, signal: options?.signal }); const res = await exec({
command: "restic",
args,
env,
signal: options?.signal,
});
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) { if (options?.signal?.aborted) {
logger.warn("Restic check was aborted by signal."); logger.warn("Restic check was aborted by signal.");
return { success: false, hasErrors: true, output: "", error: "Operation aborted" }; return {
success: false,
hasErrors: true,
output: "",
error: "Operation aborted",
};
} }
const { stdout, stderr } = res; const { stdout, stderr } = res;
@ -847,14 +1042,22 @@ const check = async (
}; };
}; };
const repairIndex = async (config: RepositoryConfig, options: { signal?: AbortSignal; organizationId: string }) => { const repairIndex = async (
config: RepositoryConfig,
options: { signal?: AbortSignal; organizationId: string },
) => {
const repoUrl = buildRepoUrl(config); const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config, options.organizationId); const env = await buildEnv(config, options.organizationId);
const args = ["repair", "index", "--repo", repoUrl]; const args = ["repair", "index", "--repo", repoUrl];
addCommonArgs(args, env, config); addCommonArgs(args, env, config);
const res = await exec({ command: "restic", args, env, signal: options?.signal }); const res = await exec({
command: "restic",
args,
env,
signal: options?.signal,
});
await cleanupTemporaryKeys(env); await cleanupTemporaryKeys(env);
if (options?.signal?.aborted) { if (options?.signal?.aborted) {
@ -894,7 +1097,13 @@ const copy = async (
RESTIC_FROM_PASSWORD_FILE: sourceEnv.RESTIC_PASSWORD_FILE, RESTIC_FROM_PASSWORD_FILE: sourceEnv.RESTIC_PASSWORD_FILE,
}; };
const args: string[] = ["--repo", destRepoUrl, "copy", "--from-repo", sourceRepoUrl]; const args: string[] = [
"--repo",
destRepoUrl,
"copy",
"--from-repo",
sourceRepoUrl,
];
if (options.tag) { if (options.tag) {
args.push("--tag", options.tag); args.push("--tag", options.tag);
@ -1023,7 +1232,12 @@ export const restic = {
}; };
export const cleanupTemporaryKeys = async (env: Record<string, string>) => { export const cleanupTemporaryKeys = async (env: Record<string, string>) => {
const keysToClean = ["_SFTP_KEY_PATH", "_SFTP_KNOWN_HOSTS_PATH", "RESTIC_CACERT", "GOOGLE_APPLICATION_CREDENTIALS"]; const keysToClean = [
"_SFTP_KEY_PATH",
"_SFTP_KNOWN_HOSTS_PATH",
"RESTIC_CACERT",
"GOOGLE_APPLICATION_CREDENTIALS",
];
for (const key of keysToClean) { for (const key of keysToClean) {
if (env[key]) { if (env[key]) {
@ -1032,7 +1246,10 @@ export const cleanupTemporaryKeys = async (env: Record<string, string>) => {
} }
// Clean up custom password files // Clean up custom password files
if (env.RESTIC_PASSWORD_FILE && env.RESTIC_PASSWORD_FILE !== RESTIC_PASS_FILE) { if (
env.RESTIC_PASSWORD_FILE &&
env.RESTIC_PASSWORD_FILE !== RESTIC_PASS_FILE
) {
await fs.unlink(env.RESTIC_PASSWORD_FILE).catch(() => {}); await fs.unlink(env.RESTIC_PASSWORD_FILE).catch(() => {});
} }
}; };