From 5a4f4641b440f5c7b806184b3ebe80739be4596f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C3=A1vn=C3=ADk?= Date: Mon, 29 Dec 2025 18:34:02 +0100 Subject: [PATCH] feat: improve config import with result tracking and idempotent repository detection - Add ImportResult type to track succeeded/warnings/errors counts across all import functions - Implement multi-layer repository duplicate detection: - URL-based check (same path/bucket/endpoint already registered) - Restic repo check (path is already a restic repository) - Name-based fallback check - Standardize logging between CLI and env var import methods with logImportSummary() - CLI exits with code 1 when import has errors - Remove shortId override for local repo migrations (use full path with isExistingRepository instead) - Update example JSON and README: - Document that local repo path is optional (defaults to /var/lib/zerobyte/repositories) - Add existing-local-repo example with isExistingRepository: true - Add S3 endpoint field to example - Expand config behavior docs to explain all repository skip conditions - Improve .gitignore to exclude all JSON except example template --- app/server/cli/commands/import-config.ts | 11 +- app/server/modules/lifecycle/config-import.ts | 183 +++++++++++++++--- .../repositories/repositories.service.ts | 18 +- examples/config-file-import/.gitignore | 4 +- examples/config-file-import/README.md | 27 ++- .../zerobyte.config.example.json | 12 +- 6 files changed, 197 insertions(+), 58 deletions(-) diff --git a/app/server/cli/commands/import-config.ts b/app/server/cli/commands/import-config.ts index ee94ef5e..8462a3bd 100644 --- a/app/server/cli/commands/import-config.ts +++ b/app/server/cli/commands/import-config.ts @@ -92,20 +92,21 @@ export const importConfigCommand = new Command("import-config") return; } - console.log("šŸš€ Starting import...\n"); - try { // Ensure database is initialized with migrations const { runDbMigrations } = await import("../../db/db"); runDbMigrations(); const { applyConfigImport } = await import("../../modules/lifecycle/config-import"); - await applyConfigImport(config); + const result = await applyConfigImport(config); - console.log("\nāœ… Import completed successfully"); + // Exit with error code if there were errors + if (result.errors > 0) { + process.exit(1); + } } catch (e) { const err = e instanceof Error ? e : new Error(String(e)); - console.error(`\nāŒ Import failed: ${err.message}`); + console.error(`āŒ Import failed: ${err.message}`); process.exit(1); } }); diff --git a/app/server/modules/lifecycle/config-import.ts b/app/server/modules/lifecycle/config-import.ts index 1f3024ba..250f38f9 100644 --- a/app/server/modules/lifecycle/config-import.ts +++ b/app/server/modules/lifecycle/config-import.ts @@ -34,6 +34,12 @@ type ImportConfig = { recoveryKey: string | null; }; +export type ImportResult = { + succeeded: number; + warnings: number; + errors: number; +}; + function interpolateEnvVars(value: unknown): unknown { if (typeof value === "string") { return value.replace(/\$\{([^}]+)\}/g, (_, v) => { @@ -94,11 +100,13 @@ function parseImportConfig(configRaw: unknown): ImportConfig { }; } -async function writeRecoveryKeyFromConfig(recoveryKey: string | null): Promise { +async function writeRecoveryKeyFromConfig(recoveryKey: string | null): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; + try { const fs = await import("node:fs/promises"); const { RESTIC_PASS_FILE } = await import("../../core/constants.js"); - if (!recoveryKey) return; + if (!recoveryKey) return result; if (typeof recoveryKey !== "string" || recoveryKey.length !== 64 || !/^[a-fA-F0-9]{64}$/.test(recoveryKey)) { throw new Error("Recovery key must be a 64-character hex string"); @@ -108,18 +116,25 @@ async function writeRecoveryKeyFromConfig(recoveryKey: string | null): Promise false, ); if (passFileExists) { - logger.info(`Restic passfile already exists at ${RESTIC_PASS_FILE}; skipping config recovery key write`); - return; + logger.warn(`Restic passfile already exists at ${RESTIC_PASS_FILE}; skipping config recovery key write`); + result.warnings++; + return result; } await fs.writeFile(RESTIC_PASS_FILE, recoveryKey, { mode: 0o600 }); logger.info(`Recovery key written from config to ${RESTIC_PASS_FILE}`); + result.succeeded++; } catch (err) { const e = err instanceof Error ? err : new Error(String(err)); logger.error(`Failed to write recovery key from config: ${e.message}`); + result.errors++; } + + return result; } -async function importVolumes(volumes: unknown[]): Promise { +async function importVolumes(volumes: unknown[]): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; + for (const v of volumes) { try { if (!isRecord(v) || typeof v.name !== "string" || !isRecord(v.config) || typeof v.config.backend !== "string") { @@ -127,6 +142,7 @@ async function importVolumes(volumes: unknown[]): Promise { } await volumeService.createVolume(v.name, v.config as BackendConfig); logger.info(`Initialized volume from config: ${v.name}`); + result.succeeded++; // If autoRemount is explicitly false, update the volume (default is true) if (v.autoRemount === false) { @@ -136,17 +152,78 @@ async function importVolumes(volumes: unknown[]): Promise { } catch (e) { const err = e instanceof Error ? e : new Error(String(e)); logger.warn(`Volume not created: ${err.message}`); + result.warnings++; } } + + return result; } -async function importRepositories(repositories: unknown[]): Promise { +async function importRepositories(repositories: unknown[]): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; const repoServiceModule = await import("../repositories/repositories.service"); + const { buildRepoUrl, restic } = await import("../../utils/restic"); + + // Get existing repositories and build sets for duplicate detection + const existingRepos = await repoServiceModule.repositoriesService.listRepositories(); + const existingNames = new Set(existingRepos.map((repo) => repo.name)); + const existingUrls = new Set(); + + for (const repo of existingRepos) { + try { + // Config fields used for URL (path, bucket, endpoint, etc.) are not encrypted + const url = buildRepoUrl(repo.config as RepositoryConfig); + existingUrls.add(url); + } catch (e) { + const err = e instanceof Error ? e : new Error(String(e)); + logger.warn(`Could not build URL for existing repository '${repo.name}': ${err.message}`); + } + } + for (const r of repositories) { try { if (!isRecord(r) || typeof r.name !== "string" || !isRecord(r.config) || typeof r.config.backend !== "string") { throw new Error("Invalid repository entry"); } + + // Skip if a repository pointing to the same location is already registered in DB + try { + const incomingUrl = buildRepoUrl(r.config as RepositoryConfig); + if (existingUrls.has(incomingUrl)) { + logger.warn(`Skipping '${r.name}': another repository is already registered for location ${incomingUrl}`); + result.warnings++; + continue; + } + } catch (e) { + const err = e instanceof Error ? e : new Error(String(e)); + logger.warn(`Could not build URL for '${r.name}' to check duplicates: ${err.message}`); + } + + // For local repos without isExistingRepository, check if the provided path is already a restic repo + // This catches the case where user forgot to set isExistingRepository: true + if (r.config.backend === "local" && !r.config.isExistingRepository) { + const isAlreadyRepo = await restic + .snapshots({ ...r.config, isExistingRepository: true } as RepositoryConfig) + .then(() => true) + .catch(() => false); + + if (isAlreadyRepo) { + logger.warn( + `Skipping '${r.name}': path '${r.config.path}' is already a restic repository. ` + + `Set "isExistingRepository": true to import it, or use a different path for a new repository.`, + ); + result.warnings++; + continue; + } + } + + // Skip if a repository with the same name already exists (fallback for repos without deterministic paths) + if (existingNames.has(r.name)) { + logger.warn(`Skipping '${r.name}': a repository with this name already exists`); + result.warnings++; + continue; + } + const compressionMode = r.compressionMode === "auto" || r.compressionMode === "off" || r.compressionMode === "max" ? r.compressionMode @@ -157,14 +234,19 @@ async function importRepositories(repositories: unknown[]): Promise { compressionMode, ); logger.info(`Initialized repository from config: ${r.name}`); + result.succeeded++; } catch (e) { const err = e instanceof Error ? e : new Error(String(e)); logger.warn(`Repository not created: ${err.message}`); + result.warnings++; } } + + return result; } -async function importNotificationDestinations(notificationDestinations: unknown[]): Promise { +async function importNotificationDestinations(notificationDestinations: unknown[]): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; const notificationsServiceModule = await import("../notifications/notifications.service"); for (const n of notificationDestinations) { try { @@ -176,6 +258,7 @@ async function importNotificationDestinations(notificationDestinations: unknown[ n.config as NotificationConfig, ); logger.info(`Initialized notification destination from config: ${n.name}`); + result.succeeded++; // If enabled is explicitly false, update the destination (default is true) if (n.enabled === false) { @@ -185,8 +268,11 @@ async function importNotificationDestinations(notificationDestinations: unknown[ } catch (e) { const err = e instanceof Error ? e : new Error(String(e)); logger.warn(`Notification destination not created: ${err.message}`); + result.warnings++; } } + + return result; } function getScheduleVolumeName(schedule: Record): string | null { @@ -261,8 +347,9 @@ async function attachScheduleNotifications( } } -async function importBackupSchedules(backupSchedules: unknown[]): Promise { - if (!Array.isArray(backupSchedules) || backupSchedules.length === 0) return; +async function importBackupSchedules(backupSchedules: unknown[]): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; + if (!Array.isArray(backupSchedules) || backupSchedules.length === 0) return result; const backupServiceModule = await import("../backups/backups.service"); const notificationsServiceModule = await import("../notifications/notifications.service"); @@ -282,28 +369,33 @@ async function importBackupSchedules(backupSchedules: unknown[]): Promise const volumeName = getScheduleVolumeName(s); if (typeof volumeName !== "string" || volumeName.length === 0) { logger.warn("Backup schedule not created: Missing volume name"); + result.warnings++; continue; } const volume = volumeByName.get(volumeName); if (!volume) { logger.warn(`Backup schedule not created: Volume '${volumeName}' not found`); + result.warnings++; continue; } const repositoryName = getScheduleRepositoryName(s); if (typeof repositoryName !== "string" || repositoryName.length === 0) { logger.warn("Backup schedule not created: Missing repository name"); + result.warnings++; continue; } const repository = repoByName.get(repositoryName); if (!repository) { logger.warn(`Backup schedule not created: Repository '${repositoryName}' not found`); + result.warnings++; continue; } const scheduleName = typeof s.name === "string" && s.name.length > 0 ? s.name : `${volumeName}-${repositoryName}`; if (typeof s.cronExpression !== "string" || s.cronExpression.length === 0) { logger.warn(`Backup schedule not created: Missing cronExpression for '${scheduleName}'`); + result.warnings++; continue; } @@ -335,9 +427,11 @@ async function importBackupSchedules(backupSchedules: unknown[]): Promise oneFileSystem: typeof s.oneFileSystem === "boolean" ? s.oneFileSystem : undefined, }); logger.info(`Initialized backup schedule from config: ${scheduleName}`); + result.succeeded++; } catch (e) { const err = e instanceof Error ? e : new Error(String(e)); logger.warn(`Backup schedule not created: ${err.message}`); + result.warnings++; continue; } @@ -354,6 +448,8 @@ async function importBackupSchedules(backupSchedules: unknown[]): Promise await attachScheduleMirrors(createdSchedule.id, s.mirrors, repoByName, backupServiceModule); } } + + return result; } async function attachScheduleMirrors( @@ -403,17 +499,20 @@ async function attachScheduleMirrors( } } -async function setupInitialUser(users: unknown[], recoveryKey: string | null): Promise { +async function setupInitialUser(users: unknown[], recoveryKey: string | null): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; + try { const { authService } = await import("../auth/auth.service"); const hasUsers = await authService.hasUsers(); - if (hasUsers) return; - if (!Array.isArray(users) || users.length === 0) return; + if (hasUsers) return result; + if (!Array.isArray(users) || users.length === 0) return result; if (users.length > 1) { logger.warn( "Multiple users provided in config. Zerobyte currently supports a single initial user; extra entries will be ignored.", ); + result.warnings++; } for (const u of users) { @@ -429,10 +528,12 @@ async function setupInitialUser(users: unknown[], recoveryKey: string | null): P typeof u.hasDownloadedResticPassword === "boolean" ? u.hasDownloadedResticPassword : Boolean(recoveryKey), }); logger.info(`User '${u.username}' imported with password hash from config.`); + result.succeeded++; break; } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); logger.warn(`User '${u.username}' not imported: ${err.message}`); + result.warnings++; } continue; } @@ -446,38 +547,70 @@ async function setupInitialUser(users: unknown[], recoveryKey: string | null): P await db.update(usersTable).set({ hasDownloadedResticPassword }).where(eq(usersTable.id, user.id)); } logger.info(`User '${u.username}' created from config.`); + result.succeeded++; break; } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); logger.warn(`User '${u.username}' not created: ${err.message}`); + result.warnings++; } continue; } logger.warn(`User '${u.username}' missing passwordHash/password; skipping`); + result.warnings++; } } catch (err) { const e = err instanceof Error ? err : new Error(String(err)); logger.error(`Automated user setup failed: ${e.message}`); + result.errors++; } + + return result; } -async function runImport(config: ImportConfig): Promise { - await writeRecoveryKeyFromConfig(config.recoveryKey); +async function runImport(config: ImportConfig): Promise { + const result: ImportResult = { succeeded: 0, warnings: 0, errors: 0 }; - await importVolumes(config.volumes); - await importRepositories(config.repositories); - await importNotificationDestinations(config.notificationDestinations); - await importBackupSchedules(config.backupSchedules); - await setupInitialUser(config.users, config.recoveryKey); + const recoveryKeyResult = await writeRecoveryKeyFromConfig(config.recoveryKey); + const volumeResult = await importVolumes(config.volumes); + const repoResult = await importRepositories(config.repositories); + const notifResult = await importNotificationDestinations(config.notificationDestinations); + const scheduleResult = await importBackupSchedules(config.backupSchedules); + const userResult = await setupInitialUser(config.users, config.recoveryKey); + + for (const r of [recoveryKeyResult, volumeResult, repoResult, notifResult, scheduleResult, userResult]) { + result.succeeded += r.succeeded; + result.warnings += r.warnings; + result.errors += r.errors; + } + + return result; +} + +function logImportSummary(result: ImportResult): void { + if (result.errors > 0) { + logger.error( + `Config import completed with ${result.errors} error(s) and ${result.warnings} warning(s), ${result.succeeded} item(s) imported`, + ); + } else if (result.warnings > 0) { + logger.warn(`Config import completed with ${result.warnings} warning(s), ${result.succeeded} item(s) imported`); + } else if (result.succeeded > 0) { + logger.info(`Config import completed successfully: ${result.succeeded} item(s) imported`); + } else { + logger.info("Config import completed: no items to import"); + } } /** * Import configuration from a raw config object (used by CLI) */ -export async function applyConfigImport(configRaw: unknown): Promise { +export async function applyConfigImport(configRaw: unknown): Promise { + logger.info("Starting config import..."); const config = parseImportConfig(configRaw); - await runImport(config); + const result = await runImport(config); + logImportSummary(result); + return result; } /** @@ -485,12 +618,16 @@ export async function applyConfigImport(configRaw: unknown): Promise { */ export async function applyConfigImportFromFile(): Promise { const configRaw = await loadConfigFromFile(); + if (configRaw === null) return; // No config file, nothing to do + + logger.info("Starting config import from file..."); const config = parseImportConfig(configRaw); try { - await runImport(config); + const result = await runImport(config); + logImportSummary(result); } catch (e) { const err = e instanceof Error ? e : new Error(String(e)); - logger.error(`Failed to initialize from config: ${err.message}`); + logger.error(`Config import failed: ${err.message}`); } } diff --git a/app/server/modules/repositories/repositories.service.ts b/app/server/modules/repositories/repositories.service.ts index 8f8c414f..2656dbd4 100644 --- a/app/server/modules/repositories/repositories.service.ts +++ b/app/server/modules/repositories/repositories.service.ts @@ -64,23 +64,7 @@ const encryptConfig = async (config: RepositoryConfig): Promise { const id = crypto.randomUUID(); - - // Determine shortId: use provided config.name for local repo migrations, otherwise generate - let shortId: string; - if (config.backend === "local" && config.name?.length) { - // User provided a name (migration scenario) - check for conflicts - shortId = config.name; - const existingByShortId = await db.query.repositoriesTable.findFirst({ - where: eq(repositoriesTable.shortId, shortId), - }); - if (existingByShortId) { - throw new ConflictError( - `A repository with shortId '${shortId}' already exists. The shortId is used as the subdirectory name for local repositories.`, - ); - } - } else { - shortId = generateShortId(); - } + const shortId = generateShortId(); let processedConfig = config; if (config.backend === "local" && !config.isExistingRepository) { diff --git a/examples/config-file-import/.gitignore b/examples/config-file-import/.gitignore index 9961ad53..481fcd55 100644 --- a/examples/config-file-import/.gitignore +++ b/examples/config-file-import/.gitignore @@ -1,2 +1,4 @@ .env -zerobyte.config.json + +*.json +!zerobyte.config.example.json diff --git a/examples/config-file-import/README.md b/examples/config-file-import/README.md index 3e37f353..16d3adbc 100644 --- a/examples/config-file-import/README.md +++ b/examples/config-file-import/README.md @@ -160,8 +160,11 @@ See the runnable example: The config file is applied on startup using a **create-only** approach: -- Resources defined in the config are only created if they don't already exist in the database -- Existing resources with the same name are **not overwritten** (a warning is logged and the config entry is skipped) +- **Volumes, notifications, schedules**: Skipped if a resource with the same name already exists +- **Repositories**: Skipped if any of these conditions are met: + - A repository pointing to the same location (path/bucket/endpoint) is already registered + - For local repos: the path is already a restic repository (set `isExistingRepository: true` to import it) + - A repository with the same name already exists - Changes made via the UI are preserved across container restarts - To update a resource from config, either modify it via the UI or delete it first @@ -303,36 +306,39 @@ For key-based authentication: ### Repository types -#### Local +#### Local (new repository) + +Creates a new restic repository. The `path` is optional and defaults to `/var/lib/zerobyte/repositories`: ```json { "name": "local-repo", "config": { - "backend": "local", - "path": "/var/lib/zerobyte/repositories" + "backend": "local" }, "compressionMode": "auto" } ``` -Note for importing existing local repositories (migration): +The actual repository will be created at `{path}/{auto-generated-id}`. -- include `config.name` and set `config.isExistingRepository: true` -- the actual restic repo is stored at `{path}/{name}` +#### Local (existing repository) + +To import an existing restic repository, set `isExistingRepository: true` and provide the **full path to the repository root**: ```json { "name": "my-local-repo", "config": { "backend": "local", - "path": "/var/lib/zerobyte/repositories", - "name": "abc123", + "path": "/var/lib/zerobyte/repositories/abc123", "isExistingRepository": true } } ``` +Note: The `path` must point directly to the restic repository root (the directory containing `config`, `data/`, `keys/`, etc.). + #### S3-compatible ```json @@ -340,6 +346,7 @@ Note for importing existing local repositories (migration): "name": "backup-repo", "config": { "backend": "s3", + "endpoint": "s3.amazonaws.com", "bucket": "mybucket", "accessKeyId": "${ACCESS_KEY_ID}", "secretAccessKey": "${SECRET_ACCESS_KEY}" diff --git a/examples/config-file-import/zerobyte.config.example.json b/examples/config-file-import/zerobyte.config.example.json index 584fa7d1..9680f6d2 100644 --- a/examples/config-file-import/zerobyte.config.example.json +++ b/examples/config-file-import/zerobyte.config.example.json @@ -64,15 +64,23 @@ { "name": "local-repo", "config": { - "backend": "local", - "path": "/var/lib/zerobyte/repositories" + "backend": "local" }, "compressionMode": "auto" }, + { + "name": "existing-local-repo", + "config": { + "backend": "local", + "path": "/var/lib/zerobyte/repositories/abc123", + "isExistingRepository": true + } + }, { "name": "s3-repo", "config": { "backend": "s3", + "endpoint": "s3.amazonaws.com", "bucket": "mybucket", "accessKeyId": "${ACCESS_KEY_ID}", "secretAccessKey": "${SECRET_ACCESS_KEY}"