diff --git a/app/server/core/constants.ts b/app/server/core/constants.ts index 003b8a39..972407a9 100644 --- a/app/server/core/constants.ts +++ b/app/server/core/constants.ts @@ -5,5 +5,3 @@ export const DATABASE_URL = process.env.DATABASE_URL || "/var/lib/zerobyte/data/ export const RESTIC_PASS_FILE = "/var/lib/zerobyte/data/restic.pass"; export const DEFAULT_EXCLUDES = [DATABASE_URL, RESTIC_PASS_FILE, REPOSITORY_BASE]; - -export const REQUIRED_MIGRATIONS = []; // ["v0.21.1"] add this once re-tagging migration is removed diff --git a/app/server/index.ts b/app/server/index.ts index f2c51b6c..28bd6df4 100644 --- a/app/server/index.ts +++ b/app/server/index.ts @@ -2,14 +2,12 @@ import { createHonoServer } from "react-router-hono-server/bun"; import * as schema from "./db/schema"; import { setSchema, runDbMigrations } from "./db/db"; import { startup } from "./modules/lifecycle/startup"; -import { retagSnapshots } from "./modules/lifecycle/migration"; import { logger } from "./utils/logger"; import { shutdown } from "./modules/lifecycle/shutdown"; -import { REQUIRED_MIGRATIONS } from "./core/constants"; -import { validateRequiredMigrations } from "./modules/lifecycle/checkpoint"; import { createApp } from "./app"; import { config } from "./core/config"; import { runCLI } from "./cli"; +import { runMigrations } from "./modules/lifecycle/migrations"; setSchema(schema); @@ -22,9 +20,7 @@ const app = createApp(); runDbMigrations(); -await retagSnapshots(); -await validateRequiredMigrations(REQUIRED_MIGRATIONS); - +await runMigrations(); await startup(); export type AppType = typeof app; diff --git a/app/server/modules/lifecycle/checkpoint.ts b/app/server/modules/lifecycle/checkpoint.ts deleted file mode 100644 index d9f80a23..00000000 --- a/app/server/modules/lifecycle/checkpoint.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { eq, sql } from "drizzle-orm"; -import { db } from "../../db/db"; -import { appMetadataTable, usersTable } from "../../db/schema"; -import { logger } from "../../utils/logger"; - -const MIGRATION_KEY_PREFIX = "migration:"; - -export const recordMigrationCheckpoint = async (version: string): Promise => { - const key = `${MIGRATION_KEY_PREFIX}${version}`; - const now = Date.now(); - - await db - .insert(appMetadataTable) - .values({ - key, - value: JSON.stringify({ completedAt: new Date().toISOString() }), - createdAt: now, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: appMetadataTable.key, - set: { - value: JSON.stringify({ completedAt: new Date().toISOString() }), - updatedAt: now, - }, - }); - - logger.info(`Recorded migration checkpoint for ${version}`); -}; - -export const hasMigrationCheckpoint = async (version: string): Promise => { - const key = `${MIGRATION_KEY_PREFIX}${version}`; - const result = await db.query.appMetadataTable.findFirst({ - where: eq(appMetadataTable.key, key), - }); - return result !== undefined; -}; - -export const validateRequiredMigrations = async (requiredVersions: string[]): Promise => { - const userCount = await db.select({ count: sql`count(*)` }).from(usersTable); - const isFreshInstall = userCount[0]?.count === 0; - - if (isFreshInstall) { - logger.info("Fresh installation detected, skipping migration checkpoint validation."); - - for (const version of requiredVersions) { - const hasCheckpoint = await hasMigrationCheckpoint(version); - if (!hasCheckpoint) { - await recordMigrationCheckpoint(version); - } - } - - return; - } - - for (const version of requiredVersions) { - const hasCheckpoint = await hasMigrationCheckpoint(version); - if (!hasCheckpoint) { - logger.error(` -================================================================================ -MIGRATION ERROR: Required migration ${version} has not been run. - -You are attempting to start a version of Zerobyte that requires migration -checkpoints from previous versions. This typically happens when you skip -versions during an upgrade. - -To fix this: -1. First upgrade to version ${version} and run the application once -2. Validate that everything is still working correctly -3. Then upgrade to the current version - -================================================================================ -`); - process.exit(1); - } - } -}; - -export const getMigrationCheckpoints = async (): Promise<{ version: string; completedAt: string }[]> => { - const results = await db.query.appMetadataTable.findMany({ - where: (table, { like }) => like(table.key, `${MIGRATION_KEY_PREFIX}%`), - }); - - return results.map((r) => ({ - version: r.key.replace(MIGRATION_KEY_PREFIX, ""), - completedAt: JSON.parse(r.value).completedAt, - })); -}; diff --git a/app/server/modules/lifecycle/migrations.ts b/app/server/modules/lifecycle/migrations.ts new file mode 100644 index 00000000..83c0edb8 --- /dev/null +++ b/app/server/modules/lifecycle/migrations.ts @@ -0,0 +1,114 @@ +import { db } from "~/server/db/db"; +import { logger } from "../../utils/logger"; +import { v00001 } from "./migrations/00001-retag-snapshots"; +import { usersTable } from "~/server/db/schema"; +import { sql } from "drizzle-orm"; +import { eq } from "drizzle-orm"; +import { appMetadataTable } from "../../db/schema"; + +const MIGRATION_KEY_PREFIX = "migration:"; + +const recordMigrationCheckpoint = async (version: string): Promise => { + const key = `${MIGRATION_KEY_PREFIX}${version}`; + const now = Date.now(); + + await db + .insert(appMetadataTable) + .values({ key, value: JSON.stringify({ completedAt: new Date().toISOString() }), createdAt: now, updatedAt: now }) + .onConflictDoUpdate({ + target: appMetadataTable.key, + set: { value: JSON.stringify({ completedAt: new Date().toISOString() }), updatedAt: now }, + }); + + logger.info(`Recorded migration checkpoint for ${version}`); +}; + +const hasMigrationCheckpoint = async (id: string): Promise => { + const key = `${MIGRATION_KEY_PREFIX}${id}`; + const result = await db.query.appMetadataTable.findFirst({ + where: eq(appMetadataTable.key, key), + }); + return result !== undefined; +}; + +type MigrationEntity = { + execute: () => Promise<{ success: boolean; errors: Array<{ name: string; error: string }> }>; + id: string; + type: "maintenance" | "critical"; + dependsOn?: string[]; +}; + +const registry: MigrationEntity[] = [v00001]; + +export const runMigrations = async () => { + const userCount = await db.select({ count: sql`count(*)` }).from(usersTable); + const isFreshInstall = userCount[0]?.count === 0; + + if (isFreshInstall) { + logger.debug("Fresh installation detected, skipping migration checkpoint validation."); + + for (const migration of registry) { + const hasCheckpoint = await hasMigrationCheckpoint(migration.id); + if (!hasCheckpoint) { + await recordMigrationCheckpoint(migration.id); + } + } + + return; + } + + for (const migration of registry) { + const alreadyMigrated = await hasMigrationCheckpoint(migration.id); + + if (alreadyMigrated) { + logger.debug(`Migration ${migration.id} already completed, skipping.`); + continue; + } + + if (migration.dependsOn) { + for (const dep of migration.dependsOn) { + const depCompleted = await hasMigrationCheckpoint(dep); + if (!depCompleted) { + const err = [ + "================================================================================", + `🚨 MIGRATION ERROR: Migration ${migration.id} depends on migration ${dep}.`, + "The application cannot start until the required migration has successfully completed.", + "Please fix the issues and restart the application.", + "", + "Seek support by opening an issue on the Zerobyte GitHub repository if you need assistance.", + "================================================================================", + ]; + err.forEach((line) => logger.error(line)); + process.exit(1); + } + } + } + + logger.info(`Running migration: ${migration.id} (${migration.type})`); + const result = await migration.execute(); + if (result.success) { + logger.info(`Migration ${migration.id} completed successfully.`); + await recordMigrationCheckpoint(migration.id); + } else { + logger.error(`Migration ${migration.id} completed with errors: ${result.errors.length} items failed.`); + for (const err of result.errors) { + logger.error(`Migration failure - ${err.name}: ${err.error}`); + } + + if (migration.type === "critical") { + const err = [ + "================================================================================", + `🚨 MIGRATION ERROR: Critical migration ${migration.id} failed.`, + "", + "The application cannot start until this migration has successfully completed.", + "", + "Please fix the issues and restart the application. Seek support by opening an issue", + "on the Zerobyte GitHub repository if you need assistance.", + "================================================================================", + ]; + err.forEach((line) => logger.error(line)); + process.exit(1); + } + } + } +}; diff --git a/app/server/modules/lifecycle/migration.ts b/app/server/modules/lifecycle/migrations/00001-retag-snapshots.ts similarity index 60% rename from app/server/modules/lifecycle/migration.ts rename to app/server/modules/lifecycle/migrations/00001-retag-snapshots.ts index 50fa9e5a..234922f8 100644 --- a/app/server/modules/lifecycle/migration.ts +++ b/app/server/modules/lifecycle/migrations/00001-retag-snapshots.ts @@ -1,60 +1,11 @@ import { eq } from "drizzle-orm"; -import { db } from "../../db/db"; -import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../db/schema"; -import { logger } from "../../utils/logger"; -import { hasMigrationCheckpoint, recordMigrationCheckpoint } from "./checkpoint"; +import { db } from "../../../db/db"; +import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../../db/schema"; +import { logger } from "../../../utils/logger"; import { toMessage } from "~/server/utils/errors"; import { safeSpawn } from "~/server/utils/spawn"; import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "~/server/utils/restic"; -const MIGRATION_VERSION = "v0.21.1"; - -interface MigrationResult { - success: boolean; - errors: Array<{ name: string; error: string }>; -} - -export class MigrationError extends Error { - version: string; - failedItems: Array<{ name: string; error: string }>; - - constructor(version: string, failedItems: Array<{ name: string; error: string }>) { - const itemNames = failedItems.map((e) => e.name).join(", "); - super(`Migration ${version} failed for: ${itemNames}`); - this.version = version; - this.failedItems = failedItems; - this.name = "MigrationError"; - } -} - -export const retagSnapshots = async () => { - const alreadyMigrated = await hasMigrationCheckpoint(MIGRATION_VERSION); - if (alreadyMigrated) { - logger.debug(`Migration ${MIGRATION_VERSION} already completed, skipping.`); - return; - } - - logger.info(`Starting snapshots retagging migration (${MIGRATION_VERSION})...`); - - const result = await migrateSnapshotsToShortIdTag(); - const allErrors = [...result.errors]; - - if (allErrors.length > 0) { - logger.error(`Migration ${MIGRATION_VERSION} completed with errors: ${allErrors.length} items failed.`); - logger.error( - `Some snapshots could not be retagged. Please check the logs for details. Fix any repository in error state and re-start zerobyte to retry the migration for failed items.`, - ); - for (const err of allErrors) { - logger.error(`Migration failure - ${err.name}: ${err.error}`); - } - - return; - } - - await recordMigrationCheckpoint(MIGRATION_VERSION); - logger.info(`Snapshots retagging migration (${MIGRATION_VERSION}) complete.`); -}; - const migrateTag = async ( oldTag: string, newTag: string, @@ -81,7 +32,7 @@ const migrateTag = async ( return null; }; -const migrateSnapshotsToShortIdTag = async (): Promise => { +const execute = async () => { const errors: Array<{ name: string; error: string }> = []; const backupSchedules = await db.query.backupSchedulesTable.findMany({}); @@ -134,3 +85,9 @@ const migrateSnapshotsToShortIdTag = async (): Promise => { return { success: errors.length === 0, errors }; }; + +export const v00001 = { + execute, + id: "00001-retag-snapshots", + type: "maintenance" as const, +}; diff --git a/app/server/utils/logger.ts b/app/server/utils/logger.ts index 6d106f60..2bc39507 100644 --- a/app/server/utils/logger.ts +++ b/app/server/utils/logger.ts @@ -27,7 +27,7 @@ const log = (level: "info" | "warn" | "error" | "debug", messages: unknown[]) => return sanitizeSensitiveData(JSON.stringify(m, null, 2)); } - return sanitizeSensitiveData(String(JSON.stringify(m))); + return sanitizeSensitiveData(String(m as string)); }); winstonLogger.log(level, stringMessages.join(" "));