refactor: implement a proper code migration system (#324)
* refactor: implement a proper code migration system * chore: rename .tsx -> .ts
This commit is contained in:
Nico
GitHub
parent
99932a8522
commit
76741d6fb8
6 changed files with 127 additions and 150 deletions
|
|
@ -5,5 +5,3 @@ export const DATABASE_URL = process.env.DATABASE_URL || "/var/lib/zerobyte/data/
|
|||
export const RESTIC_PASS_FILE = "/var/lib/zerobyte/data/restic.pass";
|
||||
|
||||
export const DEFAULT_EXCLUDES = [DATABASE_URL, RESTIC_PASS_FILE, REPOSITORY_BASE];
|
||||
|
||||
export const REQUIRED_MIGRATIONS = []; // ["v0.21.1"] add this once re-tagging migration is removed
|
||||
|
|
|
|||
|
|
@ -2,14 +2,12 @@ import { createHonoServer } from "react-router-hono-server/bun";
|
|||
import * as schema from "./db/schema";
|
||||
import { setSchema, runDbMigrations } from "./db/db";
|
||||
import { startup } from "./modules/lifecycle/startup";
|
||||
import { retagSnapshots } from "./modules/lifecycle/migration";
|
||||
import { logger } from "./utils/logger";
|
||||
import { shutdown } from "./modules/lifecycle/shutdown";
|
||||
import { REQUIRED_MIGRATIONS } from "./core/constants";
|
||||
import { validateRequiredMigrations } from "./modules/lifecycle/checkpoint";
|
||||
import { createApp } from "./app";
|
||||
import { config } from "./core/config";
|
||||
import { runCLI } from "./cli";
|
||||
import { runMigrations } from "./modules/lifecycle/migrations";
|
||||
|
||||
setSchema(schema);
|
||||
|
||||
|
|
@ -22,9 +20,7 @@ const app = createApp();
|
|||
|
||||
runDbMigrations();
|
||||
|
||||
await retagSnapshots();
|
||||
await validateRequiredMigrations(REQUIRED_MIGRATIONS);
|
||||
|
||||
await runMigrations();
|
||||
await startup();
|
||||
|
||||
export type AppType = typeof app;
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
import { eq, sql } from "drizzle-orm";
|
||||
import { db } from "../../db/db";
|
||||
import { appMetadataTable, usersTable } from "../../db/schema";
|
||||
import { logger } from "../../utils/logger";
|
||||
|
||||
const MIGRATION_KEY_PREFIX = "migration:";
|
||||
|
||||
export const recordMigrationCheckpoint = async (version: string): Promise<void> => {
|
||||
const key = `${MIGRATION_KEY_PREFIX}${version}`;
|
||||
const now = Date.now();
|
||||
|
||||
await db
|
||||
.insert(appMetadataTable)
|
||||
.values({
|
||||
key,
|
||||
value: JSON.stringify({ completedAt: new Date().toISOString() }),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: appMetadataTable.key,
|
||||
set: {
|
||||
value: JSON.stringify({ completedAt: new Date().toISOString() }),
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
|
||||
logger.info(`Recorded migration checkpoint for ${version}`);
|
||||
};
|
||||
|
||||
export const hasMigrationCheckpoint = async (version: string): Promise<boolean> => {
|
||||
const key = `${MIGRATION_KEY_PREFIX}${version}`;
|
||||
const result = await db.query.appMetadataTable.findFirst({
|
||||
where: eq(appMetadataTable.key, key),
|
||||
});
|
||||
return result !== undefined;
|
||||
};
|
||||
|
||||
export const validateRequiredMigrations = async (requiredVersions: string[]): Promise<void> => {
|
||||
const userCount = await db.select({ count: sql<number>`count(*)` }).from(usersTable);
|
||||
const isFreshInstall = userCount[0]?.count === 0;
|
||||
|
||||
if (isFreshInstall) {
|
||||
logger.info("Fresh installation detected, skipping migration checkpoint validation.");
|
||||
|
||||
for (const version of requiredVersions) {
|
||||
const hasCheckpoint = await hasMigrationCheckpoint(version);
|
||||
if (!hasCheckpoint) {
|
||||
await recordMigrationCheckpoint(version);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
for (const version of requiredVersions) {
|
||||
const hasCheckpoint = await hasMigrationCheckpoint(version);
|
||||
if (!hasCheckpoint) {
|
||||
logger.error(`
|
||||
================================================================================
|
||||
MIGRATION ERROR: Required migration ${version} has not been run.
|
||||
|
||||
You are attempting to start a version of Zerobyte that requires migration
|
||||
checkpoints from previous versions. This typically happens when you skip
|
||||
versions during an upgrade.
|
||||
|
||||
To fix this:
|
||||
1. First upgrade to version ${version} and run the application once
|
||||
2. Validate that everything is still working correctly
|
||||
3. Then upgrade to the current version
|
||||
|
||||
================================================================================
|
||||
`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const getMigrationCheckpoints = async (): Promise<{ version: string; completedAt: string }[]> => {
|
||||
const results = await db.query.appMetadataTable.findMany({
|
||||
where: (table, { like }) => like(table.key, `${MIGRATION_KEY_PREFIX}%`),
|
||||
});
|
||||
|
||||
return results.map((r) => ({
|
||||
version: r.key.replace(MIGRATION_KEY_PREFIX, ""),
|
||||
completedAt: JSON.parse(r.value).completedAt,
|
||||
}));
|
||||
};
|
||||
114
app/server/modules/lifecycle/migrations.ts
Normal file
114
app/server/modules/lifecycle/migrations.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
import { db } from "~/server/db/db";
|
||||
import { logger } from "../../utils/logger";
|
||||
import { v00001 } from "./migrations/00001-retag-snapshots";
|
||||
import { usersTable } from "~/server/db/schema";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { appMetadataTable } from "../../db/schema";
|
||||
|
||||
const MIGRATION_KEY_PREFIX = "migration:";
|
||||
|
||||
const recordMigrationCheckpoint = async (version: string): Promise<void> => {
|
||||
const key = `${MIGRATION_KEY_PREFIX}${version}`;
|
||||
const now = Date.now();
|
||||
|
||||
await db
|
||||
.insert(appMetadataTable)
|
||||
.values({ key, value: JSON.stringify({ completedAt: new Date().toISOString() }), createdAt: now, updatedAt: now })
|
||||
.onConflictDoUpdate({
|
||||
target: appMetadataTable.key,
|
||||
set: { value: JSON.stringify({ completedAt: new Date().toISOString() }), updatedAt: now },
|
||||
});
|
||||
|
||||
logger.info(`Recorded migration checkpoint for ${version}`);
|
||||
};
|
||||
|
||||
const hasMigrationCheckpoint = async (id: string): Promise<boolean> => {
|
||||
const key = `${MIGRATION_KEY_PREFIX}${id}`;
|
||||
const result = await db.query.appMetadataTable.findFirst({
|
||||
where: eq(appMetadataTable.key, key),
|
||||
});
|
||||
return result !== undefined;
|
||||
};
|
||||
|
||||
type MigrationEntity = {
|
||||
execute: () => Promise<{ success: boolean; errors: Array<{ name: string; error: string }> }>;
|
||||
id: string;
|
||||
type: "maintenance" | "critical";
|
||||
dependsOn?: string[];
|
||||
};
|
||||
|
||||
const registry: MigrationEntity[] = [v00001];
|
||||
|
||||
export const runMigrations = async () => {
|
||||
const userCount = await db.select({ count: sql<number>`count(*)` }).from(usersTable);
|
||||
const isFreshInstall = userCount[0]?.count === 0;
|
||||
|
||||
if (isFreshInstall) {
|
||||
logger.debug("Fresh installation detected, skipping migration checkpoint validation.");
|
||||
|
||||
for (const migration of registry) {
|
||||
const hasCheckpoint = await hasMigrationCheckpoint(migration.id);
|
||||
if (!hasCheckpoint) {
|
||||
await recordMigrationCheckpoint(migration.id);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
for (const migration of registry) {
|
||||
const alreadyMigrated = await hasMigrationCheckpoint(migration.id);
|
||||
|
||||
if (alreadyMigrated) {
|
||||
logger.debug(`Migration ${migration.id} already completed, skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (migration.dependsOn) {
|
||||
for (const dep of migration.dependsOn) {
|
||||
const depCompleted = await hasMigrationCheckpoint(dep);
|
||||
if (!depCompleted) {
|
||||
const err = [
|
||||
"================================================================================",
|
||||
`🚨 MIGRATION ERROR: Migration ${migration.id} depends on migration ${dep}.`,
|
||||
"The application cannot start until the required migration has successfully completed.",
|
||||
"Please fix the issues and restart the application.",
|
||||
"",
|
||||
"Seek support by opening an issue on the Zerobyte GitHub repository if you need assistance.",
|
||||
"================================================================================",
|
||||
];
|
||||
err.forEach((line) => logger.error(line));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Running migration: ${migration.id} (${migration.type})`);
|
||||
const result = await migration.execute();
|
||||
if (result.success) {
|
||||
logger.info(`Migration ${migration.id} completed successfully.`);
|
||||
await recordMigrationCheckpoint(migration.id);
|
||||
} else {
|
||||
logger.error(`Migration ${migration.id} completed with errors: ${result.errors.length} items failed.`);
|
||||
for (const err of result.errors) {
|
||||
logger.error(`Migration failure - ${err.name}: ${err.error}`);
|
||||
}
|
||||
|
||||
if (migration.type === "critical") {
|
||||
const err = [
|
||||
"================================================================================",
|
||||
`🚨 MIGRATION ERROR: Critical migration ${migration.id} failed.`,
|
||||
"",
|
||||
"The application cannot start until this migration has successfully completed.",
|
||||
"",
|
||||
"Please fix the issues and restart the application. Seek support by opening an issue",
|
||||
"on the Zerobyte GitHub repository if you need assistance.",
|
||||
"================================================================================",
|
||||
];
|
||||
err.forEach((line) => logger.error(line));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -1,60 +1,11 @@
|
|||
import { eq } from "drizzle-orm";
|
||||
import { db } from "../../db/db";
|
||||
import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../db/schema";
|
||||
import { logger } from "../../utils/logger";
|
||||
import { hasMigrationCheckpoint, recordMigrationCheckpoint } from "./checkpoint";
|
||||
import { db } from "../../../db/db";
|
||||
import { backupScheduleMirrorsTable, repositoriesTable, type Repository } from "../../../db/schema";
|
||||
import { logger } from "../../../utils/logger";
|
||||
import { toMessage } from "~/server/utils/errors";
|
||||
import { safeSpawn } from "~/server/utils/spawn";
|
||||
import { addCommonArgs, buildEnv, buildRepoUrl, cleanupTemporaryKeys } from "~/server/utils/restic";
|
||||
|
||||
const MIGRATION_VERSION = "v0.21.1";
|
||||
|
||||
interface MigrationResult {
|
||||
success: boolean;
|
||||
errors: Array<{ name: string; error: string }>;
|
||||
}
|
||||
|
||||
export class MigrationError extends Error {
|
||||
version: string;
|
||||
failedItems: Array<{ name: string; error: string }>;
|
||||
|
||||
constructor(version: string, failedItems: Array<{ name: string; error: string }>) {
|
||||
const itemNames = failedItems.map((e) => e.name).join(", ");
|
||||
super(`Migration ${version} failed for: ${itemNames}`);
|
||||
this.version = version;
|
||||
this.failedItems = failedItems;
|
||||
this.name = "MigrationError";
|
||||
}
|
||||
}
|
||||
|
||||
export const retagSnapshots = async () => {
|
||||
const alreadyMigrated = await hasMigrationCheckpoint(MIGRATION_VERSION);
|
||||
if (alreadyMigrated) {
|
||||
logger.debug(`Migration ${MIGRATION_VERSION} already completed, skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Starting snapshots retagging migration (${MIGRATION_VERSION})...`);
|
||||
|
||||
const result = await migrateSnapshotsToShortIdTag();
|
||||
const allErrors = [...result.errors];
|
||||
|
||||
if (allErrors.length > 0) {
|
||||
logger.error(`Migration ${MIGRATION_VERSION} completed with errors: ${allErrors.length} items failed.`);
|
||||
logger.error(
|
||||
`Some snapshots could not be retagged. Please check the logs for details. Fix any repository in error state and re-start zerobyte to retry the migration for failed items.`,
|
||||
);
|
||||
for (const err of allErrors) {
|
||||
logger.error(`Migration failure - ${err.name}: ${err.error}`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await recordMigrationCheckpoint(MIGRATION_VERSION);
|
||||
logger.info(`Snapshots retagging migration (${MIGRATION_VERSION}) complete.`);
|
||||
};
|
||||
|
||||
const migrateTag = async (
|
||||
oldTag: string,
|
||||
newTag: string,
|
||||
|
|
@ -81,7 +32,7 @@ const migrateTag = async (
|
|||
return null;
|
||||
};
|
||||
|
||||
const migrateSnapshotsToShortIdTag = async (): Promise<MigrationResult> => {
|
||||
const execute = async () => {
|
||||
const errors: Array<{ name: string; error: string }> = [];
|
||||
const backupSchedules = await db.query.backupSchedulesTable.findMany({});
|
||||
|
||||
|
|
@ -134,3 +85,9 @@ const migrateSnapshotsToShortIdTag = async (): Promise<MigrationResult> => {
|
|||
|
||||
return { success: errors.length === 0, errors };
|
||||
};
|
||||
|
||||
export const v00001 = {
|
||||
execute,
|
||||
id: "00001-retag-snapshots",
|
||||
type: "maintenance" as const,
|
||||
};
|
||||
|
|
@ -27,7 +27,7 @@ const log = (level: "info" | "warn" | "error" | "debug", messages: unknown[]) =>
|
|||
return sanitizeSensitiveData(JSON.stringify(m, null, 2));
|
||||
}
|
||||
|
||||
return sanitizeSensitiveData(String(JSON.stringify(m)));
|
||||
return sanitizeSensitiveData(String(m as string));
|
||||
});
|
||||
|
||||
winstonLogger.log(level, stringMessages.join(" "));
|
||||
|
|
|
|||
Loading…
Reference in a new issue