chore: final cleanup
This commit is contained in:
parent
b29500bcf4
commit
bffed92c84
10 changed files with 53 additions and 50 deletions
|
|
@ -1,6 +1,6 @@
|
||||||
SERVER_IP=localhost
|
SERVER_IP=localhost
|
||||||
DATABASE_URL=./data/zerobyte.db
|
DATABASE_URL=./data/zerobyte.db
|
||||||
RESTIC_PASS_FILE=./data/restic.pass
|
|
||||||
RESTIC_CACHE_DIR=./data/restic/cache
|
RESTIC_CACHE_DIR=./data/restic/cache
|
||||||
ZEROBYTE_REPOSITORIES_DIR=./data/repositories
|
ZEROBYTE_REPOSITORIES_DIR=./data/repositories
|
||||||
ZEROBYTE_VOLUMES_DIR=./data/volumes
|
ZEROBYTE_VOLUMES_DIR=./data/volumes
|
||||||
|
APP_SECRET=<openssl rand -hex 32>
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,2 @@
|
||||||
DATABASE_URL=:memory:
|
DATABASE_URL=:memory:
|
||||||
ZEROBYTE_APP_SECRET=8b9acd4456dd5db0a4a3c4f4e1240b2c3ae08bb59690167197425e4a25dd9a69
|
APP_SECRET=8b9acd4456dd5db0a4a3c4f4e1240b2c3ae08bb59690167197425e4a25dd9a69
|
||||||
|
|
|
||||||
|
|
@ -185,7 +185,6 @@ Zerobyte is a wrapper around Restic for backup operations. Key integration point
|
||||||
|
|
||||||
- `buildRepoUrl()` - Constructs repository URLs for different backends
|
- `buildRepoUrl()` - Constructs repository URLs for different backends
|
||||||
- `buildEnv()` - Sets environment variables (credentials, cache dir)
|
- `buildEnv()` - Sets environment variables (credentials, cache dir)
|
||||||
- `ensurePassfile()` - Manages encryption password file
|
|
||||||
- Type-safe parsing of Restic JSON output using ArkType schemas
|
- Type-safe parsing of Restic JSON output using ArkType schemas
|
||||||
|
|
||||||
**Rclone Integration** (`app/server/modules/repositories/`):
|
**Rclone Integration** (`app/server/modules/repositories/`):
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ import { eq } from "drizzle-orm";
|
||||||
import { config } from "../server/core/config";
|
import { config } from "../server/core/config";
|
||||||
import { db } from "../server/db/db";
|
import { db } from "../server/db/db";
|
||||||
import { cryptoUtils } from "../server/utils/crypto";
|
import { cryptoUtils } from "../server/utils/crypto";
|
||||||
import { organization as organizationTable, member } from "../server/db/schema";
|
import { organization as organizationTable, member, usersTable } from "../server/db/schema";
|
||||||
import { ensureOnlyOneUser } from "./auth-middlewares/only-one-user";
|
import { ensureOnlyOneUser } from "./auth-middlewares/only-one-user";
|
||||||
|
|
||||||
export type AuthMiddlewareContext = MiddlewareContext<MiddlewareOptions, AuthContext<BetterAuthOptions>>;
|
export type AuthMiddlewareContext = MiddlewareContext<MiddlewareOptions, AuthContext<BetterAuthOptions>>;
|
||||||
|
|
@ -55,25 +55,31 @@ const createBetterAuth = (secret: string) =>
|
||||||
resticPassword: await cryptoUtils.sealSecret(resticPassword),
|
resticPassword: await cryptoUtils.sealSecret(resticPassword),
|
||||||
};
|
};
|
||||||
|
|
||||||
await db.transaction(async (tx) => {
|
try {
|
||||||
const orgId = Bun.randomUUIDv7();
|
await db.transaction(async (tx) => {
|
||||||
|
const orgId = Bun.randomUUIDv7();
|
||||||
|
|
||||||
await tx.insert(organizationTable).values({
|
await tx.insert(organizationTable).values({
|
||||||
name: `${user.name}'s Workspace`,
|
name: `${user.name}'s Workspace`,
|
||||||
slug: slug,
|
slug: slug,
|
||||||
id: orgId,
|
id: orgId,
|
||||||
createdAt: new Date(),
|
createdAt: new Date(),
|
||||||
metadata,
|
metadata,
|
||||||
});
|
});
|
||||||
|
|
||||||
await tx.insert(member).values({
|
await tx.insert(member).values({
|
||||||
id: Bun.randomUUIDv7(),
|
id: Bun.randomUUIDv7(),
|
||||||
userId: user.id,
|
userId: user.id,
|
||||||
role: "owner",
|
role: "owner",
|
||||||
organizationId: orgId,
|
organizationId: orgId,
|
||||||
createdAt: new Date(),
|
createdAt: new Date(),
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
} catch {
|
||||||
|
await db.delete(usersTable).where(eq(usersTable.id, user.id));
|
||||||
|
|
||||||
|
throw new Error(`Failed to create organization for user ${user.id}`);
|
||||||
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ const envSchema = type({
|
||||||
APP_VERSION: "string = 'dev'",
|
APP_VERSION: "string = 'dev'",
|
||||||
TRUSTED_ORIGINS: "string?",
|
TRUSTED_ORIGINS: "string?",
|
||||||
DISABLE_RATE_LIMITING: 'string = "false"',
|
DISABLE_RATE_LIMITING: 'string = "false"',
|
||||||
ZEROBYTE_APP_SECRET: "32 <= string <= 256",
|
APP_SECRET: "32 <= string <= 256",
|
||||||
}).pipe((s) => ({
|
}).pipe((s) => ({
|
||||||
__prod__: s.NODE_ENV === "production",
|
__prod__: s.NODE_ENV === "production",
|
||||||
environment: s.NODE_ENV,
|
environment: s.NODE_ENV,
|
||||||
|
|
@ -46,25 +46,25 @@ const envSchema = type({
|
||||||
appVersion: s.APP_VERSION,
|
appVersion: s.APP_VERSION,
|
||||||
trustedOrigins: s.TRUSTED_ORIGINS?.split(",").map((origin) => origin.trim()),
|
trustedOrigins: s.TRUSTED_ORIGINS?.split(",").map((origin) => origin.trim()),
|
||||||
disableRateLimiting: s.DISABLE_RATE_LIMITING === "true",
|
disableRateLimiting: s.DISABLE_RATE_LIMITING === "true",
|
||||||
appSecret: s.ZEROBYTE_APP_SECRET,
|
appSecret: s.APP_SECRET,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const parseConfig = (env: unknown) => {
|
const parseConfig = (env: unknown) => {
|
||||||
const result = envSchema(env);
|
const result = envSchema(env);
|
||||||
|
|
||||||
if (result instanceof type.errors) {
|
if (result instanceof type.errors) {
|
||||||
if (!process.env.ZEROBYTE_APP_SECRET) {
|
if (!process.env.APP_SECRET) {
|
||||||
const errorMessage = [
|
const errorMessage = [
|
||||||
"",
|
"",
|
||||||
"================================================================================",
|
"================================================================================",
|
||||||
"ZEROBYTE_APP_SECRET is not configured.",
|
"APP_SECRET is not configured.",
|
||||||
"",
|
"",
|
||||||
"This secret is required for encrypting sensitive data in the database.",
|
"This secret is required for encrypting sensitive data in the database.",
|
||||||
"",
|
"",
|
||||||
"To generate a new secret, run:",
|
"To generate a new secret, run:",
|
||||||
" openssl rand -hex 32",
|
" openssl rand -hex 32",
|
||||||
"",
|
"",
|
||||||
"Then set the ZEROBYTE_APP_SECRET environment variable with the generated value.",
|
"Then set the APP_SECRET environment variable with the generated value.",
|
||||||
"",
|
"",
|
||||||
"IMPORTANT: Store this secret securely and back it up. If lost, encrypted data",
|
"IMPORTANT: Store this secret securely and back it up. If lost, encrypted data",
|
||||||
"in the database will be unrecoverable.",
|
"in the database will be unrecoverable.",
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ import { RESTIC_PASS_FILE } from "~/server/core/constants";
|
||||||
*
|
*
|
||||||
* This migration performs two critical tasks:
|
* This migration performs two critical tasks:
|
||||||
* 1. Assigns unique restic passwords to each organization (using the legacy password for existing orgs)
|
* 1. Assigns unique restic passwords to each organization (using the legacy password for existing orgs)
|
||||||
* 2. Re-keys all encrypted secrets from the legacy restic passfile to use the new ZEROBYTE_APP_SECRET
|
* 2. Re-keys all encrypted secrets from the legacy restic passfile to use the new APP_SECRET
|
||||||
*
|
*
|
||||||
* This allows per-organization encryption key isolation while ensuring
|
* This allows per-organization encryption key isolation while ensuring
|
||||||
* database encryption is decoupled from restic repository passwords.
|
* database encryption is decoupled from restic repository passwords.
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@ import { and, eq, or } from "drizzle-orm";
|
||||||
import { db } from "../../db/db";
|
import { db } from "../../db/db";
|
||||||
import { backupSchedulesTable, volumesTable } from "../../db/schema";
|
import { backupSchedulesTable, volumesTable } from "../../db/schema";
|
||||||
import { logger } from "../../utils/logger";
|
import { logger } from "../../utils/logger";
|
||||||
import { restic } from "../../utils/restic";
|
|
||||||
import { volumeService } from "../volumes/volume.service";
|
import { volumeService } from "../volumes/volume.service";
|
||||||
import { CleanupDanglingMountsJob } from "../../jobs/cleanup-dangling";
|
import { CleanupDanglingMountsJob } from "../../jobs/cleanup-dangling";
|
||||||
import { VolumeHealthCheckJob } from "../../jobs/healthchecks";
|
import { VolumeHealthCheckJob } from "../../jobs/healthchecks";
|
||||||
|
|
@ -55,10 +54,6 @@ export const startup = async () => {
|
||||||
await Scheduler.start();
|
await Scheduler.start();
|
||||||
await Scheduler.clear();
|
await Scheduler.clear();
|
||||||
|
|
||||||
await restic.ensurePassfile().catch((err) => {
|
|
||||||
logger.error(`Error ensuring restic passfile exists: ${err.message}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
await initAuth().catch((err) => {
|
await initAuth().catch((err) => {
|
||||||
logger.error(`Error initializing auth: ${toMessage(err)}`);
|
logger.error(`Error initializing auth: ${toMessage(err)}`);
|
||||||
throw err;
|
throw err;
|
||||||
|
|
|
||||||
|
|
@ -129,7 +129,9 @@ const createRepository = async (name: string, config: RepositoryConfig, compress
|
||||||
}
|
}
|
||||||
|
|
||||||
const errorMessage = toMessage(error);
|
const errorMessage = toMessage(error);
|
||||||
await db.delete(repositoriesTable).where(and(eq(repositoriesTable.id, id), eq(repositoriesTable.organizationId, organizationId)));
|
await db
|
||||||
|
.delete(repositoriesTable)
|
||||||
|
.where(and(eq(repositoriesTable.id, id), eq(repositoriesTable.organizationId, organizationId)));
|
||||||
|
|
||||||
throw new InternalServerError(`Failed to initialize repository: ${errorMessage}`);
|
throw new InternalServerError(`Failed to initialize repository: ${errorMessage}`);
|
||||||
};
|
};
|
||||||
|
|
@ -155,7 +157,9 @@ const deleteRepository = async (id: string) => {
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.delete(repositoriesTable)
|
.delete(repositoriesTable)
|
||||||
.where(and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)));
|
.where(
|
||||||
|
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)),
|
||||||
|
);
|
||||||
|
|
||||||
cache.delByPrefix(`snapshots:${repository.id}:`);
|
cache.delByPrefix(`snapshots:${repository.id}:`);
|
||||||
cache.delByPrefix(`ls:${repository.id}:`);
|
cache.delByPrefix(`ls:${repository.id}:`);
|
||||||
|
|
@ -330,7 +334,9 @@ const checkHealth = async (repositoryId: string) => {
|
||||||
lastChecked: Date.now(),
|
lastChecked: Date.now(),
|
||||||
lastError: error,
|
lastError: error,
|
||||||
})
|
})
|
||||||
.where(and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)));
|
.where(
|
||||||
|
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)),
|
||||||
|
);
|
||||||
|
|
||||||
return { lastError: error };
|
return { lastError: error };
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -420,7 +426,9 @@ const doctorRepository = async (id: string) => {
|
||||||
lastChecked: Date.now(),
|
lastChecked: Date.now(),
|
||||||
lastError: doctorError,
|
lastError: doctorError,
|
||||||
})
|
})
|
||||||
.where(and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)));
|
.where(
|
||||||
|
and(eq(repositoriesTable.id, repository.id), eq(repositoriesTable.organizationId, repository.organizationId)),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: doctorSucceeded,
|
success: doctorSucceeded,
|
||||||
|
|
@ -466,7 +474,11 @@ const deleteSnapshots = async (id: string, snapshotIds: string[]) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const tagSnapshots = async (id: string, snapshotIds: string[], tags: { add?: string[]; remove?: string[]; set?: string[] }) => {
|
const tagSnapshots = async (
|
||||||
|
id: string,
|
||||||
|
snapshotIds: string[],
|
||||||
|
tags: { add?: string[]; remove?: string[]; set?: string[] },
|
||||||
|
) => {
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
const repository = await findRepository(id);
|
const repository = await findRepository(id);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -64,17 +64,6 @@ const snapshotInfoSchema = type({
|
||||||
}).optional(),
|
}).optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
const ensurePassfile = async () => {
|
|
||||||
await fs.mkdir(path.dirname(RESTIC_PASS_FILE), { recursive: true });
|
|
||||||
|
|
||||||
try {
|
|
||||||
await fs.access(RESTIC_PASS_FILE);
|
|
||||||
} catch {
|
|
||||||
logger.info("Restic passfile not found, creating a new one...");
|
|
||||||
await fs.writeFile(RESTIC_PASS_FILE, crypto.randomBytes(32).toString("hex"), { mode: 0o600 });
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const buildRepoUrl = (config: RepositoryConfig): string => {
|
export const buildRepoUrl = (config: RepositoryConfig): string => {
|
||||||
switch (config.backend) {
|
switch (config.backend) {
|
||||||
case "local":
|
case "local":
|
||||||
|
|
@ -239,8 +228,6 @@ export const buildEnv = async (config: RepositoryConfig, organizationId: string)
|
||||||
};
|
};
|
||||||
|
|
||||||
const init = async (config: RepositoryConfig, organizationId: string) => {
|
const init = async (config: RepositoryConfig, organizationId: string) => {
|
||||||
await ensurePassfile();
|
|
||||||
|
|
||||||
const repoUrl = buildRepoUrl(config);
|
const repoUrl = buildRepoUrl(config);
|
||||||
|
|
||||||
logger.info(`Initializing restic repository at ${repoUrl}...`);
|
logger.info(`Initializing restic repository at ${repoUrl}...`);
|
||||||
|
|
@ -956,7 +943,6 @@ export const addCommonArgs = (
|
||||||
};
|
};
|
||||||
|
|
||||||
export const restic = {
|
export const restic = {
|
||||||
ensurePassfile,
|
|
||||||
init,
|
init,
|
||||||
backup,
|
backup,
|
||||||
restore,
|
restore,
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ services:
|
||||||
- SYS_ADMIN
|
- SYS_ADMIN
|
||||||
environment:
|
environment:
|
||||||
- NODE_ENV=development
|
- NODE_ENV=development
|
||||||
|
- APP_SECRET=94bad4678ce84a60b9789bd2114a6bf780aeb38df426f7352c941c66e25d5c2b
|
||||||
ports:
|
ports:
|
||||||
- "4096:4096"
|
- "4096:4096"
|
||||||
volumes:
|
volumes:
|
||||||
|
|
@ -21,6 +22,7 @@ services:
|
||||||
- ~/.config/rclone:/root/.config/rclone
|
- ~/.config/rclone:/root/.config/rclone
|
||||||
|
|
||||||
zerobyte-prod:
|
zerobyte-prod:
|
||||||
|
# image: ghcr.io/nicotsx/zerobyte:v0.22.0
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
|
|
@ -35,6 +37,8 @@ services:
|
||||||
- SYS_ADMIN
|
- SYS_ADMIN
|
||||||
ports:
|
ports:
|
||||||
- "4096:4096"
|
- "4096:4096"
|
||||||
|
environment:
|
||||||
|
- APP_SECRET=94bad4678ce84a60b9789bd2114a6bf780aeb38df426f7352c941c66e25d5c2b
|
||||||
volumes:
|
volumes:
|
||||||
- /etc/localtime:/etc/localtime:ro
|
- /etc/localtime:/etc/localtime:ro
|
||||||
- /var/lib/zerobyte:/var/lib/zerobyte
|
- /var/lib/zerobyte:/var/lib/zerobyte
|
||||||
|
|
@ -49,6 +53,7 @@ services:
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
- DISABLE_RATE_LIMITING=true
|
- DISABLE_RATE_LIMITING=true
|
||||||
|
- APP_SECRET=94bad4678ce84a60b9789bd2114a6bf780aeb38df426f7352c941c66e25d5c2b
|
||||||
devices:
|
devices:
|
||||||
- /dev/fuse:/dev/fuse
|
- /dev/fuse:/dev/fuse
|
||||||
cap_add:
|
cap_add:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue