diff --git a/Dockerfile b/Dockerfile index 1893ad0e..7129ae2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -97,6 +97,7 @@ COPY . . RUN bun run build RUN bun build apps/agent/src/index.ts --outfile .output/agent/index.mjs --target bun +RUN bun run build:backend-integration FROM base AS production @@ -123,3 +124,7 @@ COPY ./LICENSE ./LICENSE.md EXPOSE 4096 CMD ["bun", ".output/server/index.mjs"] + +FROM production AS backend-integration + +CMD ["bun", ".output/backend-integration/index.js"] diff --git a/app/test/backend-integration/README.md b/app/test/backend-integration/README.md new file mode 100644 index 00000000..0a3e1291 --- /dev/null +++ b/app/test/backend-integration/README.md @@ -0,0 +1,26 @@ +# Backend Integration + +This runner executes isolated source-level integration tests for Zerobyte volume backends and repository backends inside a Docker container that shares the same runtime tooling as the main image. + +## What it verifies + +For each scenario the runner will: + +1. Mount the configured volume backend into an isolated temp workspace. +2. Read fixture files directly from the mounted filesystem. +3. Verify mounted file ownership and permission bits. +4. Run a restic backup directly against the configured repository backend. +5. Inspect the created snapshot with `restic ls` and verify snapshot metadata. +6. Restore the snapshot into an isolated temp directory. +7. Verify restored content, ownership, and permission bits. +8. Unmount and clean up local test artifacts. + +## Bootstrap a Debian target + +This folder includes `setup-target.sh`, which connects to a VM host and configures: + +- NFS export at `/srv/zerobyte-backend-integration/fixtures` +- Samba share `///zerobyte-backend-integration` +- WebDAV endpoint at `http:///zerobyte-backend-integration` +- SFTP access for both fixture reads and a reusable restic repository +- A generated local keypair, `known_hosts`, password files, and a ready-to-run config under `artifacts/192.168.2.41/` diff --git a/app/test/backend-integration/artifacts/.gitignore b/app/test/backend-integration/artifacts/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/app/test/backend-integration/artifacts/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/app/test/backend-integration/index.ts b/app/test/backend-integration/index.ts new file mode 100644 index 00000000..5346ee00 --- /dev/null +++ b/app/test/backend-integration/index.ts @@ -0,0 +1,18 @@ +function ensureIntegrationEnv(): void { + process.env.NODE_ENV ??= "production"; + process.env.LOG_LEVEL ??= "warn"; + process.env.BASE_URL ??= "http://localhost:4096"; + process.env.TRUSTED_ORIGINS ??= process.env.BASE_URL; + process.env.APP_SECRET ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +} + +async function main(): Promise { + ensureIntegrationEnv(); + + const { runBackendIntegration } = await import("./runner"); + await runBackendIntegration(); +} + +await main(); + +export {}; diff --git a/app/test/backend-integration/run.sh b/app/test/backend-integration/run.sh new file mode 100755 index 00000000..ebf0d406 --- /dev/null +++ b/app/test/backend-integration/run.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +default_config_path="$script_dir/artifacts/192.168.2.41/config.generated.json" +config_path="${1:-$default_config_path}" +image_tag="zerobyte-backend-integration" + +if [[ ! -f "$config_path" ]]; then + if [[ "$config_path" == "$default_config_path" ]]; then + echo "Generated config not found: $config_path" >&2 + echo "Run the target bootstrap first:" >&2 + echo " bash app/test/backend-integration/setup-debian-target.sh" >&2 + else + echo "Config file not found: $config_path" >&2 + fi + exit 1 +fi + +if command -v realpath >/dev/null 2>&1; then + config_path="$(realpath "$config_path")" +else + config_dir="$(cd "$(dirname "$config_path")" && pwd)" + config_path="$config_dir/$(basename "$config_path")" +fi + +docker build --target backend-integration -t "$image_tag" "$repo_root" +docker run --rm \ + --cap-add SYS_ADMIN \ + --device /dev/fuse:/dev/fuse \ + -e ZEROBYTE_INTEGRATION_CONFIG=/config/config.json \ + -v "$config_path:/config/config.json:ro" \ + "$image_tag" diff --git a/app/test/backend-integration/runner.ts b/app/test/backend-integration/runner.ts new file mode 100644 index 00000000..d253a31f --- /dev/null +++ b/app/test/backend-integration/runner.ts @@ -0,0 +1,318 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { Effect } from "effect"; +import { createRestic } from "@zerobyte/core/restic/server"; +import { RCLONE_CONFIG_FILE } from "~/server/core/constants"; +import type { VolumeBackend } from "~/server/modules/backends/backend"; +import { makeDirectoryBackend } from "~/server/modules/backends/directory/directory-backend"; +import { makeNfsBackend } from "~/server/modules/backends/nfs/nfs-backend"; +import { makeRcloneBackend } from "~/server/modules/backends/rclone/rclone-backend"; +import { makeSftpBackend } from "~/server/modules/backends/sftp/sftp-backend"; +import { makeSmbBackend } from "~/server/modules/backends/smb/smb-backend"; +import { makeWebdavBackend } from "~/server/modules/backends/webdav/webdav-backend"; +import { configSchema, type IntegrationScenario } from "./runner/config"; +import { + formatError, + logScenario, + printRunSummary, + type IntegrationReport, + type ScenarioReport, + type StageReport, +} from "./runner/reporting"; +import { verifyFilesystemEntries, verifySnapshotEntries } from "./runner/verification"; + +const WORKDIR_ROOT = "/tmp/zerobyte-integration"; +const INTEGRATION_ORGANIZATION_ID = "backend-integration"; + +function createVolumeBackend(config: IntegrationScenario["volume"], mountPath: string): VolumeBackend { + switch (config.backend) { + case "nfs": + return makeNfsBackend(config, mountPath); + case "smb": + return makeSmbBackend(config, mountPath); + case "directory": + return makeDirectoryBackend(config, mountPath); + case "webdav": + return makeWebdavBackend(config, mountPath); + case "rclone": + return makeRcloneBackend(config, mountPath); + case "sftp": + return makeSftpBackend(config, mountPath); + } +} + +function createResticSession(scenarioWorkspace: string): { + organizationId: string; + client: ReturnType; +} { + const password = crypto.randomBytes(32).toString("hex"); + const cacheDir = path.join(scenarioWorkspace, "restic-cache"); + const passFile = path.join(scenarioWorkspace, "restic.pass"); + + return { + organizationId: INTEGRATION_ORGANIZATION_ID, + client: createRestic({ + resolveSecret: async (value: string) => value, + getOrganizationResticPassword: async () => password, + resticCacheDir: cacheDir, + resticPassFile: passFile, + defaultExcludes: [], + rcloneConfigFile: RCLONE_CONFIG_FILE, + }), + }; +} + +async function runStage(stages: StageReport[], name: string, fn: () => Promise): Promise { + const startedAt = Date.now(); + + try { + const result = await fn(); + stages.push({ + name, + status: "passed", + durationMs: Date.now() - startedAt, + }); + return result; + } catch (error) { + stages.push({ + name, + status: "failed", + durationMs: Date.now() - startedAt, + error: formatError(error), + }); + throw error; + } +} + +async function mountBackend(backend: VolumeBackend) { + const mountResult = await backend.mount(); + if (mountResult.status !== "mounted") { + throw new Error(mountResult.error ?? `Mount returned ${mountResult.status}`); + } + + const healthResult = await backend.checkHealth(); + if (healthResult.status !== "mounted") { + throw new Error(healthResult.error ?? `Health check returned ${healthResult.status}`); + } +} + +async function backupSnapshot( + resticClient: ReturnType, + repositoryConfig: IntegrationScenario["repository"], + fixtureRootPath: string, + organizationId: string, + backupOptions: IntegrationScenario["backup"], + uniqueBackupTag: string, +) { + const execution = await Effect.runPromise( + resticClient.backup(repositoryConfig, fixtureRootPath, { + organizationId, + compressionMode: backupOptions?.compressionMode, + customResticParams: backupOptions?.customResticParams, + tags: [...(backupOptions?.tags ?? []), uniqueBackupTag], + }), + ); + + if (execution.exitCode !== 0) { + throw new Error(`restic backup returned exit code ${execution.exitCode}`); + } + + if (!execution.result?.snapshot_id) { + throw new Error("restic backup completed without a snapshot id"); + } + + if (execution.warningDetails) { + throw new Error(`restic backup reported warnings: ${execution.warningDetails}`); + } + + return execution.result.snapshot_id; +} + +async function verifySnapshot( + resticClient: ReturnType, + repositoryConfig: IntegrationScenario["repository"], + organizationId: string, + snapshotId: string, + uniqueBackupTag: string, + fixtureRootPath: string, + scenario: IntegrationScenario, +) { + const snapshots = await resticClient.snapshots(repositoryConfig, { + organizationId, + tags: [uniqueBackupTag], + }); + + const snapshot = snapshots.find((candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId); + if (!snapshot) { + throw new Error(`Unable to find snapshot ${snapshotId} by integration tag ${uniqueBackupTag}`); + } + + const res = await resticClient.ls(repositoryConfig, snapshotId, organizationId, undefined, {}); + + await verifySnapshotEntries(fixtureRootPath, res.nodes, scenario.expectedEntries); +} + +async function restoreSnapshot( + resticClient: ReturnType, + repositoryConfig: IntegrationScenario["repository"], + snapshotId: string, + restoreTarget: string, + organizationId: string, + fixtureRootPath: string, + restoreOptions: IntegrationScenario["restore"], +) { + await fs.mkdir(restoreTarget, { recursive: true }); + await resticClient.restore(repositoryConfig, snapshotId, restoreTarget, { + organizationId, + basePath: fixtureRootPath, + excludeXattr: restoreOptions?.excludeXattr, + overwrite: restoreOptions?.overwrite, + }); +} + +async function cleanupScenario(backend: VolumeBackend, restoreTarget: string, scenarioWorkspace: string) { + await backend.unmount(); + await fs.rm(restoreTarget, { recursive: true, force: true }); + await fs.rm(scenarioWorkspace, { recursive: true, force: true }); +} + +async function runScenario(scenario: IntegrationScenario, runId: string): Promise { + const startedAt = Date.now(); + const stages: StageReport[] = []; + const scenarioWorkspace = path.join(WORKDIR_ROOT, `${scenario.id}-${runId}`); + const mountPath = path.join(scenarioWorkspace, "volume", "_data"); + const restoreTarget = path.join(scenarioWorkspace, "restore"); + + const uniqueBackupTag = `zerobyte-integration-${scenario.id}-${runId}`; + const { client: resticClient, organizationId } = createResticSession(scenarioWorkspace); + const backend = createVolumeBackend(scenario.volume, mountPath); + + let snapshotId = ""; + let status: ScenarioReport["status"] = "passed"; + let errorMessage: string | undefined; + + await fs.mkdir(scenarioWorkspace, { recursive: true }); + logScenario(scenario.id, "running..."); + + try { + await runStage(stages, "mount", async () => { + await mountBackend(backend); + }); + + const fixtureRootPath = path.resolve(mountPath, scenario.fixtureRoot); + await runStage(stages, "verify-mounted-source", async () => { + await verifyFilesystemEntries(fixtureRootPath, scenario.expectedEntries, "mounted source"); + }); + + await runStage(stages, "init-repository", async () => { + if (scenario.repository.isExistingRepository) return; + + const initResult = await resticClient.init(scenario.repository, organizationId, undefined); + if (!initResult.success) { + throw new Error(initResult.error ?? "restic init failed"); + } + }); + + snapshotId = await runStage(stages, "backup", async () => { + return await backupSnapshot( + resticClient, + scenario.repository, + fixtureRootPath, + organizationId, + scenario.backup, + uniqueBackupTag, + ); + }); + + if (!snapshotId) { + throw new Error("restic backup completed without a snapshot id"); + } + + await runStage(stages, "inspect-snapshot", async () => { + await verifySnapshot( + resticClient, + scenario.repository, + organizationId, + snapshotId, + uniqueBackupTag, + fixtureRootPath, + scenario, + ); + }); + + await runStage(stages, "restore", async () => { + await restoreSnapshot( + resticClient, + scenario.repository, + snapshotId, + restoreTarget, + organizationId, + fixtureRootPath, + scenario.restore, + ); + }); + + await runStage(stages, "verify-restore", async () => { + await verifyFilesystemEntries(restoreTarget, scenario.expectedEntries, "restored output"); + }); + + logScenario(scenario.id, "passed"); + } catch (error) { + status = "failed"; + errorMessage = formatError(error); + logScenario(scenario.id, `failed: ${errorMessage}`); + } finally { + await runStage(stages, "cleanup", async () => { + await cleanupScenario(backend, restoreTarget, scenarioWorkspace); + }); + } + + return { + id: scenario.id, + volumeBackend: scenario.volume.backend, + repositoryBackend: scenario.repository.backend, + status, + durationMs: Date.now() - startedAt, + stages, + snapshotId, + error: errorMessage, + }; +} + +export async function runBackendIntegration(): Promise { + const runStartedAt = Date.now(); + const runId = crypto.randomBytes(6).toString("hex"); + + await fs.mkdir(WORKDIR_ROOT, { recursive: true }); + + const content = await fs.readFile("/config/config.json", "utf8"); + const { scenarios } = configSchema.parse(JSON.parse(content)); + + process.stdout.write( + `Running ${scenarios.length} integration scenario${scenarios.length === 1 ? "" : "s"} (run ${runId})\n`, + ); + + const scenarioReports: ScenarioReport[] = []; + for (const scenario of scenarios) { + scenarioReports.push(await runScenario(scenario, runId)); + } + + const passed = scenarioReports.filter((scenario) => scenario.status === "passed").length; + const failed = scenarioReports.length - passed; + const report: IntegrationReport = { + runId, + startedAt: new Date(runStartedAt).toISOString(), + finishedAt: new Date().toISOString(), + durationMs: Date.now() - runStartedAt, + passed, + failed, + scenarios: scenarioReports, + }; + + printRunSummary(report); + + if (failed > 0) { + process.exitCode = 1; + } +} diff --git a/app/test/backend-integration/runner/config.ts b/app/test/backend-integration/runner/config.ts new file mode 100644 index 00000000..60bd3556 --- /dev/null +++ b/app/test/backend-integration/runner/config.ts @@ -0,0 +1,84 @@ +import { z } from "zod"; +import { + COMPRESSION_MODES, + OVERWRITE_MODES, + repositoryConfigSchema, + type RepositoryConfig, +} from "@zerobyte/core/restic"; +import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; + +const modeSchema = z + .union([z.number(), z.string().transform((value) => Number.parseInt(value, 8))]) + .transform((value) => value & 0o7777); + +const entryTypeSchema = z.enum(["file", "directory", "symlink"]); + +const expectedEntrySchema = z + .object({ + path: z.string().min(1), + type: entryTypeSchema, + uid: z.number().optional(), + gid: z.number().optional(), + mode: modeSchema.optional(), + sha256: z.string().optional(), + text: z.string().optional(), + linkTarget: z.string().min(1).optional(), + }) + .superRefine((value, ctx) => { + if (value.sha256 && value.text) { + ctx.addIssue({ + code: "custom", + message: "Specify either sha256 or text, not both", + path: ["sha256"], + }); + } + + if (value.type !== "file" && (value.sha256 || value.text)) { + ctx.addIssue({ + code: "custom", + message: "Only file entries can define sha256 or text expectations", + path: ["type"], + }); + } + + if (value.type !== "symlink" && value.linkTarget) { + ctx.addIssue({ + code: "custom", + message: "Only symlink entries can define linkTarget", + path: ["linkTarget"], + }); + } + }); + +const backupOptionsSchema = z.object({ + compressionMode: z.enum(COMPRESSION_MODES).optional(), + tags: z.array(z.string().min(1)).optional(), + customResticParams: z.array(z.string().min(1)).optional(), +}); + +const restoreOptionsSchema = z.object({ + excludeXattr: z.array(z.string().min(1)).optional(), + overwrite: z.enum(OVERWRITE_MODES).optional(), +}); + +const scenarioSchema = z.object({ + id: z.string().min(1), + description: z.string().optional(), + volume: volumeConfigSchema, + repository: repositoryConfigSchema, + fixtureRoot: z.string().min(1).default("."), + expectedEntries: z.array(expectedEntrySchema).min(1), + backup: backupOptionsSchema.optional(), + restore: restoreOptionsSchema.optional(), +}); + +export const configSchema = z.object({ + version: z.literal(1).default(1), + scenarios: z.array(scenarioSchema).min(1), +}); + +export type ExpectedEntry = z.infer; +export type IntegrationScenario = z.infer & { + volume: BackendConfig; + repository: RepositoryConfig; +}; diff --git a/app/test/backend-integration/runner/reporting.ts b/app/test/backend-integration/runner/reporting.ts new file mode 100644 index 00000000..cdde2696 --- /dev/null +++ b/app/test/backend-integration/runner/reporting.ts @@ -0,0 +1,82 @@ +export type StageReport = { + name: string; + status: "passed" | "failed"; + durationMs: number; + error?: string; +}; + +export type ScenarioReport = { + id: string; + volumeBackend: string; + repositoryBackend: string; + status: "passed" | "failed"; + durationMs: number; + stages: StageReport[]; + snapshotId?: string; + error?: string; +}; + +export type IntegrationReport = { + runId: string; + startedAt: string; + finishedAt: string; + durationMs: number; + passed: number; + failed: number; + scenarios: ScenarioReport[]; +}; + +export function formatError(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + + return String(error); +} + +function log(message: string): void { + process.stdout.write(`${message}\n`); +} + +export function logScenario(scenarioId: string, message: string): void { + log(`[${scenarioId}] ${message}`); +} + +function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + + if (durationMs < 10_000) { + return `${(durationMs / 1000).toFixed(1)}s`; + } + + return `${(durationMs / 1000).toFixed(0)}s`; +} + +function formatScenarioResult(scenario: ScenarioReport, labelWidth: number): string { + const status = scenario.status === "passed" ? "✔︎ PASS" : "⨯ FAIL"; + const id = scenario.id.padEnd(labelWidth); + return `${status} ${id} ${formatDuration(scenario.durationMs)} ${scenario.volumeBackend} -> ${scenario.repositoryBackend}`; +} + +export function printRunSummary(report: IntegrationReport): void { + const labelWidth = report.scenarios.reduce((max, scenario) => Math.max(max, scenario.id.length), 4); + + log(""); + log("Scenario Results"); + + for (const scenario of report.scenarios) { + log(formatScenarioResult(scenario, labelWidth)); + + const failedStage = scenario.stages.find((stage) => stage.status === "failed"); + if (failedStage?.error) { + log(` ${failedStage.name}: ${failedStage.error}`); + } + } + + log(""); + log( + `${report.failed === 0 ? "PASS" : "FAIL"} ${report.passed}/${report.scenarios.length} scenarios passed in ${formatDuration(report.durationMs)} (run ${report.runId})`, + ); +} diff --git a/app/test/backend-integration/runner/verification.ts b/app/test/backend-integration/runner/verification.ts new file mode 100644 index 00000000..0ee42272 --- /dev/null +++ b/app/test/backend-integration/runner/verification.ts @@ -0,0 +1,150 @@ +import crypto from "node:crypto"; +import type { Stats } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { ExpectedEntry } from "./config"; + +export type SnapshotNode = { + name: string; + type: string; + path: string; + uid?: number; + gid?: number; + mode?: number; + size?: number; +}; + +function normalizeRelativePath(input: string): string { + const normalized = path.posix.normalize(input); + if (normalized === "") { + return "."; + } + + return normalized; +} + +function normalizeEntryType(type: string): string { + if (type === "dir" || type === "directory") { + return "directory"; + } + + return type; +} + +async function readSha256(filePath: string): Promise { + const digest = crypto.createHash("sha256"); + digest.update(await fs.readFile(filePath)); + return digest.digest("hex"); +} + +function assertMetadata( + entryLabel: string, + expected: ExpectedEntry, + actual: { uid?: number; gid?: number; mode?: number }, +) { + if (expected.uid && actual.uid !== expected.uid) { + throw new Error(`${entryLabel} uid mismatch: expected ${expected.uid}, got ${String(actual.uid)}`); + } + + if (expected.gid && actual.gid !== expected.gid) { + throw new Error(`${entryLabel} gid mismatch: expected ${expected.gid}, got ${String(actual.gid)}`); + } + + if (!expected.mode) { + return; + } + + if (!actual.mode) { + throw new Error(`${entryLabel} mode is missing`); + } + + const permissionBits = actual.mode & 0o7777; + if (permissionBits !== expected.mode) { + throw new Error( + `${entryLabel} mode mismatch: expected ${expected.mode.toString(8)}, got ${permissionBits.toString(8)}`, + ); + } +} + +function getFilesystemEntryType(stats: Stats): ExpectedEntry["type"] { + if (stats.isDirectory()) { + return "directory"; + } + + if (stats.isSymbolicLink()) { + return "symlink"; + } + + return "file"; +} + +export async function verifyFilesystemEntries(basePath: string, expectedEntries: ExpectedEntry[], label: string) { + for (const expected of expectedEntries) { + const targetPath = path.resolve(basePath, expected.path); + const stats = await fs.lstat(targetPath); + + const actualType = getFilesystemEntryType(stats); + if (actualType !== expected.type) { + throw new Error(`${label}: ${expected.path} type mismatch: expected ${expected.type}, got ${actualType}`); + } + + assertMetadata(`${label}: ${expected.path}`, expected, stats); + + if (expected.type === "file" && expected.text !== undefined) { + const actualText = await fs.readFile(targetPath, "utf8"); + if (actualText !== expected.text) { + throw new Error(`${label}: ${expected.path} text content mismatch`); + } + } + + if (expected.type === "file" && expected.sha256 !== undefined) { + const actualSha = await readSha256(targetPath); + if (actualSha !== expected.sha256.toLowerCase()) { + throw new Error(`${label}: ${expected.path} sha256 mismatch`); + } + } + + if (expected.type !== "symlink" || expected.linkTarget === undefined) { + continue; + } + + const actualTarget = await fs.readlink(targetPath); + if (actualTarget !== expected.linkTarget) { + throw new Error( + `${label}: ${expected.path} symlink target mismatch: expected ${expected.linkTarget}, got ${actualTarget}`, + ); + } + } +} + +export async function verifySnapshotEntries( + snapshotRootPath: string, + nodes: SnapshotNode[], + expectedEntries: ExpectedEntry[], +) { + const nodesByRelativePath = new Map(); + + for (const node of nodes) { + const relativePath = path.posix.relative(snapshotRootPath, node.path); + if (!relativePath || relativePath === "." || relativePath === ".." || relativePath.startsWith("../")) { + continue; + } + + nodesByRelativePath.set(normalizeRelativePath(relativePath), node); + } + + for (const expected of expectedEntries) { + const normalizedExpectedPath = normalizeRelativePath(expected.path); + const node = nodesByRelativePath.get(normalizedExpectedPath); + if (!node) { + throw new Error(`snapshot: missing ${expected.path}`); + } + + const actualType = normalizeEntryType(node.type); + if (actualType !== expected.type) { + throw new Error(`snapshot: ${expected.path} type mismatch: expected ${expected.type}, got ${actualType}`); + } + + assertMetadata(`snapshot: ${expected.path}`, expected, node); + } +} diff --git a/app/test/backend-integration/setup-target.sh b/app/test/backend-integration/setup-target.sh new file mode 100755 index 00000000..d59eb1af --- /dev/null +++ b/app/test/backend-integration/setup-target.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TARGET_HOST="192.168.2.41" +TARGET="root@$TARGET_HOST" +FIXTURE_UID="1000" +FIXTURE_GID="1000" + +ARTIFACTS_DIR="$SCRIPT_DIR/artifacts/$TARGET_HOST" +KEY_PATH="$ARTIFACTS_DIR/zerobyte-sftp-ed25519" +KNOWN_HOSTS_PATH="$ARTIFACTS_DIR/known_hosts" +CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json" + +SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt" +SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt" +WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt" +RESTIC_PASSWORD_FILE="$ARTIFACTS_DIR/restic-password.txt" + +read_or_create_secret() { + local file_path="$1" + + if [[ -f "$file_path" ]]; then + cat "$file_path" + else + openssl rand -hex 12 >"$file_path" + chmod 600 "$file_path" + cat "$file_path" + fi +} + +mkdir -p "$ARTIFACTS_DIR" +chmod 700 "$ARTIFACTS_DIR" + +SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")" +SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")" +WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")" +RESTIC_PASSWORD="$(read_or_create_secret "$RESTIC_PASSWORD_FILE")" + +if [[ ! -f "$KEY_PATH" || ! -f "$KEY_PATH.pub" ]]; then + ssh-keygen -q -t ed25519 -N "" -C "zerobyte-backend-integration@$TARGET_HOST" -f "$KEY_PATH" + chmod 600 "$KEY_PATH" +fi + +PUBLIC_KEY_BASE64="$(base64 <"$KEY_PATH.pub" | tr -d '\n')" + +ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" "$RESTIC_PASSWORD" "$PUBLIC_KEY_BASE64" <<'REMOTE' +set -euo pipefail + +fixture_uid="$1" +fixture_gid="$2" +smb_password="$3" +sftp_password="$4" +webdav_password="$5" +restic_password="$6" +public_key="$(printf '%s' "$7" | base64 -d)" + +export DEBIAN_FRONTEND=noninteractive + +write_file() { + local file_path="$1" + cat >"$file_path" +} + +apt-get update +apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server restic rpcbind samba + +id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp +id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb + +install -d -m 0755 /srv/zerobyte-backend-integration/fixtures/case-a/docs +printf 'hello from zerobyte integration\n' >/srv/zerobyte-backend-integration/fixtures/case-a/hello.txt +printf 'fixture documentation\n' >/srv/zerobyte-backend-integration/fixtures/case-a/docs/readme.md +chown -R "$fixture_uid:$fixture_gid" /srv/zerobyte-backend-integration/fixtures +find /srv/zerobyte-backend-integration/fixtures -type d -exec chmod 0755 {} + +find /srv/zerobyte-backend-integration/fixtures -type f -exec chmod 0644 {} + + +install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /srv/zerobyte-backend-integration/restic-repo +install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp +install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp/.ssh +printf '%s\n' "$public_key" >/home/zerobyte-sftp/.ssh/authorized_keys +chown zerobyte-sftp:zerobyte-sftp /home/zerobyte-sftp/.ssh/authorized_keys +chmod 0600 /home/zerobyte-sftp/.ssh/authorized_keys + +printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb >/dev/null +smbpasswd -e zerobyte-smb >/dev/null +printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd +passwd -u zerobyte-sftp >/dev/null 2>&1 || true +htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null + +if [[ ! -f /srv/zerobyte-backend-integration/restic-repo/config ]]; then + password_file="$(mktemp)" + printf '%s\n' "$restic_password" >"$password_file" + chown zerobyte-sftp:zerobyte-sftp "$password_file" + chmod 0600 "$password_file" + su -s /bin/sh -c "restic init --repo /srv/zerobyte-backend-integration/restic-repo --password-file '$password_file'" zerobyte-sftp + rm -f "$password_file" +fi + +write_file /etc/exports <<'EOF' +/srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure) +EOF +exportfs -ra +systemctl unmask rpcbind rpcbind.socket >/dev/null 2>&1 +systemctl start rpcbind.socket +systemctl start rpcbind +systemctl start proc-fs-nfsd.mount +systemctl restart nfs-kernel-server + +write_file /etc/samba/smb.conf <<'EOF' +[zerobyte-backend-integration] + path = /srv/zerobyte-backend-integration/fixtures + browseable = yes + read only = yes + guest ok = no + valid users = zerobyte-smb +EOF + +install -d -o www-data -g www-data -m 0755 /var/lib/dav +a2enmod dav dav_fs auth_basic >/dev/null +printf 'ServerName localhost\n' >/etc/apache2/conf-available/zerobyte-backend-integration-servername.conf +a2enconf zerobyte-backend-integration-servername >/dev/null +write_file /etc/apache2/sites-available/zerobyte-backend-integration-dav.conf <<'EOF' +Alias /zerobyte-backend-integration /srv/zerobyte-backend-integration/fixtures + +DAVLockDB /var/lib/dav/lockdb + + + DAV On + AuthType Basic + AuthName "Zerobyte Backend Integration WebDAV" + AuthUserFile /etc/apache2/zerobyte-backend-integration.htpasswd + Require valid-user + + + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + +EOF +a2ensite zerobyte-backend-integration-dav >/dev/null +apache2ctl configtest + +install -d -m 0755 /etc/ssh/sshd_config.d +write_file /etc/ssh/sshd_config.d/zerobyte-backend-integration.conf <<'EOF' +Match User zerobyte-sftp + PasswordAuthentication yes + PubkeyAuthentication yes + PermitTTY no + X11Forwarding no + AllowTcpForwarding no + ForceCommand internal-sftp +EOF +sshd -t + +systemctl restart apache2 +systemctl restart smbd +systemctl restart ssh +REMOTE + +ssh-keyscan "$TARGET_HOST" >"$KNOWN_HOSTS_PATH" 2>/dev/null + +INTEGRATION_HOST="$TARGET_HOST" \ + FIXTURE_UID="$FIXTURE_UID" \ + FIXTURE_GID="$FIXTURE_GID" \ + SMB_PASSWORD="$SMB_PASSWORD" \ + WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \ + RESTIC_PASSWORD="$RESTIC_PASSWORD" \ + SFTP_KEY_PATH="$KEY_PATH" \ + KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \ + CONFIG_PATH="$CONFIG_PATH" \ + bun run "$SCRIPT_DIR/write-generated-config.ts" + +echo "Provisioned $TARGET" +echo "Generated config: $CONFIG_PATH" +echo "Artifacts: $ARTIFACTS_DIR" diff --git a/app/test/backend-integration/write-generated-config.ts b/app/test/backend-integration/write-generated-config.ts new file mode 100644 index 00000000..c2030e8c --- /dev/null +++ b/app/test/backend-integration/write-generated-config.ts @@ -0,0 +1,130 @@ +import fs from "node:fs"; + +function getRequiredEnv(name: string) { + return process.env[name]!; +} + +function getRequiredNumberEnv(name: string) { + return Number.parseInt(getRequiredEnv(name), 10); +} + +const host = getRequiredEnv("INTEGRATION_HOST"); +const fixtureUid = getRequiredNumberEnv("FIXTURE_UID"); +const fixtureGid = getRequiredNumberEnv("FIXTURE_GID"); +const smbPassword = getRequiredEnv("SMB_PASSWORD"); +const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD"); +const resticPassword = getRequiredEnv("RESTIC_PASSWORD"); +const privateKey = fs.readFileSync(getRequiredEnv("SFTP_KEY_PATH"), "utf8"); +const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8"); +const configPath = getRequiredEnv("CONFIG_PATH"); + +const fileText = "hello from zerobyte integration\n"; +const readmeText = "fixture documentation\n"; + +const nfsEntries = [ + { path: "hello.txt", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: fileText }, + { path: "docs", type: "directory", uid: fixtureUid, gid: fixtureGid, mode: "0755" }, + { path: "docs/readme.md", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: readmeText }, +]; + +const contentOnlyEntries = [ + { path: "hello.txt", type: "file", text: fileText }, + { path: "docs", type: "directory" }, + { path: "docs/readme.md", type: "file", text: readmeText }, +]; + +const config = { + version: 1, + scenarios: [ + { + id: "nfs-local-repo", + volume: { + backend: "nfs", + server: host, + exportPath: "/srv/zerobyte-backend-integration/fixtures", + port: 2049, + version: "4.1", + readOnly: true, + }, + repository: { backend: "local", path: "repo-nfs" }, + fixtureRoot: "case-a", + expectedEntries: nfsEntries, + }, + { + id: "smb-local-repo", + volume: { + backend: "smb", + server: host, + share: "zerobyte-backend-integration", + username: "zerobyte-smb", + password: smbPassword, + vers: "3.0", + port: 445, + readOnly: true, + }, + repository: { backend: "local", path: "repo-smb" }, + fixtureRoot: "case-a", + expectedEntries: nfsEntries, + }, + { + id: "sftp-local-repo", + volume: { + backend: "sftp", + host, + port: 22, + username: "zerobyte-sftp", + privateKey, + path: "/srv/zerobyte-backend-integration/fixtures", + readOnly: true, + skipHostKeyCheck: false, + knownHosts, + }, + repository: { backend: "local", path: "repo-sftp-volume" }, + fixtureRoot: "case-a", + expectedEntries: contentOnlyEntries, + }, + { + id: "webdav-local-repo", + volume: { + backend: "webdav", + server: host, + path: "/zerobyte-backend-integration", + username: "zerobyte-webdav", + password: webdavPassword, + port: 80, + readOnly: true, + ssl: false, + }, + repository: { backend: "local", path: "repo-webdav" }, + fixtureRoot: "case-a", + expectedEntries: contentOnlyEntries, + }, + { + id: "nfs-sftp-repository", + volume: { + backend: "nfs", + server: host, + exportPath: "/srv/zerobyte-backend-integration/fixtures", + port: 2049, + version: "4.1", + readOnly: true, + }, + repository: { + backend: "sftp", + host, + port: 22, + user: "zerobyte-sftp", + path: "/srv/zerobyte-backend-integration/restic-repo", + privateKey, + skipHostKeyCheck: false, + knownHosts, + isExistingRepository: true, + customPassword: resticPassword, + }, + fixtureRoot: "case-a", + expectedEntries: nfsEntries, + }, + ], +}; + +fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); diff --git a/package.json b/package.json index 2a995a89..9620d97c 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,9 @@ "check": "vp check", "dev": "NODE_ENV=development bunx --bun vite", "build": "bunx --bun vite build", + "build:backend-integration": "bun build app/test/backend-integration/index.ts --outdir .output/backend-integration --target bun", "start": "bun run .output/server/index.mjs", + "test:integration:backends": "bash app/test/backend-integration/run.sh", "preview": "bunx --bun vite preview", "cli:dev": "bun run app/server/cli/main.ts", "cli": "ZEROBYTE_CLI=1 bun .output/server/_ssr/ssr.mjs", diff --git a/packages/core/src/node/spawn.ts b/packages/core/src/node/spawn.ts index 02cfafcf..383a5e4a 100644 --- a/packages/core/src/node/spawn.ts +++ b/packages/core/src/node/spawn.ts @@ -17,6 +17,7 @@ export const safeExec = async ({ command, args = [], env = {}, ...rest }: ExecPr const { stdout, stderr } = await promisify(execFile)(command, args, { ...options, ...rest, + shell: false, encoding: "utf8", }); @@ -78,6 +79,7 @@ export function safeSpawn(params: SafeSpawnParams): Promise { return new Promise((resolve) => { const child = spawn(command, args, { env: { ...process.env, ...env }, + shell: false, signal: signal, stdio: ["ignore", "pipe", "pipe"], });