diff --git a/app/test/backend-integration/setup-target.sh b/app/test/backend-integration/setup-target.sh index e19dab25..4aec65c9 100755 --- a/app/test/backend-integration/setup-target.sh +++ b/app/test/backend-integration/setup-target.sh @@ -13,7 +13,6 @@ CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json" SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt" SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt" -WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt" read_or_create_secret() { local file_path="$1" @@ -32,16 +31,14 @@ chmod 700 "$ARTIFACTS_DIR" SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")" SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")" -WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")" -ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" <<'REMOTE' +ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" <<'REMOTE' set -euo pipefail fixture_uid="$1" fixture_gid="$2" smb_password="$3" sftp_password="$4" -webdav_password="$5" legacy_sshd_dir="/etc/ssh/zerobyte-backend-integration-legacy" export DEBIAN_FRONTEND=noninteractive @@ -52,7 +49,7 @@ write_file() { } apt-get update -apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server rpcbind samba +apt-get install -y nfs-kernel-server openssh-server rpcbind samba id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb @@ -68,7 +65,6 @@ printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb smbpasswd -e zerobyte-smb >/dev/null printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd passwd -u zerobyte-sftp >/dev/null 2>&1 || true -htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null write_file /etc/exports <<'EOF' /srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure) @@ -89,32 +85,6 @@ write_file /etc/samba/smb.conf <<'EOF' valid users = zerobyte-smb EOF -install -d -o www-data -g www-data -m 0755 /var/lib/dav -a2enmod dav dav_fs auth_basic >/dev/null -printf 'ServerName localhost\n' >/etc/apache2/conf-available/zerobyte-backend-integration-servername.conf -a2enconf zerobyte-backend-integration-servername >/dev/null -write_file /etc/apache2/sites-available/zerobyte-backend-integration-dav.conf <<'EOF' -Alias /zerobyte-backend-integration /srv/zerobyte-backend-integration/fixtures - -DAVLockDB /var/lib/dav/lockdb - - - DAV On - AuthType Basic - AuthName "Zerobyte Backend Integration WebDAV" - AuthUserFile /etc/apache2/zerobyte-backend-integration.htpasswd - Require valid-user - - - - Options Indexes FollowSymLinks - AllowOverride None - Require all granted - -EOF -a2ensite zerobyte-backend-integration-dav >/dev/null -apache2ctl configtest - install -d -m 0700 "$legacy_sshd_dir" if [[ ! -f "$legacy_sshd_dir/ssh_host_rsa_key" ]]; then ssh-keygen -q -t rsa -b 2048 -N "" -f "$legacy_sshd_dir/ssh_host_rsa_key" @@ -169,7 +139,6 @@ EOF systemctl daemon-reload systemctl enable --now zerobyte-backend-integration-legacy-sshd.service -systemctl restart apache2 systemctl restart smbd systemctl restart ssh systemctl restart zerobyte-backend-integration-legacy-sshd.service @@ -194,7 +163,6 @@ INTEGRATION_HOST="$TARGET_HOST" \ FIXTURE_GID="$FIXTURE_GID" \ SMB_PASSWORD="$SMB_PASSWORD" \ SFTP_PASSWORD="$SFTP_PASSWORD" \ - WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \ KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \ CONFIG_PATH="$CONFIG_PATH" \ bun run "$SCRIPT_DIR/write-generated-config.ts" diff --git a/app/test/backend-integration/write-generated-config.ts b/app/test/backend-integration/write-generated-config.ts index 56d8c234..0ad35ed0 100644 --- a/app/test/backend-integration/write-generated-config.ts +++ b/app/test/backend-integration/write-generated-config.ts @@ -13,7 +13,6 @@ const fixtureUid = getRequiredNumberEnv("FIXTURE_UID"); const fixtureGid = getRequiredNumberEnv("FIXTURE_GID"); const smbPassword = getRequiredEnv("SMB_PASSWORD"); const sftpPassword = getRequiredEnv("SFTP_PASSWORD"); -const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD"); const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8"); const configPath = getRequiredEnv("CONFIG_PATH"); @@ -90,22 +89,6 @@ const config = { fixtureRoot: "case-a", expectedEntries: contentOnlyEntries, }, - { - id: "webdav-local-repo", - volume: { - backend: "webdav", - server: host, - path: "/zerobyte-backend-integration", - username: "zerobyte-webdav", - password: webdavPassword, - port: 80, - readOnly: true, - ssl: false, - }, - repository: { backend: "local", path: "repo-webdav" }, - fixtureRoot: "case-a", - expectedEntries: contentOnlyEntries, - }, ], }; diff --git a/app/test/integration/infra/docker-compose.yml b/app/test/integration/infra/docker-compose.yml index 44f288a9..9d2014ba 100644 --- a/app/test/integration/infra/docker-compose.yml +++ b/app/test/integration/infra/docker-compose.yml @@ -39,6 +39,26 @@ services: timeout: 5s retries: 30 + webdav: + build: + context: ./webdav + environment: + WEBDAV_USER: "zerobyte-webdav" + WEBDAV_PASSWORD: "zerobyte-webdav-password" + healthcheck: + test: + [ + "CMD", + "curl", + "-fsS", + "-u", + "zerobyte-webdav:zerobyte-webdav-password", + "http://localhost/zerobyte-integration/case-a/hello.txt", + ] + interval: 1s + timeout: 5s + retries: 30 + integration: build: context: ../../../.. @@ -50,6 +70,8 @@ services: condition: service_completed_successfully sftp: condition: service_healthy + webdav: + condition: service_healthy cap_add: - SYS_ADMIN devices: diff --git a/app/test/integration/infra/webdav/Dockerfile b/app/test/integration/infra/webdav/Dockerfile new file mode 100644 index 00000000..f1b50346 --- /dev/null +++ b/app/test/integration/infra/webdav/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3.21 + +RUN apk add --no-cache apache2 apache2-utils apache2-webdav curl + +COPY entrypoint.sh /usr/local/bin/zerobyte-webdav-entrypoint +RUN chmod +x /usr/local/bin/zerobyte-webdav-entrypoint + +EXPOSE 80 + +ENTRYPOINT ["/usr/local/bin/zerobyte-webdav-entrypoint"] diff --git a/app/test/integration/infra/webdav/entrypoint.sh b/app/test/integration/infra/webdav/entrypoint.sh new file mode 100644 index 00000000..1e64bdc9 --- /dev/null +++ b/app/test/integration/infra/webdav/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu + +WEBDAV_USER="${WEBDAV_USER:-zerobyte-webdav}" +WEBDAV_PASSWORD="${WEBDAV_PASSWORD:-zerobyte-webdav-password}" +SERVICE_ROOT="/srv/zerobyte-integration" +FIXTURE_ROOT="$SERVICE_ROOT/fixtures" +LOCK_ROOT="/var/lib/zerobyte-webdav" + +install -d -m 0755 "$FIXTURE_ROOT/case-a/docs" +printf 'hello from zerobyte integration\n' >"$FIXTURE_ROOT/case-a/hello.txt" +printf 'fixture documentation\n' >"$FIXTURE_ROOT/case-a/docs/readme.md" +chown -R apache:apache "$SERVICE_ROOT" + +install -d -o apache -g apache -m 0755 "$LOCK_ROOT" +htpasswd -bc /etc/apache2/zerobyte-webdav.htpasswd "$WEBDAV_USER" "$WEBDAV_PASSWORD" >/dev/null + +cat >/etc/apache2/conf.d/zerobyte-webdav.conf < + DAV On + AuthType Basic + AuthName "Zerobyte Integration WebDAV" + AuthUserFile /etc/apache2/zerobyte-webdav.htpasswd + Require valid-user + + + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + +EOF + +httpd -t +exec httpd -D FOREGROUND diff --git a/app/test/integration/run.sh b/app/test/integration/run.sh index f490f256..8dd4cbeb 100644 --- a/app/test/integration/run.sh +++ b/app/test/integration/run.sh @@ -8,11 +8,17 @@ compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' ' artifacts_dir="$script_dir/artifacts" sftp_artifacts_dir="$artifacts_dir/sftp" compose_file="$script_dir/infra/docker-compose.yml" +docker_output_log="$artifacts_dir/docker-output.log" +compose=(docker compose -f "$compose_file" -p "$compose_project") mkdir -p "$artifacts_dir" +if [[ -d "$artifacts_dir/runs" ]]; then + chmod -R u+rwX "$artifacts_dir/runs" || true +fi rm -rf "$artifacts_dir/runs" rm -rf "$sftp_artifacts_dir" rm -f "$artifacts_dir/compose.log" +rm -f "$docker_output_log" mkdir -p "$artifacts_dir/runs" mkdir -p "$sftp_artifacts_dir" @@ -20,19 +26,29 @@ ssh-keygen -q -t ed25519 -N "" -f "$sftp_artifacts_dir/id_ed25519" chmod 600 "$sftp_artifacts_dir/id_ed25519" chmod 644 "$sftp_artifacts_dir/id_ed25519.pub" -docker build --target runtime-tools -t "$base_image" "$repo_root" +docker build --progress quiet --target runtime-tools -t "$base_image" "$repo_root" >"$docker_output_log" 2>&1 exit_code=0 -docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup || exit_code=$? +"${compose[@]}" up --build --quiet-build --no-color --detach rustfs >>"$docker_output_log" 2>&1 || exit_code=$? if [[ "$exit_code" -eq 0 ]]; then - docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from integration integration || exit_code=$? + "${compose[@]}" up --build --quiet-build --no-color --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup >>"$docker_output_log" 2>&1 || exit_code=$? +fi + +if [[ "$exit_code" -eq 0 ]]; then + "${compose[@]}" up --build --quiet-build --no-color --detach --wait sftp webdav >>"$docker_output_log" 2>&1 || exit_code=$? +fi + +if [[ "$exit_code" -eq 0 ]]; then + "${compose[@]}" run --rm --no-deps --build --quiet-build integration 2>>"$docker_output_log" || exit_code=$? fi if [[ "$exit_code" -ne 0 ]]; then - docker compose -f "$compose_file" -p "$compose_project" logs --no-color >"$artifacts_dir/compose.log" || true + "${compose[@]}" logs --no-color >"$artifacts_dir/compose.log" || true + echo "Integration Docker logs: $artifacts_dir/compose.log" >&2 + echo "Integration Docker command output: $docker_output_log" >&2 fi -docker compose -f "$compose_file" -p "$compose_project" down --volumes --remove-orphans || true +"${compose[@]}" down --volumes --remove-orphans >>"$docker_output_log" 2>&1 || true exit "$exit_code" diff --git a/app/test/integration/src/helpers/fixture.ts b/app/test/integration/src/helpers/fixture.ts index 9f3982f4..1949b47b 100644 --- a/app/test/integration/src/helpers/fixture.ts +++ b/app/test/integration/src/helpers/fixture.ts @@ -70,7 +70,7 @@ export const createScenarioFixture = async (workspace: string, scenarioId: strin return fixture; }; -export const createStaticSftpFixture = (sourceRoot: string): ScenarioFixture => ({ +export const createStaticVolumeFixture = (sourceRoot: string): ScenarioFixture => ({ sourceRoot, entries: [ { diff --git a/app/test/integration/src/helpers/webdav.ts b/app/test/integration/src/helpers/webdav.ts new file mode 100644 index 00000000..f5b0bc27 --- /dev/null +++ b/app/test/integration/src/helpers/webdav.ts @@ -0,0 +1,18 @@ +import type { BackendConfig } from "@zerobyte/contracts/volumes"; + +export const WEBDAV_HOST = "webdav"; +export const WEBDAV_PORT = 80; +export const WEBDAV_USERNAME = "zerobyte-webdav"; +export const WEBDAV_PASSWORD = "zerobyte-webdav-password"; +export const WEBDAV_FIXTURE_ROOT = "/zerobyte-integration"; + +export const buildWebdavVolumeConfig = (): BackendConfig => ({ + backend: "webdav", + server: WEBDAV_HOST, + path: WEBDAV_FIXTURE_ROOT, + username: WEBDAV_USERNAME, + password: WEBDAV_PASSWORD, + port: WEBDAV_PORT, + readOnly: true, + ssl: false, +}); diff --git a/app/test/integration/src/volume-backends.test.ts b/app/test/integration/src/volume-backends.test.ts index 8c141d8d..68f096b7 100644 --- a/app/test/integration/src/volume-backends.test.ts +++ b/app/test/integration/src/volume-backends.test.ts @@ -1,14 +1,15 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; -import type { BackendConfig } from "@zerobyte/contracts/volumes"; import type { RepositoryConfig } from "@zerobyte/core/restic"; import { Effect } from "effect"; import { expect, test } from "vitest"; import { makeSftpBackend } from "../../../../apps/agent/src/volume-host/backends/sftp"; +import { makeWebdavBackend } from "../../../../apps/agent/src/volume-host/backends/webdav"; +import type { VolumeBackend } from "../../../../apps/agent/src/volume-host/types"; import { INTEGRATION_ORGANIZATION_ID, INTEGRATION_RUNS_DIR } from "./constants"; import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions"; -import { createStaticSftpFixture } from "./helpers/fixture"; +import { createStaticVolumeFixture } from "./helpers/fixture"; import { createIntegrationRestic } from "./helpers/restic"; import { buildSftpPasswordVolumeConfig, @@ -16,26 +17,58 @@ import { readSftpPrivateKey, scanSftpKnownHosts, } from "./helpers/sftp"; +import { buildWebdavVolumeConfig } from "./helpers/webdav"; -type SftpVolumeScenario = { +type VolumeScenario = { id: string; name: string; - createVolumeConfig: (runtime: { privateKey: string; knownHosts: string }) => BackendConfig; + createBackend: (mountPath: string) => Promise; }; -const scenarios: SftpVolumeScenario[] = [ +const scenarios: VolumeScenario[] = [ { id: "sftp-local-repo", name: "SFTP volume with private key auth and local repository", - createVolumeConfig: ({ privateKey, knownHosts }) => buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts }), + createBackend: async (mountPath) => { + const knownHosts = await scanSftpKnownHosts(); + const privateKey = await readSftpPrivateKey(); + const config = buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts }); + return makeSftpBackend(config, mountPath); + }, }, { id: "sftp-password-local-repo", name: "SFTP volume with password auth and local repository", - createVolumeConfig: ({ knownHosts }) => buildSftpPasswordVolumeConfig({ knownHosts }), + createBackend: async (mountPath) => { + const knownHosts = await scanSftpKnownHosts(); + const config = buildSftpPasswordVolumeConfig({ knownHosts }); + return makeSftpBackend(config, mountPath); + }, + }, + { + id: "webdav-local-repo", + name: "WebDAV volume with local repository", + createBackend: async (mountPath) => makeWebdavBackend(buildWebdavVolumeConfig(), mountPath), }, ]; +const makeDirectoriesWritable = async (root: string): Promise => { + await fs.chmod(root, 0o700).catch(() => {}); + + const entries = await fs.readdir(root, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }); + + await Promise.all( + entries.map(async (entry) => { + if (entry.isDirectory()) { + await makeDirectoriesWritable(path.join(root, entry.name)); + } + }), + ); +}; + test.concurrent.each(scenarios)("$name can backup and restore static fixture data", async (scenario) => { const runId = crypto.randomUUID(); const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`); @@ -47,17 +80,15 @@ test.concurrent.each(scenarios)("$name can backup and restore static fixture dat const repositoryConfig: RepositoryConfig = { backend: "local", path: repositoryPath }; const restic = createIntegrationRestic(workspace, resticPassword); - let backend: ReturnType | undefined; + let backend: VolumeBackend | undefined; let passed = false; let unmountFailed = false; + let cleanupError: Error | undefined; try { await fs.mkdir(workspace, { recursive: true }); - const knownHosts = await scanSftpKnownHosts(); - const privateKey = await readSftpPrivateKey(); - const volumeConfig = scenario.createVolumeConfig({ privateKey, knownHosts }); - backend = makeSftpBackend(volumeConfig, mountPath); + backend = await scenario.createBackend(mountPath); const mountResult = await backend.mount(); expect(mountResult.status).toBe("mounted"); @@ -65,7 +96,7 @@ test.concurrent.each(scenarios)("$name can backup and restore static fixture dat const healthResult = await backend.checkHealth(); expect(healthResult.status).toBe("mounted"); - const fixture = createStaticSftpFixture(path.join(mountPath, "case-a")); + const fixture = createStaticVolumeFixture(path.join(mountPath, "case-a")); await assertFixtureSourceExists(fixture); const initResult = await Effect.runPromise( @@ -131,18 +162,23 @@ test.concurrent.each(scenarios)("$name can backup and restore static fixture dat const unmountResult = await backend.unmount(); if (unmountResult.status === "error") { unmountFailed = true; - console.error(`Failed to unmount SFTP volume at ${mountPath}: ${unmountResult.error}`); + console.error(`Failed to unmount ${scenario.id} volume at ${mountPath}: ${unmountResult.error}`); } } if (passed && !unmountFailed) { + await makeDirectoriesWritable(workspace); await fs.rm(workspace, { recursive: true, force: true }); } else { console.error(`Integration scenario artifacts retained in ${workspace}`); } if (passed && unmountFailed) { - throw new Error(`Failed to unmount SFTP volume at ${mountPath}`); + cleanupError = new Error(`Failed to unmount ${scenario.id} volume at ${mountPath}`); } } + + if (cleanupError) { + throw cleanupError; + } });