danvics/zerobyte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

test: automate webdav integration

Browse source
This commit is contained in:
Nicolas Meienberger 2026-06-02 17:30:49 +02:00
parent 756ecbddcd
commit 036382d82d
No known key found for this signature in database
9 changed files with 167 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
app/test/backend-integration/setup-target.sh
View file

@ -13,7 +13,6 @@ CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json"
SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt"
SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt"
WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt"
read_or_create_secret() {
local file_path="$1"
@ -32,16 +31,14 @@ chmod 700 "$ARTIFACTS_DIR"
SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")"
SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")"
WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")"
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" <<'REMOTE'
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" <<'REMOTE'
set -euo pipefail
fixture_uid="$1"
fixture_gid="$2"
smb_password="$3"
sftp_password="$4"
webdav_password="$5"
legacy_sshd_dir="/etc/ssh/zerobyte-backend-integration-legacy"
export DEBIAN_FRONTEND=noninteractive
@ -52,7 +49,7 @@ write_file() {
}
apt-get update
apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server rpcbind samba
apt-get install -y nfs-kernel-server openssh-server rpcbind samba
id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp
id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb
@ -68,7 +65,6 @@ printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb
smbpasswd -e zerobyte-smb >/dev/null
printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd
passwd -u zerobyte-sftp >/dev/null 2>&1 || true
htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null
write_file /etc/exports <<'EOF'
/srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure)
@ -89,32 +85,6 @@ write_file /etc/samba/smb.conf <<'EOF'
valid users = zerobyte-smb
EOF
install -d -o www-data -g www-data -m 0755 /var/lib/dav
a2enmod dav dav_fs auth_basic >/dev/null
printf 'ServerName localhost\n' >/etc/apache2/conf-available/zerobyte-backend-integration-servername.conf
a2enconf zerobyte-backend-integration-servername >/dev/null
write_file /etc/apache2/sites-available/zerobyte-backend-integration-dav.conf <<'EOF'
Alias /zerobyte-backend-integration /srv/zerobyte-backend-integration/fixtures
DAVLockDB /var/lib/dav/lockdb
<Location /zerobyte-backend-integration>
DAV On
AuthType Basic
AuthName "Zerobyte Backend Integration WebDAV"
AuthUserFile /etc/apache2/zerobyte-backend-integration.htpasswd
Require valid-user
</Location>
<Directory /srv/zerobyte-backend-integration/fixtures>
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
EOF
a2ensite zerobyte-backend-integration-dav >/dev/null
apache2ctl configtest
install -d -m 0700 "$legacy_sshd_dir"
if [[ ! -f "$legacy_sshd_dir/ssh_host_rsa_key" ]]; then
ssh-keygen -q -t rsa -b 2048 -N "" -f "$legacy_sshd_dir/ssh_host_rsa_key"
@ -169,7 +139,6 @@ EOF
systemctl daemon-reload
systemctl enable --now zerobyte-backend-integration-legacy-sshd.service
systemctl restart apache2
systemctl restart smbd
systemctl restart ssh
systemctl restart zerobyte-backend-integration-legacy-sshd.service
@ -194,7 +163,6 @@ INTEGRATION_HOST="$TARGET_HOST" \
FIXTURE_GID="$FIXTURE_GID" \
SMB_PASSWORD="$SMB_PASSWORD" \
SFTP_PASSWORD="$SFTP_PASSWORD" \
WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \
KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \
CONFIG_PATH="$CONFIG_PATH" \
bun run "$SCRIPT_DIR/write-generated-config.ts"

17
app/test/backend-integration/write-generated-config.ts
View file

@ -13,7 +13,6 @@ const fixtureUid = getRequiredNumberEnv("FIXTURE_UID");
const fixtureGid = getRequiredNumberEnv("FIXTURE_GID");
const smbPassword = getRequiredEnv("SMB_PASSWORD");
const sftpPassword = getRequiredEnv("SFTP_PASSWORD");
const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD");
const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8");
const configPath = getRequiredEnv("CONFIG_PATH");
@ -90,22 +89,6 @@ const config = {
fixtureRoot: "case-a",
expectedEntries: contentOnlyEntries,
},
{
id: "webdav-local-repo",
volume: {
backend: "webdav",
server: host,
path: "/zerobyte-backend-integration",
username: "zerobyte-webdav",
password: webdavPassword,
port: 80,
readOnly: true,
ssl: false,
},
repository: { backend: "local", path: "repo-webdav" },
fixtureRoot: "case-a",
expectedEntries: contentOnlyEntries,
},
],
};

22
app/test/integration/infra/docker-compose.yml
View file

@ -39,6 +39,26 @@ services:
timeout: 5s
retries: 30
webdav:
build:
context: ./webdav
environment:
WEBDAV_USER: "zerobyte-webdav"
WEBDAV_PASSWORD: "zerobyte-webdav-password"
healthcheck:
test:
[
"CMD",
"curl",
"-fsS",
"-u",
"zerobyte-webdav:zerobyte-webdav-password",
"http://localhost/zerobyte-integration/case-a/hello.txt",
]
interval: 1s
timeout: 5s
retries: 30
integration:
build:
context: ../../../..
@ -50,6 +70,8 @@ services:
condition: service_completed_successfully
sftp:
condition: service_healthy
webdav:
condition: service_healthy
cap_add:
- SYS_ADMIN
devices:

10
app/test/integration/infra/webdav/Dockerfile Normal file
View file

@ -0,0 +1,10 @@
FROM alpine:3.21
RUN apk add --no-cache apache2 apache2-utils apache2-webdav curl
COPY entrypoint.sh /usr/local/bin/zerobyte-webdav-entrypoint
RUN chmod +x /usr/local/bin/zerobyte-webdav-entrypoint
EXPOSE 80
ENTRYPOINT ["/usr/local/bin/zerobyte-webdav-entrypoint"]

42
app/test/integration/infra/webdav/entrypoint.sh Normal file
View file

@ -0,0 +1,42 @@
#!/bin/sh
set -eu
WEBDAV_USER="${WEBDAV_USER:-zerobyte-webdav}"
WEBDAV_PASSWORD="${WEBDAV_PASSWORD:-zerobyte-webdav-password}"
SERVICE_ROOT="/srv/zerobyte-integration"
FIXTURE_ROOT="$SERVICE_ROOT/fixtures"
LOCK_ROOT="/var/lib/zerobyte-webdav"
install -d -m 0755 "$FIXTURE_ROOT/case-a/docs"
printf 'hello from zerobyte integration\n' >"$FIXTURE_ROOT/case-a/hello.txt"
printf 'fixture documentation\n' >"$FIXTURE_ROOT/case-a/docs/readme.md"
chown -R apache:apache "$SERVICE_ROOT"
install -d -o apache -g apache -m 0755 "$LOCK_ROOT"
htpasswd -bc /etc/apache2/zerobyte-webdav.htpasswd "$WEBDAV_USER" "$WEBDAV_PASSWORD" >/dev/null
cat >/etc/apache2/conf.d/zerobyte-webdav.conf <<EOF
ServerName localhost
ErrorLog /proc/self/fd/2
CustomLog /proc/self/fd/1 combined
DAVLockDB $LOCK_ROOT/lockdb
Alias /zerobyte-integration $FIXTURE_ROOT
<Location /zerobyte-integration>
DAV On
AuthType Basic
AuthName "Zerobyte Integration WebDAV"
AuthUserFile /etc/apache2/zerobyte-webdav.htpasswd
Require valid-user
</Location>
<Directory $FIXTURE_ROOT>
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
EOF
httpd -t
exec httpd -D FOREGROUND

26
app/test/integration/run.sh
View file

@ -8,11 +8,17 @@ compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' '
artifacts_dir="$script_dir/artifacts"
sftp_artifacts_dir="$artifacts_dir/sftp"
compose_file="$script_dir/infra/docker-compose.yml"
docker_output_log="$artifacts_dir/docker-output.log"
compose=(docker compose -f "$compose_file" -p "$compose_project")
mkdir -p "$artifacts_dir"
if [[ -d "$artifacts_dir/runs" ]]; then
chmod -R u+rwX "$artifacts_dir/runs" || true
fi
rm -rf "$artifacts_dir/runs"
rm -rf "$sftp_artifacts_dir"
rm -f "$artifacts_dir/compose.log"
rm -f "$docker_output_log"
mkdir -p "$artifacts_dir/runs"
mkdir -p "$sftp_artifacts_dir"
@ -20,19 +26,29 @@ ssh-keygen -q -t ed25519 -N "" -f "$sftp_artifacts_dir/id_ed25519"
chmod 600 "$sftp_artifacts_dir/id_ed25519"
chmod 644 "$sftp_artifacts_dir/id_ed25519.pub"
docker build --target runtime-tools -t "$base_image" "$repo_root"
docker build --progress quiet --target runtime-tools -t "$base_image" "$repo_root" >"$docker_output_log" 2>&1
exit_code=0
docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup || exit_code=$?
"${compose[@]}" up --build --quiet-build --no-color --detach rustfs >>"$docker_output_log" 2>&1 || exit_code=$?
if [[ "$exit_code" -eq 0 ]]; then
docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from integration integration || exit_code=$?
"${compose[@]}" up --build --quiet-build --no-color --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup >>"$docker_output_log" 2>&1 || exit_code=$?
fi
if [[ "$exit_code" -eq 0 ]]; then
"${compose[@]}" up --build --quiet-build --no-color --detach --wait sftp webdav >>"$docker_output_log" 2>&1 || exit_code=$?
fi
if [[ "$exit_code" -eq 0 ]]; then
"${compose[@]}" run --rm --no-deps --build --quiet-build integration 2>>"$docker_output_log" || exit_code=$?
fi
if [[ "$exit_code" -ne 0 ]]; then
docker compose -f "$compose_file" -p "$compose_project" logs --no-color >"$artifacts_dir/compose.log" || true
"${compose[@]}" logs --no-color >"$artifacts_dir/compose.log" || true
echo "Integration Docker logs: $artifacts_dir/compose.log" >&2
echo "Integration Docker command output: $docker_output_log" >&2
fi
docker compose -f "$compose_file" -p "$compose_project" down --volumes --remove-orphans || true
"${compose[@]}" down --volumes --remove-orphans >>"$docker_output_log" 2>&1 || true
exit "$exit_code"

2
app/test/integration/src/helpers/fixture.ts
View file

@ -70,7 +70,7 @@ export const createScenarioFixture = async (workspace: string, scenarioId: strin
return fixture;
};
export const createStaticSftpFixture = (sourceRoot: string): ScenarioFixture => ({
export const createStaticVolumeFixture = (sourceRoot: string): ScenarioFixture => ({
sourceRoot,
entries: [
{

18
app/test/integration/src/helpers/webdav.ts Normal file
View file

@ -0,0 +1,18 @@
import type { BackendConfig } from "@zerobyte/contracts/volumes";
export const WEBDAV_HOST = "webdav";
export const WEBDAV_PORT = 80;
export const WEBDAV_USERNAME = "zerobyte-webdav";
export const WEBDAV_PASSWORD = "zerobyte-webdav-password";
export const WEBDAV_FIXTURE_ROOT = "/zerobyte-integration";
export const buildWebdavVolumeConfig = (): BackendConfig => ({
backend: "webdav",
server: WEBDAV_HOST,
path: WEBDAV_FIXTURE_ROOT,
username: WEBDAV_USERNAME,
password: WEBDAV_PASSWORD,
port: WEBDAV_PORT,
readOnly: true,
ssl: false,
});

66
app/test/integration/src/volume-backends.test.ts
View file

@ -1,14 +1,15 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import type { BackendConfig } from "@zerobyte/contracts/volumes";
import type { RepositoryConfig } from "@zerobyte/core/restic";
import { Effect } from "effect";
import { expect, test } from "vitest";
import { makeSftpBackend } from "../../../../apps/agent/src/volume-host/backends/sftp";
import { makeWebdavBackend } from "../../../../apps/agent/src/volume-host/backends/webdav";
import type { VolumeBackend } from "../../../../apps/agent/src/volume-host/types";
import { INTEGRATION_ORGANIZATION_ID, INTEGRATION_RUNS_DIR } from "./constants";
import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
import { createStaticSftpFixture } from "./helpers/fixture";
import { createStaticVolumeFixture } from "./helpers/fixture";
import { createIntegrationRestic } from "./helpers/restic";
import {
buildSftpPasswordVolumeConfig,
@ -16,26 +17,58 @@ import {
readSftpPrivateKey,
scanSftpKnownHosts,
} from "./helpers/sftp";
import { buildWebdavVolumeConfig } from "./helpers/webdav";
type SftpVolumeScenario = {
type VolumeScenario = {
id: string;
name: string;
createVolumeConfig: (runtime: { privateKey: string; knownHosts: string }) => BackendConfig;
createBackend: (mountPath: string) => Promise<VolumeBackend>;
};
const scenarios: SftpVolumeScenario[] = [
const scenarios: VolumeScenario[] = [
{
id: "sftp-local-repo",
name: "SFTP volume with private key auth and local repository",
createVolumeConfig: ({ privateKey, knownHosts }) => buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts }),
createBackend: async (mountPath) => {
const knownHosts = await scanSftpKnownHosts();
const privateKey = await readSftpPrivateKey();
const config = buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts });
return makeSftpBackend(config, mountPath);
},
},
{
id: "sftp-password-local-repo",
name: "SFTP volume with password auth and local repository",
createVolumeConfig: ({ knownHosts }) => buildSftpPasswordVolumeConfig({ knownHosts }),
createBackend: async (mountPath) => {
const knownHosts = await scanSftpKnownHosts();
const config = buildSftpPasswordVolumeConfig({ knownHosts });
return makeSftpBackend(config, mountPath);
},
},
{
id: "webdav-local-repo",
name: "WebDAV volume with local repository",
createBackend: async (mountPath) => makeWebdavBackend(buildWebdavVolumeConfig(), mountPath),
},
];
const makeDirectoriesWritable = async (root: string): Promise<void> => {
await fs.chmod(root, 0o700).catch(() => {});
const entries = await fs.readdir(root, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return [];
throw error;
});
await Promise.all(
entries.map(async (entry) => {
if (entry.isDirectory()) {
await makeDirectoriesWritable(path.join(root, entry.name));
}
}),
);
};
test.concurrent.each(scenarios)("$name can backup and restore static fixture data", async (scenario) => {
const runId = crypto.randomUUID();
const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`);
@ -47,17 +80,15 @@ test.concurrent.each(scenarios)("$name can backup and restore static fixture dat
const repositoryConfig: RepositoryConfig = { backend: "local", path: repositoryPath };
const restic = createIntegrationRestic(workspace, resticPassword);
let backend: ReturnType<typeof makeSftpBackend> | undefined;
let backend: VolumeBackend | undefined;
let passed = false;
let unmountFailed = false;
let cleanupError: Error | undefined;
try {
await fs.mkdir(workspace, { recursive: true });
const knownHosts = await scanSftpKnownHosts();
const privateKey = await readSftpPrivateKey();
const volumeConfig = scenario.createVolumeConfig({ privateKey, knownHosts });
backend = makeSftpBackend(volumeConfig, mountPath);
backend = await scenario.createBackend(mountPath);
const mountResult = await backend.mount();
expect(mountResult.status).toBe("mounted");
@ -65,7 +96,7 @@ test.concurrent.each(scenarios)("$name can backup and restore static fixture dat
const healthResult = await backend.checkHealth();
expect(healthResult.status).toBe("mounted");
const fixture = createStaticSftpFixture(path.join(mountPath, "case-a"));
const fixture = createStaticVolumeFixture(path.join(mountPath, "case-a"));
await assertFixtureSourceExists(fixture);
const initResult = await Effect.runPromise(
@ -131,18 +162,23 @@ test.concurrent.each(scenarios)("$name can backup and restore static fixture dat
const unmountResult = await backend.unmount();
if (unmountResult.status === "error") {
unmountFailed = true;
console.error(`Failed to unmount SFTP volume at ${mountPath}: ${unmountResult.error}`);
console.error(`Failed to unmount ${scenario.id} volume at ${mountPath}: ${unmountResult.error}`);
}
}
if (passed && !unmountFailed) {
await makeDirectoriesWritable(workspace);
await fs.rm(workspace, { recursive: true, force: true });
} else {
console.error(`Integration scenario artifacts retained in ${workspace}`);
}
if (passed && unmountFailed) {
throw new Error(`Failed to unmount SFTP volume at ${mountPath}`);
cleanupError = new Error(`Failed to unmount ${scenario.id} volume at ${mountPath}`);
}
}
if (cleanupError) {
throw cleanupError;
}
});