test: backend integration (#889)
* test: backend integration * docs: mounted shares acls * feat: smb expose real ACLs when available * fix: re-init repo on setup * chore: add missing @hono/standard-validator package * chore: add happy-dom dev dep
This commit is contained in:
parent
3142cb026a
commit
19a0781667
26 changed files with 1541 additions and 414 deletions
|
|
@ -10,7 +10,7 @@ ENV VITE_RESTIC_VERSION=${RESTIC_VERSION} \
|
||||||
|
|
||||||
RUN apk update --no-cache && \
|
RUN apk update --no-cache && \
|
||||||
apk upgrade --no-cache && \
|
apk upgrade --no-cache && \
|
||||||
apk add --no-cache davfs2=1.6.1-r2 openssh-client fuse3 sshfs tini tzdata
|
apk add --no-cache acl attr cifs-utils davfs2=1.6.1-r2 openssh-client fuse3 sshfs tini tzdata
|
||||||
|
|
||||||
ENTRYPOINT ["/sbin/tini", "-s", "--"]
|
ENTRYPOINT ["/sbin/tini", "-s", "--"]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -66,7 +66,7 @@ type Props = {
|
||||||
const defaultValuesForType = {
|
const defaultValuesForType = {
|
||||||
directory: { backend: "directory" as const, path: "/" },
|
directory: { backend: "directory" as const, path: "/" },
|
||||||
nfs: { backend: "nfs" as const, port: 2049, version: "4.1" as const },
|
nfs: { backend: "nfs" as const, port: 2049, version: "4.1" as const },
|
||||||
smb: { backend: "smb" as const, port: 445, vers: "3.0" as const },
|
smb: { backend: "smb" as const, port: 445, vers: "3.0" as const, mapToContainerUidGid: false },
|
||||||
webdav: { backend: "webdav" as const, port: 80, ssl: false, path: "/webdav" },
|
webdav: { backend: "webdav" as const, port: 80, ssl: false, path: "/webdav" },
|
||||||
rclone: { backend: "rclone" as const, path: "/" },
|
rclone: { backend: "rclone" as const, path: "/" },
|
||||||
sftp: { backend: "sftp" as const, port: 22, path: "/", skipHostKeyCheck: false },
|
sftp: { backend: "sftp" as const, port: 22, path: "/", skipHostKeyCheck: false },
|
||||||
|
|
|
||||||
|
|
@ -168,6 +168,32 @@ export const SMBForm = ({ form }: Props) => {
|
||||||
</FormItem>
|
</FormItem>
|
||||||
)}
|
)}
|
||||||
/>
|
/>
|
||||||
|
<FormField
|
||||||
|
control={form.control}
|
||||||
|
name="mapToContainerUidGid"
|
||||||
|
defaultValue={false}
|
||||||
|
render={({ field }) => (
|
||||||
|
<FormItem>
|
||||||
|
<FormLabel>Ownership Mapping</FormLabel>
|
||||||
|
<FormControl>
|
||||||
|
<div className="flex items-center space-x-2">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={field.value ?? false}
|
||||||
|
onChange={(e) => field.onChange(e.target.checked)}
|
||||||
|
className="rounded border-gray-300"
|
||||||
|
/>
|
||||||
|
<span className="text-sm">Map all files to container user/group</span>
|
||||||
|
</div>
|
||||||
|
</FormControl>
|
||||||
|
<FormDescription>
|
||||||
|
Keep the old behavior by forcing the SMB mount to present every file and directory as owned by the
|
||||||
|
container user and group instead of using server reported ownership.
|
||||||
|
</FormDescription>
|
||||||
|
<FormMessage />
|
||||||
|
</FormItem>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
<FormField
|
<FormField
|
||||||
control={form.control}
|
control={form.control}
|
||||||
name="readOnly"
|
name="readOnly"
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import { v00002 } from "./migrations/00002-isolate-restic-passwords";
|
||||||
import { v00003 } from "./migrations/00003-assign-organization";
|
import { v00003 } from "./migrations/00003-assign-organization";
|
||||||
import { v00004 } from "./migrations/00004-concat-path-name";
|
import { v00004 } from "./migrations/00004-concat-path-name";
|
||||||
import { v00005 } from "./migrations/00005-split-backup-include-paths";
|
import { v00005 } from "./migrations/00005-split-backup-include-paths";
|
||||||
|
import { v00006 } from "./migrations/00006-map-smb-files-to-container-uid-gid";
|
||||||
import { sql } from "drizzle-orm";
|
import { sql } from "drizzle-orm";
|
||||||
import { appMetadataTable, usersTable } from "../../db/schema";
|
import { appMetadataTable, usersTable } from "../../db/schema";
|
||||||
import { db } from "../../db/db";
|
import { db } from "../../db/db";
|
||||||
|
|
@ -38,7 +39,7 @@ type MigrationEntity = {
|
||||||
dependsOn?: string[];
|
dependsOn?: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
const registry: MigrationEntity[] = [v00001, v00002, v00003, v00004, v00005];
|
const registry: MigrationEntity[] = [v00001, v00002, v00003, v00004, v00005, v00006];
|
||||||
|
|
||||||
export const runMigrations = async () => {
|
export const runMigrations = async () => {
|
||||||
const userCount = await db.select({ count: sql<number>`count(*)` }).from(usersTable);
|
const userCount = await db.select({ count: sql<number>`count(*)` }).from(usersTable);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
import { eq } from "drizzle-orm";
|
||||||
|
import { logger } from "@zerobyte/core/node";
|
||||||
|
import { db } from "../../../db/db";
|
||||||
|
import { volumesTable } from "../../../db/schema";
|
||||||
|
import { toMessage } from "~/server/utils/errors";
|
||||||
|
|
||||||
|
const execute = async () => {
|
||||||
|
const errors: Array<{ name: string; error: string }> = [];
|
||||||
|
const volumes = await db.query.volumesTable.findMany();
|
||||||
|
let migratedCount = 0;
|
||||||
|
|
||||||
|
for (const volume of volumes) {
|
||||||
|
if (volume.type !== "smb" || volume.config.backend !== "smb" || volume.config.mapToContainerUidGid !== undefined) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await db
|
||||||
|
.update(volumesTable)
|
||||||
|
.set({
|
||||||
|
config: { ...volume.config, mapToContainerUidGid: true },
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
})
|
||||||
|
.where(eq(volumesTable.id, volume.id));
|
||||||
|
|
||||||
|
migratedCount += 1;
|
||||||
|
} catch (error) {
|
||||||
|
errors.push({
|
||||||
|
name: `volume:${volume.id}`,
|
||||||
|
error: toMessage(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Migration 00006-map-smb-files-to-container-uid-gid updated ${migratedCount} SMB volumes.`);
|
||||||
|
|
||||||
|
return { success: errors.length === 0, errors };
|
||||||
|
};
|
||||||
|
|
||||||
|
export const v00006 = {
|
||||||
|
execute,
|
||||||
|
id: "00006-map-smb-files-to-container-uid-gid",
|
||||||
|
type: "maintenance" as const,
|
||||||
|
};
|
||||||
47
app/test/backend-integration/Dockerfile
Normal file
47
app/test/backend-integration/Dockerfile
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
FROM oven/bun:1.3.13-alpine@sha256:4de475389889577f346c636f956b42a5c31501b654664e9ae5726f94d7bb5349
|
||||||
|
|
||||||
|
ARG RESTIC_VERSION="0.18.1"
|
||||||
|
ARG RCLONE_VERSION="1.74.0"
|
||||||
|
ARG SHOUTRRR_VERSION="0.14.3"
|
||||||
|
ARG TARGETARCH
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN apk update --no-cache && \
|
||||||
|
apk upgrade --no-cache && \
|
||||||
|
apk add --no-cache acl attr cifs-utils curl bzip2 unzip tar davfs2=1.6.1-r2 openssh-client fuse3 sshfs tini tzdata
|
||||||
|
|
||||||
|
COPY ./package.json ./bun.lock ./
|
||||||
|
COPY ./packages/core/package.json ./packages/core/package.json
|
||||||
|
COPY ./packages/contracts/package.json ./packages/contracts/package.json
|
||||||
|
COPY ./apps/agent/package.json ./apps/agent/package.json
|
||||||
|
COPY ./apps/docs/package.json ./apps/docs/package.json
|
||||||
|
|
||||||
|
RUN VITE_GIT_HOOKS=0 bun install --frozen-lockfile
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN bun run build:backend-integration
|
||||||
|
|
||||||
|
RUN echo "Building for ${TARGETARCH}" && if [ "${TARGETARCH}" = "arm64" ]; then \
|
||||||
|
curl -fL -o restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_arm64.bz2"; \
|
||||||
|
curl -fL -o rclone.zip "https://github.com/rclone/rclone/releases/download/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-arm64.zip"; \
|
||||||
|
unzip rclone.zip; \
|
||||||
|
curl -fL -o shoutrrr.tar.gz "https://github.com/nicholas-fedor/shoutrrr/releases/download/v${SHOUTRRR_VERSION}/shoutrrr_linux_arm64v8_${SHOUTRRR_VERSION}.tar.gz"; \
|
||||||
|
elif [ "${TARGETARCH}" = "amd64" ]; then \
|
||||||
|
curl -fL -o restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_amd64.bz2"; \
|
||||||
|
curl -fL -o rclone.zip "https://github.com/rclone/rclone/releases/download/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip"; \
|
||||||
|
unzip rclone.zip; \
|
||||||
|
curl -fL -o shoutrrr.tar.gz "https://github.com/nicholas-fedor/shoutrrr/releases/download/v${SHOUTRRR_VERSION}/shoutrrr_linux_amd64_${SHOUTRRR_VERSION}.tar.gz"; \
|
||||||
|
else \
|
||||||
|
echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
RUN bzip2 -d restic.bz2 && install -m 0755 restic /usr/local/bin/restic
|
||||||
|
RUN mv rclone-v*-linux-*/rclone /usr/local/bin/rclone && chmod +x /usr/local/bin/rclone
|
||||||
|
RUN tar -xzf shoutrrr.tar.gz && install -m 0755 shoutrrr /usr/local/bin/shoutrrr
|
||||||
|
|
||||||
|
ENTRYPOINT ["/sbin/tini", "-s", "--"]
|
||||||
|
|
||||||
|
CMD ["bun", ".output/backend-integration/index.js"]
|
||||||
26
app/test/backend-integration/README.md
Normal file
26
app/test/backend-integration/README.md
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
# Backend Integration
|
||||||
|
|
||||||
|
This runner executes isolated source-level integration tests for Zerobyte volume backends and repository backends inside a Docker container that shares the same runtime tooling as the main image.
|
||||||
|
|
||||||
|
## What it verifies
|
||||||
|
|
||||||
|
For each scenario the runner will:
|
||||||
|
|
||||||
|
1. Mount the configured volume backend into an isolated temp workspace.
|
||||||
|
2. Read fixture files directly from the mounted filesystem.
|
||||||
|
3. Verify mounted file ownership and permission bits.
|
||||||
|
4. Run a restic backup directly against the configured repository backend.
|
||||||
|
5. Inspect the created snapshot with `restic ls` and verify snapshot metadata.
|
||||||
|
6. Restore the snapshot into an isolated temp directory.
|
||||||
|
7. Verify restored content, ownership, and permission bits.
|
||||||
|
8. Unmount and clean up local test artifacts.
|
||||||
|
|
||||||
|
## Bootstrap a Debian target
|
||||||
|
|
||||||
|
This folder includes `setup-target.sh`, which connects to a VM host and configures:
|
||||||
|
|
||||||
|
- NFS export at `/srv/zerobyte-backend-integration/fixtures`
|
||||||
|
- Samba share `//<host>/zerobyte-backend-integration`
|
||||||
|
- WebDAV endpoint at `http://<host>/zerobyte-backend-integration`
|
||||||
|
- SFTP access for both fixture reads and a reusable restic repository
|
||||||
|
- A generated local keypair, `known_hosts`, password files, and a ready-to-run config under `artifacts/192.168.2.41/`
|
||||||
2
app/test/backend-integration/artifacts/.gitignore
vendored
Normal file
2
app/test/backend-integration/artifacts/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
*
|
||||||
|
!.gitignore
|
||||||
18
app/test/backend-integration/index.ts
Normal file
18
app/test/backend-integration/index.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
function ensureIntegrationEnv(): void {
|
||||||
|
process.env.NODE_ENV ??= "production";
|
||||||
|
process.env.LOG_LEVEL ??= "warn";
|
||||||
|
process.env.BASE_URL ??= "http://localhost:4096";
|
||||||
|
process.env.TRUSTED_ORIGINS ??= process.env.BASE_URL;
|
||||||
|
process.env.APP_SECRET ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main(): Promise<void> {
|
||||||
|
ensureIntegrationEnv();
|
||||||
|
|
||||||
|
const { runBackendIntegration } = await import("./runner");
|
||||||
|
await runBackendIntegration();
|
||||||
|
}
|
||||||
|
|
||||||
|
await main();
|
||||||
|
|
||||||
|
export {};
|
||||||
34
app/test/backend-integration/run.sh
Executable file
34
app/test/backend-integration/run.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
script_dir="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
repo_root="$(cd "$script_dir/../../.." && pwd)"
|
||||||
|
default_config_path="$script_dir/artifacts/192.168.2.41/config.generated.json"
|
||||||
|
config_path="${1:-$default_config_path}"
|
||||||
|
image_tag="zerobyte-backend-integration"
|
||||||
|
|
||||||
|
if [[ ! -f "$config_path" ]]; then
|
||||||
|
if [[ "$config_path" == "$default_config_path" ]]; then
|
||||||
|
echo "Generated config not found: $config_path" >&2
|
||||||
|
echo "Run the target bootstrap first:" >&2
|
||||||
|
echo " bash app/test/backend-integration/setup-target.sh" >&2
|
||||||
|
else
|
||||||
|
echo "Config file not found: $config_path" >&2
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v realpath >/dev/null 2>&1; then
|
||||||
|
config_path="$(realpath "$config_path")"
|
||||||
|
else
|
||||||
|
config_dir="$(cd "$(dirname "$config_path")" && pwd)"
|
||||||
|
config_path="$config_dir/$(basename "$config_path")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker build -f "$script_dir/Dockerfile" -t "$image_tag" "$repo_root"
|
||||||
|
docker run --rm \
|
||||||
|
--cap-add SYS_ADMIN \
|
||||||
|
--device /dev/fuse:/dev/fuse \
|
||||||
|
-e ZEROBYTE_INTEGRATION_CONFIG=/config/config.json \
|
||||||
|
-v "$config_path:/config/config.json:ro" \
|
||||||
|
"$image_tag"
|
||||||
350
app/test/backend-integration/runner.ts
Normal file
350
app/test/backend-integration/runner.ts
Normal file
|
|
@ -0,0 +1,350 @@
|
||||||
|
import crypto from "node:crypto";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import { Effect } from "effect";
|
||||||
|
import type { VolumeBackend } from "../../../apps/agent/src/volume-host/types";
|
||||||
|
import { makeDirectoryBackend } from "../../../apps/agent/src/volume-host/backends/directory";
|
||||||
|
import { makeNfsBackend } from "../../../apps/agent/src/volume-host/backends/nfs";
|
||||||
|
import { makeRcloneBackend } from "../../../apps/agent/src/volume-host/backends/rclone";
|
||||||
|
import { makeSftpBackend } from "../../../apps/agent/src/volume-host/backends/sftp";
|
||||||
|
import { makeSmbBackend } from "../../../apps/agent/src/volume-host/backends/smb";
|
||||||
|
import { makeWebdavBackend } from "../../../apps/agent/src/volume-host/backends/webdav";
|
||||||
|
import { createRestic } from "@zerobyte/core/restic/server";
|
||||||
|
import { RCLONE_CONFIG_FILE } from "~/server/core/constants";
|
||||||
|
import { configSchema, type IntegrationScenario } from "./runner/config";
|
||||||
|
import {
|
||||||
|
formatError,
|
||||||
|
logScenario,
|
||||||
|
printRunSummary,
|
||||||
|
type IntegrationReport,
|
||||||
|
type ScenarioReport,
|
||||||
|
type StageReport,
|
||||||
|
} from "./runner/reporting";
|
||||||
|
import { verifyFilesystemEntries, verifySnapshotEntries } from "./runner/verification";
|
||||||
|
|
||||||
|
const WORKDIR_ROOT = "/tmp/zerobyte-integration";
|
||||||
|
const INTEGRATION_ORGANIZATION_ID = "backend-integration";
|
||||||
|
|
||||||
|
function createVolumeBackend(config: IntegrationScenario["volume"], mountPath: string): VolumeBackend {
|
||||||
|
switch (config.backend) {
|
||||||
|
case "nfs":
|
||||||
|
return makeNfsBackend(config, mountPath);
|
||||||
|
case "smb":
|
||||||
|
return makeSmbBackend(config, mountPath);
|
||||||
|
case "directory":
|
||||||
|
return makeDirectoryBackend(config, mountPath);
|
||||||
|
case "webdav":
|
||||||
|
return makeWebdavBackend(config, mountPath);
|
||||||
|
case "rclone":
|
||||||
|
return makeRcloneBackend(config, mountPath);
|
||||||
|
case "sftp":
|
||||||
|
return makeSftpBackend(config, mountPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createResticSession(scenarioWorkspace: string): {
|
||||||
|
organizationId: string;
|
||||||
|
client: ReturnType<typeof createRestic>;
|
||||||
|
} {
|
||||||
|
const password = crypto.randomBytes(32).toString("hex");
|
||||||
|
const cacheDir = path.join(scenarioWorkspace, "restic-cache");
|
||||||
|
const passFile = path.join(scenarioWorkspace, "restic.pass");
|
||||||
|
|
||||||
|
return {
|
||||||
|
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||||
|
client: createRestic({
|
||||||
|
resolveSecret: async (value: string) => value,
|
||||||
|
getOrganizationResticPassword: async () => password,
|
||||||
|
resticCacheDir: cacheDir,
|
||||||
|
resticPassFile: passFile,
|
||||||
|
defaultExcludes: [],
|
||||||
|
rcloneConfigFile: RCLONE_CONFIG_FILE,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runStage<T>(stages: StageReport[], name: string, fn: () => Promise<T>): Promise<T> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await fn();
|
||||||
|
stages.push({
|
||||||
|
name,
|
||||||
|
status: "passed",
|
||||||
|
durationMs: Date.now() - startedAt,
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
stages.push({
|
||||||
|
name,
|
||||||
|
status: "failed",
|
||||||
|
durationMs: Date.now() - startedAt,
|
||||||
|
error: formatError(error),
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function mountBackend(backend: VolumeBackend) {
|
||||||
|
const mountResult = await backend.mount();
|
||||||
|
if (mountResult.status !== "mounted") {
|
||||||
|
throw new Error(mountResult.error ?? `Mount returned ${mountResult.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const healthResult = await backend.checkHealth();
|
||||||
|
if (healthResult.status !== "mounted") {
|
||||||
|
throw new Error(healthResult.error ?? `Health check returned ${healthResult.status}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function backupSnapshot(
|
||||||
|
resticClient: ReturnType<typeof createRestic>,
|
||||||
|
repositoryConfig: IntegrationScenario["repository"],
|
||||||
|
fixtureRootPath: string,
|
||||||
|
organizationId: string,
|
||||||
|
backupOptions: IntegrationScenario["backup"],
|
||||||
|
uniqueBackupTag: string,
|
||||||
|
) {
|
||||||
|
const execution = await Effect.runPromise(
|
||||||
|
resticClient.backup(repositoryConfig, fixtureRootPath, {
|
||||||
|
organizationId,
|
||||||
|
compressionMode: backupOptions?.compressionMode,
|
||||||
|
customResticParams: backupOptions?.customResticParams,
|
||||||
|
tags: [...(backupOptions?.tags ?? []), uniqueBackupTag],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (execution.exitCode !== 0) {
|
||||||
|
throw new Error(`restic backup returned exit code ${execution.exitCode}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!execution.result?.snapshot_id) {
|
||||||
|
throw new Error("restic backup completed without a snapshot id");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (execution.warningDetails) {
|
||||||
|
throw new Error(`restic backup reported warnings: ${execution.warningDetails}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return execution.result.snapshot_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verifySnapshot(
|
||||||
|
resticClient: ReturnType<typeof createRestic>,
|
||||||
|
repositoryConfig: IntegrationScenario["repository"],
|
||||||
|
organizationId: string,
|
||||||
|
snapshotId: string,
|
||||||
|
uniqueBackupTag: string,
|
||||||
|
fixtureRootPath: string,
|
||||||
|
scenario: IntegrationScenario,
|
||||||
|
) {
|
||||||
|
const snapshots = await resticClient.snapshots(repositoryConfig, {
|
||||||
|
organizationId,
|
||||||
|
tags: [uniqueBackupTag],
|
||||||
|
});
|
||||||
|
|
||||||
|
const snapshot = snapshots.find((candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId);
|
||||||
|
if (!snapshot) {
|
||||||
|
throw new Error(`Unable to find snapshot ${snapshotId} by integration tag ${uniqueBackupTag}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await resticClient.ls(repositoryConfig, snapshotId, organizationId, undefined, {});
|
||||||
|
|
||||||
|
await verifySnapshotEntries(fixtureRootPath, res.nodes, scenario.expectedEntries);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restoreSnapshot(
|
||||||
|
resticClient: ReturnType<typeof createRestic>,
|
||||||
|
repositoryConfig: IntegrationScenario["repository"],
|
||||||
|
snapshotId: string,
|
||||||
|
restoreTarget: string,
|
||||||
|
organizationId: string,
|
||||||
|
fixtureRootPath: string,
|
||||||
|
restoreOptions: IntegrationScenario["restore"],
|
||||||
|
) {
|
||||||
|
await fs.mkdir(restoreTarget, { recursive: true });
|
||||||
|
await resticClient.restore(repositoryConfig, snapshotId, restoreTarget, {
|
||||||
|
organizationId,
|
||||||
|
basePath: fixtureRootPath,
|
||||||
|
excludeXattr: restoreOptions?.excludeXattr,
|
||||||
|
overwrite: restoreOptions?.overwrite,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cleanupScenario(backend: VolumeBackend, restoreTarget: string, scenarioWorkspace: string) {
|
||||||
|
const cleanupErrors: string[] = [];
|
||||||
|
const unmountResult = await backend.unmount();
|
||||||
|
const didUnmount = unmountResult.status !== "error";
|
||||||
|
|
||||||
|
if (!didUnmount) {
|
||||||
|
cleanupErrors.push(unmountResult.error ?? `Unmount returned ${unmountResult.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.rm(restoreTarget, { recursive: true, force: true }).catch((error) => {
|
||||||
|
cleanupErrors.push(`Failed to remove restore target: ${formatError(error)}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (didUnmount) {
|
||||||
|
await fs.rm(scenarioWorkspace, { recursive: true, force: true }).catch((error) => {
|
||||||
|
cleanupErrors.push(`Failed to remove scenario workspace: ${formatError(error)}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cleanupErrors.length > 0) {
|
||||||
|
throw new Error(cleanupErrors.join("; "));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runScenario(scenario: IntegrationScenario, runId: string): Promise<ScenarioReport> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const stages: StageReport[] = [];
|
||||||
|
const scenarioWorkspace = path.join(WORKDIR_ROOT, `${scenario.id}-${runId}`);
|
||||||
|
const mountPath = path.join(scenarioWorkspace, "volume", "_data");
|
||||||
|
const restoreTarget = path.join(scenarioWorkspace, "restore");
|
||||||
|
|
||||||
|
const uniqueBackupTag = `zerobyte-integration-${scenario.id}-${runId}`;
|
||||||
|
const { client: resticClient, organizationId } = createResticSession(scenarioWorkspace);
|
||||||
|
const backend = createVolumeBackend(scenario.volume, mountPath);
|
||||||
|
|
||||||
|
let snapshotId = "";
|
||||||
|
let status: ScenarioReport["status"] = "passed";
|
||||||
|
let errorMessage: string | undefined;
|
||||||
|
|
||||||
|
await fs.mkdir(scenarioWorkspace, { recursive: true });
|
||||||
|
logScenario(scenario.id, "running...");
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runStage(stages, "mount", async () => {
|
||||||
|
await mountBackend(backend);
|
||||||
|
});
|
||||||
|
|
||||||
|
const fixtureRootPath = path.resolve(mountPath, scenario.fixtureRoot);
|
||||||
|
await runStage(stages, "verify-mounted-source", async () => {
|
||||||
|
await verifyFilesystemEntries(fixtureRootPath, scenario.expectedEntries, "mounted source");
|
||||||
|
});
|
||||||
|
|
||||||
|
await runStage(stages, "init-repository", async () => {
|
||||||
|
if (scenario.repository.isExistingRepository) return;
|
||||||
|
|
||||||
|
const initResult = await resticClient.init(scenario.repository, organizationId, undefined);
|
||||||
|
if (!initResult.success) {
|
||||||
|
throw new Error(initResult.error ?? "restic init failed");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
snapshotId = await runStage(stages, "backup", async () => {
|
||||||
|
return await backupSnapshot(
|
||||||
|
resticClient,
|
||||||
|
scenario.repository,
|
||||||
|
fixtureRootPath,
|
||||||
|
organizationId,
|
||||||
|
scenario.backup,
|
||||||
|
uniqueBackupTag,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!snapshotId) {
|
||||||
|
throw new Error("restic backup completed without a snapshot id");
|
||||||
|
}
|
||||||
|
|
||||||
|
await runStage(stages, "inspect-snapshot", async () => {
|
||||||
|
await verifySnapshot(
|
||||||
|
resticClient,
|
||||||
|
scenario.repository,
|
||||||
|
organizationId,
|
||||||
|
snapshotId,
|
||||||
|
uniqueBackupTag,
|
||||||
|
fixtureRootPath,
|
||||||
|
scenario,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
await runStage(stages, "restore", async () => {
|
||||||
|
await restoreSnapshot(
|
||||||
|
resticClient,
|
||||||
|
scenario.repository,
|
||||||
|
snapshotId,
|
||||||
|
restoreTarget,
|
||||||
|
organizationId,
|
||||||
|
fixtureRootPath,
|
||||||
|
scenario.restore,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
await runStage(stages, "verify-restore", async () => {
|
||||||
|
await verifyFilesystemEntries(restoreTarget, scenario.expectedEntries, "restored output");
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
status = "failed";
|
||||||
|
errorMessage = formatError(error);
|
||||||
|
logScenario(scenario.id, `failed: ${errorMessage}`);
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
await runStage(stages, "cleanup", async () => {
|
||||||
|
await cleanupScenario(backend, restoreTarget, scenarioWorkspace);
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
status = "failed";
|
||||||
|
const cleanupError = formatError(error);
|
||||||
|
|
||||||
|
if (!errorMessage) {
|
||||||
|
errorMessage = cleanupError;
|
||||||
|
logScenario(scenario.id, `failed: ${cleanupError}`);
|
||||||
|
} else {
|
||||||
|
logScenario(scenario.id, `cleanup failed: ${cleanupError}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status === "passed") {
|
||||||
|
logScenario(scenario.id, "passed");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: scenario.id,
|
||||||
|
volumeBackend: scenario.volume.backend,
|
||||||
|
repositoryBackend: scenario.repository.backend,
|
||||||
|
status,
|
||||||
|
durationMs: Date.now() - startedAt,
|
||||||
|
stages,
|
||||||
|
snapshotId,
|
||||||
|
error: errorMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runBackendIntegration(): Promise<void> {
|
||||||
|
const runStartedAt = Date.now();
|
||||||
|
const runId = crypto.randomBytes(6).toString("hex");
|
||||||
|
|
||||||
|
await fs.mkdir(WORKDIR_ROOT, { recursive: true });
|
||||||
|
|
||||||
|
const content = await fs.readFile("/config/config.json", "utf8");
|
||||||
|
const { scenarios } = configSchema.parse(JSON.parse(content));
|
||||||
|
|
||||||
|
process.stdout.write(
|
||||||
|
`Running ${scenarios.length} integration scenario${scenarios.length === 1 ? "" : "s"} (run ${runId})\n`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const scenarioReports: ScenarioReport[] = [];
|
||||||
|
for (const scenario of scenarios) {
|
||||||
|
scenarioReports.push(await runScenario(scenario, runId));
|
||||||
|
}
|
||||||
|
|
||||||
|
const passed = scenarioReports.filter((scenario) => scenario.status === "passed").length;
|
||||||
|
const failed = scenarioReports.length - passed;
|
||||||
|
const report: IntegrationReport = {
|
||||||
|
runId,
|
||||||
|
startedAt: new Date(runStartedAt).toISOString(),
|
||||||
|
finishedAt: new Date().toISOString(),
|
||||||
|
durationMs: Date.now() - runStartedAt,
|
||||||
|
passed,
|
||||||
|
failed,
|
||||||
|
scenarios: scenarioReports,
|
||||||
|
};
|
||||||
|
|
||||||
|
printRunSummary(report);
|
||||||
|
|
||||||
|
if (failed > 0) {
|
||||||
|
process.exitCode = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
84
app/test/backend-integration/runner/config.ts
Normal file
84
app/test/backend-integration/runner/config.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
import { z } from "zod";
|
||||||
|
import {
|
||||||
|
COMPRESSION_MODES,
|
||||||
|
OVERWRITE_MODES,
|
||||||
|
repositoryConfigSchema,
|
||||||
|
type RepositoryConfig,
|
||||||
|
} from "@zerobyte/core/restic";
|
||||||
|
import { volumeConfigSchema, type BackendConfig } from "@zerobyte/contracts/volumes";
|
||||||
|
|
||||||
|
const modeSchema = z
|
||||||
|
.union([z.number(), z.string().transform((value) => Number.parseInt(value, 8))])
|
||||||
|
.transform((value) => value & 0o7777);
|
||||||
|
|
||||||
|
const entryTypeSchema = z.enum(["file", "directory", "symlink"]);
|
||||||
|
|
||||||
|
const expectedEntrySchema = z
|
||||||
|
.object({
|
||||||
|
path: z.string().min(1),
|
||||||
|
type: entryTypeSchema,
|
||||||
|
uid: z.number().optional(),
|
||||||
|
gid: z.number().optional(),
|
||||||
|
mode: modeSchema.optional(),
|
||||||
|
sha256: z.string().optional(),
|
||||||
|
text: z.string().optional(),
|
||||||
|
linkTarget: z.string().min(1).optional(),
|
||||||
|
})
|
||||||
|
.superRefine((value, ctx) => {
|
||||||
|
if (value.sha256 && value.text) {
|
||||||
|
ctx.addIssue({
|
||||||
|
code: "custom",
|
||||||
|
message: "Specify either sha256 or text, not both",
|
||||||
|
path: ["sha256"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value.type !== "file" && (value.sha256 || value.text)) {
|
||||||
|
ctx.addIssue({
|
||||||
|
code: "custom",
|
||||||
|
message: "Only file entries can define sha256 or text expectations",
|
||||||
|
path: ["type"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value.type !== "symlink" && value.linkTarget) {
|
||||||
|
ctx.addIssue({
|
||||||
|
code: "custom",
|
||||||
|
message: "Only symlink entries can define linkTarget",
|
||||||
|
path: ["linkTarget"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const backupOptionsSchema = z.object({
|
||||||
|
compressionMode: z.enum(COMPRESSION_MODES).optional(),
|
||||||
|
tags: z.array(z.string().min(1)).optional(),
|
||||||
|
customResticParams: z.array(z.string().min(1)).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const restoreOptionsSchema = z.object({
|
||||||
|
excludeXattr: z.array(z.string().min(1)).optional(),
|
||||||
|
overwrite: z.enum(OVERWRITE_MODES).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const scenarioSchema = z.object({
|
||||||
|
id: z.string().min(1),
|
||||||
|
description: z.string().optional(),
|
||||||
|
volume: volumeConfigSchema,
|
||||||
|
repository: repositoryConfigSchema,
|
||||||
|
fixtureRoot: z.string().min(1).default("."),
|
||||||
|
expectedEntries: z.array(expectedEntrySchema).min(1),
|
||||||
|
backup: backupOptionsSchema.optional(),
|
||||||
|
restore: restoreOptionsSchema.optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const configSchema = z.object({
|
||||||
|
version: z.literal(1).default(1),
|
||||||
|
scenarios: z.array(scenarioSchema).min(1),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ExpectedEntry = z.infer<typeof expectedEntrySchema>;
|
||||||
|
export type IntegrationScenario = z.infer<typeof scenarioSchema> & {
|
||||||
|
volume: BackendConfig;
|
||||||
|
repository: RepositoryConfig;
|
||||||
|
};
|
||||||
82
app/test/backend-integration/runner/reporting.ts
Normal file
82
app/test/backend-integration/runner/reporting.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
export type StageReport = {
|
||||||
|
name: string;
|
||||||
|
status: "passed" | "failed";
|
||||||
|
durationMs: number;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ScenarioReport = {
|
||||||
|
id: string;
|
||||||
|
volumeBackend: string;
|
||||||
|
repositoryBackend: string;
|
||||||
|
status: "passed" | "failed";
|
||||||
|
durationMs: number;
|
||||||
|
stages: StageReport[];
|
||||||
|
snapshotId?: string;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type IntegrationReport = {
|
||||||
|
runId: string;
|
||||||
|
startedAt: string;
|
||||||
|
finishedAt: string;
|
||||||
|
durationMs: number;
|
||||||
|
passed: number;
|
||||||
|
failed: number;
|
||||||
|
scenarios: ScenarioReport[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export function formatError(error: unknown): string {
|
||||||
|
if (error instanceof Error) {
|
||||||
|
return error.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
return String(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
function log(message: string): void {
|
||||||
|
process.stdout.write(`${message}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function logScenario(scenarioId: string, message: string): void {
|
||||||
|
log(`[${scenarioId}] ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDuration(durationMs: number): string {
|
||||||
|
if (durationMs < 1000) {
|
||||||
|
return `${durationMs}ms`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (durationMs < 10_000) {
|
||||||
|
return `${(durationMs / 1000).toFixed(1)}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${(durationMs / 1000).toFixed(0)}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatScenarioResult(scenario: ScenarioReport, labelWidth: number): string {
|
||||||
|
const status = scenario.status === "passed" ? "✔︎ PASS" : "⨯ FAIL";
|
||||||
|
const id = scenario.id.padEnd(labelWidth);
|
||||||
|
return `${status} ${id} ${formatDuration(scenario.durationMs)} ${scenario.volumeBackend} -> ${scenario.repositoryBackend}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function printRunSummary(report: IntegrationReport): void {
|
||||||
|
const labelWidth = report.scenarios.reduce((max, scenario) => Math.max(max, scenario.id.length), 4);
|
||||||
|
|
||||||
|
log("");
|
||||||
|
log("Scenario Results");
|
||||||
|
|
||||||
|
for (const scenario of report.scenarios) {
|
||||||
|
log(formatScenarioResult(scenario, labelWidth));
|
||||||
|
|
||||||
|
const failedStage = scenario.stages.find((stage) => stage.status === "failed");
|
||||||
|
if (failedStage?.error) {
|
||||||
|
log(` ${failedStage.name}: ${failedStage.error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log("");
|
||||||
|
log(
|
||||||
|
`${report.failed === 0 ? "PASS" : "FAIL"} ${report.passed}/${report.scenarios.length} scenarios passed in ${formatDuration(report.durationMs)} (run ${report.runId})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
150
app/test/backend-integration/runner/verification.ts
Normal file
150
app/test/backend-integration/runner/verification.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
import crypto from "node:crypto";
|
||||||
|
import type { Stats } from "node:fs";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import type { ExpectedEntry } from "./config";
|
||||||
|
|
||||||
|
export type SnapshotNode = {
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
path: string;
|
||||||
|
uid?: number;
|
||||||
|
gid?: number;
|
||||||
|
mode?: number;
|
||||||
|
size?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
function normalizeRelativePath(input: string): string {
|
||||||
|
const normalized = path.posix.normalize(input);
|
||||||
|
if (normalized === "") {
|
||||||
|
return ".";
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeEntryType(type: string): string {
|
||||||
|
if (type === "dir" || type === "directory") {
|
||||||
|
return "directory";
|
||||||
|
}
|
||||||
|
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readSha256(filePath: string): Promise<string> {
|
||||||
|
const digest = crypto.createHash("sha256");
|
||||||
|
digest.update(await fs.readFile(filePath));
|
||||||
|
return digest.digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertMetadata(
|
||||||
|
entryLabel: string,
|
||||||
|
expected: ExpectedEntry,
|
||||||
|
actual: { uid?: number; gid?: number; mode?: number },
|
||||||
|
) {
|
||||||
|
if (expected.uid !== undefined && actual.uid !== expected.uid) {
|
||||||
|
throw new Error(`${entryLabel} uid mismatch: expected ${expected.uid}, got ${String(actual.uid)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expected.gid !== undefined && actual.gid !== expected.gid) {
|
||||||
|
throw new Error(`${entryLabel} gid mismatch: expected ${expected.gid}, got ${String(actual.gid)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expected.mode === undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (actual.mode === undefined) {
|
||||||
|
throw new Error(`${entryLabel} mode is missing`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const permissionBits = actual.mode & 0o7777;
|
||||||
|
if (permissionBits !== expected.mode) {
|
||||||
|
throw new Error(
|
||||||
|
`${entryLabel} mode mismatch: expected ${expected.mode.toString(8)}, got ${permissionBits.toString(8)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFilesystemEntryType(stats: Stats): ExpectedEntry["type"] {
|
||||||
|
if (stats.isDirectory()) {
|
||||||
|
return "directory";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (stats.isSymbolicLink()) {
|
||||||
|
return "symlink";
|
||||||
|
}
|
||||||
|
|
||||||
|
return "file";
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function verifyFilesystemEntries(basePath: string, expectedEntries: ExpectedEntry[], label: string) {
|
||||||
|
for (const expected of expectedEntries) {
|
||||||
|
const targetPath = path.resolve(basePath, expected.path);
|
||||||
|
const stats = await fs.lstat(targetPath);
|
||||||
|
|
||||||
|
const actualType = getFilesystemEntryType(stats);
|
||||||
|
if (actualType !== expected.type) {
|
||||||
|
throw new Error(`${label}: ${expected.path} type mismatch: expected ${expected.type}, got ${actualType}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
assertMetadata(`${label}: ${expected.path}`, expected, stats);
|
||||||
|
|
||||||
|
if (expected.type === "file" && expected.text !== undefined) {
|
||||||
|
const actualText = await fs.readFile(targetPath, "utf8");
|
||||||
|
if (actualText !== expected.text) {
|
||||||
|
throw new Error(`${label}: ${expected.path} text content mismatch`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expected.type === "file" && expected.sha256 !== undefined) {
|
||||||
|
const actualSha = await readSha256(targetPath);
|
||||||
|
if (actualSha !== expected.sha256.toLowerCase()) {
|
||||||
|
throw new Error(`${label}: ${expected.path} sha256 mismatch`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (expected.type !== "symlink" || expected.linkTarget === undefined) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const actualTarget = await fs.readlink(targetPath);
|
||||||
|
if (actualTarget !== expected.linkTarget) {
|
||||||
|
throw new Error(
|
||||||
|
`${label}: ${expected.path} symlink target mismatch: expected ${expected.linkTarget}, got ${actualTarget}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function verifySnapshotEntries(
|
||||||
|
snapshotRootPath: string,
|
||||||
|
nodes: SnapshotNode[],
|
||||||
|
expectedEntries: ExpectedEntry[],
|
||||||
|
) {
|
||||||
|
const nodesByRelativePath = new Map<string, SnapshotNode>();
|
||||||
|
|
||||||
|
for (const node of nodes) {
|
||||||
|
const relativePath = path.posix.relative(snapshotRootPath, node.path);
|
||||||
|
if (!relativePath || relativePath === "." || relativePath === ".." || relativePath.startsWith("../")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
nodesByRelativePath.set(normalizeRelativePath(relativePath), node);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const expected of expectedEntries) {
|
||||||
|
const normalizedExpectedPath = normalizeRelativePath(expected.path);
|
||||||
|
const node = nodesByRelativePath.get(normalizedExpectedPath);
|
||||||
|
if (!node) {
|
||||||
|
throw new Error(`snapshot: missing ${expected.path}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const actualType = normalizeEntryType(node.type);
|
||||||
|
if (actualType !== expected.type) {
|
||||||
|
throw new Error(`snapshot: ${expected.path} type mismatch: expected ${expected.type}, got ${actualType}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
assertMetadata(`snapshot: ${expected.path}`, expected, node);
|
||||||
|
}
|
||||||
|
}
|
||||||
193
app/test/backend-integration/setup-target.sh
Executable file
193
app/test/backend-integration/setup-target.sh
Executable file
|
|
@ -0,0 +1,193 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
TARGET_HOST="192.168.2.41"
|
||||||
|
TARGET="root@$TARGET_HOST"
|
||||||
|
FIXTURE_UID="1000"
|
||||||
|
FIXTURE_GID="1000"
|
||||||
|
|
||||||
|
ARTIFACTS_DIR="$SCRIPT_DIR/artifacts/$TARGET_HOST"
|
||||||
|
KEY_PATH="$ARTIFACTS_DIR/zerobyte-sftp-ed25519"
|
||||||
|
KNOWN_HOSTS_PATH="$ARTIFACTS_DIR/known_hosts"
|
||||||
|
CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json"
|
||||||
|
|
||||||
|
SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt"
|
||||||
|
SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt"
|
||||||
|
WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt"
|
||||||
|
RESTIC_PASSWORD_FILE="$ARTIFACTS_DIR/restic-password.txt"
|
||||||
|
|
||||||
|
read_or_create_secret() {
|
||||||
|
local file_path="$1"
|
||||||
|
|
||||||
|
if [[ -f "$file_path" ]]; then
|
||||||
|
cat "$file_path"
|
||||||
|
else
|
||||||
|
openssl rand -hex 12 >"$file_path"
|
||||||
|
chmod 600 "$file_path"
|
||||||
|
cat "$file_path"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdir -p "$ARTIFACTS_DIR"
|
||||||
|
chmod 700 "$ARTIFACTS_DIR"
|
||||||
|
|
||||||
|
SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")"
|
||||||
|
SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")"
|
||||||
|
WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")"
|
||||||
|
RESTIC_PASSWORD="$(read_or_create_secret "$RESTIC_PASSWORD_FILE")"
|
||||||
|
|
||||||
|
if [[ ! -f "$KEY_PATH" || ! -f "$KEY_PATH.pub" ]]; then
|
||||||
|
ssh-keygen -q -t ed25519 -N "" -C "zerobyte-backend-integration@$TARGET_HOST" -f "$KEY_PATH"
|
||||||
|
chmod 600 "$KEY_PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
|
PUBLIC_KEY_BASE64="$(base64 <"$KEY_PATH.pub" | tr -d '\n')"
|
||||||
|
|
||||||
|
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" "$RESTIC_PASSWORD" "$PUBLIC_KEY_BASE64" <<'REMOTE'
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
fixture_uid="$1"
|
||||||
|
fixture_gid="$2"
|
||||||
|
smb_password="$3"
|
||||||
|
sftp_password="$4"
|
||||||
|
webdav_password="$5"
|
||||||
|
restic_password="$6"
|
||||||
|
public_key="$(printf '%s' "$7" | base64 -d)"
|
||||||
|
repo_path="/srv/zerobyte-backend-integration/restic-repo"
|
||||||
|
repo_password_fingerprint_path="$repo_path/.zerobyte-password-sha256"
|
||||||
|
repo_password_fingerprint="$(printf '%s' "$restic_password" | sha256sum | cut -d' ' -f1)"
|
||||||
|
|
||||||
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
write_file() {
|
||||||
|
local file_path="$1"
|
||||||
|
cat >"$file_path"
|
||||||
|
}
|
||||||
|
|
||||||
|
initialize_restic_repo() {
|
||||||
|
local password_file
|
||||||
|
|
||||||
|
rm -rf "$repo_path"
|
||||||
|
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 "$repo_path"
|
||||||
|
|
||||||
|
password_file="$(mktemp)"
|
||||||
|
printf '%s\n' "$restic_password" >"$password_file"
|
||||||
|
chown zerobyte-sftp:zerobyte-sftp "$password_file"
|
||||||
|
chmod 0600 "$password_file"
|
||||||
|
su -s /bin/sh -c "restic init --repo '$repo_path' --password-file '$password_file'" zerobyte-sftp
|
||||||
|
rm -f "$password_file"
|
||||||
|
|
||||||
|
printf '%s\n' "$repo_password_fingerprint" >"$repo_password_fingerprint_path"
|
||||||
|
chmod 0600 "$repo_password_fingerprint_path"
|
||||||
|
}
|
||||||
|
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server restic rpcbind samba
|
||||||
|
|
||||||
|
id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp
|
||||||
|
id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb
|
||||||
|
|
||||||
|
install -d -m 0755 /srv/zerobyte-backend-integration/fixtures/case-a/docs
|
||||||
|
printf 'hello from zerobyte integration\n' >/srv/zerobyte-backend-integration/fixtures/case-a/hello.txt
|
||||||
|
printf 'fixture documentation\n' >/srv/zerobyte-backend-integration/fixtures/case-a/docs/readme.md
|
||||||
|
chown -R "$fixture_uid:$fixture_gid" /srv/zerobyte-backend-integration/fixtures
|
||||||
|
find /srv/zerobyte-backend-integration/fixtures -type d -exec chmod 0755 {} +
|
||||||
|
find /srv/zerobyte-backend-integration/fixtures -type f -exec chmod 0644 {} +
|
||||||
|
|
||||||
|
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp
|
||||||
|
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp/.ssh
|
||||||
|
printf '%s\n' "$public_key" >/home/zerobyte-sftp/.ssh/authorized_keys
|
||||||
|
chown zerobyte-sftp:zerobyte-sftp /home/zerobyte-sftp/.ssh/authorized_keys
|
||||||
|
chmod 0600 /home/zerobyte-sftp/.ssh/authorized_keys
|
||||||
|
|
||||||
|
printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb >/dev/null
|
||||||
|
smbpasswd -e zerobyte-smb >/dev/null
|
||||||
|
printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd
|
||||||
|
passwd -u zerobyte-sftp >/dev/null 2>&1 || true
|
||||||
|
htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null
|
||||||
|
|
||||||
|
if [[ ! -f "$repo_path/config" ]]; then
|
||||||
|
initialize_restic_repo
|
||||||
|
elif [[ ! -f "$repo_password_fingerprint_path" ]] || [[ "$(cat "$repo_password_fingerprint_path")" != "$repo_password_fingerprint" ]]; then
|
||||||
|
initialize_restic_repo
|
||||||
|
fi
|
||||||
|
|
||||||
|
write_file /etc/exports <<'EOF'
|
||||||
|
/srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure)
|
||||||
|
EOF
|
||||||
|
exportfs -ra
|
||||||
|
systemctl unmask rpcbind rpcbind.socket >/dev/null 2>&1
|
||||||
|
systemctl start rpcbind.socket
|
||||||
|
systemctl start rpcbind
|
||||||
|
systemctl start proc-fs-nfsd.mount
|
||||||
|
systemctl restart nfs-kernel-server
|
||||||
|
|
||||||
|
write_file /etc/samba/smb.conf <<'EOF'
|
||||||
|
[zerobyte-backend-integration]
|
||||||
|
path = /srv/zerobyte-backend-integration/fixtures
|
||||||
|
browseable = yes
|
||||||
|
read only = yes
|
||||||
|
guest ok = no
|
||||||
|
valid users = zerobyte-smb
|
||||||
|
EOF
|
||||||
|
|
||||||
|
install -d -o www-data -g www-data -m 0755 /var/lib/dav
|
||||||
|
a2enmod dav dav_fs auth_basic >/dev/null
|
||||||
|
printf 'ServerName localhost\n' >/etc/apache2/conf-available/zerobyte-backend-integration-servername.conf
|
||||||
|
a2enconf zerobyte-backend-integration-servername >/dev/null
|
||||||
|
write_file /etc/apache2/sites-available/zerobyte-backend-integration-dav.conf <<'EOF'
|
||||||
|
Alias /zerobyte-backend-integration /srv/zerobyte-backend-integration/fixtures
|
||||||
|
|
||||||
|
DAVLockDB /var/lib/dav/lockdb
|
||||||
|
|
||||||
|
<Location /zerobyte-backend-integration>
|
||||||
|
DAV On
|
||||||
|
AuthType Basic
|
||||||
|
AuthName "Zerobyte Backend Integration WebDAV"
|
||||||
|
AuthUserFile /etc/apache2/zerobyte-backend-integration.htpasswd
|
||||||
|
Require valid-user
|
||||||
|
</Location>
|
||||||
|
|
||||||
|
<Directory /srv/zerobyte-backend-integration/fixtures>
|
||||||
|
Options Indexes FollowSymLinks
|
||||||
|
AllowOverride None
|
||||||
|
Require all granted
|
||||||
|
</Directory>
|
||||||
|
EOF
|
||||||
|
a2ensite zerobyte-backend-integration-dav >/dev/null
|
||||||
|
apache2ctl configtest
|
||||||
|
|
||||||
|
install -d -m 0755 /etc/ssh/sshd_config.d
|
||||||
|
write_file /etc/ssh/sshd_config.d/zerobyte-backend-integration.conf <<'EOF'
|
||||||
|
Match User zerobyte-sftp
|
||||||
|
PasswordAuthentication yes
|
||||||
|
PubkeyAuthentication yes
|
||||||
|
PermitTTY no
|
||||||
|
X11Forwarding no
|
||||||
|
AllowTcpForwarding no
|
||||||
|
ForceCommand internal-sftp
|
||||||
|
EOF
|
||||||
|
sshd -t
|
||||||
|
|
||||||
|
systemctl restart apache2
|
||||||
|
systemctl restart smbd
|
||||||
|
systemctl restart ssh
|
||||||
|
REMOTE
|
||||||
|
|
||||||
|
ssh-keyscan "$TARGET_HOST" >"$KNOWN_HOSTS_PATH" 2>/dev/null
|
||||||
|
|
||||||
|
INTEGRATION_HOST="$TARGET_HOST" \
|
||||||
|
FIXTURE_UID="$FIXTURE_UID" \
|
||||||
|
FIXTURE_GID="$FIXTURE_GID" \
|
||||||
|
SMB_PASSWORD="$SMB_PASSWORD" \
|
||||||
|
WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \
|
||||||
|
RESTIC_PASSWORD="$RESTIC_PASSWORD" \
|
||||||
|
SFTP_KEY_PATH="$KEY_PATH" \
|
||||||
|
KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \
|
||||||
|
CONFIG_PATH="$CONFIG_PATH" \
|
||||||
|
bun run "$SCRIPT_DIR/write-generated-config.ts"
|
||||||
|
|
||||||
|
echo "Provisioned $TARGET"
|
||||||
|
echo "Generated config: $CONFIG_PATH"
|
||||||
|
echo "Artifacts: $ARTIFACTS_DIR"
|
||||||
137
app/test/backend-integration/write-generated-config.ts
Normal file
137
app/test/backend-integration/write-generated-config.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
||||||
|
import fs from "node:fs";
|
||||||
|
|
||||||
|
function getRequiredEnv(name: string) {
|
||||||
|
return process.env[name]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getRequiredNumberEnv(name: string) {
|
||||||
|
return Number.parseInt(getRequiredEnv(name), 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = getRequiredEnv("INTEGRATION_HOST");
|
||||||
|
const fixtureUid = getRequiredNumberEnv("FIXTURE_UID");
|
||||||
|
const fixtureGid = getRequiredNumberEnv("FIXTURE_GID");
|
||||||
|
const smbPassword = getRequiredEnv("SMB_PASSWORD");
|
||||||
|
const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD");
|
||||||
|
const resticPassword = getRequiredEnv("RESTIC_PASSWORD");
|
||||||
|
const privateKey = fs.readFileSync(getRequiredEnv("SFTP_KEY_PATH"), "utf8");
|
||||||
|
const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8");
|
||||||
|
const configPath = getRequiredEnv("CONFIG_PATH");
|
||||||
|
|
||||||
|
const fileText = "hello from zerobyte integration\n";
|
||||||
|
const readmeText = "fixture documentation\n";
|
||||||
|
|
||||||
|
const nfsEntries = [
|
||||||
|
{ path: "hello.txt", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: fileText },
|
||||||
|
{ path: "docs", type: "directory", uid: fixtureUid, gid: fixtureGid, mode: "0755" },
|
||||||
|
{ path: "docs/readme.md", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: readmeText },
|
||||||
|
];
|
||||||
|
|
||||||
|
const smbEntries = [
|
||||||
|
{ path: "hello.txt", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: fileText },
|
||||||
|
{ path: "docs", type: "directory", uid: fixtureUid, gid: fixtureGid, mode: "0755" },
|
||||||
|
{ path: "docs/readme.md", type: "file", uid: fixtureUid, gid: fixtureGid, mode: "0644", text: readmeText },
|
||||||
|
];
|
||||||
|
|
||||||
|
const contentOnlyEntries = [
|
||||||
|
{ path: "hello.txt", type: "file", text: fileText },
|
||||||
|
{ path: "docs", type: "directory" },
|
||||||
|
{ path: "docs/readme.md", type: "file", text: readmeText },
|
||||||
|
];
|
||||||
|
|
||||||
|
const config = {
|
||||||
|
version: 1,
|
||||||
|
scenarios: [
|
||||||
|
{
|
||||||
|
id: "nfs-local-repo",
|
||||||
|
volume: {
|
||||||
|
backend: "nfs",
|
||||||
|
server: host,
|
||||||
|
exportPath: "/srv/zerobyte-backend-integration/fixtures",
|
||||||
|
port: 2049,
|
||||||
|
version: "4.1",
|
||||||
|
readOnly: true,
|
||||||
|
},
|
||||||
|
repository: { backend: "local", path: "repo-nfs" },
|
||||||
|
fixtureRoot: "case-a",
|
||||||
|
expectedEntries: nfsEntries,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "smb-local-repo",
|
||||||
|
volume: {
|
||||||
|
backend: "smb",
|
||||||
|
server: host,
|
||||||
|
share: "zerobyte-backend-integration",
|
||||||
|
username: "zerobyte-smb",
|
||||||
|
password: smbPassword,
|
||||||
|
mapToContainerUidGid: false,
|
||||||
|
vers: "3.0",
|
||||||
|
port: 445,
|
||||||
|
readOnly: true,
|
||||||
|
},
|
||||||
|
repository: { backend: "local", path: "repo-smb" },
|
||||||
|
fixtureRoot: "case-a",
|
||||||
|
expectedEntries: smbEntries,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "sftp-local-repo",
|
||||||
|
volume: {
|
||||||
|
backend: "sftp",
|
||||||
|
host,
|
||||||
|
port: 22,
|
||||||
|
username: "zerobyte-sftp",
|
||||||
|
privateKey,
|
||||||
|
path: "/srv/zerobyte-backend-integration/fixtures",
|
||||||
|
readOnly: true,
|
||||||
|
skipHostKeyCheck: false,
|
||||||
|
knownHosts,
|
||||||
|
},
|
||||||
|
repository: { backend: "local", path: "repo-sftp-volume" },
|
||||||
|
fixtureRoot: "case-a",
|
||||||
|
expectedEntries: contentOnlyEntries,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "webdav-local-repo",
|
||||||
|
volume: {
|
||||||
|
backend: "webdav",
|
||||||
|
server: host,
|
||||||
|
path: "/zerobyte-backend-integration",
|
||||||
|
username: "zerobyte-webdav",
|
||||||
|
password: webdavPassword,
|
||||||
|
port: 80,
|
||||||
|
readOnly: true,
|
||||||
|
ssl: false,
|
||||||
|
},
|
||||||
|
repository: { backend: "local", path: "repo-webdav" },
|
||||||
|
fixtureRoot: "case-a",
|
||||||
|
expectedEntries: contentOnlyEntries,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "nfs-sftp-repository",
|
||||||
|
volume: {
|
||||||
|
backend: "nfs",
|
||||||
|
server: host,
|
||||||
|
exportPath: "/srv/zerobyte-backend-integration/fixtures",
|
||||||
|
port: 2049,
|
||||||
|
version: "4.1",
|
||||||
|
readOnly: true,
|
||||||
|
},
|
||||||
|
repository: {
|
||||||
|
backend: "sftp",
|
||||||
|
host,
|
||||||
|
port: 22,
|
||||||
|
user: "zerobyte-sftp",
|
||||||
|
path: "/srv/zerobyte-backend-integration/restic-repo",
|
||||||
|
privateKey,
|
||||||
|
skipHostKeyCheck: false,
|
||||||
|
knownHosts,
|
||||||
|
isExistingRepository: true,
|
||||||
|
customPassword: resticPassword,
|
||||||
|
},
|
||||||
|
fixtureRoot: "case-a",
|
||||||
|
expectedEntries: nfsEntries,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`);
|
||||||
|
|
@ -9,6 +9,19 @@ import { getMountForPath } from "../fs";
|
||||||
import type { VolumeBackend } from "../types";
|
import type { VolumeBackend } from "../types";
|
||||||
import { assertMounted, executeMount, executeUnmount } from "./utils";
|
import { assertMounted, executeMount, executeUnmount } from "./utils";
|
||||||
|
|
||||||
|
const isUnsupportedAclMountOptionError = (message: string) =>
|
||||||
|
/invalid argument|unknown mount option|unrecognized mount option|bad option/i.test(message);
|
||||||
|
|
||||||
|
const toSmbMountError = (error: unknown, usingContainerMapping: boolean) => {
|
||||||
|
const message = toMessage(error);
|
||||||
|
|
||||||
|
if (usingContainerMapping || !isUnsupportedAclMountOptionError(message)) {
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${message} Your host/kernel may not support cifsacl, idsfromsid, or modefromsid. Enable "Map all files to container user/group" to fall back to the old uid/gid mapping behavior.`;
|
||||||
|
};
|
||||||
|
|
||||||
const checkHealth = async (mountPath: string) => {
|
const checkHealth = async (mountPath: string) => {
|
||||||
const run = async () => {
|
const run = async () => {
|
||||||
await assertMounted(mountPath, (fstype) => fstype === "cifs");
|
await assertMounted(mountPath, (fstype) => fstype === "cifs");
|
||||||
|
|
@ -79,8 +92,15 @@ const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
|
|
||||||
const run = async () => {
|
const run = async () => {
|
||||||
await fs.mkdir(mountPath, { recursive: true });
|
await fs.mkdir(mountPath, { recursive: true });
|
||||||
const { uid, gid } = os.userInfo();
|
const usingContainerMapping = config.mapToContainerUidGid ?? true;
|
||||||
const options = [`port=${config.port}`, `uid=${uid}`, `gid=${gid}`, "iocharset=utf8"];
|
const options = [`port=${config.port}`, "iocharset=utf8"];
|
||||||
|
|
||||||
|
if (usingContainerMapping) {
|
||||||
|
const { uid, gid } = os.userInfo();
|
||||||
|
options.push(`uid=${uid}`, `gid=${gid}`);
|
||||||
|
} else {
|
||||||
|
options.push("cifsacl", "idsfromsid", "modefromsid");
|
||||||
|
}
|
||||||
|
|
||||||
if (config.guest) {
|
if (config.guest) {
|
||||||
options.push("username=guest", "password=");
|
options.push("username=guest", "password=");
|
||||||
|
|
@ -102,8 +122,9 @@ const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
try {
|
try {
|
||||||
await executeMount(args);
|
await executeMount(args);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(`SMB mount failed: ${toMessage(error)}`);
|
const message = toSmbMountError(error, usingContainerMapping);
|
||||||
throw error;
|
logger.error(`SMB mount failed: ${message}`);
|
||||||
|
throw new Error(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`SMB volume at ${mountPath} mounted successfully.`);
|
logger.info(`SMB volume at ${mountPath} mounted successfully.`);
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,10 @@ A volume tells Zerobyte *where* your data lives. When you create a volume, you p
|
||||||
|
|
||||||
Volumes support a range of protocols, from simple local directories to NFS shares, Windows/Samba file servers, WebDAV endpoints, SFTP connections, and cloud storage via rclone. Once a volume is mounted, you can browse its contents directly in the UI, assign it to one or more backup jobs, and let Zerobyte handle the rest.
|
Volumes support a range of protocols, from simple local directories to NFS shares, Windows/Samba file servers, WebDAV endpoints, SFTP connections, and cloud storage via rclone. Once a volume is mounted, you can browse its contents directly in the UI, assign it to one or more backup jobs, and let Zerobyte handle the rest.
|
||||||
|
|
||||||
|
<Callout type="warn">
|
||||||
|
Mounted remote volumes can expose translated metadata instead of the source system's original ownership and ACL model. Before relying on NFS, SMB, WebDAV, SFTP, or rclone volumes for metadata-sensitive backups, read [Mounted Shares, ACLs, and Metadata Fidelity](/docs/guides/mounted-shares-and-acls).
|
||||||
|
</Callout>
|
||||||
|
|
||||||
## Supported volume types
|
## Supported volume types
|
||||||
|
|
||||||
Zerobyte supports six volume types. Each one is configured through the web UI when you create or edit a volume.
|
Zerobyte supports six volume types. Each one is configured through the web UI when you create or edit a volume.
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
"3-2-1-backup-strategy",
|
"3-2-1-backup-strategy",
|
||||||
"restoring",
|
"restoring",
|
||||||
"backup-webhooks",
|
"backup-webhooks",
|
||||||
|
"mounted-shares-and-acls",
|
||||||
"notifications",
|
"notifications",
|
||||||
"recovery-key-and-repository-passwords",
|
"recovery-key-and-repository-passwords",
|
||||||
"repository-maintenance",
|
"repository-maintenance",
|
||||||
|
|
|
||||||
182
apps/docs/content/docs/guides/mounted-shares-and-acls.mdx
Normal file
182
apps/docs/content/docs/guides/mounted-shares-and-acls.mdx
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
---
|
||||||
|
title: Mounted Shares and Permissions
|
||||||
|
description: A simple guide to what mounted volumes preserve well, what they simplify, and when a local directory is the better fit
|
||||||
|
---
|
||||||
|
|
||||||
|
Mounted volumes are still very useful in Zerobyte. For many people, backing up the file contents is the main goal, and mounted shares work well for that.
|
||||||
|
|
||||||
|
Where things get a little more nuanced is permissions and other filesystem details.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
Backing up a mounted share does **not** remove permissions from the source.
|
||||||
|
|
||||||
|
The main question is simpler than it sounds: how much of the source system's permission information can the mounted view actually show to the backup?
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## The short version
|
||||||
|
|
||||||
|
- If your main goal is recovering files and folders, mounted volumes are often perfectly fine.
|
||||||
|
- Zerobyte backs up the view that the mounted filesystem presents inside the container.
|
||||||
|
- Some backends show permissions and ownership very well.
|
||||||
|
- Some backends show a simplified view.
|
||||||
|
- If you want the closest thing to a true replica, a local **Directory** volume is the best choice.
|
||||||
|
|
||||||
|
## Why this matters
|
||||||
|
|
||||||
|
When Zerobyte backs up a remote volume, it does not talk directly to every storage system in its native format.
|
||||||
|
|
||||||
|
Instead:
|
||||||
|
|
||||||
|
1. Zerobyte mounts the volume inside the Linux container.
|
||||||
|
2. Linux sees that mounted volume as a normal filesystem.
|
||||||
|
3. Restic backs up what Linux can see there.
|
||||||
|
|
||||||
|
That means the backup is only as detailed as the mounted view.
|
||||||
|
|
||||||
|
If the mount shows the real owner, permissions, and advanced permission rules, Restic can back them up.
|
||||||
|
|
||||||
|
If the mount shows a simplified version, the snapshot will also contain that simplified version.
|
||||||
|
|
||||||
|
## What might be simplified
|
||||||
|
|
||||||
|
Depending on the backend, the mounted view may simplify:
|
||||||
|
|
||||||
|
- file owner
|
||||||
|
- group
|
||||||
|
- read, write, and execute permissions
|
||||||
|
- advanced permission rules
|
||||||
|
- extra filesystem metadata
|
||||||
|
|
||||||
|
This is why a backup can be excellent for file contents while being less exact for permission details.
|
||||||
|
|
||||||
|
## Editing files on a mounted share
|
||||||
|
|
||||||
|
This is also useful to know outside of backups.
|
||||||
|
|
||||||
|
- Editing an existing file often keeps that file's existing server-side permissions.
|
||||||
|
- Deleting a file and creating it again usually creates a new remote file.
|
||||||
|
- A new file usually gets the folder's default or inherited permissions, not necessarily the old file's exact ones.
|
||||||
|
- Some editors save by replacing the file behind the scenes, so they may behave more like recreate than edit-in-place.
|
||||||
|
|
||||||
|
For many users this is not a problem. It mainly matters when you are trying to keep permissions exactly the same.
|
||||||
|
|
||||||
|
Common examples where permissions matter more:
|
||||||
|
|
||||||
|
- a shared company folder where different teams should see different subfolders
|
||||||
|
- user home directories where private files should stay private after restore
|
||||||
|
- website or app files where the service depends on the right owner or executable permissions
|
||||||
|
- server scripts, cron jobs, or deployment hooks that need to remain executable
|
||||||
|
- business shares with inherited folder rules, such as finance or HR documents
|
||||||
|
|
||||||
|
## Backend comparison
|
||||||
|
|
||||||
|
| Backend | Good fit for | Permission fidelity | What to keep in mind |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| **Directory** | Local data and bind-mounted host paths | **Best** | Best choice when you want the backup and restore to stay as close as possible to the original filesystem |
|
||||||
|
| **NFS** | Linux and NAS shares | **Usually good** | Often a strong remote option, but ownership mapping can still vary between systems |
|
||||||
|
| **SMB/CIFS** | Windows shares and Samba servers | **Mixed to good** | Often the best remote option when permission details matter, but still not a perfect copy of the original Windows permission model |
|
||||||
|
| **WebDAV** | Nextcloud, ownCloud, and content-focused remote access | **Limited** | Great for backing up files, less reliable for exact permission replication |
|
||||||
|
| **SFTP** | Secure access to remote Linux servers | **Limited** | Good for content backup, but advanced permission details are often reduced |
|
||||||
|
| **Rclone** | Cloud providers and object storage | **Lowest** | Best treated as a content source, not as a full filesystem replica |
|
||||||
|
|
||||||
|
## What each backend is best at
|
||||||
|
|
||||||
|
### Directory
|
||||||
|
|
||||||
|
This is the easiest option to trust when you care about both file contents and permissions.
|
||||||
|
|
||||||
|
Why it works so well:
|
||||||
|
|
||||||
|
- Zerobyte sees the filesystem directly
|
||||||
|
- there is no extra network protocol translating the data
|
||||||
|
- restores are usually the most predictable
|
||||||
|
|
||||||
|
If your goal is the closest thing to a true replica, choose **Directory** whenever you can.
|
||||||
|
|
||||||
|
### NFS
|
||||||
|
|
||||||
|
NFS is often a very good fit for Linux-style file sharing.
|
||||||
|
|
||||||
|
It is usually a strong choice when:
|
||||||
|
|
||||||
|
- the share comes from a Linux or NAS system
|
||||||
|
- ownership matches well between systems
|
||||||
|
- you want a remote option that still feels fairly native
|
||||||
|
|
||||||
|
It can still vary depending on how the server is configured, so it is good to test with a few real files first.
|
||||||
|
|
||||||
|
### SMB/CIFS
|
||||||
|
|
||||||
|
SMB is a good choice for Windows shares and Samba servers.
|
||||||
|
|
||||||
|
Compared with other remote options, it often gives Linux a richer view of remote permissions. That makes it the most promising remote choice when those details matter.
|
||||||
|
|
||||||
|
Still, it helps to think of SMB as "often good" rather than "guaranteed exact."
|
||||||
|
|
||||||
|
### WebDAV
|
||||||
|
|
||||||
|
WebDAV is very convenient and widely supported.
|
||||||
|
|
||||||
|
It is usually best for:
|
||||||
|
|
||||||
|
- backing up file contents
|
||||||
|
- reaching services like Nextcloud and ownCloud
|
||||||
|
- simple remote access
|
||||||
|
|
||||||
|
It is usually **not** the best choice if you want a very exact copy of the original permission model.
|
||||||
|
|
||||||
|
### SFTP
|
||||||
|
|
||||||
|
SFTP is a solid option when you need secure remote access over SSH.
|
||||||
|
|
||||||
|
It is usually good for:
|
||||||
|
|
||||||
|
- backing up files from remote Linux servers
|
||||||
|
- simple remote access with passwords or keys
|
||||||
|
- content-focused backups
|
||||||
|
|
||||||
|
Like WebDAV, it is usually better for file contents than for exact permission replication.
|
||||||
|
|
||||||
|
### Rclone
|
||||||
|
|
||||||
|
Rclone is excellent for connecting to many cloud providers.
|
||||||
|
|
||||||
|
It is usually best for:
|
||||||
|
|
||||||
|
- backing up data stored in cloud services
|
||||||
|
- content-focused backup jobs
|
||||||
|
- providers that are not traditional filesystems
|
||||||
|
|
||||||
|
It is the least suitable option when you want the mounted view to behave like a normal local filesystem.
|
||||||
|
|
||||||
|
## Best option for true replication
|
||||||
|
|
||||||
|
If by "true replication" you mean:
|
||||||
|
|
||||||
|
- same files
|
||||||
|
- same ownership
|
||||||
|
- same permissions
|
||||||
|
- same advanced permission rules
|
||||||
|
- restore behavior that stays very close to the original
|
||||||
|
|
||||||
|
then the best option is still **Directory**.
|
||||||
|
|
||||||
|
If the original data lives somewhere else, the most reliable approach is usually one of these:
|
||||||
|
|
||||||
|
1. Run Zerobyte close to the data and back it up as a local directory there.
|
||||||
|
2. Mount the source on the host first, then bind-mount that local path into Zerobyte as a Directory volume.
|
||||||
|
3. Use remote mounted backends mainly for content backup, not for exact replication.
|
||||||
|
|
||||||
|
## Practical advice
|
||||||
|
|
||||||
|
1. If you mainly care about recovering files, most mounted backends are fine.
|
||||||
|
2. If you care a lot about permissions too, prefer **Directory** first.
|
||||||
|
3. After **Directory**, **NFS** is often the next best choice.
|
||||||
|
4. **SMB** can also be a good option, especially for Windows and Samba shares, but it is worth testing first.
|
||||||
|
5. Treat **WebDAV**, **SFTP**, and **rclone** as content-first backends.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
The message here is not "avoid mounted shares." They are useful and often exactly the right choice.
|
||||||
|
|
||||||
|
The simpler rule is: if file contents matter most, mounted shares are often great. If you want the closest possible replica of the original filesystem, a local **Directory** volume is still the best option.
|
||||||
|
</Callout>
|
||||||
|
|
@ -187,7 +187,9 @@ That works well on a normal local Linux filesystem. It gets more complicated whe
|
||||||
These destinations may let Zerobyte write file contents while still rejecting `chown`, `chmod`, timestamp, or xattr updates.
|
These destinations may let Zerobyte write file contents while still rejecting `chown`, `chmod`, timestamp, or xattr updates.
|
||||||
|
|
||||||
<Callout type="info">
|
<Callout type="info">
|
||||||
In Zerobyte today, SFTP, SMB, and WebDAV mounts pass the container process `uid` and `gid` to the mount command. If the container runs as root, those mounts can appear as `0:0` inside the container even when the remote system stores ownership differently. A backup taken from that mounted view can therefore record synthetic ownership in the snapshot.
|
Mounted destinations can expose translated metadata. In Zerobyte today, WebDAV and SFTP mounts still pass the container process `uid` and `gid` to the mount command, and SMB mounts still depend on what the Linux CIFS client can represent from the remote server. A backup taken from any mounted view records what Linux can see there, which may not be the remote system's full ACL model.
|
||||||
|
|
||||||
|
For the practical implications and backend-by-backend tradeoffs, see [Mounted Shares and Permissions](/docs/guides/mounted-shares-and-acls).
|
||||||
</Callout>
|
</Callout>
|
||||||
|
|
||||||
## What Zerobyte can and cannot change
|
## What Zerobyte can and cannot change
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,8 @@ services:
|
||||||
|
|
||||||
<Callout type="warn">
|
<Callout type="warn">
|
||||||
**Security Note**: The `SYS_ADMIN` capability and `/dev/fuse` device are required for mounting remote filesystems (NFS, SMB, WebDAV, SFTP). If you only need local directory backups, see the [Simplified Installation](#simplified-installation-no-remote-mounts) section below.
|
**Security Note**: The `SYS_ADMIN` capability and `/dev/fuse` device are required for mounting remote filesystems (NFS, SMB, WebDAV, SFTP). If you only need local directory backups, see the [Simplified Installation](#simplified-installation-no-remote-mounts) section below.
|
||||||
|
|
||||||
|
Remote mounts are convenient, but they can expose translated ownership, permissions, and ACL metadata instead of the source system's original view. Read [Mounted Shares and Permissions](/docs/guides/mounted-shares-and-acls) before using mounted volumes for metadata-sensitive backups.
|
||||||
</Callout>
|
</Callout>
|
||||||
|
|
||||||
### 2. Configure Environment Variables
|
### 2. Configure Environment Variables
|
||||||
|
|
@ -218,6 +220,8 @@ services:
|
||||||
|
|
||||||
<Callout type="info">
|
<Callout type="info">
|
||||||
If you need remote mount capabilities later, you can update your `docker-compose.yml` to add back the `cap_add: SYS_ADMIN` and `devices: /dev/fuse:/dev/fuse` directives.
|
If you need remote mount capabilities later, you can update your `docker-compose.yml` to add back the `cap_add: SYS_ADMIN` and `devices: /dev/fuse:/dev/fuse` directives.
|
||||||
|
|
||||||
|
If your goal is the closest thing to true replication, prefer local bind-mounted directories whenever possible. See [Mounted Shares and Permissions](/docs/guides/mounted-shares-and-acls) for the practical tradeoffs.
|
||||||
</Callout>
|
</Callout>
|
||||||
|
|
||||||
## Mounting Local Directories
|
## Mounting Local Directories
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,8 @@
|
||||||
"dev": "NODE_ENV=development bunx --bun vite",
|
"dev": "NODE_ENV=development bunx --bun vite",
|
||||||
"build": "bunx --bun vite build",
|
"build": "bunx --bun vite build",
|
||||||
"start": "bun run .output/server/index.mjs",
|
"start": "bun run .output/server/index.mjs",
|
||||||
|
"build:backend-integration": "bun build app/test/backend-integration/index.ts --outdir .output/backend-integration --target bun",
|
||||||
|
"test:integration:backends": "bash app/test/backend-integration/run.sh",
|
||||||
"preview": "bunx --bun vite preview",
|
"preview": "bunx --bun vite preview",
|
||||||
"cli:dev": "bun run app/server/cli/main.ts",
|
"cli:dev": "bun run app/server/cli/main.ts",
|
||||||
"cli": "ZEROBYTE_CLI=1 bun .output/server/_ssr/ssr.mjs",
|
"cli": "ZEROBYTE_CLI=1 bun .output/server/_ssr/ssr.mjs",
|
||||||
|
|
@ -38,6 +40,7 @@
|
||||||
"@dnd-kit/core": "^6.3.1",
|
"@dnd-kit/core": "^6.3.1",
|
||||||
"@dnd-kit/sortable": "^10.0.0",
|
"@dnd-kit/sortable": "^10.0.0",
|
||||||
"@dnd-kit/utilities": "^3.2.2",
|
"@dnd-kit/utilities": "^3.2.2",
|
||||||
|
"@hono/standard-validator": "^0.2.0",
|
||||||
"@hookform/resolvers": "^5.2.2",
|
"@hookform/resolvers": "^5.2.2",
|
||||||
"@inquirer/prompts": "^8.4.3",
|
"@inquirer/prompts": "^8.4.3",
|
||||||
"@radix-ui/react-alert-dialog": "^1.1.15",
|
"@radix-ui/react-alert-dialog": "^1.1.15",
|
||||||
|
|
@ -122,6 +125,7 @@
|
||||||
"dotenv-cli": "^11.0.0",
|
"dotenv-cli": "^11.0.0",
|
||||||
"drizzle-kit": "^1.0.0-rc.2-e38a2ba",
|
"drizzle-kit": "^1.0.0-rc.2-e38a2ba",
|
||||||
"fast-check": "^4.8.0",
|
"fast-check": "^4.8.0",
|
||||||
|
"happy-dom": "^20.9.0",
|
||||||
"msw": "^2.14.6",
|
"msw": "^2.14.6",
|
||||||
"nitro": "^3.0.1-alpha.2",
|
"nitro": "^3.0.1-alpha.2",
|
||||||
"tailwindcss": "^4.3.0",
|
"tailwindcss": "^4.3.0",
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ export const smbConfigSchema = z.object({
|
||||||
username: z.string().optional(),
|
username: z.string().optional(),
|
||||||
password: z.string().optional(),
|
password: z.string().optional(),
|
||||||
guest: z.boolean().optional(),
|
guest: z.boolean().optional(),
|
||||||
|
mapToContainerUidGid: z.boolean().default(false),
|
||||||
vers: z.enum(["1.0", "2.0", "2.1", "3.0", "auto"]).default("auto"),
|
vers: z.enum(["1.0", "2.0", "2.1", "3.0", "auto"]).default("auto"),
|
||||||
domain: z.string().optional(),
|
domain: z.string().optional(),
|
||||||
port: z
|
port: z
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ export const safeExec = async ({ command, args = [], env = {}, ...rest }: ExecPr
|
||||||
const { stdout, stderr } = await promisify(execFile)(command, args, {
|
const { stdout, stderr } = await promisify(execFile)(command, args, {
|
||||||
...options,
|
...options,
|
||||||
...rest,
|
...rest,
|
||||||
|
shell: false,
|
||||||
encoding: "utf8",
|
encoding: "utf8",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -78,6 +79,7 @@ export function safeSpawn(params: SafeSpawnParams): Promise<SpawnResult> {
|
||||||
return new Promise<SpawnResult>((resolve) => {
|
return new Promise<SpawnResult>((resolve) => {
|
||||||
const child = spawn(command, args, {
|
const child = spawn(command, args, {
|
||||||
env: { ...process.env, ...env },
|
env: { ...process.env, ...env },
|
||||||
|
shell: false,
|
||||||
signal: signal,
|
signal: signal,
|
||||||
stdio: ["ignore", "pipe", "pipe"],
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue