Compare commits

...

5 commits

Author SHA1 Message Date
Nicolas Meienberger
82c3eafc1c
fix: agent runtime duplication 2026-05-05 21:29:31 +02:00
Nicolas Meienberger
35a52554de
refactor(agent): move volume lifecycle cleanup to agent 2026-05-05 21:09:03 +02:00
Nicolas Meienberger
188888ff9a
fix: wait for agent to be ready before startup 2026-05-05 20:49:35 +02:00
Nicolas Meienberger
a5d0cc986c
refactor: move all volume operational code to agent app 2026-05-05 20:49:32 +02:00
Nicolas Meienberger
0ce3f79ff1
feat: agent routing depending on volume configured 2026-05-05 20:49:28 +02:00
51 changed files with 4637 additions and 1274 deletions

View file

@ -0,0 +1,2 @@
ALTER TABLE `volumes_table` ADD `agent_id` text DEFAULT 'local' NOT NULL;--> statement-breakpoint
CREATE INDEX `volumes_table_agent_id_idx` ON `volumes_table` (`agent_id`);

File diff suppressed because it is too large Load diff

View file

@ -13,4 +13,4 @@ CREATE TABLE `agents_table` (
); );
--> statement-breakpoint --> statement-breakpoint
CREATE INDEX `agents_table_organization_id_idx` ON `agents_table` (`organization_id`);--> statement-breakpoint CREATE INDEX `agents_table_organization_id_idx` ON `agents_table` (`organization_id`);--> statement-breakpoint
CREATE INDEX `agents_table_status_idx` ON `agents_table` (`status`); CREATE INDEX `agents_table_status_idx` ON `agents_table` (`status`);

View file

@ -13,6 +13,7 @@ import type { BackupWebhooks } from "@zerobyte/core/backup-hooks";
import type { BackendConfig, BackendStatus, BackendType } from "~/schemas/volumes"; import type { BackendConfig, BackendStatus, BackendType } from "~/schemas/volumes";
import type { NotificationConfig, NotificationType } from "~/schemas/notifications"; import type { NotificationConfig, NotificationType } from "~/schemas/notifications";
import type { ShortId } from "~/server/utils/branded"; import type { ShortId } from "~/server/utils/branded";
import { LOCAL_AGENT_ID } from "../modules/agents/constants";
/** /**
* Users Table * Users Table
@ -253,12 +254,14 @@ export const volumesTable = sqliteTable(
.default(sql`(unixepoch() * 1000)`), .default(sql`(unixepoch() * 1000)`),
config: text("config", { mode: "json" }).$type<BackendConfig>().notNull(), config: text("config", { mode: "json" }).$type<BackendConfig>().notNull(),
autoRemount: int("auto_remount", { mode: "boolean" }).notNull().default(true), autoRemount: int("auto_remount", { mode: "boolean" }).notNull().default(true),
agentId: text("agent_id").notNull().default(LOCAL_AGENT_ID),
organizationId: text("organization_id") organizationId: text("organization_id")
.notNull() .notNull()
.references(() => organization.id, { onDelete: "cascade" }), .references(() => organization.id, { onDelete: "cascade" }),
}, },
(table) => [ (table) => [
unique().on(table.name, table.organizationId), unique().on(table.name, table.organizationId),
index("volumes_table_agent_id_idx").on(table.agentId),
uniqueIndex("volumes_table_org_provisioning_id_uidx").on(table.organizationId, table.provisioningId), uniqueIndex("volumes_table_org_provisioning_id_uidx").on(table.organizationId, table.provisioningId),
], ],
); );

View file

@ -0,0 +1,18 @@
import { afterEach, expect, test, vi } from "vitest";
import { config } from "../../core/config";
import { CleanupDanglingMountsJob } from "../cleanup-dangling";
import * as mountinfo from "../../utils/mountinfo";
afterEach(() => {
config.flags.enableLocalAgent = true;
vi.restoreAllMocks();
});
test("skips controller-local mount inspection when local volume execution is agent-owned", async () => {
config.flags.enableLocalAgent = true;
const readMountInfo = vi.spyOn(mountinfo, "readMountInfo");
await new CleanupDanglingMountsJob().run();
expect(readMountInfo).not.toHaveBeenCalled();
});

View file

@ -5,14 +5,21 @@ import { volumeService } from "../modules/volumes/volume.service";
import { readMountInfo } from "../utils/mountinfo"; import { readMountInfo } from "../utils/mountinfo";
import { getVolumePath } from "../modules/volumes/helpers"; import { getVolumePath } from "../modules/volumes/helpers";
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import { executeUnmount } from "../modules/backends/utils/backend-utils"; import { executeUnmount } from "../../../apps/agent/src/volume-host/backends/utils";
import { toMessage } from "../utils/errors"; import { toMessage } from "../utils/errors";
import { VOLUME_MOUNT_BASE } from "../core/constants"; import { VOLUME_MOUNT_BASE } from "../core/constants";
import { db } from "../db/db"; import { db } from "../db/db";
import { withContext } from "../core/request-context"; import { withContext } from "../core/request-context";
import { config } from "../core/config";
import { LOCAL_AGENT_ID } from "../modules/agents/constants";
export class CleanupDanglingMountsJob extends Job { export class CleanupDanglingMountsJob extends Job {
async run() { async run() {
if (config.flags.enableLocalAgent) {
logger.debug("Skipping controller-local dangling mount cleanup because local volume execution is agent-owned.");
return { done: true, timestamp: new Date() };
}
const organizations = await db.query.organization.findMany({}); const organizations = await db.query.organization.findMany({});
if (organizations.length === 0) { if (organizations.length === 0) {
logger.warn("No organizations found; skipping dangling mount cleanup to avoid false positives."); logger.warn("No organizations found; skipping dangling mount cleanup to avoid false positives.");
@ -22,7 +29,7 @@ export class CleanupDanglingMountsJob extends Job {
const allVolumes = []; const allVolumes = [];
for (const org of organizations) { for (const org of organizations) {
const volumes = await withContext({ organizationId: org.id }, async () => volumeService.listVolumes()); const volumes = await withContext({ organizationId: org.id }, async () => volumeService.listVolumes());
allVolumes.push(...volumes); allVolumes.push(...volumes.filter((volume) => volume.agentId === LOCAL_AGENT_ID));
} }
const allSystemMounts = await readMountInfo(); const allSystemMounts = await readMountInfo();

View file

@ -4,7 +4,7 @@ import { fromAny, fromPartial } from "@total-typescript/shoehorn";
import { Effect } from "effect"; import { Effect } from "effect";
import { agentManager, type ProcessWithAgentRuntime } from "../agents-manager"; import { agentManager, type ProcessWithAgentRuntime } from "../agents-manager";
import type { AgentManagerRuntime } from "../controller/server"; import type { AgentManagerRuntime } from "../controller/server";
import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol"; import type { BackupRunPayload, VolumeCommand } from "@zerobyte/contracts/agent-protocol";
const setAgentRuntime = (agentManagerRuntime: Partial<AgentManagerRuntime> | null) => { const setAgentRuntime = (agentManagerRuntime: Partial<AgentManagerRuntime> | null) => {
(process as ProcessWithAgentRuntime).__zerobyteAgentRuntime = { (process as ProcessWithAgentRuntime).__zerobyteAgentRuntime = {
@ -46,3 +46,32 @@ test("cancelBackup resolves a running backup when the cancel command cannot be d
scheduleId: "schedule-1", scheduleId: "schedule-1",
}); });
}); });
test("runVolumeCommand sends the command to the selected agent", async () => {
const runVolumeCommand = vi.fn(() =>
Effect.succeed({
commandId: "command-1",
status: "success" as const,
command: { name: "volume.mount" as const, result: { status: "mounted" as const } },
}),
);
setAgentRuntime({ runVolumeCommand });
const command = fromPartial<VolumeCommand>({ name: "volume.mount", volume: { agentId: "agent-1" } });
await expect(agentManager.runVolumeCommand("agent-1", command)).resolves.toEqual({
name: "volume.mount",
result: { status: "mounted" },
});
expect(runVolumeCommand).toHaveBeenCalledWith("agent-1", command);
});
test("runVolumeCommand fails when the selected agent is unavailable", async () => {
setAgentRuntime(null);
const command = fromPartial<VolumeCommand>({ name: "volume.mount", volume: { agentId: "agent-1" } });
await expect(agentManager.runVolumeCommand("agent-1", command)).rejects.toThrow(
"Volume agent agent-1 is not connected",
);
});

View file

@ -1,16 +1,23 @@
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol"; import type { BackupRunPayload, VolumeCommand, VolumeCommandResult } from "@zerobyte/contracts/agent-protocol";
import { Effect } from "effect"; import { Effect } from "effect";
import { config } from "../../core/config"; import { config } from "../../core/config";
import { createAgentManagerRuntime, type AgentManagerEvent } from "./controller/server"; import { createAgentManagerRuntime, type AgentManagerEvent } from "./controller/server";
import { LOCAL_AGENT_ID } from "./constants";
import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process"; import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process";
import type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state"; import {
import { createAgentRuntimeState } from "./helpers/runtime-state"; createAgentRuntimeState,
type AgentRuntimeState,
type BackupExecutionProgress,
type BackupExecutionResult,
} from "./helpers/runtime-state";
import { getDevAgentRuntimeState } from "./helpers/runtime-state.dev"; import { getDevAgentRuntimeState } from "./helpers/runtime-state.dev";
export type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state"; export type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
export type { ProcessWithAgentRuntime } from "./helpers/runtime-state.dev"; export type { ProcessWithAgentRuntime } from "./helpers/runtime-state.dev";
const productionRuntimeState = createAgentRuntimeState(); type ProcessWithProductionAgentRuntime = NodeJS.Process & {
__zerobyteProductionAgentRuntime?: AgentRuntimeState;
};
type AgentRunBackupRequest = { type AgentRunBackupRequest = {
scheduleId: number; scheduleId: number;
@ -19,7 +26,18 @@ type AgentRunBackupRequest = {
onProgress: (progress: BackupExecutionProgress) => void; onProgress: (progress: BackupExecutionProgress) => void;
}; };
const getAgentRuntimeState = () => (config.__prod__ ? productionRuntimeState : getDevAgentRuntimeState()); const getProductionAgentRuntimeState = () => {
// Nitro production builds can bundle startup plugins and API handlers into separate chunks.
// Keep the live controller on process so both chunks see the same agent sessions.
const runtimeProcess = process as ProcessWithProductionAgentRuntime;
if (!runtimeProcess.__zerobyteProductionAgentRuntime) {
runtimeProcess.__zerobyteProductionAgentRuntime = createAgentRuntimeState();
}
return runtimeProcess.__zerobyteProductionAgentRuntime;
};
const getAgentRuntimeState = () => (config.__prod__ ? getProductionAgentRuntimeState() : getDevAgentRuntimeState());
const getAgentManagerRuntime = () => getAgentRuntimeState().agentManager; const getAgentManagerRuntime = () => getAgentRuntimeState().agentManager;
const getActiveBackupsByScheduleId = () => getAgentRuntimeState().activeBackupsByScheduleId; const getActiveBackupsByScheduleId = () => getAgentRuntimeState().activeBackupsByScheduleId;
const getActiveBackupScheduleIdsByJobId = () => getAgentRuntimeState().activeBackupScheduleIdsByJobId; const getActiveBackupScheduleIdsByJobId = () => getAgentRuntimeState().activeBackupScheduleIdsByJobId;
@ -266,10 +284,36 @@ export const agentManager = {
cancelBackup: async (agentId: string, scheduleId: number) => { cancelBackup: async (agentId: string, scheduleId: number) => {
return requestBackupCancellation(agentId, scheduleId); return requestBackupCancellation(agentId, scheduleId);
}, },
runVolumeCommand: async (agentId: string, command: VolumeCommand): Promise<VolumeCommandResult> => {
const runtime = getAgentManagerRuntime();
if (!runtime) {
throw new Error(`Volume agent ${agentId} is not connected`);
}
const response = await Effect.runPromise(runtime.runVolumeCommand(agentId, command));
if (!response) {
throw new Error(`Failed to send volume command ${command.name} to agent ${agentId}`);
}
if (response.status === "error") {
throw new Error(response.error);
}
return response.command;
},
}; };
export const startLocalAgent = async () => { export const startLocalAgent = async () => {
await spawnLocalAgentProcess(getAgentRuntimeState()); const runtime = getAgentRuntimeState();
await spawnLocalAgentProcess(runtime);
if (!runtime.agentManager) {
return;
}
if (!(await runtime.agentManager.waitForAgentReady(LOCAL_AGENT_ID))) {
throw new Error("Local agent did not become ready before startup");
}
}; };
// fallow-ignore-next-line unused-export // fallow-ignore-next-line unused-export

View file

@ -1,7 +1,13 @@
import { Data, Effect, Exit, Fiber, Scope } from "effect"; import { Data, Effect, Exit, Fiber, Scope } from "effect";
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils"; import { toMessage } from "@zerobyte/core/utils";
import type { AgentMessage, BackupCancelPayload, BackupRunPayload } from "@zerobyte/contracts/agent-protocol"; import type {
AgentMessage,
BackupCancelPayload,
BackupRunPayload,
VolumeCommand,
VolumeCommandResponsePayload,
} from "@zerobyte/contracts/agent-protocol";
import { import {
createControllerAgentSession, createControllerAgentSession,
type AgentConnectionData, type AgentConnectionData,
@ -40,6 +46,7 @@ class StopAgentManagerServerError extends Data.TaggedError("StopAgentManagerServ
export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) => void) { export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) => void) {
let sessions = new Map<string, ControllerAgentSessionHandle>(); let sessions = new Map<string, ControllerAgentSessionHandle>();
let runtimeScope: Scope.CloseableScope | null = null; let runtimeScope: Scope.CloseableScope | null = null;
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const closeSession = (sessionHandle: ControllerAgentSessionHandle) => const closeSession = (sessionHandle: ControllerAgentSessionHandle) =>
Effect.gen(function* () { Effect.gen(function* () {
@ -73,6 +80,11 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
const getSessionHandle = (agentId: string) => sessions.get(agentId); const getSessionHandle = (agentId: string) => sessions.get(agentId);
const getSession = (agentId: string) => getSessionHandle(agentId)?.session; const getSession = (agentId: string) => getSessionHandle(agentId)?.session;
const isAgentReady = (agentId: string) => {
const session = getSession(agentId);
return !!session && Effect.runSync(session.isReady());
};
const handleSessionEvent = (params: { agentId: string; agentName: string; sessionId: string }) => { const handleSessionEvent = (params: { agentId: string; agentName: string; sessionId: string }) => {
const { agentId, agentName } = params; const { agentId, agentName } = params;
@ -284,6 +296,19 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
return { return {
start, start,
waitForAgentReady: async (agentId: string, timeoutMs = 10_000) => {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (isAgentReady(agentId)) {
return true;
}
await sleep(50);
}
return isAgentReady(agentId);
},
sendBackup: (agentId: string, payload: BackupRunPayload) => sendBackup: (agentId: string, payload: BackupRunPayload) =>
Effect.gen(function* () { Effect.gen(function* () {
const session = getSession(agentId); const session = getSession(agentId);
@ -319,10 +344,30 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
logger.warn(`Cannot cancel backup command. Agent ${agentId} is no longer accepting commands.`); logger.warn(`Cannot cancel backup command. Agent ${agentId} is no longer accepting commands.`);
return false; return false;
} }
logger.info(`Sent backup cancel for command ${payload.jobId} to agent ${agentId}`); logger.info(`Sent backup cancel for command ${payload.jobId} to agent ${agentId}`);
return true; return true;
}), }),
runVolumeCommand: (
agentId: string,
command: VolumeCommand,
): Effect.Effect<VolumeCommandResponsePayload | null, Error> =>
Effect.gen(function* () {
const session = getSession(agentId);
if (!session) {
yield* logger.effect.warn(`Cannot send volume command ${command.name}. Agent ${agentId} is not connected.`);
return null;
}
if (!(yield* session.isReady())) {
yield* logger.effect.warn(`Cannot send volume command ${command.name}. Agent ${agentId} is not ready.`);
return null;
}
const result = yield* session.runVolumeCommand(command);
yield* logger.effect.info(`Completed volume command ${command.name} on agent ${agentId}`);
return result;
}),
stop, stop,
}; };
} }

View file

@ -7,6 +7,8 @@ import {
type BackupCancelPayload, type BackupCancelPayload,
type BackupRunPayload, type BackupRunPayload,
type ControllerWireMessage, type ControllerWireMessage,
type VolumeCommand,
type VolumeCommandResponsePayload,
} from "@zerobyte/contracts/agent-protocol"; } from "@zerobyte/contracts/agent-protocol";
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils"; import { toMessage } from "@zerobyte/core/utils";
@ -27,13 +29,24 @@ type SessionState = {
lastPongAt: number | null; lastPongAt: number | null;
}; };
export type ControllerAgentSessionEvent = AgentMessage | { type: "agent.disconnected" }; type PendingVolumeCommand = {
resolve: (payload: VolumeCommandResponsePayload) => void;
reject: (error: Error) => void;
timeout: ReturnType<typeof setTimeout>;
};
export type ControllerAgentSessionEvent =
| Exclude<AgentMessage, { type: "volume.commandResult" }>
| {
type: "agent.disconnected";
};
export type ControllerAgentSession = { export type ControllerAgentSession = {
readonly connectionId: string; readonly connectionId: string;
handleMessage: (data: string) => Effect.Effect<void>; handleMessage: (data: string) => Effect.Effect<void>;
sendBackup: (payload: BackupRunPayload) => Effect.Effect<boolean>; sendBackup: (payload: BackupRunPayload) => Effect.Effect<boolean>;
sendBackupCancel: (payload: BackupCancelPayload) => Effect.Effect<boolean>; sendBackupCancel: (payload: BackupCancelPayload) => Effect.Effect<boolean>;
runVolumeCommand: (command: VolumeCommand) => Effect.Effect<VolumeCommandResponsePayload, Error>;
isReady: () => Effect.Effect<boolean>; isReady: () => Effect.Effect<boolean>;
run: Effect.Effect<void, never, Scope.Scope>; run: Effect.Effect<void, never, Scope.Scope>;
}; };
@ -45,6 +58,7 @@ export const createControllerAgentSession = (
Effect.gen(function* () { Effect.gen(function* () {
let isClosed = false; let isClosed = false;
const outboundQueue = yield* Queue.bounded<ControllerWireMessage>(64); const outboundQueue = yield* Queue.bounded<ControllerWireMessage>(64);
const pendingVolumeCommands = new Map<string, PendingVolumeCommand>();
const state = yield* Ref.make<SessionState>({ const state = yield* Ref.make<SessionState>({
isReady: false, isReady: false,
lastSeenAt: null, lastSeenAt: null,
@ -63,9 +77,18 @@ export const createControllerAgentSession = (
const updateState = (update: (current: SessionState) => SessionState) => Ref.update(state, update); const updateState = (update: (current: SessionState) => SessionState) => Ref.update(state, update);
const rejectPendingVolumeCommands = () => {
for (const [commandId, pending] of pendingVolumeCommands) {
clearTimeout(pending.timeout);
pending.reject(new Error(`Agent session closed before volume command ${commandId} completed`));
}
pendingVolumeCommands.clear();
};
const releaseSession = Effect.gen(function* () { const releaseSession = Effect.gen(function* () {
const disconnectedAt = Date.now(); const disconnectedAt = Date.now();
yield* updateState((current) => ({ ...current, isReady: false, lastSeenAt: disconnectedAt })); yield* updateState((current) => ({ ...current, isReady: false, lastSeenAt: disconnectedAt }));
yield* Effect.sync(rejectPendingVolumeCommands);
yield* onEvent({ type: "agent.disconnected" }); yield* onEvent({ type: "agent.disconnected" });
yield* Queue.shutdown(outboundQueue); yield* Queue.shutdown(outboundQueue);
@ -129,20 +152,43 @@ export const createControllerAgentSession = (
return yield* Effect.never; return yield* Effect.never;
}); });
const handleVolumeCommandResult = (payload: VolumeCommandResponsePayload) => {
const pending = pendingVolumeCommands.get(payload.commandId);
if (!pending) {
logger.warn(`Received response for unknown volume command ${payload.commandId}`);
return;
}
pendingVolumeCommands.delete(payload.commandId);
clearTimeout(pending.timeout);
pending.resolve(payload);
};
const handleAgentMessage = (message: AgentMessage) => const handleAgentMessage = (message: AgentMessage) =>
Effect.gen(function* () { Effect.gen(function* () {
if (message.type === "agent.ready") { switch (message.type) {
const readyAt = Date.now(); case "agent.ready": {
yield* updateState((current) => ({ ...current, isReady: true, lastSeenAt: readyAt })); const readyAt = Date.now();
yield* logger.effect.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`); yield* updateState((current) => ({ ...current, isReady: true, lastSeenAt: readyAt }));
yield* logger.effect.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
yield* onEvent(message);
break;
}
case "heartbeat.pong": {
const seenAt = Date.now();
yield* updateState((current) => ({ ...current, lastSeenAt: seenAt, lastPongAt: message.payload.sentAt }));
yield* onEvent(message);
break;
}
case "volume.commandResult": {
yield* Effect.sync(() => handleVolumeCommandResult(message.payload));
break;
}
default: {
yield* onEvent(message);
break;
}
} }
if (message.type === "heartbeat.pong") {
const seenAt = Date.now();
yield* updateState((current) => ({ ...current, lastSeenAt: seenAt, lastPongAt: message.payload.sentAt }));
}
yield* onEvent(message);
}); });
return { return {
@ -166,6 +212,37 @@ export const createControllerAgentSession = (
}, },
sendBackup: (payload) => offerOutbound(createControllerMessage("backup.run", payload)), sendBackup: (payload) => offerOutbound(createControllerMessage("backup.run", payload)),
sendBackupCancel: (payload) => offerOutbound(createControllerMessage("backup.cancel", payload)), sendBackupCancel: (payload) => offerOutbound(createControllerMessage("backup.cancel", payload)),
runVolumeCommand: (command) => {
return Effect.tryPromise({
try: async () => {
const commandId = Bun.randomUUIDv7();
const response = new Promise<VolumeCommandResponsePayload>((resolve, reject) => {
const timeout = setTimeout(() => {
pendingVolumeCommands.delete(commandId);
reject(new Error(`Volume command ${command.name} timed out`));
}, 60_000);
pendingVolumeCommands.set(commandId, { resolve, reject, timeout });
});
const queued = await Effect.runPromise(
offerOutbound(createControllerMessage("volume.command", { commandId, command })),
);
if (!queued) {
const pending = pendingVolumeCommands.get(commandId);
if (pending) {
clearTimeout(pending.timeout);
pendingVolumeCommands.delete(commandId);
}
throw new Error(`Failed to queue volume command ${command.name}`);
}
return response;
},
catch: (error) => (error instanceof Error ? error : new Error(toMessage(error))),
});
},
isReady: () => Ref.get(state).pipe(Effect.map((current) => current.isReady)), isReady: () => Ref.get(state).pipe(Effect.map((current) => current.isReady)),
run, run,
}; };

View file

@ -1,46 +0,0 @@
import type { BackendStatus } from "~/schemas/volumes";
import type { Volume } from "../../db/schema";
import { getVolumePath } from "../volumes/helpers";
import { makeDirectoryBackend } from "./directory/directory-backend";
import { makeNfsBackend } from "./nfs/nfs-backend";
import { makeRcloneBackend } from "./rclone/rclone-backend";
import { makeSmbBackend } from "./smb/smb-backend";
import { makeWebdavBackend } from "./webdav/webdav-backend";
import { makeSftpBackend } from "./sftp/sftp-backend";
type OperationResult = {
error?: string;
status: BackendStatus;
};
export type VolumeBackend = {
mount: () => Promise<OperationResult>;
unmount: () => Promise<OperationResult>;
checkHealth: () => Promise<OperationResult>;
};
export const createVolumeBackend = (volume: Volume, mountPath = getVolumePath(volume)): VolumeBackend => {
switch (volume.config.backend) {
case "nfs": {
return makeNfsBackend(volume.config, mountPath);
}
case "smb": {
return makeSmbBackend(volume.config, mountPath);
}
case "directory": {
return makeDirectoryBackend(volume.config, mountPath);
}
case "webdav": {
return makeWebdavBackend(volume.config, mountPath);
}
case "rclone": {
return makeRcloneBackend(volume.config, mountPath);
}
case "sftp": {
return makeSftpBackend(volume.config, mountPath);
}
default: {
throw new Error("Unsupported backend");
}
}
};

View file

@ -1,135 +0,0 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
import { OPERATION_TIMEOUT } from "../../../core/constants";
import { toMessage } from "../../../utils/errors";
import { logger } from "@zerobyte/core/node";
import { getMountForPath } from "../../../utils/mountinfo";
import { withTimeout } from "../../../utils/timeout";
import type { VolumeBackend } from "../backend";
import { assertMounted, executeMount, executeUnmount } from "../utils/backend-utils";
const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting volume ${path}...`);
if (config.backend !== "nfs") {
logger.error("Provided config is not for NFS backend");
return {
status: BACKEND_STATUS.error,
error: "Provided config is not for NFS backend",
};
}
if (os.platform() !== "linux") {
logger.error("NFS mounting is only supported on Linux hosts.");
return {
status: BACKEND_STATUS.error,
error: "NFS mounting is only supported on Linux hosts.",
};
}
const { status } = await checkHealth(path);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
await unmount(path);
}
const run = async () => {
await fs.mkdir(path, { recursive: true });
const source = `${config.server}:${config.exportPath}`;
const options = [`vers=${config.version}`, `port=${config.port}`];
if (config.version === "3") {
options.push("nolock");
}
if (config.readOnly) {
options.push("ro");
}
const args = ["-t", "nfs", "-o", options.join(","), source, path];
logger.debug(`Mounting volume ${path}...`);
logger.info(`Executing mount: mount ${args.join(" ")}`);
try {
await executeMount(args);
} catch (error) {
logger.warn(`Initial NFS mount failed, retrying with -i flag: ${toMessage(error)}`);
// Fallback with -i flag if the first mount fails using the mount helper
await executeMount(["-i", ...args]);
}
logger.info(`NFS volume at ${path} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS mount");
} catch (err) {
logger.error("Error mounting NFS volume", { error: toMessage(err) });
return { status: BACKEND_STATUS.error, error: toMessage(err) };
}
};
const unmount = async (path: string) => {
if (os.platform() !== "linux") {
logger.error("NFS unmounting is only supported on Linux hosts.");
return {
status: BACKEND_STATUS.error,
error: "NFS unmounting is only supported on Linux hosts.",
};
}
const run = async () => {
const mount = await getMountForPath(path);
if (!mount || mount.mountPoint !== path) {
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
return { status: BACKEND_STATUS.unmounted };
}
await executeUnmount(path);
await fs.rmdir(path).catch(() => {});
logger.info(`NFS volume at ${path} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS unmount");
} catch (err) {
logger.error("Error unmounting NFS volume", {
path,
error: toMessage(err),
});
return { status: BACKEND_STATUS.error, error: toMessage(err) };
}
};
const checkHealth = async (path: string) => {
const run = async () => {
await assertMounted(path, (fstype) => fstype.startsWith("nfs"));
logger.debug(`NFS volume at ${path} is healthy and mounted.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("NFS volume health check failed:", message);
}
return { status: BACKEND_STATUS.error, error: message };
}
};
export const makeNfsBackend = (config: BackendConfig, path: string): VolumeBackend => ({
mount: () => mount(config, path),
unmount: () => unmount(path),
checkHealth: () => checkHealth(path),
});

View file

@ -1,131 +0,0 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { OPERATION_TIMEOUT, RCLONE_CONFIG_FILE } from "../../../core/constants";
import { toMessage } from "../../../utils/errors";
import { logger } from "@zerobyte/core/node";
import { getMountForPath } from "../../../utils/mountinfo";
import { withTimeout } from "../../../utils/timeout";
import type { VolumeBackend } from "../backend";
import { assertMounted, executeUnmount } from "../utils/backend-utils";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
import { safeExec } from "@zerobyte/core/node";
import { config as zbConfig } from "~/server/core/config";
const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting rclone volume ${path}...`);
if (config.backend !== "rclone") {
logger.error("Provided config is not for rclone backend");
return { status: BACKEND_STATUS.error, error: "Provided config is not for rclone backend" };
}
if (os.platform() !== "linux") {
logger.error("Rclone mounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "Rclone mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(path);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
await unmount(path);
}
const run = async () => {
await fs.mkdir(path, { recursive: true });
const remotePath = `${config.remote}:${config.path}`;
const args = ["mount", remotePath, path, "--daemon"];
if (config.readOnly) {
args.push("--read-only");
}
args.push("--vfs-cache-mode", "writes");
args.push("--allow-non-empty");
args.push("--allow-other");
logger.debug(`Mounting rclone volume ${path}...`);
logger.info(`Executing rclone: rclone ${args.join(" ")}`);
const result = await safeExec({
command: "rclone",
args,
env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
timeout: zbConfig.serverIdleTimeout * 1000,
});
if (result.exitCode !== 0) {
const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error";
throw new Error(`Failed to mount rclone volume: ${errorMsg}`);
}
logger.info(`Rclone volume at ${path} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), zbConfig.serverIdleTimeout * 1000, "Rclone mount");
} catch (error) {
const errorMsg = toMessage(error);
logger.error("Error mounting rclone volume", { error: errorMsg });
return { status: BACKEND_STATUS.error, error: errorMsg };
}
};
const unmount = async (path: string) => {
if (os.platform() !== "linux") {
logger.error("Rclone unmounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "Rclone unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(path);
if (!mount || mount.mountPoint !== path) {
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
return { status: BACKEND_STATUS.unmounted };
}
await executeUnmount(path);
await fs.rmdir(path).catch(() => {});
logger.info(`Rclone volume at ${path} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone unmount");
} catch (error) {
logger.error("Error unmounting rclone volume", { path, error: toMessage(error) });
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const checkHealth = async (path: string) => {
const run = async () => {
await assertMounted(path, (fstype) => fstype.includes("rclone"));
logger.debug(`Rclone volume at ${path} is healthy and mounted.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("Rclone volume health check failed:", message);
}
return { status: BACKEND_STATUS.error, error: message };
}
};
export const makeRcloneBackend = (config: BackendConfig, path: string): VolumeBackend => ({
mount: () => mount(config, path),
unmount: () => unmount(path),
checkHealth: () => checkHealth(path),
});

View file

@ -1,212 +0,0 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { spawn } from "node:child_process";
import { OPERATION_TIMEOUT, SSH_KEYS_DIR } from "../../../core/constants";
import { cryptoUtils } from "../../../utils/crypto";
import { toMessage } from "../../../utils/errors";
import { logger, FILE_MODES, writeFileWithMode } from "@zerobyte/core/node";
import { getMountForPath } from "../../../utils/mountinfo";
import { withTimeout } from "../../../utils/timeout";
import type { VolumeBackend } from "../backend";
import { executeUnmount } from "../utils/backend-utils";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const getPrivateKeyPath = (mountPath: string) => {
const name = path.basename(mountPath);
return path.join(SSH_KEYS_DIR, `${name}.key`);
};
const getKnownHostsPath = (mountPath: string) => {
const name = path.basename(mountPath);
return path.join(SSH_KEYS_DIR, `${name}.known_hosts`);
};
const mount = async (config: BackendConfig, mountPath: string) => {
logger.debug(`Mounting SFTP volume ${mountPath}...`);
if (config.backend !== "sftp") {
logger.error("Provided config is not for SFTP backend");
return { status: BACKEND_STATUS.error, error: "Provided config is not for SFTP backend" };
}
if (os.platform() !== "linux") {
logger.error("SFTP mounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "SFTP mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(mountPath);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
await unmount(mountPath);
}
const run = async () => {
await fs.mkdir(mountPath, { recursive: true });
await fs.mkdir(SSH_KEYS_DIR, { recursive: true });
const { uid, gid } = os.userInfo();
const options = [
"reconnect",
"ServerAliveInterval=15",
"ServerAliveCountMax=3",
"allow_other",
`uid=${uid}`,
`gid=${gid}`,
];
if (config.skipHostKeyCheck) {
options.push("StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null");
} else if (config.knownHosts) {
const knownHostsPath = getKnownHostsPath(mountPath);
await writeFileWithMode(knownHostsPath, config.knownHosts, FILE_MODES.ownerReadWrite);
options.push(`UserKnownHostsFile=${knownHostsPath}`, "StrictHostKeyChecking=yes");
} else {
options.push("StrictHostKeyChecking=yes");
}
if (config.readOnly) {
options.push("ro");
}
if (config.port) {
options.push(`port=${config.port}`);
}
const keyPath = getPrivateKeyPath(mountPath);
if (config.privateKey) {
const decryptedKey = await cryptoUtils.resolveSecret(config.privateKey);
let normalizedKey = decryptedKey.replace(/\r\n/g, "\n");
if (!normalizedKey.endsWith("\n")) {
normalizedKey += "\n";
}
await writeFileWithMode(keyPath, normalizedKey, FILE_MODES.ownerReadWrite);
options.push(`IdentityFile=${keyPath}`);
}
const source = `${config.username}@${config.host}:${config.path || ""}`;
const args = [source, mountPath, "-o", options.join(",")];
logger.debug(`Mounting SFTP volume ${mountPath}...`);
const runSshfs = async (mountArgs: string[], password?: string) => {
return new Promise<void>((resolve, reject) => {
const child = spawn("sshfs", mountArgs, { stdio: ["pipe", "pipe", "pipe"] });
let stdout = "";
let stderr = "";
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (data) => {
stdout += data;
});
child.stderr.on("data", (data) => {
stderr += data;
});
child.on("error", (error) => {
reject(new Error(`Failed to start sshfs: ${error.message}`));
});
child.on("close", (code) => {
if (code === 0) {
resolve();
return;
}
const errorMsg = stderr.trim() || stdout.trim() || "Unknown error";
reject(new Error(`Failed to mount SFTP volume: ${errorMsg}`));
});
if (password) {
child.stdin.write(password);
}
child.stdin.end();
});
};
if (config.password) {
const password = await cryptoUtils.resolveSecret(config.password);
args.push("-o", "password_stdin");
logger.info(`Executing sshfs: sshfs ${args.join(" ")}`);
await runSshfs(args, password);
} else {
logger.info(`Executing sshfs: sshfs ${args.join(" ")}`);
await runSshfs(args);
}
logger.info(`SFTP volume at ${mountPath} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT * 2, "SFTP mount");
} catch (error) {
const errorMsg = toMessage(error);
logger.error("Error mounting SFTP volume", { error: errorMsg });
return { status: BACKEND_STATUS.error, error: errorMsg };
}
};
const unmount = async (mountPath: string) => {
if (os.platform() !== "linux") {
logger.error("SFTP unmounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "SFTP unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
} else {
await executeUnmount(mountPath);
}
const keyPath = getPrivateKeyPath(mountPath);
await fs.unlink(keyPath).catch(() => {});
const knownHostsPath = getKnownHostsPath(mountPath);
await fs.unlink(knownHostsPath).catch(() => {});
await fs.rmdir(mountPath).catch(() => {});
logger.info(`SFTP volume at ${mountPath} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SFTP unmount");
} catch (error) {
logger.error("Error unmounting SFTP volume", { mountPath, error: toMessage(error) });
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const checkHealth = async (mountPath: string) => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
return { status: BACKEND_STATUS.unmounted };
}
if (mount.fstype !== "fuse.sshfs") {
return {
status: BACKEND_STATUS.error,
error: `Invalid filesystem type: ${mount.fstype} (expected fuse.sshfs)`,
};
}
return { status: BACKEND_STATUS.mounted };
};
export const makeSftpBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
mount: () => mount(config, mountPath),
unmount: () => unmount(mountPath),
checkHealth: () => checkHealth(mountPath),
});

View file

@ -1,141 +0,0 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { OPERATION_TIMEOUT } from "../../../core/constants";
import { cryptoUtils } from "../../../utils/crypto";
import { toMessage } from "../../../utils/errors";
import { logger } from "@zerobyte/core/node";
import { getMountForPath } from "../../../utils/mountinfo";
import { withTimeout } from "../../../utils/timeout";
import type { VolumeBackend } from "../backend";
import { assertMounted, executeMount, executeUnmount } from "../utils/backend-utils";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting SMB volume ${path}...`);
if (config.backend !== "smb") {
logger.error("Provided config is not for SMB backend");
return { status: BACKEND_STATUS.error, error: "Provided config is not for SMB backend" };
}
if (os.platform() !== "linux") {
logger.error("SMB mounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "SMB mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(path);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
await unmount(path);
}
const run = async () => {
await fs.mkdir(path, { recursive: true });
const source = `//${config.server}/${config.share}`;
const { uid, gid } = os.userInfo();
const options = [`port=${config.port}`, `uid=${uid}`, `gid=${gid}`, "iocharset=utf8"];
if (config.guest) {
options.push("username=guest", "password=");
} else {
const password = await cryptoUtils.resolveSecret(config.password ?? "");
const safePassword = password.replace(/\\/g, "\\\\").replace(/,/g, "\\,");
options.push(`username=${config.username ?? "user"}`, `password=${safePassword}`);
}
if (config.domain) {
options.push(`domain=${config.domain}`);
}
if (config.vers && config.vers !== "auto") {
options.push(`vers=${config.vers}`);
}
if (config.readOnly) {
options.push("ro");
}
const args = ["-t", "cifs", "-o", options.join(","), source, path];
logger.debug(`Mounting SMB volume ${path}...`);
logger.info(`Executing SMB mount for ${source} at ${path}`);
try {
await executeMount(args);
} catch (error) {
logger.error(`SMB mount failed: ${toMessage(error)}`);
throw error;
}
logger.info(`SMB volume at ${path} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB mount");
} catch (error) {
logger.error("Error mounting SMB volume", { error: toMessage(error) });
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const unmount = async (path: string) => {
if (os.platform() !== "linux") {
logger.error("SMB unmounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "SMB unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(path);
if (!mount || mount.mountPoint !== path) {
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
return { status: BACKEND_STATUS.unmounted };
}
await executeUnmount(path);
await fs.rmdir(path).catch(() => {});
logger.info(`SMB volume at ${path} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB unmount");
} catch (error) {
logger.error("Error unmounting SMB volume", { path, error: toMessage(error) });
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const checkHealth = async (path: string) => {
const run = async () => {
await assertMounted(path, (fstype) => fstype === "cifs");
logger.debug(`SMB volume at ${path} is healthy and mounted.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("SMB volume health check failed:", message);
}
return { status: BACKEND_STATUS.error, error: message };
}
};
export const makeSmbBackend = (config: BackendConfig, path: string): VolumeBackend => ({
mount: () => mount(config, path),
unmount: () => unmount(path),
checkHealth: () => checkHealth(path),
});

View file

@ -1,59 +0,0 @@
import { afterEach, describe, expect, test, vi } from "vitest";
vi.mock("node:fs/promises", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs/promises")>();
return {
...actual,
access: vi.fn(actual.access),
};
});
vi.mock("../../../../utils/mountinfo", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../../../utils/mountinfo")>();
return {
...actual,
getMountForPath: vi.fn(actual.getMountForPath),
};
});
import * as fs from "node:fs/promises";
import * as mountinfo from "../../../../utils/mountinfo";
import { assertMounted } from "../backend-utils";
afterEach(() => {
vi.restoreAllMocks();
});
describe("assertMountedFilesystem", () => {
test("throws when the path is not accessible", async () => {
vi.mocked(fs.access).mockRejectedValueOnce(new Error("missing"));
await expect(assertMounted("/tmp/volume", (fstype) => fstype.startsWith("nfs"))).rejects.toThrow(
"Volume is not mounted",
);
});
test("throws when the mount filesystem does not match", async () => {
vi.mocked(fs.access).mockResolvedValueOnce(undefined);
vi.mocked(mountinfo.getMountForPath).mockResolvedValueOnce({
mountPoint: "/tmp/volume",
fstype: "cifs",
});
await expect(assertMounted("/tmp/volume", (fstype) => fstype.startsWith("nfs"))).rejects.toThrow(
"Path /tmp/volume is not mounted as correct fstype (found cifs).",
);
});
test("accepts a matching mounted filesystem", async () => {
vi.mocked(fs.access).mockResolvedValueOnce(undefined);
vi.mocked(mountinfo.getMountForPath).mockResolvedValueOnce({
mountPoint: "/tmp/volume",
fstype: "nfs4",
});
await expect(assertMounted("/tmp/volume", (fstype) => fstype.startsWith("nfs"))).resolves.toBeUndefined();
});
});

View file

@ -1,159 +0,0 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { OPERATION_TIMEOUT } from "../../../core/constants";
import { cryptoUtils } from "../../../utils/crypto";
import { toMessage } from "../../../utils/errors";
import { logger } from "@zerobyte/core/node";
import { getMountForPath } from "../../../utils/mountinfo";
import { withTimeout } from "../../../utils/timeout";
import type { VolumeBackend } from "../backend";
import { assertMounted, executeMount, executeUnmount } from "../utils/backend-utils";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting WebDAV volume ${path}...`);
if (config.backend !== "webdav") {
logger.error("Provided config is not for WebDAV backend");
return { status: BACKEND_STATUS.error, error: "Provided config is not for WebDAV backend" };
}
if (os.platform() !== "linux") {
logger.error("WebDAV mounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "WebDAV mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(path);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
await unmount(path);
}
const run = async () => {
await fs.mkdir(path, { recursive: true }).catch((err) => {
logger.warn(`Failed to create directory ${path}: ${err.message}`);
});
const protocol = config.ssl ? "https" : "http";
const defaultPort = config.ssl ? 443 : 80;
const port = config.port !== defaultPort ? `:${config.port}` : "";
const source = `${protocol}://${config.server}${port}${config.path}`;
const { uid, gid } = os.userInfo();
const options = config.readOnly
? [`uid=${uid}`, `gid=${gid}`, "file_mode=0444", "dir_mode=0555", "ro"]
: [`uid=${uid}`, `gid=${gid}`, "file_mode=0664", "dir_mode=0775"];
if (config.username && config.password) {
const password = await cryptoUtils.resolveSecret(config.password);
const secretsFile = "/etc/davfs2/secrets";
const entry = [source, config.username, password].map((value) => value.replace(/[\r\n\t\s]+/g, " ")).join(" ");
const secretsContent = `${entry}\n`;
await fs.appendFile(secretsFile, secretsContent, { mode: 0o600 });
}
logger.debug(`Mounting WebDAV volume ${path}...`);
const args = ["-t", "davfs", "-o", options.join(","), source, path];
try {
await executeMount(args);
} catch (error) {
logger.warn(`Initial WebDAV mount failed, retrying with -i flag: ${toMessage(error)}`);
// Fallback with -i flag if the first mount fails using the mount helper
await executeMount(["-i", ...args]);
}
logger.info(`WebDAV volume at ${path} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV mount");
} catch (error) {
const errorMsg = toMessage(error);
if (errorMsg.includes("already mounted")) {
return { status: BACKEND_STATUS.mounted };
}
logger.error("Error mounting WebDAV volume", { error: errorMsg });
if (errorMsg.includes("option") && errorMsg.includes("requires argument")) {
return {
status: BACKEND_STATUS.error,
error: "Invalid mount options. Please check your WebDAV server configuration.",
};
} else if (errorMsg.includes("connection refused") || errorMsg.includes("Connection refused")) {
return {
status: BACKEND_STATUS.error,
error: "Cannot connect to WebDAV server. Please check the server address and port.",
};
} else if (errorMsg.includes("unauthorized") || errorMsg.includes("Unauthorized")) {
return {
status: BACKEND_STATUS.error,
error: "Authentication failed. Please check your username and password.",
};
}
return { status: BACKEND_STATUS.error, error: errorMsg };
}
};
const unmount = async (path: string) => {
if (os.platform() !== "linux") {
logger.error("WebDAV unmounting is only supported on Linux hosts.");
return { status: BACKEND_STATUS.error, error: "WebDAV unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(path);
if (!mount || mount.mountPoint !== path) {
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
return { status: BACKEND_STATUS.unmounted };
}
await executeUnmount(path);
await fs.rmdir(path).catch(() => {});
logger.info(`WebDAV volume at ${path} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV unmount");
} catch (error) {
logger.error("Error unmounting WebDAV volume", { path, error: toMessage(error) });
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const checkHealth = async (path: string) => {
const run = async () => {
await assertMounted(path, (fstype) => fstype === "fuse" || fstype === "davfs");
logger.debug(`WebDAV volume at ${path} is healthy and mounted.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("WebDAV volume health check failed:", message);
}
return { status: BACKEND_STATUS.error, error: message };
}
};
export const makeWebdavBackend = (config: BackendConfig, path: string): VolumeBackend => ({
mount: () => mount(config, path),
unmount: () => unmount(path),
checkHealth: () => checkHealth(path),
});

View file

@ -421,6 +421,50 @@ describe("backup execution - validation failures", () => {
}); });
}); });
describe("backup execution - routing", () => {
test("fails local repository backups on non-local volume agents", async () => {
const { runBackupMock } = setup();
const volume = await createTestVolume({ agentId: "agent-remote" });
const repository = await createTestRepository();
const schedule = await createTestBackupSchedule({
volumeId: volume.id,
repositoryId: repository.id,
});
await backupsService.executeBackup(schedule.id);
const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
expect(updatedSchedule.lastBackupStatus).toBe("error");
expect(updatedSchedule.lastBackupError).toBe(
`Local repository "${repository.name}" can only be used with the local agent`,
);
expect(runBackupMock).not.toHaveBeenCalled();
});
test("routes remote repository backups through the owning volume agent", async () => {
const { runBackupMock } = setup();
const volume = await createTestVolume({ agentId: "agent-remote" });
const repository = await createTestRepository({
type: "s3",
config: {
backend: "s3",
endpoint: "https://s3.amazonaws.com",
bucket: "bucket-name",
accessKeyId: "access-key",
secretAccessKey: "secret-key",
},
});
const schedule = await createTestBackupSchedule({
volumeId: volume.id,
repositoryId: repository.id,
});
await backupsService.executeBackup(schedule.id);
expect(runBackupMock).toHaveBeenCalledWith("agent-remote", expect.objectContaining({ scheduleId: schedule.id }));
});
});
describe("stop backup", () => { describe("stop backup", () => {
test("should keep restic warning details when backup completes with read errors", async () => { test("should keep restic warning details when backup completes with read errors", async () => {
const { resticBackupMock } = setup(); const { resticBackupMock } = setup();

View file

@ -10,6 +10,7 @@ import { getVolumePath } from "../volumes/helpers";
import { decryptRepositoryConfig } from "../repositories/repository-config-secrets"; import { decryptRepositoryConfig } from "../repositories/repository-config-secrets";
import { createBackupOptions } from "./backup.helpers"; import { createBackupOptions } from "./backup.helpers";
import { toErrorDetails } from "../../utils/errors"; import { toErrorDetails } from "../../utils/errors";
import { BadRequestError } from "http-errors-enhanced";
const FUSE_VOLUME_BACKENDS = new Set<Volume["type"]>(["rclone", "sftp", "webdav"]); const FUSE_VOLUME_BACKENDS = new Set<Volume["type"]>(["rclone", "sftp", "webdav"]);
const IGNORE_INODE_FLAG = "--ignore-inode"; const IGNORE_INODE_FLAG = "--ignore-inode";
@ -23,7 +24,17 @@ type BackupExecutionRequest = {
onProgress: (progress: BackupExecutionProgress) => void; onProgress: (progress: BackupExecutionProgress) => void;
}; };
const activeControllersByScheduleId = new Map<number, AbortController>(); export type { BackupExecutionResult } from "../agents/agents-manager";
const activeControllersByScheduleId = new Map<number, { abortController: AbortController; agentId: string | null }>();
const getBackupExecutionAgentId = (volume: Volume, repository: Repository) => {
if (repository.type === "local" && volume.agentId !== LOCAL_AGENT_ID) {
throw new BadRequestError(`Local repository "${repository.name}" can only be used with the local agent`);
}
return volume.agentId;
};
const createBackupRunPayload = async ({ const createBackupRunPayload = async ({
jobId, jobId,
@ -93,17 +104,17 @@ const executeBackupWithoutAgent = async (
export const backupExecutor = { export const backupExecutor = {
track: (scheduleId: number) => { track: (scheduleId: number) => {
const abortController = new AbortController(); const abortController = new AbortController();
activeControllersByScheduleId.set(scheduleId, abortController); activeControllersByScheduleId.set(scheduleId, { abortController, agentId: null });
return abortController; return abortController;
}, },
untrack: (scheduleId: number, abortController: AbortController) => { untrack: (scheduleId: number, abortController: AbortController) => {
if (activeControllersByScheduleId.get(scheduleId) === abortController) { if (activeControllersByScheduleId.get(scheduleId)?.abortController === abortController) {
activeControllersByScheduleId.delete(scheduleId); activeControllersByScheduleId.delete(scheduleId);
} }
}, },
execute: async (request: Omit<BackupExecutionRequest, "jobId">) => { execute: async (request: Omit<BackupExecutionRequest, "jobId">) => {
const trackedAbortController = activeControllersByScheduleId.get(request.scheduleId); const trackedExecution = activeControllersByScheduleId.get(request.scheduleId);
if (!trackedAbortController || trackedAbortController.signal !== request.signal) { if (!trackedExecution || trackedExecution.abortController.signal !== request.signal) {
throw new Error(`Backup execution for schedule ${request.scheduleId} was not tracked`); throw new Error(`Backup execution for schedule ${request.scheduleId} was not tracked`);
} }
@ -119,27 +130,38 @@ export const backupExecutor = {
throw request.signal.reason || new Error("Operation aborted"); throw request.signal.reason || new Error("Operation aborted");
} }
const executionResult = await agentManager.runBackup(LOCAL_AGENT_ID, { const executionAgentId = getBackupExecutionAgentId(request.volume, request.repository);
trackedExecution.agentId = executionAgentId;
const executionResult = await agentManager.runBackup(executionAgentId, {
scheduleId: request.scheduleId, scheduleId: request.scheduleId,
payload, payload,
signal: request.signal, signal: request.signal,
onProgress: request.onProgress, onProgress: request.onProgress,
}); });
if (executionResult.status === "unavailable" && !config.flags.enableLocalAgent) { if (
executionResult.status === "unavailable" &&
executionAgentId === LOCAL_AGENT_ID &&
!config.flags.enableLocalAgent
) {
return executeBackupWithoutAgent(payload, request); return executeBackupWithoutAgent(payload, request);
} }
return executionResult; return executionResult;
}, },
cancel: async (scheduleId: number) => { cancel: async (scheduleId: number) => {
const abortController = activeControllersByScheduleId.get(scheduleId); const trackedExecution = activeControllersByScheduleId.get(scheduleId);
if (!abortController) { if (!trackedExecution) {
return false; return false;
} }
abortController.abort(); trackedExecution.abortController.abort();
await agentManager.cancelBackup(LOCAL_AGENT_ID, scheduleId); if (!trackedExecution.agentId) {
return true;
}
await agentManager.cancelBackup(trackedExecution.agentId, scheduleId);
return true; return true;
}, },
}; };

View file

@ -29,7 +29,6 @@ import { copyToMirrors, runForget, syncSnapshotsToMirror } from "./helpers/backu
import { restic } from "../../core/restic"; import { restic } from "../../core/restic";
import { mirrorQueries } from "./backups.queries"; import { mirrorQueries } from "./backups.queries";
import { toMessage } from "../../utils/errors"; import { toMessage } from "../../utils/errors";
const listSchedules = async () => { const listSchedules = async () => {
const organizationId = getOrganizationId(); const organizationId = getOrganizationId();
const schedules = await db.query.backupSchedulesTable.findMany({ const schedules = await db.query.backupSchedulesTable.findMany({

View file

@ -1,9 +1,10 @@
import { afterEach, describe, expect, test, vi } from "vitest"; import { afterEach, describe, expect, test, vi } from "vitest";
import { Scheduler } from "../../../core/scheduler"; import { Scheduler } from "../../../core/scheduler";
import * as backendModule from "../../backends/backend";
import type { VolumeBackend } from "../../backends/backend";
import * as bootstrapModule from "../bootstrap"; import * as bootstrapModule from "../bootstrap";
import { agentManager } from "../../agents/agents-manager";
import { createTestVolume } from "~/test/helpers/volume"; import { createTestVolume } from "~/test/helpers/volume";
import { config } from "~/server/core/config";
import { db } from "~/server/db/db";
const loadShutdownModule = async () => { const loadShutdownModule = async () => {
const moduleUrl = new URL("../shutdown.ts", import.meta.url); const moduleUrl = new URL("../shutdown.ts", import.meta.url);
@ -12,11 +13,12 @@ const loadShutdownModule = async () => {
}; };
afterEach(() => { afterEach(() => {
config.flags.enableLocalAgent = true;
vi.restoreAllMocks(); vi.restoreAllMocks();
}); });
describe("shutdown", () => { describe("shutdown", () => {
test("stops the agent runtime before unmounting mounted volumes", async () => { test("does not unmount agent-owned volumes during controller shutdown", async () => {
const events: string[] = []; const events: string[] = [];
const stopScheduler = vi.fn(async () => { const stopScheduler = vi.fn(async () => {
events.push("scheduler.stop"); events.push("scheduler.stop");
@ -24,35 +26,51 @@ describe("shutdown", () => {
const stopApplicationRuntime = vi.fn(async () => { const stopApplicationRuntime = vi.fn(async () => {
events.push("agents.stop"); events.push("agents.stop");
}); });
const unmountVolume = vi.fn(async () => { const runVolumeCommand = vi.spyOn(agentManager, "runVolumeCommand");
events.push("backend.unmount");
return { status: "unmounted" as const };
});
await createTestVolume({ const volume = await createTestVolume({
name: "Shutdown test volume", name: "Shutdown test volume",
config: { config: {
backend: "directory", backend: "directory",
path: "/Applications", path: "/Applications",
}, },
status: "mounted", status: "mounted",
agentId: "agent-1",
}); });
vi.spyOn(Scheduler, "stop").mockImplementation(stopScheduler); vi.spyOn(Scheduler, "stop").mockImplementation(stopScheduler);
vi.spyOn(bootstrapModule, "stopApplicationRuntime").mockImplementation(stopApplicationRuntime); vi.spyOn(bootstrapModule, "stopApplicationRuntime").mockImplementation(stopApplicationRuntime);
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(
() =>
({
mount: async () => ({ status: "mounted" as const }),
unmount: unmountVolume,
checkHealth: async () => ({ status: "mounted" as const }),
}) satisfies VolumeBackend,
);
const { shutdown } = await loadShutdownModule(); const { shutdown } = await loadShutdownModule();
await shutdown(); await shutdown();
expect(events).toEqual(["scheduler.stop", "agents.stop", "backend.unmount"]); expect(events).toEqual(["scheduler.stop", "agents.stop"]);
expect(runVolumeCommand).not.toHaveBeenCalled();
});
test("keeps legacy controller-local fallback unmount on shutdown", async () => {
config.flags.enableLocalAgent = false;
const events: string[] = [];
vi.spyOn(Scheduler, "stop").mockImplementation(async () => {
events.push("scheduler.stop");
});
vi.spyOn(bootstrapModule, "stopApplicationRuntime").mockImplementation(async () => {
events.push("agents.stop");
});
const volume = await createTestVolume({
name: "Fallback shutdown test volume",
config: { backend: "directory", path: "/Applications" },
status: "mounted",
});
const { shutdown } = await loadShutdownModule();
await shutdown();
const updated = await db.query.volumesTable.findFirst({ where: { id: volume.id } });
expect(events).toEqual(["scheduler.stop", "agents.stop"]);
expect(updated?.status).toBe("unmounted");
}); });
}); });

View file

@ -1,21 +1,29 @@
import { Scheduler } from "../../core/scheduler"; import { Scheduler } from "../../core/scheduler";
import { db } from "../../db/db"; import { db } from "../../db/db";
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import { createVolumeBackend } from "../backends/backend";
import { stopApplicationRuntime } from "./bootstrap"; import { stopApplicationRuntime } from "./bootstrap";
import { withContext } from "../../core/request-context";
import { volumeService } from "../volumes/volume.service";
import { toMessage } from "../../utils/errors";
import { config } from "../../core/config";
import { LOCAL_AGENT_ID } from "../agents/constants";
export const shutdown = async () => { export const shutdown = async () => {
await Scheduler.stop(); await Scheduler.stop();
await stopApplicationRuntime();
const volumes = await db.query.volumesTable.findMany({ if (!config.flags.enableLocalAgent) {
where: { status: "mounted" }, const volumes = await db.query.volumesTable.findMany({
}); where: { AND: [{ status: "mounted" }, { agentId: LOCAL_AGENT_ID }] },
});
for (const volume of volumes) { for (const volume of volumes) {
const backend = createVolumeBackend(volume); const { status, error } = await withContext({ organizationId: volume.organizationId }, () =>
const { status, error } = await backend.unmount(); volumeService.unmountVolume(volume.shortId),
).catch((error) => ({ status: "error" as const, error: toMessage(error) }));
logger.info(`Volume ${volume.name} unmount status: ${status}${error ? `, error: ${error}` : ""}`); logger.info(`Volume ${volume.name} unmount status: ${status}${error ? `, error: ${error}` : ""}`);
}
} }
await stopApplicationRuntime();
}; };

View file

@ -17,6 +17,7 @@ import { backupsService } from "../backups/backups.service";
import { config } from "~/server/core/config"; import { config } from "~/server/core/config";
import { syncProvisionedResources } from "../provisioning/provisioning"; import { syncProvisionedResources } from "../provisioning/provisioning";
import { toMessage } from "~/server/utils/errors"; import { toMessage } from "~/server/utils/errors";
import { LOCAL_AGENT_ID } from "../agents/constants";
const ensureLatestConfigurationSchema = async () => { const ensureLatestConfigurationSchema = async () => {
const volumes = await db.query.volumesTable.findMany({}); const volumes = await db.query.volumesTable.findMany({});
@ -71,23 +72,30 @@ export const startup = async () => {
logger.warn(`Removed ${deletedSchedules} orphaned backup schedule(s) during startup`); logger.warn(`Removed ${deletedSchedules} orphaned backup schedule(s) during startup`);
} }
const volumes = await db.query.volumesTable.findMany({ if (!config.flags.enableLocalAgent) {
where: { const volumes = await db.query.volumesTable.findMany({
OR: [ where: {
{ status: "mounted" }, AND: [
{ { agentId: LOCAL_AGENT_ID },
AND: [{ autoRemount: true }, { status: "error" }], {
}, OR: [
], { status: "mounted" },
}, {
}); AND: [{ autoRemount: true }, { status: "error" }],
},
for (const volume of volumes) { ],
await withContext({ organizationId: volume.organizationId }, async () => { },
await volumeService.mountVolume(volume.shortId).catch((err) => { ],
logger.error(`Error auto-remounting volume ${volume.name} on startup: ${err.message}`); },
});
}); });
for (const volume of volumes) {
await withContext({ organizationId: volume.organizationId }, async () => {
await volumeService.mountVolume(volume.shortId).catch((err) => {
logger.error(`Error auto-remounting volume ${volume.name} on startup: ${err.message}`);
});
});
}
} }
await db await db
@ -102,7 +110,9 @@ export const startup = async () => {
logger.error(`Failed to update stuck backup schedules on startup: ${err.message}`); logger.error(`Failed to update stuck backup schedules on startup: ${err.message}`);
}); });
Scheduler.build(CleanupDanglingMountsJob).schedule("0 * * * *"); if (!config.flags.enableLocalAgent) {
Scheduler.build(CleanupDanglingMountsJob).schedule("0 * * * *");
}
Scheduler.build(VolumeHealthCheckJob).schedule("*/30 * * * *"); Scheduler.build(VolumeHealthCheckJob).schedule("*/30 * * * *");
Scheduler.build(RepositoryHealthCheckJob).schedule("50 12 * * *"); Scheduler.build(RepositoryHealthCheckJob).schedule("50 12 * * *");
Scheduler.build(BackupExecutionJob).schedule("* * * * *"); Scheduler.build(BackupExecutionJob).schedule("* * * * *");

View file

@ -13,6 +13,7 @@ import { config as appConfig } from "~/server/core/config";
import { restic } from "~/server/core/restic"; import { restic } from "~/server/core/restic";
import { db } from "~/server/db/db"; import { db } from "~/server/db/db";
import { repositoriesTable, volumesTable } from "~/server/db/schema"; import { repositoriesTable, volumesTable } from "~/server/db/schema";
import { LOCAL_AGENT_ID } from "~/server/modules/agents/constants";
import { mapRepositoryConfigSecrets } from "~/server/modules/repositories/repository-config-secrets"; import { mapRepositoryConfigSecrets } from "~/server/modules/repositories/repository-config-secrets";
import { mapVolumeConfigSecrets } from "~/server/modules/volumes/volume-config-secrets"; import { mapVolumeConfigSecrets } from "~/server/modules/volumes/volume-config-secrets";
import { BACKEND_TYPES, volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { BACKEND_TYPES, volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
@ -154,7 +155,6 @@ const syncProvisionedRepositories = async (repositories: ProvisionedRepository[]
const existing = existingRepositories.find((r) => r.provisioningId === provisioningId); const existing = existingRepositories.find((r) => r.provisioningId === provisioningId);
const encryptedConfig = await encryptProvisionedRepositoryConfig(repository.config); const encryptedConfig = await encryptProvisionedRepositoryConfig(repository.config);
if (!existing) { if (!existing) {
const id = Bun.randomUUIDv7(); const id = Bun.randomUUIDv7();
@ -228,6 +228,7 @@ const syncProvisionedVolumes = async (volumes: ProvisionedVolume[]) => {
type: volume.backend, type: volume.backend,
config: await encryptProvisionedVolumeConfig(volume.config), config: await encryptProvisionedVolumeConfig(volume.config),
autoRemount: volume.autoRemount, autoRemount: volume.autoRemount,
agentId: LOCAL_AGENT_ID,
status: volume.autoRemount ? "mounted" : "unmounted", status: volume.autoRemount ? "mounted" : "unmounted",
organizationId: volume.organizationId, organizationId: volume.organizationId,
}); });
@ -239,6 +240,7 @@ const syncProvisionedVolumes = async (volumes: ProvisionedVolume[]) => {
type: volume.backend, type: volume.backend,
config: await encryptProvisionedVolumeConfig(volume.config), config: await encryptProvisionedVolumeConfig(volume.config),
autoRemount: volume.autoRemount, autoRemount: volume.autoRemount,
agentId: LOCAL_AGENT_ID,
organizationId: volume.organizationId, organizationId: volume.organizationId,
updatedAt: Date.now(), updatedAt: Date.now(),
}; };

View file

@ -73,7 +73,6 @@ const createRepository = async (name: string, config: RepositoryConfig, compress
const organizationId = getOrganizationId(); const organizationId = getOrganizationId();
const id = Bun.randomUUIDv7(); const id = Bun.randomUUIDv7();
const shortId = generateShortId(); const shortId = generateShortId();
if (config.backend === "local" && !config.isExistingRepository) { if (config.backend === "local" && !config.isExistingRepository) {
config.path = `${config.path}/${shortId}`; config.path = `${config.path}/${shortId}`;
} }
@ -717,7 +716,6 @@ const updateRepository = async (shortId: ShortId, updates: UpdateRepositoryBody)
const decryptedExisting = await decryptRepositoryConfig(existingConfig); const decryptedExisting = await decryptRepositoryConfig(existingConfig);
const configChanged = updates.config && JSON.stringify(decryptedExisting) !== JSON.stringify(parsedConfig); const configChanged = updates.config && JSON.stringify(decryptedExisting) !== JSON.stringify(parsedConfig);
const encryptedConfig = updates.config ? await encryptRepositoryConfig(parsedConfig) : existingConfig; const encryptedConfig = updates.config ? await encryptRepositoryConfig(parsedConfig) : existingConfig;
const updatedAt = Date.now(); const updatedAt = Date.now();
const updatePayload = { const updatePayload = {
name: newName, name: newName,

View file

@ -1,19 +1,26 @@
import { afterEach, describe, expect, test, vi } from "vitest"; import { afterEach, describe, expect, test, vi } from "vitest";
const agentManagerMock = vi.hoisted(() => ({
runVolumeCommand: vi.fn(),
}));
vi.mock("../../agents/agents-manager", () => ({
agentManager: agentManagerMock,
}));
import { volumeService } from "../volume.service"; import { volumeService } from "../volume.service";
import { db } from "~/server/db/db"; import { db } from "~/server/db/db";
import { volumesTable } from "~/server/db/schema"; import { volumesTable } from "~/server/db/schema";
import { randomUUID } from "node:crypto"; import { randomUUID } from "node:crypto";
import * as fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { createTestSession } from "~/test/helpers/auth"; import { createTestSession } from "~/test/helpers/auth";
import { withContext } from "~/server/core/request-context"; import { withContext } from "~/server/core/request-context";
import { asShortId } from "~/server/utils/branded"; import { asShortId } from "~/server/utils/branded";
import { createTestVolume } from "~/test/helpers/volume"; import { createTestVolume } from "~/test/helpers/volume";
import * as backendModule from "../../backends/backend"; import { config } from "~/server/core/config";
afterEach(() => { afterEach(() => {
config.flags.enableLocalAgent = false;
vi.restoreAllMocks(); vi.restoreAllMocks();
agentManagerMock.runVolumeCommand.mockReset();
}); });
describe("volumeService.getVolume", () => { describe("volumeService.getVolume", () => {
@ -98,13 +105,7 @@ describe("volumeService.getVolume", () => {
describe("volumeService.listFiles security", () => { describe("volumeService.listFiles security", () => {
test("should reject traversal outside the volume root in listFiles", async () => { test("should reject traversal outside the volume root in listFiles", async () => {
const { organizationId, user } = await createTestSession(); const { organizationId, user } = await createTestSession();
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-vol-svc-")); agentManagerMock.runVolumeCommand.mockRejectedValue(new Error("Invalid path"));
const volumePath = path.join(tempRoot, "vol");
const secretPath = path.join(tempRoot, "volume-secret");
await fs.mkdir(volumePath, { recursive: true });
await fs.mkdir(secretPath, { recursive: true });
await fs.writeFile(path.join(secretPath, "secret.txt"), "top secret", "utf-8");
const [volume] = await db const [volume] = await db
.insert(volumesTable) .insert(volumesTable)
@ -113,44 +114,40 @@ describe("volumeService.listFiles security", () => {
name: `test-vol-${randomUUID().slice(0, 8)}`, name: `test-vol-${randomUUID().slice(0, 8)}`,
type: "directory", type: "directory",
status: "mounted", status: "mounted",
config: { backend: "directory", path: volumePath }, config: { backend: "directory", path: "/tmp/volume" },
autoRemount: true, autoRemount: true,
organizationId, organizationId,
}) })
.returning(); .returning();
try { await withContext({ organizationId, userId: user.id }, async () => {
await withContext({ organizationId, userId: user.id }, async () => { await expect(volumeService.listFiles(volume.shortId, "../volume-secret")).rejects.toThrow("Invalid path");
const traversalPath = `../${path.basename(secretPath)}`; });
await expect(volumeService.listFiles(volume.shortId, traversalPath)).rejects.toThrow("Invalid path");
});
} finally {
await fs.rm(tempRoot, { recursive: true, force: true });
}
}); });
}); });
describe("volumeService.mountVolume", () => { describe("volumeService.mountVolume", () => {
test("unmounts any existing mount before mounting", async () => { test("routes unmount and mount to the owning agent before updating state", async () => {
const { organizationId, user } = await createTestSession(); const { organizationId, user } = await createTestSession();
const volume = await createTestVolume({ organizationId, status: "mounted" }); const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
const unmount = vi.fn().mockResolvedValue({ status: "unmounted" }); agentManagerMock.runVolumeCommand
const mount = vi.fn().mockResolvedValue({ status: "mounted" }); .mockResolvedValueOnce({ name: "volume.unmount", result: { status: "unmounted" } })
.mockResolvedValueOnce({ name: "volume.mount", result: { status: "mounted" } });
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({
mount,
unmount,
checkHealth: vi.fn().mockResolvedValue({ status: "mounted" }),
}));
await withContext({ organizationId, userId: user.id }, async () => { await withContext({ organizationId, userId: user.id }, async () => {
const result = await volumeService.mountVolume(volume.shortId); const result = await volumeService.mountVolume(volume.shortId);
expect(result.status).toBe("mounted"); expect(result.status).toBe("mounted");
expect(unmount).toHaveBeenCalledOnce(); expect(agentManagerMock.runVolumeCommand).toHaveBeenNthCalledWith(
expect(mount).toHaveBeenCalledOnce(); 1,
expect(unmount.mock.invocationCallOrder[0]).toBeLessThan(mount.mock.invocationCallOrder[0]); volume.agentId,
expect.objectContaining({ name: "volume.unmount", volume: expect.objectContaining({ id: volume.id }) }),
);
expect(agentManagerMock.runVolumeCommand).toHaveBeenNthCalledWith(
2,
volume.agentId,
expect.objectContaining({ name: "volume.mount", volume: expect.objectContaining({ id: volume.id }) }),
);
}); });
}); });
}); });
@ -158,15 +155,8 @@ describe("volumeService.mountVolume", () => {
describe("volumeService.ensureHealthyVolume", () => { describe("volumeService.ensureHealthyVolume", () => {
test("returns ready when the mounted volume passes its health check", async () => { test("returns ready when the mounted volume passes its health check", async () => {
const { organizationId, user } = await createTestSession(); const { organizationId, user } = await createTestSession();
const volume = await createTestVolume({ organizationId, status: "mounted" }); const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
const mount = vi.fn().mockResolvedValue({ status: "mounted" }); agentManagerMock.runVolumeCommand.mockResolvedValue({ name: "volume.checkHealth", result: { status: "mounted" } });
const checkHealth = vi.fn().mockResolvedValue({ status: "mounted" });
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({
mount,
unmount: vi.fn().mockResolvedValue({ status: "unmounted" }),
checkHealth,
}));
await withContext({ organizationId, userId: user.id }, async () => { await withContext({ organizationId, userId: user.id }, async () => {
const result = await volumeService.ensureHealthyVolume(volume.shortId); const result = await volumeService.ensureHealthyVolume(volume.shortId);
@ -176,22 +166,21 @@ describe("volumeService.ensureHealthyVolume", () => {
volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }), volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }),
remounted: false, remounted: false,
}); });
expect(checkHealth).toHaveBeenCalledOnce(); expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledOnce();
expect(mount).not.toHaveBeenCalled(); expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
volume.agentId,
expect.objectContaining({ name: "volume.checkHealth", volume: expect.objectContaining({ id: volume.id }) }),
);
}); });
}); });
test("auto-remounts when the mounted volume fails its health check", async () => { test("auto-remounts when the mounted volume fails its health check", async () => {
const { organizationId, user } = await createTestSession(); const { organizationId, user } = await createTestSession();
const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: true }); const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: true, agentId: "agent-1" });
const mount = vi.fn().mockResolvedValue({ status: "mounted" }); agentManagerMock.runVolumeCommand
const checkHealth = vi.fn().mockResolvedValue({ status: "error", error: "stale mount" }); .mockResolvedValueOnce({ name: "volume.checkHealth", result: { status: "error", error: "stale mount" } })
.mockResolvedValueOnce({ name: "volume.unmount", result: { status: "unmounted" } })
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({ .mockResolvedValueOnce({ name: "volume.mount", result: { status: "mounted" } });
mount,
unmount: vi.fn().mockResolvedValue({ status: "unmounted" }),
checkHealth,
}));
await withContext({ organizationId, userId: user.id }, async () => { await withContext({ organizationId, userId: user.id }, async () => {
const result = await volumeService.ensureHealthyVolume(volume.shortId); const result = await volumeService.ensureHealthyVolume(volume.shortId);
@ -201,8 +190,7 @@ describe("volumeService.ensureHealthyVolume", () => {
volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }), volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }),
remounted: true, remounted: true,
}); });
expect(checkHealth).toHaveBeenCalledOnce(); expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledTimes(3);
expect(mount).toHaveBeenCalledOnce();
const updatedVolume = await db.query.volumesTable.findFirst({ where: { id: volume.id } }); const updatedVolume = await db.query.volumesTable.findFirst({ where: { id: volume.id } });
expect(updatedVolume?.status).toBe("mounted"); expect(updatedVolume?.status).toBe("mounted");
@ -212,15 +200,16 @@ describe("volumeService.ensureHealthyVolume", () => {
test("returns not ready when the health check fails and auto-remount is disabled", async () => { test("returns not ready when the health check fails and auto-remount is disabled", async () => {
const { organizationId, user } = await createTestSession(); const { organizationId, user } = await createTestSession();
const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: false }); const volume = await createTestVolume({
const mount = vi.fn().mockResolvedValue({ status: "mounted" }); organizationId,
const checkHealth = vi.fn().mockResolvedValue({ status: "error", error: "stale mount" }); status: "mounted",
autoRemount: false,
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({ agentId: "agent-1",
mount, });
unmount: vi.fn().mockResolvedValue({ status: "unmounted" }), agentManagerMock.runVolumeCommand.mockResolvedValue({
checkHealth, name: "volume.checkHealth",
})); result: { status: "error", error: "stale mount" },
});
await withContext({ organizationId, userId: user.id }, async () => { await withContext({ organizationId, userId: user.id }, async () => {
const result = await volumeService.ensureHealthyVolume(volume.shortId); const result = await volumeService.ensureHealthyVolume(volume.shortId);
@ -230,54 +219,17 @@ describe("volumeService.ensureHealthyVolume", () => {
volume: expect.objectContaining({ id: volume.id, status: "error", lastError: "stale mount" }), volume: expect.objectContaining({ id: volume.id, status: "error", lastError: "stale mount" }),
reason: "stale mount", reason: "stale mount",
}); });
expect(checkHealth).toHaveBeenCalledOnce(); expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledOnce();
expect(mount).not.toHaveBeenCalled();
}); });
}); });
}); });
describe("volumeService.testConnection", () => { describe("volumeService.testConnection", () => {
test("uses an isolated temp mount path for backend test connections", async () => { test("routes test connections to the local agent", async () => {
const mount = vi.fn().mockResolvedValue({ status: "mounted" }); config.flags.enableLocalAgent = true;
const unmount = vi.fn().mockResolvedValue({ status: "unmounted" }); agentManagerMock.runVolumeCommand.mockResolvedValue({
const createVolumeBackendSpy = vi.spyOn(backendModule, "createVolumeBackend").mockReturnValue({ name: "volume.testConnection",
mount, result: { success: true, message: "Connection successful" },
unmount,
checkHealth: vi.fn(),
});
await volumeService.testConnection({
backend: "nfs",
server: "127.0.0.1",
exportPath: "/exports/test",
version: "4",
port: 2049,
readOnly: false,
});
expect(createVolumeBackendSpy).toHaveBeenCalledOnce();
const [, mountPath] = createVolumeBackendSpy.mock.calls[0];
expect(mountPath).toEqual(expect.stringContaining(`${path.sep}zerobyte-test-`));
await expect(fs.access(mountPath as string)).rejects.toThrow();
expect(mount).toHaveBeenCalledOnce();
expect(unmount).toHaveBeenCalledOnce();
});
test("does not fail when backend unmount already removed the temp mount path", async () => {
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
let mountPath: string | undefined;
const unmount = vi.fn().mockImplementation(async () => {
await fs.rm(mountPath!, { recursive: true, force: true });
return { status: "unmounted" };
});
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation((_volume, tempPath) => {
mountPath = tempPath;
return {
mount,
unmount,
checkHealth: vi.fn(),
};
}); });
await expect( await expect(
@ -294,9 +246,9 @@ describe("volumeService.testConnection", () => {
message: "Connection successful", message: "Connection successful",
}); });
expect(mountPath).toEqual(expect.stringContaining(`${path.sep}zerobyte-test-`)); expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
await expect(fs.access(mountPath as string)).rejects.toThrow(); "local",
expect(mount).toHaveBeenCalledOnce(); expect.objectContaining({ name: "volume.testConnection" }),
expect(unmount).toHaveBeenCalledOnce(); );
}); });
}); });

View file

@ -32,3 +32,7 @@ export const mapVolumeConfigSecrets = async (
export const encryptVolumeConfig = async (config: BackendConfig): Promise<BackendConfig> => { export const encryptVolumeConfig = async (config: BackendConfig): Promise<BackendConfig> => {
return await mapVolumeConfigSecrets(config, cryptoUtils.sealSecret); return await mapVolumeConfigSecrets(config, cryptoUtils.sealSecret);
}; };
export const decryptVolumeConfig = async (config: BackendConfig): Promise<BackendConfig> => {
return await mapVolumeConfigSecrets(config, cryptoUtils.resolveSecret);
};

View file

@ -1,37 +1,39 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { and, eq } from "drizzle-orm"; import { and, eq } from "drizzle-orm";
import { BadRequestError, InternalServerError, NotFoundError } from "http-errors-enhanced"; import { BadRequestError, InternalServerError, NotFoundError } from "http-errors-enhanced";
import { db } from "../../db/db"; import { db } from "../../db/db";
import { volumesTable } from "../../db/schema"; import { volumesTable } from "../../db/schema";
import { toMessage } from "../../utils/errors"; import { toMessage } from "../../utils/errors";
import { generateShortId } from "../../utils/id"; import { generateShortId } from "../../utils/id";
import { getStatFs, type StatFs } from "../../utils/mountinfo"; import type { StatFs } from "../../utils/mountinfo";
import { withTimeout } from "../../utils/timeout"; import { withTimeout } from "../../utils/timeout";
import { createVolumeBackend } from "../backends/backend"; import { config } from "../../core/config";
import { LOCAL_AGENT_ID } from "../agents/constants";
import { agentManager } from "../agents/agents-manager";
import type { UpdateVolumeBody } from "./volume.dto"; import type { UpdateVolumeBody } from "./volume.dto";
import { getVolumePath } from "./helpers";
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import { serverEvents } from "../../core/events"; import { serverEvents } from "../../core/events";
import type { Volume } from "../../db/schema"; import type { Volume } from "../../db/schema";
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
import { getOrganizationId } from "~/server/core/request-context"; import { getOrganizationId } from "~/server/core/request-context";
import { isNodeJSErrnoException } from "~/server/utils/fs"; import { type ShortId } from "~/server/utils/branded";
import { asShortId, type ShortId } from "~/server/utils/branded"; import { decryptVolumeConfig, encryptVolumeConfig } from "./volume-config-secrets";
import { encryptVolumeConfig } from "./volume-config-secrets"; import type { VolumeCommand, VolumeCommandResult } from "@zerobyte/contracts/agent-protocol";
import {
createVolumeBackend,
getStatFs,
getVolumePath,
type AgentVolume,
type BackendConfig as HostBackendConfig,
} from "../../../../apps/agent/src/volume-host";
import {
browseFilesystem as browseHostFilesystem,
listVolumeFiles,
testVolumeConnection,
} from "../../../../apps/agent/src/volume-host/operations";
type EnsureHealthyVolumeResult = type EnsureHealthyVolumeResult =
| { | { ready: true; volume: Volume; remounted: boolean }
ready: true; | { ready: false; volume: Volume; reason: string };
volume: Volume;
remounted: boolean;
}
| {
ready: false;
volume: Volume;
reason: string;
};
const listVolumes = async () => { const listVolumes = async () => {
const organizationId = getOrganizationId(); const organizationId = getOrganizationId();
@ -52,6 +54,68 @@ const findVolume = async (shortId: ShortId) => {
}); });
}; };
const runVolumeCommand = async <TCommand extends VolumeCommand>(agentId: string, command: TCommand) => {
const result = await agentManager.runVolumeCommand(agentId, command);
if (result.name !== command.name) {
throw new InternalServerError(`Unexpected agent response for ${command.name}`);
}
return result as Extract<VolumeCommandResult, { name: TCommand["name"] }>;
};
const volumeForAgent = async (volume: Volume): Promise<Volume> => ({
...volume,
config: await decryptVolumeConfig(volume.config),
});
const volumeForHost = async (volume: Volume): Promise<AgentVolume> => ({
...volume,
shortId: volume.shortId,
config: (await decryptVolumeConfig(volume.config)) as HostBackendConfig,
provisioningId: volume.provisioningId ?? null,
});
// Transitional fallback: older controller-only installs do not run the supervised local agent.
// Keep all controller-local host execution behind this predicate so the fallback is easy to delete
// once volume operations always go through the local agent.
const shouldRunViaAgent = (volume: Volume) => volume.agentId !== LOCAL_AGENT_ID || config.flags.enableLocalAgent;
const shouldUseControllerLocalVolumeFallback = (volume: Volume) => !shouldRunViaAgent(volume);
const runVolumeBackendCommand = async (
volume: Volume,
name: "volume.mount" | "volume.unmount" | "volume.checkHealth",
) => {
if (shouldUseControllerLocalVolumeFallback(volume)) {
const backend = createVolumeBackend(await volumeForHost(volume));
switch (name) {
case "volume.mount":
return backend.mount();
case "volume.unmount":
return backend.unmount();
case "volume.checkHealth":
return backend.checkHealth();
}
}
const command = await runVolumeCommand(volume.agentId, {
name,
volume: await volumeForAgent(volume),
});
return command.result;
};
const mapAgentFileError = (error: unknown) => {
const message = toMessage(error);
if (message === "Invalid path") {
throw new BadRequestError("Invalid path");
}
if (message === "Directory not found") {
throw new NotFoundError("Directory not found");
}
throw error;
};
const createVolume = async (name: string, backendConfig: BackendConfig) => { const createVolume = async (name: string, backendConfig: BackendConfig) => {
const organizationId = getOrganizationId(); const organizationId = getOrganizationId();
const trimmedName = name.trim(); const trimmedName = name.trim();
@ -70,6 +134,7 @@ const createVolume = async (name: string, backendConfig: BackendConfig) => {
name: trimmedName, name: trimmedName,
config: encryptedConfig, config: encryptedConfig,
type: backendConfig.backend, type: backendConfig.backend,
agentId: LOCAL_AGENT_ID,
organizationId, organizationId,
}) })
.returning(); .returning();
@ -78,8 +143,7 @@ const createVolume = async (name: string, backendConfig: BackendConfig) => {
throw new InternalServerError("Failed to create volume"); throw new InternalServerError("Failed to create volume");
} }
const backend = createVolumeBackend(created); const { error, status } = await runVolumeBackendCommand(created, "volume.mount");
const { error, status } = await backend.mount();
await db await db
.update(volumesTable) .update(volumesTable)
@ -97,8 +161,7 @@ const deleteVolume = async (shortId: ShortId) => {
throw new NotFoundError("Volume not found"); throw new NotFoundError("Volume not found");
} }
const backend = createVolumeBackend(volume); await runVolumeBackendCommand(volume, "volume.unmount");
await backend.unmount();
await db await db
.delete(volumesTable) .delete(volumesTable)
.where(and(eq(volumesTable.id, volume.id), eq(volumesTable.organizationId, organizationId))); .where(and(eq(volumesTable.id, volume.id), eq(volumesTable.organizationId, organizationId)));
@ -112,9 +175,8 @@ const mountVolume = async (shortId: ShortId) => {
throw new NotFoundError("Volume not found"); throw new NotFoundError("Volume not found");
} }
const backend = createVolumeBackend(volume); await runVolumeBackendCommand(volume, "volume.unmount");
await backend.unmount(); const { error, status } = await runVolumeBackendCommand(volume, "volume.mount");
const { error, status } = await backend.mount();
await db await db
.update(volumesTable) .update(volumesTable)
@ -136,8 +198,7 @@ const unmountVolume = async (shortId: ShortId) => {
throw new NotFoundError("Volume not found"); throw new NotFoundError("Volume not found");
} }
const backend = createVolumeBackend(volume); const { status, error } = await runVolumeBackendCommand(volume, "volume.unmount");
const { status, error } = await backend.unmount();
await db await db
.update(volumesTable) .update(volumesTable)
@ -160,7 +221,15 @@ const getVolume = async (shortId: ShortId) => {
let statfs: Partial<StatFs> = {}; let statfs: Partial<StatFs> = {};
if (volume.status === "mounted") { if (volume.status === "mounted") {
statfs = await withTimeout(getStatFs(getVolumePath(volume)), 1000, "getStatFs").catch((error) => { statfs = await withTimeout(
!shouldUseControllerLocalVolumeFallback(volume)
? runVolumeCommand(volume.agentId, { name: "volume.statfs", volume: await volumeForAgent(volume) }).then(
(command) => command.result,
)
: volumeForHost(volume).then((hostVolume) => getStatFs(getVolumePath(hostVolume))),
1000,
"volume.statfs",
).catch((error) => {
logger.warn(`Failed to get statfs for volume ${volume.name}: ${toMessage(error)}`); logger.warn(`Failed to get statfs for volume ${volume.name}: ${toMessage(error)}`);
return {}; return {};
}); });
@ -188,8 +257,7 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => {
if (configChanged) { if (configChanged) {
logger.debug("Unmounting existing volume before applying new config"); logger.debug("Unmounting existing volume before applying new config");
const backend = createVolumeBackend(existing); await runVolumeBackendCommand(existing, "volume.unmount");
await backend.unmount();
} }
const newConfigResult = volumeConfigSchema.safeParse(volumeData.config || existing.config); const newConfigResult = volumeConfigSchema.safeParse(volumeData.config || existing.config);
@ -217,8 +285,7 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => {
} }
if (configChanged) { if (configChanged) {
const backend = createVolumeBackend(updated); const { error, status } = await runVolumeBackendCommand(updated, "volume.mount");
const { error, status } = await backend.mount();
await db await db
.update(volumesTable) .update(volumesTable)
.set({ status, lastError: error ?? null, lastHealthCheck: Date.now() }) .set({ status, lastError: error ?? null, lastHealthCheck: Date.now() })
@ -231,38 +298,12 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => {
}; };
const testConnection = async (backendConfig: BackendConfig) => { const testConnection = async (backendConfig: BackendConfig) => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-test-")); if (!config.flags.enableLocalAgent) {
try { return await testVolumeConnection(backendConfig as HostBackendConfig);
const encryptedConfig = await encryptVolumeConfig(backendConfig);
const mockVolume = {
id: 0,
shortId: asShortId("test"),
name: "test-connection",
config: encryptedConfig,
createdAt: Date.now(),
updatedAt: Date.now(),
lastHealthCheck: Date.now(),
type: encryptedConfig.backend,
status: "unmounted" as const,
lastError: null,
provisioningId: null,
autoRemount: true,
organizationId: "test-org",
};
const backend = createVolumeBackend(mockVolume, tempDir);
const { error } = await backend.mount();
await backend.unmount();
return {
success: !error,
message: error ? toMessage(error) : "Connection successful",
};
} finally {
await fs.rm(tempDir, { recursive: true, force: true });
} }
const command = await runVolumeCommand(LOCAL_AGENT_ID, { name: "volume.testConnection", backendConfig });
return command.result;
}; };
const checkHealth = async (shortId: ShortId) => { const checkHealth = async (shortId: ShortId) => {
@ -273,8 +314,7 @@ const checkHealth = async (shortId: ShortId) => {
throw new NotFoundError("Volume not found"); throw new NotFoundError("Volume not found");
} }
const backend = createVolumeBackend(volume); const { error, status } = await runVolumeBackendCommand(volume, "volume.checkHealth");
const { error, status } = await backend.checkHealth();
if (status !== volume.status) { if (status !== volume.status) {
serverEvents.emit("volume:status_changed", { organizationId, volumeName: volume.name, status }); serverEvents.emit("volume:status_changed", { organizationId, volumeName: volume.name, status });
@ -342,7 +382,6 @@ const ensureHealthyVolume = async (shortId: ShortId): Promise<EnsureHealthyVolum
}; };
const DEFAULT_PAGE_SIZE = 500; const DEFAULT_PAGE_SIZE = 500;
const MAX_PAGE_SIZE = 500;
const listFiles = async (shortId: ShortId, subPath?: string, offset: number = 0, limit: number = DEFAULT_PAGE_SIZE) => { const listFiles = async (shortId: ShortId, subPath?: string, offset: number = 0, limit: number = DEFAULT_PAGE_SIZE) => {
const volume = await findVolume(shortId); const volume = await findVolume(shortId);
@ -355,110 +394,33 @@ const listFiles = async (shortId: ShortId, subPath?: string, offset: number = 0,
throw new InternalServerError("Volume is not mounted"); throw new InternalServerError("Volume is not mounted");
} }
const volumePath = getVolumePath(volume);
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
const normalizedPath = path.normalize(requestedPath);
const relative = path.relative(volumePath, normalizedPath);
if (relative.startsWith("..") || path.isAbsolute(relative)) {
throw new BadRequestError("Invalid path");
}
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
const startOffset = Math.max(offset, 0);
try { try {
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true }); if (shouldUseControllerLocalVolumeFallback(volume)) {
return await listVolumeFiles(await volumeForHost(volume), subPath, offset, limit);
dirents.sort((a, b) => {
const aIsDir = a.isDirectory();
const bIsDir = b.isDirectory();
if (aIsDir === bIsDir) {
return a.name.localeCompare(b.name);
}
return aIsDir ? -1 : 1;
});
const total = dirents.length;
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
const entries = (
await Promise.all(
paginatedDirents.map(async (dirent) => {
const fullPath = path.join(normalizedPath, dirent.name);
try {
const stats = await fs.stat(fullPath);
const relativePath = path.relative(volumePath, fullPath);
return {
name: dirent.name,
path: `/${relativePath}`,
type: dirent.isDirectory() ? ("directory" as const) : ("file" as const),
size: dirent.isFile() ? stats.size : undefined,
modifiedAt: stats.mtimeMs,
};
} catch {
return null;
}
}),
)
).filter((e) => e !== null);
return {
files: entries,
path: subPath || "/",
offset: startOffset,
limit: pageSize,
total,
hasMore: startOffset + pageSize < total,
};
} catch (error) {
if (isNodeJSErrnoException(error) && error.code === "ENOENT") {
throw new NotFoundError("Directory not found");
} }
const command = await runVolumeCommand(volume.agentId, {
name: "volume.listFiles",
volume: await volumeForAgent(volume),
subPath,
offset,
limit,
});
return command.result;
} catch (error) {
mapAgentFileError(error);
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`); throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
} }
}; };
const browseFilesystem = async (browsePath: string) => { const browseFilesystem = async (browsePath: string) => {
const normalizedPath = path.normalize(browsePath);
try { try {
const entries = await fs.readdir(normalizedPath, { withFileTypes: true }); if (!config.flags.enableLocalAgent) {
return await browseHostFilesystem(browsePath);
}
const directories = await Promise.all( const command = await runVolumeCommand(LOCAL_AGENT_ID, { name: "filesystem.browse", path: browsePath });
entries return command.result;
.filter((entry) => entry.isDirectory())
.map(async (entry) => {
const fullPath = path.join(normalizedPath, entry.name);
try {
const stats = await fs.stat(fullPath);
return {
name: entry.name,
path: fullPath,
type: "directory" as const,
size: undefined,
modifiedAt: stats.mtimeMs,
};
} catch {
return {
name: entry.name,
path: fullPath,
type: "directory" as const,
size: undefined,
modifiedAt: undefined,
};
}
}),
);
return {
directories: directories.sort((a, b) => a.name.localeCompare(b.name)),
path: normalizedPath,
};
} catch (error) { } catch (error) {
throw new InternalServerError(`Failed to browse filesystem: ${toMessage(error)}`); throw new InternalServerError(`Failed to browse filesystem: ${toMessage(error)}`);
} }

View file

@ -1,6 +1,7 @@
import { db } from "~/server/db/db"; import { db } from "~/server/db/db";
import { faker } from "@faker-js/faker"; import { faker } from "@faker-js/faker";
import { volumesTable, type VolumeInsert } from "~/server/db/schema"; import { volumesTable, type VolumeInsert } from "~/server/db/schema";
import { LOCAL_AGENT_ID } from "~/server/modules/agents/constants";
import { ensureTestOrganization, TEST_ORG_ID } from "./organization"; import { ensureTestOrganization, TEST_ORG_ID } from "./organization";
import { generateShortId } from "~/server/utils/id"; import { generateShortId } from "~/server/utils/id";
@ -17,6 +18,7 @@ export const createTestVolume = async (overrides: Partial<VolumeInsert> = {}) =>
autoRemount: true, autoRemount: true,
shortId: generateShortId(), shortId: generateShortId(),
type: "directory", type: "directory",
agentId: LOCAL_AGENT_ID,
organizationId: TEST_ORG_ID, organizationId: TEST_ORG_ID,
...overrides, ...overrides,
}; };

View file

@ -3,6 +3,7 @@ import { handleBackupCancelCommand } from "./backup-cancel";
import { handleBackupRunCommand } from "./backup-run"; import { handleBackupRunCommand } from "./backup-run";
import type { ControllerCommandContext } from "../context"; import type { ControllerCommandContext } from "../context";
import { handleHeartbeatPingCommand } from "./heartbeat-ping"; import { handleHeartbeatPingCommand } from "./heartbeat-ping";
import { handleVolumeCommand } from "./volume";
export const handleControllerCommand = (context: ControllerCommandContext, message: ControllerMessage) => { export const handleControllerCommand = (context: ControllerCommandContext, message: ControllerMessage) => {
switch (message.type) { switch (message.type) {
@ -12,6 +13,9 @@ export const handleControllerCommand = (context: ControllerCommandContext, messa
case "backup.cancel": { case "backup.cancel": {
return handleBackupCancelCommand(context, message.payload); return handleBackupCancelCommand(context, message.payload);
} }
case "volume.command": {
return handleVolumeCommand(context, message.payload);
}
case "heartbeat.ping": { case "heartbeat.ping": {
return handleHeartbeatPingCommand(context, message.payload); return handleHeartbeatPingCommand(context, message.payload);
} }

View file

@ -0,0 +1,127 @@
import { afterEach, expect, test, vi } from "vitest";
import { Effect } from "effect";
import { fromPartial } from "@total-typescript/shoehorn";
import {
parseAgentMessage,
type AgentWireMessage,
type VolumeCommandPayload,
} from "@zerobyte/contracts/agent-protocol";
import type { ControllerCommandContext } from "../context";
const volumeHostMock = vi.hoisted(() => ({
createVolumeBackend: vi.fn(),
getStatFs: vi.fn(),
getVolumePath: vi.fn(),
}));
const operationsMock = vi.hoisted(() => ({
browseFilesystem: vi.fn(),
listVolumeFiles: vi.fn(),
testVolumeConnection: vi.fn(),
}));
vi.mock("../volume-host", () => volumeHostMock);
vi.mock("../volume-host/operations", () => operationsMock);
import { handleVolumeCommand } from "./volume";
afterEach(() => {
vi.restoreAllMocks();
volumeHostMock.createVolumeBackend.mockReset();
volumeHostMock.getStatFs.mockReset();
volumeHostMock.getVolumePath.mockReset();
operationsMock.browseFilesystem.mockReset();
operationsMock.listVolumeFiles.mockReset();
operationsMock.testVolumeConnection.mockReset();
});
const runVolumeCommand = async (payload: VolumeCommandPayload) => {
const outboundMessages: string[] = [];
const context = fromPartial<ControllerCommandContext>({
offerOutbound: (message: AgentWireMessage) =>
Effect.sync(() => {
outboundMessages.push(message);
return true;
}),
});
await Effect.runPromise(handleVolumeCommand(context, payload));
return outboundMessages.map((message) => parseAgentMessage(message));
};
test("runs backend-backed volume commands on the agent host", async () => {
const mount = vi.fn(async () => ({ status: "mounted" as const }));
volumeHostMock.createVolumeBackend.mockReturnValue({ mount });
const messages = await runVolumeCommand(
fromPartial<VolumeCommandPayload>({
commandId: "command-1",
command: {
name: "volume.mount",
volume: { id: 1, config: { backend: "directory", path: "/tmp/source" }, provisioningId: undefined },
},
}),
);
expect(volumeHostMock.createVolumeBackend).toHaveBeenCalledWith(
expect.objectContaining({ id: 1, config: { backend: "directory", path: "/tmp/source" }, provisioningId: null }),
);
expect(mount).toHaveBeenCalledOnce();
expect(messages[0]?.success).toBe(true);
if (messages[0]?.success && messages[0].data.type === "volume.commandResult") {
expect(messages[0].data.payload).toEqual({
commandId: "command-1",
status: "success",
command: { name: "volume.mount", result: { status: "mounted" } },
});
}
});
test("returns command errors without throwing", async () => {
operationsMock.browseFilesystem.mockRejectedValue(new Error("permission denied"));
const messages = await runVolumeCommand({
commandId: "command-2",
command: { name: "filesystem.browse", path: "/root" },
});
expect(messages[0]?.success).toBe(true);
if (messages[0]?.success && messages[0].data.type === "volume.commandResult") {
expect(messages[0].data.payload).toEqual({
commandId: "command-2",
status: "error",
error: "permission denied",
});
}
});
test("routes file listing commands to host operations", async () => {
operationsMock.listVolumeFiles.mockResolvedValue({
files: [],
path: "/logs",
offset: 0,
limit: 10,
total: 0,
hasMore: false,
});
await runVolumeCommand(
fromPartial<VolumeCommandPayload>({
commandId: "command-3",
command: {
name: "volume.listFiles",
volume: { id: 1, config: { backend: "directory", path: "/tmp/source" } },
subPath: "/logs",
offset: 0,
limit: 10,
},
}),
);
expect(operationsMock.listVolumeFiles).toHaveBeenCalledWith(
expect.objectContaining({ id: 1, config: { backend: "directory", path: "/tmp/source" } }),
"/logs",
0,
10,
);
});

View file

@ -0,0 +1,76 @@
import { Effect } from "effect";
import {
createAgentMessage,
type VolumeCommand,
type VolumeCommandPayload,
type VolumeCommandResult,
} from "@zerobyte/contracts/agent-protocol";
import { toMessage } from "@zerobyte/core/utils";
import { createVolumeBackend, getStatFs, getVolumePath, type AgentVolume, type BackendConfig } from "../volume-host";
import { browseFilesystem, listVolumeFiles, testVolumeConnection } from "../volume-host/operations";
import type { ControllerCommandContext } from "../context";
type VolumeBackedCommand = Extract<VolumeCommand, { volume: unknown }>;
const asVolume = (volume: VolumeBackedCommand["volume"]): AgentVolume => ({
...volume,
config: volume.config as BackendConfig,
provisioningId: volume.provisioningId ?? null,
});
const runBackendOperation = async (
command: Extract<VolumeCommand, { volume: unknown }>,
operation: "mount" | "unmount" | "checkHealth",
) => {
const backend = createVolumeBackend(asVolume(command.volume));
return backend[operation]();
};
const executeVolumeCommand = async (command: VolumeCommand): Promise<VolumeCommandResult> => {
switch (command.name) {
case "volume.mount":
return { name: command.name, result: await runBackendOperation(command, "mount") };
case "volume.unmount":
return { name: command.name, result: await runBackendOperation(command, "unmount") };
case "volume.checkHealth":
return { name: command.name, result: await runBackendOperation(command, "checkHealth") };
case "volume.statfs":
return { name: command.name, result: await getStatFs(getVolumePath(asVolume(command.volume))) };
case "volume.listFiles":
return {
name: command.name,
result: await listVolumeFiles(asVolume(command.volume), command.subPath, command.offset, command.limit),
};
case "volume.testConnection":
return { name: command.name, result: await testVolumeConnection(command.backendConfig as BackendConfig) };
case "filesystem.browse":
return { name: command.name, result: await browseFilesystem(command.path) };
}
};
export const handleVolumeCommand = (context: ControllerCommandContext, payload: VolumeCommandPayload) => {
return Effect.promise(async () => {
try {
const command = await executeVolumeCommand(payload.command);
await Effect.runPromise(
context.offerOutbound(
createAgentMessage("volume.commandResult", {
commandId: payload.commandId,
status: "success",
command,
}),
),
);
} catch (error) {
await Effect.runPromise(
context.offerOutbound(
createAgentMessage("volume.commandResult", {
commandId: payload.commandId,
status: "error",
error: toMessage(error),
}),
),
);
}
});
};

View file

@ -1,14 +1,32 @@
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import { createControllerSession, type ControllerSession } from "./controller-session"; import { createControllerSession, type ControllerSession } from "./controller-session";
import { cleanupDanglingVolumeMountDirectories } from "./volume-host/cleanup";
const controllerUrl = process.env.ZEROBYTE_CONTROLLER_URL; const controllerUrl = process.env.ZEROBYTE_CONTROLLER_URL;
const agentToken = process.env.ZEROBYTE_AGENT_TOKEN; const agentToken = process.env.ZEROBYTE_AGENT_TOKEN;
const RECONNECT_DELAY_MS = 1000; const RECONNECT_DELAY_MS = 1000;
const VOLUME_CLEANUP_INTERVAL_MS = 60 * 60 * 1000;
export class Agent { export class Agent {
private ws: WebSocket | null = null; private ws: WebSocket | null = null;
private controllerSession: ControllerSession | null = null; private controllerSession: ControllerSession | null = null;
private reconnectTimeout: ReturnType<typeof setTimeout> | null = null; private reconnectTimeout: ReturnType<typeof setTimeout> | null = null;
private volumeCleanupInterval: ReturnType<typeof setInterval> | null = null;
private runVolumeCleanup() {
void cleanupDanglingVolumeMountDirectories().catch((error) => {
logger.warn(`Agent volume cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
});
}
private startVolumeCleanup() {
if (this.volumeCleanupInterval) {
return;
}
this.runVolumeCleanup();
this.volumeCleanupInterval = setInterval(() => this.runVolumeCleanup(), VOLUME_CLEANUP_INTERVAL_MS);
}
private scheduleReconnect() { private scheduleReconnect() {
if (this.reconnectTimeout) { if (this.reconnectTimeout) {
@ -22,6 +40,8 @@ export class Agent {
} }
connect() { connect() {
this.startVolumeCleanup();
if (this.reconnectTimeout) { if (this.reconnectTimeout) {
clearTimeout(this.reconnectTimeout); clearTimeout(this.reconnectTimeout);
this.reconnectTimeout = null; this.reconnectTimeout = null;

View file

@ -0,0 +1,49 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { afterEach, expect, test, vi } from "vitest";
let tempRoot: string | undefined;
let mockMountPoints: string[] = [];
afterEach(async () => {
vi.doUnmock("./constants");
vi.doUnmock("./fs");
vi.resetModules();
mockMountPoints = [];
if (tempRoot) {
await fs.rm(tempRoot, { recursive: true, force: true });
tempRoot = undefined;
}
});
const loadCleanup = async () => {
vi.resetModules();
tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-agent-cleanup-"));
vi.doMock("./constants", () => ({ VOLUME_MOUNT_BASE: tempRoot }));
vi.doMock("./fs", () => ({
readMountInfo: async () => mockMountPoints.map((mountPoint) => ({ mountPoint, fstype: "fuse.sshfs" })),
}));
return import("../cleanup");
};
test("removes stale volume directories that are not mounted on the agent host", async () => {
const { cleanupDanglingVolumeMountDirectories } = await loadCleanup();
await fs.mkdir(path.join(tempRoot!, "stale-volume", "_data"), { recursive: true });
await cleanupDanglingVolumeMountDirectories();
await expect(fs.access(path.join(tempRoot!, "stale-volume"))).rejects.toThrow();
});
test("keeps volume directories that are still mounted on the agent host", async () => {
const { cleanupDanglingVolumeMountDirectories } = await loadCleanup();
const localVolumeDir = path.join(tempRoot!, "mounted-volume");
mockMountPoints = [path.join(localVolumeDir, "_data")];
await fs.mkdir(path.join(localVolumeDir, "_data"), { recursive: true });
await cleanupDanglingVolumeMountDirectories();
await expect(fs.access(localVolumeDir)).resolves.toBeNull();
});

View file

@ -0,0 +1,66 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { afterEach, expect, test } from "vitest";
import { listVolumeFiles } from "../operations";
import type { AgentVolume } from "../types";
let tempRoot: string | undefined;
afterEach(async () => {
if (tempRoot) {
await fs.rm(tempRoot, { recursive: true, force: true });
tempRoot = undefined;
}
});
const createDirectoryVolume = async (): Promise<AgentVolume> => {
tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-volume-ops-"));
return {
id: 1,
shortId: "volume-1",
name: "Test volume",
config: { backend: "directory", path: tempRoot },
createdAt: Date.now(),
updatedAt: Date.now(),
lastHealthCheck: Date.now(),
type: "directory",
status: "mounted",
lastError: null,
provisioningId: null,
autoRemount: true,
agentId: "local",
organizationId: "org-1",
};
};
test("listVolumeFiles returns sorted paginated entries inside the volume", async () => {
const volume = await createDirectoryVolume();
await fs.mkdir(path.join(tempRoot!, "z-dir"));
await fs.mkdir(path.join(tempRoot!, "a-dir"));
await fs.writeFile(path.join(tempRoot!, "b-file.txt"), "hello");
const result = await listVolumeFiles(volume, undefined, 1, 2);
expect(result).toMatchObject({
path: "/",
offset: 1,
limit: 2,
total: 3,
hasMore: false,
});
expect(result.files.map((entry) => entry.name)).toEqual(["z-dir", "b-file.txt"]);
expect(result.files[1]).toMatchObject({ path: "/b-file.txt", type: "file", size: 5 });
});
test("listVolumeFiles rejects traversal outside the volume", async () => {
const volume = await createDirectoryVolume();
await expect(listVolumeFiles(volume, "../outside", 0, 10)).rejects.toThrow("Invalid path");
});
test("listVolumeFiles reports missing directories consistently", async () => {
const volume = await createDirectoryVolume();
await expect(listVolumeFiles(volume, "missing", 0, 10)).rejects.toThrow("Directory not found");
});

View file

@ -1,12 +1,11 @@
import * as fs from "node:fs/promises"; import * as fs from "node:fs/promises";
import { toMessage } from "../../../utils/errors";
import { logger } from "@zerobyte/core/node"; import { logger } from "@zerobyte/core/node";
import type { VolumeBackend } from "../backend"; import { toMessage } from "@zerobyte/core/utils";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes"; import type { BackendConfig, VolumeBackend } from "../types";
const mount = async (config: BackendConfig, _volumePath: string) => { const mount = async (config: BackendConfig, volumePath: string) => {
if (config.backend !== "directory") { if (config.backend !== "directory") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" }; return { status: "error" as const, error: "Invalid backend type" };
} }
logger.info("Mounting directory volume from:", config.path); logger.info("Mounting directory volume from:", config.path);
@ -16,33 +15,33 @@ const mount = async (config: BackendConfig, _volumePath: string) => {
const stats = await fs.stat(config.path); const stats = await fs.stat(config.path);
if (!stats.isDirectory()) { if (!stats.isDirectory()) {
return { status: BACKEND_STATUS.error, error: "Path is not a directory" }; return { status: "error" as const, error: "Path is not a directory" };
} }
return { status: BACKEND_STATUS.mounted }; return { status: "mounted" as const };
} catch (error) { } catch (error) {
logger.error("Failed to mount directory volume:", error); logger.error("Failed to mount directory volume:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) }; return { status: "error" as const, error: toMessage(error) };
} }
}; };
const unmount = async () => { const unmount = async () => {
logger.info("Cannot unmount directory volume."); logger.info("Cannot unmount directory volume.");
return { status: BACKEND_STATUS.unmounted }; return { status: "unmounted" as const };
}; };
const checkHealth = async (config: BackendConfig) => { const checkHealth = async (config: BackendConfig) => {
if (config.backend !== "directory") { if (config.backend !== "directory") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" }; return { status: "error" as const, error: "Invalid backend type" };
} }
try { try {
await fs.access(config.path); await fs.access(config.path);
return { status: BACKEND_STATUS.mounted }; return { status: "mounted" as const };
} catch (error) { } catch (error) {
logger.error("Directory health check failed:", error); logger.error("Directory health check failed:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) }; return { status: "error" as const, error: toMessage(error) };
} }
}; };

View file

@ -0,0 +1,112 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { logger } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils";
import { OPERATION_TIMEOUT } from "../constants";
import { withTimeout } from "../timeout";
import { getMountForPath } from "../fs";
import type { BackendConfig, VolumeBackend } from "../types";
import { assertMounted, executeMount, executeUnmount } from "./utils";
const checkHealth = async (mountPath: string) => {
const run = async () => {
await assertMounted(mountPath, (fstype) => fstype.startsWith("nfs"));
logger.debug(`NFS volume at ${mountPath} is healthy and mounted.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("NFS volume health check failed:", message);
}
return { status: "error" as const, error: message };
}
};
const unmount = async (mountPath: string) => {
if (os.platform() !== "linux") {
logger.error("NFS unmounting is only supported on Linux hosts.");
return { status: "error" as const, error: "NFS unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
return { status: "unmounted" as const };
}
await executeUnmount(mountPath);
await fs.rmdir(mountPath).catch(() => {});
logger.info(`NFS volume at ${mountPath} unmounted successfully.`);
return { status: "unmounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS unmount");
} catch (error) {
logger.error("Error unmounting NFS volume", { mountPath, error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
const mount = async (config: BackendConfig, mountPath: string) => {
logger.debug(`Mounting volume ${mountPath}...`);
if (config.backend !== "nfs") {
logger.error("Provided config is not for NFS backend");
return { status: "error" as const, error: "Provided config is not for NFS backend" };
}
if (os.platform() !== "linux") {
logger.error("NFS mounting is only supported on Linux hosts.");
return { status: "error" as const, error: "NFS mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(mountPath);
if (status === "mounted") return { status: "mounted" as const };
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
await unmount(mountPath);
}
const run = async () => {
await fs.mkdir(mountPath, { recursive: true });
const options = [`vers=${config.version}`, `port=${config.port}`];
if (config.version === "3") options.push("nolock");
if (config.readOnly) options.push("ro");
const args = ["-t", "nfs", "-o", options.join(","), `${config.server}:${config.exportPath}`, mountPath];
logger.debug(`Mounting volume ${mountPath}...`);
logger.info(`Executing mount: mount ${args.join(" ")}`);
try {
await executeMount(args);
} catch (error) {
logger.warn(`Initial NFS mount failed, retrying with -i flag: ${toMessage(error)}`);
await executeMount(["-i", ...args]);
}
logger.info(`NFS volume at ${mountPath} mounted successfully.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS mount");
} catch (error) {
logger.error("Error mounting NFS volume", { error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
export const makeNfsBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
mount: () => mount(config, mountPath),
unmount: () => unmount(mountPath),
checkHealth: () => checkHealth(mountPath),
});

View file

@ -0,0 +1,124 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { logger, safeExec } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils";
import { OPERATION_TIMEOUT, RCLONE_CONFIG_FILE, RCLONE_TIMEOUT } from "../constants";
import { withTimeout } from "../timeout";
import { getMountForPath } from "../fs";
import type { BackendConfig, VolumeBackend } from "../types";
import { assertMounted, executeUnmount } from "./utils";
const checkHealth = async (mountPath: string) => {
const run = async () => {
await assertMounted(mountPath, (fstype) => fstype.includes("rclone"));
logger.debug(`Rclone volume at ${mountPath} is healthy and mounted.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("Rclone volume health check failed:", message);
}
return { status: "error" as const, error: message };
}
};
const unmount = async (mountPath: string) => {
if (os.platform() !== "linux") {
logger.error("Rclone unmounting is only supported on Linux hosts.");
return { status: "error" as const, error: "Rclone unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
return { status: "unmounted" as const };
}
await executeUnmount(mountPath);
await fs.rmdir(mountPath).catch(() => {});
logger.info(`Rclone volume at ${mountPath} unmounted successfully.`);
return { status: "unmounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone unmount");
} catch (error) {
logger.error("Error unmounting rclone volume", { mountPath, error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
const mount = async (config: BackendConfig, mountPath: string) => {
logger.debug(`Mounting rclone volume ${mountPath}...`);
if (config.backend !== "rclone") {
logger.error("Provided config is not for rclone backend");
return { status: "error" as const, error: "Provided config is not for rclone backend" };
}
if (os.platform() !== "linux") {
logger.error("Rclone mounting is only supported on Linux hosts.");
return { status: "error" as const, error: "Rclone mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(mountPath);
if (status === "mounted") return { status: "mounted" as const };
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
await unmount(mountPath);
}
const run = async () => {
await fs.mkdir(mountPath, { recursive: true });
const args = [
"mount",
`${config.remote}:${config.path}`,
mountPath,
"--daemon",
"--vfs-cache-mode",
"writes",
"--allow-non-empty",
"--allow-other",
];
if (config.readOnly) args.push("--read-only");
logger.debug(`Mounting rclone volume ${mountPath}...`);
logger.info(`Executing rclone: rclone ${args.join(" ")}`);
const result = await safeExec({
command: "rclone",
args,
env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
timeout: RCLONE_TIMEOUT,
});
if (result.exitCode !== 0) {
const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error";
throw new Error(`Failed to mount rclone volume: ${errorMsg}`);
}
logger.info(`Rclone volume at ${mountPath} mounted successfully.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), RCLONE_TIMEOUT, "Rclone mount");
} catch (error) {
const errorMsg = toMessage(error);
logger.error("Error mounting rclone volume", { error: errorMsg });
return { status: "error" as const, error: errorMsg };
}
};
export const makeRcloneBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
mount: () => mount(config, mountPath),
unmount: () => unmount(mountPath),
checkHealth: () => checkHealth(mountPath),
});

View file

@ -0,0 +1,161 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { spawn } from "node:child_process";
import { FILE_MODES, logger, writeFileWithMode } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils";
import { OPERATION_TIMEOUT, SSH_KEYS_DIR } from "../constants";
import { getMountForPath } from "../fs";
import { withTimeout } from "../timeout";
import type { BackendConfig, VolumeBackend } from "../types";
import { executeUnmount } from "./utils";
const getPrivateKeyPath = (mountPath: string) => path.join(SSH_KEYS_DIR, `${path.basename(mountPath)}.key`);
const getKnownHostsPath = (mountPath: string) => path.join(SSH_KEYS_DIR, `${path.basename(mountPath)}.known_hosts`);
const runSshfs = async (args: string[], password?: string) =>
new Promise<void>((resolve, reject) => {
const child = spawn("sshfs", args, { stdio: ["pipe", "pipe", "pipe"] });
let stdout = "";
let stderr = "";
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (data) => {
stdout += data;
});
child.stderr.on("data", (data) => {
stderr += data;
});
child.on("error", (error) => {
reject(new Error(`Failed to start sshfs: ${error.message}`));
});
child.on("close", (code) => {
if (code === 0) {
resolve();
return;
}
const errorMsg = stderr.trim() || stdout.trim() || "Unknown error";
reject(new Error(`Failed to mount SFTP volume: ${errorMsg}`));
});
if (password) child.stdin.write(password);
child.stdin.end();
});
const checkHealth = async (mountPath: string) => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) return { status: "unmounted" as const };
if (mount.fstype !== "fuse.sshfs") {
return { status: "error" as const, error: `Invalid filesystem type: ${mount.fstype} (expected fuse.sshfs)` };
}
return { status: "mounted" as const };
};
const unmount = async (mountPath: string) => {
if (os.platform() !== "linux") {
logger.error("SFTP unmounting is only supported on Linux hosts.");
return { status: "error" as const, error: "SFTP unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
} else {
await executeUnmount(mountPath);
}
await fs.unlink(getPrivateKeyPath(mountPath)).catch(() => {});
await fs.unlink(getKnownHostsPath(mountPath)).catch(() => {});
await fs.rmdir(mountPath).catch(() => {});
logger.info(`SFTP volume at ${mountPath} unmounted successfully.`);
return { status: "unmounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SFTP unmount");
} catch (error) {
logger.error("Error unmounting SFTP volume", { mountPath, error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
const mount = async (config: BackendConfig, mountPath: string) => {
logger.debug(`Mounting SFTP volume ${mountPath}...`);
if (config.backend !== "sftp") {
logger.error("Provided config is not for SFTP backend");
return { status: "error" as const, error: "Provided config is not for SFTP backend" };
}
if (os.platform() !== "linux") {
logger.error("SFTP mounting is only supported on Linux hosts.");
return { status: "error" as const, error: "SFTP mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(mountPath);
if (status === "mounted") return { status: "mounted" as const };
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
await unmount(mountPath);
}
const run = async () => {
await fs.mkdir(mountPath, { recursive: true });
await fs.mkdir(SSH_KEYS_DIR, { recursive: true });
const { uid, gid } = os.userInfo();
const options = [
"reconnect",
"ServerAliveInterval=15",
"ServerAliveCountMax=3",
"allow_other",
`uid=${uid}`,
`gid=${gid}`,
];
if (config.skipHostKeyCheck) {
options.push("StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null");
} else if (config.knownHosts) {
await writeFileWithMode(getKnownHostsPath(mountPath), config.knownHosts, FILE_MODES.ownerReadWrite);
options.push(`UserKnownHostsFile=${getKnownHostsPath(mountPath)}`, "StrictHostKeyChecking=yes");
} else {
options.push("StrictHostKeyChecking=yes");
}
if (config.readOnly) options.push("ro");
if (config.port) options.push(`port=${config.port}`);
if (config.privateKey) {
let key = config.privateKey.replace(/\r\n/g, "\n");
if (!key.endsWith("\n")) key += "\n";
await writeFileWithMode(getPrivateKeyPath(mountPath), key, FILE_MODES.ownerReadWrite);
options.push(`IdentityFile=${getPrivateKeyPath(mountPath)}`);
}
const args = [`${config.username}@${config.host}:${config.path || ""}`, mountPath, "-o", options.join(",")];
if (config.password) args.push("-o", "password_stdin");
logger.info(`Executing sshfs: sshfs ${args.join(" ")}`);
await runSshfs(args, config.password);
logger.info(`SFTP volume at ${mountPath} mounted successfully.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT * 2, "SFTP mount");
} catch (error) {
logger.error("Error mounting SFTP volume", { error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
export const makeSftpBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
mount: () => mount(config, mountPath),
unmount: () => unmount(mountPath),
checkHealth: () => checkHealth(mountPath),
});

View file

@ -0,0 +1,124 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { logger } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils";
import { OPERATION_TIMEOUT } from "../constants";
import { withTimeout } from "../timeout";
import { getMountForPath } from "../fs";
import type { BackendConfig, VolumeBackend } from "../types";
import { assertMounted, executeMount, executeUnmount } from "./utils";
const checkHealth = async (mountPath: string) => {
const run = async () => {
await assertMounted(mountPath, (fstype) => fstype === "cifs");
logger.debug(`SMB volume at ${mountPath} is healthy and mounted.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("SMB volume health check failed:", message);
}
return { status: "error" as const, error: message };
}
};
const unmount = async (mountPath: string) => {
if (os.platform() !== "linux") {
logger.error("SMB unmounting is only supported on Linux hosts.");
return { status: "error" as const, error: "SMB unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
return { status: "unmounted" as const };
}
await executeUnmount(mountPath);
await fs.rmdir(mountPath).catch(() => {});
logger.info(`SMB volume at ${mountPath} unmounted successfully.`);
return { status: "unmounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB unmount");
} catch (error) {
logger.error("Error unmounting SMB volume", { mountPath, error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
const mount = async (config: BackendConfig, mountPath: string) => {
logger.debug(`Mounting SMB volume ${mountPath}...`);
if (config.backend !== "smb") {
logger.error("Provided config is not for SMB backend");
return { status: "error" as const, error: "Provided config is not for SMB backend" };
}
if (os.platform() !== "linux") {
logger.error("SMB mounting is only supported on Linux hosts.");
return { status: "error" as const, error: "SMB mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(mountPath);
if (status === "mounted") return { status: "mounted" as const };
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
await unmount(mountPath);
}
const run = async () => {
await fs.mkdir(mountPath, { recursive: true });
const { uid, gid } = os.userInfo();
const options = [`port=${config.port}`, `uid=${uid}`, `gid=${gid}`, "iocharset=utf8"];
if (config.guest) {
options.push("username=guest", "password=");
} else {
const safePassword = (config.password ?? "").replace(/\\/g, "\\\\").replace(/,/g, "\\,");
options.push(`username=${config.username ?? "user"}`, `password=${safePassword}`);
}
if (config.domain) options.push(`domain=${config.domain}`);
if (config.vers && config.vers !== "auto") options.push(`vers=${config.vers}`);
if (config.readOnly) options.push("ro");
const source = `//${config.server}/${config.share}`;
const args = ["-t", "cifs", "-o", options.join(","), source, mountPath];
logger.debug(`Mounting SMB volume ${mountPath}...`);
logger.info(`Executing SMB mount for ${source} at ${mountPath}`);
try {
await executeMount(args);
} catch (error) {
logger.error(`SMB mount failed: ${toMessage(error)}`);
throw error;
}
logger.info(`SMB volume at ${mountPath} mounted successfully.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB mount");
} catch (error) {
logger.error("Error mounting SMB volume", { error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
export const makeSmbBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
mount: () => mount(config, mountPath),
unmount: () => unmount(mountPath),
checkHealth: () => checkHealth(mountPath),
});

View file

@ -1,7 +1,6 @@
import * as fs from "node:fs/promises"; import * as fs from "node:fs/promises";
import { logger } from "@zerobyte/core/node"; import { logger, safeExec } from "@zerobyte/core/node";
import { safeExec } from "@zerobyte/core/node"; import { getMountForPath } from "../fs";
import { getMountForPath } from "../../../utils/mountinfo";
export const executeMount = async (args: string[]): Promise<void> => { export const executeMount = async (args: string[]): Promise<void> => {
const shouldBeVerbose = process.env.LOG_LEVEL === "debug" || process.env.NODE_ENV !== "production"; const shouldBeVerbose = process.env.LOG_LEVEL === "debug" || process.env.NODE_ENV !== "production";
@ -10,7 +9,6 @@ export const executeMount = async (args: string[]): Promise<void> => {
logger.debug(`Executing mount ${effectiveArgs.join(" ")}`); logger.debug(`Executing mount ${effectiveArgs.join(" ")}`);
const result = await safeExec({ command: "mount", args: effectiveArgs, timeout: 10000 }); const result = await safeExec({ command: "mount", args: effectiveArgs, timeout: 10000 });
const stdout = result.stdout.toString().trim(); const stdout = result.stdout.toString().trim();
const stderr = result.stderr.toString().trim(); const stderr = result.stderr.toString().trim();
@ -26,37 +24,29 @@ export const executeMount = async (args: string[]): Promise<void> => {
throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr || stdout || "unknown error"}`); throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr || stdout || "unknown error"}`);
}; };
export const executeUnmount = async (path: string): Promise<void> => { export const executeUnmount = async (mountPath: string): Promise<void> => {
let stderr: string | undefined; logger.debug(`Executing umount -l ${mountPath}`);
const result = await safeExec({ command: "umount", args: ["-l", mountPath], timeout: 10000 });
logger.debug(`Executing umount -l ${path}`); const stderr = result.stderr.toString();
const result = await safeExec({ command: "umount", args: ["-l", path], timeout: 10000 });
stderr = result.stderr.toString();
if (stderr?.trim()) {
logger.warn(stderr.trim());
}
if (stderr.trim()) logger.warn(stderr.trim());
if (result.exitCode !== 0) { if (result.exitCode !== 0) {
throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr?.trim()}`); throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr.trim()}`);
} }
}; };
export const assertMounted = async (path: string, isExpectedFilesystem: (fstype: string) => boolean) => { export const assertMounted = async (mountPath: string, isExpectedFilesystem: (fstype: string) => boolean) => {
try { try {
await fs.access(path); await fs.access(mountPath);
} catch { } catch {
throw new Error("Volume is not mounted"); throw new Error("Volume is not mounted");
} }
const mount = await getMountForPath(path); const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
if (!mount || mount.mountPoint !== path) {
throw new Error("Volume is not mounted"); throw new Error("Volume is not mounted");
} }
if (!isExpectedFilesystem(mount.fstype)) { if (!isExpectedFilesystem(mount.fstype)) {
throw new Error(`Path ${path} is not mounted as correct fstype (found ${mount.fstype}).`); throw new Error(`Path ${mountPath} is not mounted as correct fstype (found ${mount.fstype}).`);
} }
}; };

View file

@ -0,0 +1,144 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import { logger } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils";
import { OPERATION_TIMEOUT } from "../constants";
import { withTimeout } from "../timeout";
import { getMountForPath } from "../fs";
import type { BackendConfig, VolumeBackend } from "../types";
import { assertMounted, executeMount, executeUnmount } from "./utils";
const checkHealth = async (mountPath: string) => {
const run = async () => {
await assertMounted(mountPath, (fstype) => fstype === "fuse" || fstype === "davfs");
logger.debug(`WebDAV volume at ${mountPath} is healthy and mounted.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV health check");
} catch (error) {
const message = toMessage(error);
if (message !== "Volume is not mounted") {
logger.error("WebDAV volume health check failed:", message);
}
return { status: "error" as const, error: message };
}
};
const unmount = async (mountPath: string) => {
if (os.platform() !== "linux") {
logger.error("WebDAV unmounting is only supported on Linux hosts.");
return { status: "error" as const, error: "WebDAV unmounting is only supported on Linux hosts." };
}
const run = async () => {
const mount = await getMountForPath(mountPath);
if (!mount || mount.mountPoint !== mountPath) {
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
return { status: "unmounted" as const };
}
await executeUnmount(mountPath);
await fs.rmdir(mountPath).catch(() => {});
logger.info(`WebDAV volume at ${mountPath} unmounted successfully.`);
return { status: "unmounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV unmount");
} catch (error) {
logger.error("Error unmounting WebDAV volume", { mountPath, error: toMessage(error) });
return { status: "error" as const, error: toMessage(error) };
}
};
const mount = async (config: BackendConfig, mountPath: string) => {
logger.debug(`Mounting WebDAV volume ${mountPath}...`);
if (config.backend !== "webdav") {
logger.error("Provided config is not for WebDAV backend");
return { status: "error" as const, error: "Provided config is not for WebDAV backend" };
}
if (os.platform() !== "linux") {
logger.error("WebDAV mounting is only supported on Linux hosts.");
return { status: "error" as const, error: "WebDAV mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(mountPath);
if (status === "mounted") return { status: "mounted" as const };
if (status === "error") {
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
await unmount(mountPath);
}
const run = async () => {
await fs.mkdir(mountPath, { recursive: true }).catch((error) => {
logger.warn(`Failed to create directory ${mountPath}: ${toMessage(error)}`);
});
const protocol = config.ssl ? "https" : "http";
const defaultPort = config.ssl ? 443 : 80;
const source = `${protocol}://${config.server}${config.port !== defaultPort ? `:${config.port}` : ""}${config.path}`;
const { uid, gid } = os.userInfo();
const options = config.readOnly
? [`uid=${uid}`, `gid=${gid}`, "file_mode=0444", "dir_mode=0555", "ro"]
: [`uid=${uid}`, `gid=${gid}`, "file_mode=0664", "dir_mode=0775"];
if (config.username && config.password) {
const entry = [source, config.username, config.password]
.map((value) => value.replace(/[\r\n\t\s]+/g, " "))
.join(" ");
await fs.appendFile("/etc/davfs2/secrets", `${entry}\n`, { mode: 0o600 });
}
logger.debug(`Mounting WebDAV volume ${mountPath}...`);
const args = ["-t", "davfs", "-o", options.join(","), source, mountPath];
try {
await executeMount(args);
} catch (error) {
logger.warn(`Initial WebDAV mount failed, retrying with -i flag: ${toMessage(error)}`);
await executeMount(["-i", ...args]);
}
logger.info(`WebDAV volume at ${mountPath} mounted successfully.`);
return { status: "mounted" as const };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV mount");
} catch (error) {
const message = toMessage(error);
if (message.includes("already mounted")) return { status: "mounted" as const };
logger.error("Error mounting WebDAV volume", { error: message });
if (message.includes("option") && message.includes("requires argument")) {
return {
status: "error" as const,
error: "Invalid mount options. Please check your WebDAV server configuration.",
};
}
if (message.includes("connection refused") || message.includes("Connection refused")) {
return {
status: "error" as const,
error: "Cannot connect to WebDAV server. Please check the server address and port.",
};
}
if (message.includes("unauthorized") || message.includes("Unauthorized")) {
return { status: "error" as const, error: "Authentication failed. Please check your username and password." };
}
return { status: "error" as const, error: message };
}
};
export const makeWebdavBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
mount: () => mount(config, mountPath),
unmount: () => unmount(mountPath),
checkHealth: () => checkHealth(mountPath),
});

View file

@ -0,0 +1,28 @@
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { logger } from "@zerobyte/core/node";
import { toMessage } from "@zerobyte/core/utils";
import { VOLUME_MOUNT_BASE } from "./constants";
import { readMountInfo } from "./fs";
export const cleanupDanglingVolumeMountDirectories = async () => {
const mounts = await readMountInfo().catch((error) => {
logger.warn(`Failed to read mount info for volume cleanup: ${toMessage(error)}`);
return [];
});
const mountedPaths = new Set(mounts.map((mount) => mount.mountPoint));
const volumeDirs = await fs.readdir(VOLUME_MOUNT_BASE).catch(() => []);
for (const dir of volumeDirs) {
const mountPath = path.join(VOLUME_MOUNT_BASE, dir, "_data");
if (mountedPaths.has(mountPath)) {
continue;
}
const fullPath = path.join(VOLUME_MOUNT_BASE, dir);
logger.info(`Removing stale volume mount directory at ${fullPath}`);
await fs.rm(fullPath, { recursive: true, force: true }).catch((error) => {
logger.warn(`Failed to remove stale volume mount directory ${fullPath}: ${toMessage(error)}`);
});
}
};

View file

@ -0,0 +1,9 @@
import * as path from "node:path";
export const OPERATION_TIMEOUT = 5000;
export const VOLUME_MOUNT_BASE = process.env.ZEROBYTE_VOLUMES_DIR || "/var/lib/zerobyte/volumes";
export const SSH_KEYS_DIR = "/var/lib/zerobyte/ssh";
export const RCLONE_CONFIG_DIR = process.env.RCLONE_CONFIG_DIR || "/root/.config/rclone";
export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf");
const serverIdleTimeout = Number(process.env.SERVER_IDLE_TIMEOUT ?? 60);
export const RCLONE_TIMEOUT = (Number.isFinite(serverIdleTimeout) ? serverIdleTimeout : 60) * 1000;

View file

@ -0,0 +1,66 @@
import * as fs from "node:fs/promises";
import { isPathWithin } from "@zerobyte/core/utils";
type MountInfo = {
mountPoint: string;
fstype: string;
};
const unescapeMount = (value: string) =>
value.replace(/\\([0-7]{3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8)));
export const readMountInfo = async (): Promise<MountInfo[]> => {
const text = await fs.readFile("/proc/self/mountinfo", "utf-8");
const result: MountInfo[] = [];
for (const line of text.split("\n")) {
if (!line) continue;
const sep = line.indexOf(" - ");
if (sep === -1) continue;
const left = line.slice(0, sep).split(" ");
const right = line.slice(sep + 3).split(" ");
const mpRaw = left[4];
const fstype = right[0];
if (!mpRaw || !fstype) continue;
result.push({ mountPoint: unescapeMount(mpRaw), fstype });
}
return result;
};
export const getMountForPath = async (targetPath: string): Promise<MountInfo | undefined> => {
const mounts = await readMountInfo();
let best: MountInfo | undefined;
for (const mount of mounts) {
if (!isPathWithin(mount.mountPoint, targetPath)) continue;
if (!best || mount.mountPoint.length > best.mountPoint.length) {
best = mount;
}
}
return best;
};
export const getStatFs = async (mountPoint: string) => {
const stat = await fs.statfs(mountPoint, { bigint: true });
const unit = stat.bsize > 0n ? stat.bsize : 1n;
const blocks = stat.blocks > 0n ? stat.blocks : 0n;
let bfree = stat.bfree > 0n ? stat.bfree : 0n;
if (bfree > blocks) bfree = blocks;
const bavail = stat.bavail > 0n ? stat.bavail : 0n;
const max = BigInt(Number.MAX_SAFE_INTEGER);
const toNumber = (value: bigint) => (value > max ? Number.MAX_SAFE_INTEGER : Number(value));
return {
total: toNumber(blocks * unit),
used: toNumber((blocks - bfree) * unit),
free: toNumber(bavail * unit),
};
};
export const isNodeJSErrnoException = (error: unknown): error is NodeJS.ErrnoException => {
return error instanceof Error && "code" in error;
};

View file

@ -0,0 +1,31 @@
import { makeDirectoryBackend } from "./backends/directory";
import { makeNfsBackend } from "./backends/nfs";
import { makeRcloneBackend } from "./backends/rclone";
import { makeSftpBackend } from "./backends/sftp";
import { makeSmbBackend } from "./backends/smb";
import { makeWebdavBackend } from "./backends/webdav";
import { getVolumePath } from "./paths";
import type { AgentVolume, VolumeBackend } from "./types";
export { getStatFs, isNodeJSErrnoException } from "./fs";
export { getVolumePath } from "./paths";
export type { AgentVolume, BackendConfig, VolumeBackend } from "./types";
export const createVolumeBackend = (volume: AgentVolume, mountPath = getVolumePath(volume)): VolumeBackend => {
switch (volume.config.backend) {
case "directory":
return makeDirectoryBackend(volume.config, mountPath);
case "nfs":
return makeNfsBackend(volume.config, mountPath);
case "smb":
return makeSmbBackend(volume.config, mountPath);
case "webdav":
return makeWebdavBackend(volume.config, mountPath);
case "rclone":
return makeRcloneBackend(volume.config, mountPath);
case "sftp":
return makeSftpBackend(volume.config, mountPath);
}
throw new Error("Unsupported backend");
};

View file

@ -0,0 +1,156 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { toMessage } from "@zerobyte/core/utils";
import { createVolumeBackend, getVolumePath, isNodeJSErrnoException } from ".";
import type { AgentVolume, BackendConfig } from "./types";
const DEFAULT_PAGE_SIZE = 500;
const MAX_PAGE_SIZE = 500;
export const listVolumeFiles = async (
volume: AgentVolume,
subPath?: string,
offset: number = 0,
limit: number = DEFAULT_PAGE_SIZE,
) => {
const volumePath = getVolumePath(volume);
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
const normalizedPath = path.normalize(requestedPath);
const relative = path.relative(volumePath, normalizedPath);
if (relative.startsWith("..") || path.isAbsolute(relative)) {
throw new Error("Invalid path");
}
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
const startOffset = Math.max(offset, 0);
try {
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
dirents.sort((a, b) => {
const aIsDir = a.isDirectory();
const bIsDir = b.isDirectory();
if (aIsDir === bIsDir) {
return a.name.localeCompare(b.name);
}
return aIsDir ? -1 : 1;
});
const total = dirents.length;
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
const entries = (
await Promise.all(
paginatedDirents.map(async (dirent) => {
const fullPath = path.join(normalizedPath, dirent.name);
try {
const stats = await fs.stat(fullPath);
const relativePath = path.relative(volumePath, fullPath);
return {
name: dirent.name,
path: `/${relativePath}`,
type: dirent.isDirectory() ? ("directory" as const) : ("file" as const),
size: dirent.isFile() ? stats.size : undefined,
modifiedAt: stats.mtimeMs,
};
} catch {
return null;
}
}),
)
).filter((entry) => entry !== null);
return {
files: entries,
path: subPath || "/",
offset: startOffset,
limit: pageSize,
total,
hasMore: startOffset + pageSize < total,
};
} catch (error) {
if (isNodeJSErrnoException(error) && error.code === "ENOENT") {
throw new Error("Directory not found");
}
if (toMessage(error) === "Invalid path") {
throw error;
}
throw new Error(`Failed to list files: ${toMessage(error)}`);
}
};
export const browseFilesystem = async (browsePath: string) => {
const normalizedPath = path.normalize(browsePath);
const entries = await fs.readdir(normalizedPath, { withFileTypes: true });
const directories = await Promise.all(
entries
.filter((entry) => entry.isDirectory())
.map(async (entry) => {
const fullPath = path.join(normalizedPath, entry.name);
try {
const stats = await fs.stat(fullPath);
return {
name: entry.name,
path: fullPath,
type: "directory" as const,
size: undefined,
modifiedAt: stats.mtimeMs,
};
} catch {
return {
name: entry.name,
path: fullPath,
type: "directory" as const,
size: undefined,
modifiedAt: undefined,
};
}
}),
);
return {
directories: directories.sort((a, b) => a.name.localeCompare(b.name)),
path: normalizedPath,
};
};
export const testVolumeConnection = async (backendConfig: BackendConfig) => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-test-"));
try {
const mockVolume: AgentVolume = {
id: 0,
shortId: "test",
name: "test-connection",
config: backendConfig,
createdAt: Date.now(),
updatedAt: Date.now(),
lastHealthCheck: Date.now(),
type: backendConfig.backend,
status: "unmounted",
lastError: null,
provisioningId: null,
autoRemount: true,
agentId: "local",
organizationId: "test-org",
};
const backend = createVolumeBackend(mockVolume, tempDir);
const { error } = await backend.mount();
await backend.unmount();
return {
success: !error,
message: error ? toMessage(error) : "Connection successful",
};
} finally {
await fs.rm(tempDir, { recursive: true, force: true });
}
};

View file

@ -0,0 +1,10 @@
import { VOLUME_MOUNT_BASE } from "./constants";
import type { AgentVolume } from "./types";
export const getVolumePath = (volume: AgentVolume) => {
if (volume.config.backend === "directory") {
return volume.config.path;
}
return `${VOLUME_MOUNT_BASE}/${volume.shortId}/_data`;
};

View file

@ -0,0 +1,21 @@
class TimeoutError extends Error {
code = "ETIMEOUT";
constructor(message: string) {
super(message);
this.name = "TimeoutError";
}
}
export const withTimeout = async <T>(promise: Promise<T>, ms: number, label: string): Promise<T> => {
let timeout: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timeout = setTimeout(() => reject(new TimeoutError(`${label} timed out after ${ms}ms`)), ms);
}),
]);
} finally {
if (timeout) clearTimeout(timeout);
}
};

View file

@ -0,0 +1,67 @@
export type BackendStatus = "mounted" | "unmounted" | "error";
type BaseConfig = { backend: string; readOnly?: boolean };
export type BackendConfig =
| (BaseConfig & { backend: "directory"; path: string })
| (BaseConfig & { backend: "nfs"; server: string; exportPath: string; port: number; version: "3" | "4" | "4.1" })
| (BaseConfig & {
backend: "smb";
server: string;
share: string;
username?: string;
password?: string;
guest?: boolean;
vers?: "1.0" | "2.0" | "2.1" | "3.0" | "auto";
domain?: string;
port: number;
})
| (BaseConfig & {
backend: "webdav";
server: string;
path: string;
username?: string;
password?: string;
port: number;
ssl?: boolean;
})
| (BaseConfig & { backend: "rclone"; remote: string; path: string })
| (BaseConfig & {
backend: "sftp";
host: string;
port: number;
username: string;
password?: string;
privateKey?: string;
path: string;
skipHostKeyCheck?: boolean;
knownHosts?: string;
});
export type AgentVolume = {
id: number;
shortId: string;
name: string;
config: BackendConfig;
createdAt: number;
updatedAt: number;
lastHealthCheck: number;
type: string;
status: BackendStatus;
lastError: string | null;
provisioningId?: string | null;
autoRemount: boolean;
agentId: string;
organizationId: string;
};
export type OperationResult = {
status: BackendStatus;
error?: string;
};
export type VolumeBackend = {
mount: () => Promise<OperationResult>;
unmount: () => Promise<OperationResult>;
checkHealth: () => Promise<OperationResult>;
};

View file

@ -51,6 +51,111 @@ const backupCancelSchema = z.object({
payload: z.object({ jobId: z.string(), scheduleId: z.string() }), payload: z.object({ jobId: z.string(), scheduleId: z.string() }),
}); });
const backendStatusSchema = z.enum(["mounted", "unmounted", "error"]);
const volumeSchema = z.object({
id: z.number(),
shortId: z.string(),
name: z.string(),
path: z.string().nullable().optional(),
config: z.record(z.string(), z.unknown()).and(z.object({ backend: z.string() })),
createdAt: z.number(),
updatedAt: z.number(),
lastHealthCheck: z.number(),
type: z.string(),
status: backendStatusSchema,
lastError: z.string().nullable(),
provisioningId: z.string().nullable().optional(),
autoRemount: z.boolean(),
agentId: z.string(),
organizationId: z.string(),
});
const volumeOperationResultSchema = z.object({
status: backendStatusSchema,
error: z.string().optional(),
});
const statfsSchema = z.object({
total: z.number().optional(),
used: z.number().optional(),
free: z.number().optional(),
});
const fileEntrySchema = z.object({
name: z.string(),
path: z.string(),
type: z.enum(["directory", "file"]),
size: z.number().optional(),
modifiedAt: z.number().optional(),
});
const directoryEntrySchema = z.object({
name: z.string(),
path: z.string(),
type: z.literal("directory"),
size: z.undefined().optional(),
modifiedAt: z.number().optional(),
});
const volumeCommandSchema = z.discriminatedUnion("name", [
z.object({ name: z.literal("volume.mount"), volume: volumeSchema }),
z.object({ name: z.literal("volume.unmount"), volume: volumeSchema }),
z.object({ name: z.literal("volume.checkHealth"), volume: volumeSchema }),
z.object({ name: z.literal("volume.statfs"), volume: volumeSchema }),
z.object({
name: z.literal("volume.listFiles"),
volume: volumeSchema,
subPath: z.string().optional(),
offset: z.number(),
limit: z.number(),
}),
z.object({ name: z.literal("volume.testConnection"), backendConfig: z.record(z.string(), z.unknown()) }),
z.object({ name: z.literal("filesystem.browse"), path: z.string() }),
]);
const volumeCommandRequestSchema = z.object({
type: z.literal("volume.command"),
payload: z.object({
commandId: z.string(),
command: volumeCommandSchema,
}),
});
const volumeCommandResultSchema = z.discriminatedUnion("name", [
z.object({ name: z.literal("volume.mount"), result: volumeOperationResultSchema }),
z.object({ name: z.literal("volume.unmount"), result: volumeOperationResultSchema }),
z.object({ name: z.literal("volume.checkHealth"), result: volumeOperationResultSchema }),
z.object({ name: z.literal("volume.statfs"), result: statfsSchema }),
z.object({
name: z.literal("volume.listFiles"),
result: z.object({
files: z.array(fileEntrySchema),
path: z.string(),
offset: z.number(),
limit: z.number(),
total: z.number(),
hasMore: z.boolean(),
}),
}),
z.object({
name: z.literal("volume.testConnection"),
result: z.object({ success: z.boolean(), message: z.string() }),
}),
z.object({
name: z.literal("filesystem.browse"),
result: z.object({ directories: z.array(directoryEntrySchema), path: z.string() }),
}),
]);
const volumeCommandResponseSchema = z.object({
type: z.literal("volume.commandResult"),
payload: z.discriminatedUnion("status", [
z.object({ commandId: z.string(), status: z.literal("success"), command: volumeCommandResultSchema }),
z.object({ commandId: z.string(), status: z.literal("error"), error: z.string() }),
]),
});
const heartbeatPingSchema = z.object({ const heartbeatPingSchema = z.object({
type: z.literal("heartbeat.ping"), type: z.literal("heartbeat.ping"),
payload: z.object({ sentAt: z.number() }), payload: z.object({ sentAt: z.number() }),
@ -113,6 +218,7 @@ const heartbeatPongSchema = z.object({
const controllerMessageSchema = z.discriminatedUnion("type", [ const controllerMessageSchema = z.discriminatedUnion("type", [
backupRunSchema, backupRunSchema,
backupCancelSchema, backupCancelSchema,
volumeCommandRequestSchema,
heartbeatPingSchema, heartbeatPingSchema,
]); ]);
const agentMessageSchema = z.discriminatedUnion("type", [ const agentMessageSchema = z.discriminatedUnion("type", [
@ -122,6 +228,7 @@ const agentMessageSchema = z.discriminatedUnion("type", [
backupCompletedSchema, backupCompletedSchema,
backupFailedSchema, backupFailedSchema,
backupCancelledSchema, backupCancelledSchema,
volumeCommandResponseSchema,
heartbeatPongSchema, heartbeatPongSchema,
]); ]);
@ -132,6 +239,10 @@ export type BackupProgressPayload = z.infer<typeof backupProgressSchema>["payloa
export type BackupCompletedPayload = z.infer<typeof backupCompletedSchema>["payload"]; export type BackupCompletedPayload = z.infer<typeof backupCompletedSchema>["payload"];
export type BackupFailedPayload = z.infer<typeof backupFailedSchema>["payload"]; export type BackupFailedPayload = z.infer<typeof backupFailedSchema>["payload"];
export type BackupCancelledPayload = z.infer<typeof backupCancelledSchema>["payload"]; export type BackupCancelledPayload = z.infer<typeof backupCancelledSchema>["payload"];
export type VolumeCommandPayload = z.infer<typeof volumeCommandRequestSchema>["payload"];
export type VolumeCommand = z.infer<typeof volumeCommandSchema>;
export type VolumeCommandResult = z.infer<typeof volumeCommandResultSchema>;
export type VolumeCommandResponsePayload = z.infer<typeof volumeCommandResponseSchema>["payload"];
export type ControllerMessage = z.infer<typeof controllerMessageSchema>; export type ControllerMessage = z.infer<typeof controllerMessageSchema>;
export type AgentMessage = z.infer<typeof agentMessageSchema>; export type AgentMessage = z.infer<typeof agentMessageSchema>;