Compare commits
5 commits
main
...
04-16-feat
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
82c3eafc1c | ||
|
|
35a52554de | ||
|
|
188888ff9a | ||
|
|
a5d0cc986c | ||
|
|
0ce3f79ff1 |
51 changed files with 4637 additions and 1274 deletions
|
|
@ -0,0 +1,2 @@
|
||||||
|
ALTER TABLE `volumes_table` ADD `agent_id` text DEFAULT 'local' NOT NULL;--> statement-breakpoint
|
||||||
|
CREATE INDEX `volumes_table_agent_id_idx` ON `volumes_table` (`agent_id`);
|
||||||
2505
app/drizzle/20260416123510_supreme_stone_men/snapshot.json
Normal file
2505
app/drizzle/20260416123510_supreme_stone_men/snapshot.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -13,4 +13,4 @@ CREATE TABLE `agents_table` (
|
||||||
);
|
);
|
||||||
--> statement-breakpoint
|
--> statement-breakpoint
|
||||||
CREATE INDEX `agents_table_organization_id_idx` ON `agents_table` (`organization_id`);--> statement-breakpoint
|
CREATE INDEX `agents_table_organization_id_idx` ON `agents_table` (`organization_id`);--> statement-breakpoint
|
||||||
CREATE INDEX `agents_table_status_idx` ON `agents_table` (`status`);
|
CREATE INDEX `agents_table_status_idx` ON `agents_table` (`status`);
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ import type { BackupWebhooks } from "@zerobyte/core/backup-hooks";
|
||||||
import type { BackendConfig, BackendStatus, BackendType } from "~/schemas/volumes";
|
import type { BackendConfig, BackendStatus, BackendType } from "~/schemas/volumes";
|
||||||
import type { NotificationConfig, NotificationType } from "~/schemas/notifications";
|
import type { NotificationConfig, NotificationType } from "~/schemas/notifications";
|
||||||
import type { ShortId } from "~/server/utils/branded";
|
import type { ShortId } from "~/server/utils/branded";
|
||||||
|
import { LOCAL_AGENT_ID } from "../modules/agents/constants";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Users Table
|
* Users Table
|
||||||
|
|
@ -253,12 +254,14 @@ export const volumesTable = sqliteTable(
|
||||||
.default(sql`(unixepoch() * 1000)`),
|
.default(sql`(unixepoch() * 1000)`),
|
||||||
config: text("config", { mode: "json" }).$type<BackendConfig>().notNull(),
|
config: text("config", { mode: "json" }).$type<BackendConfig>().notNull(),
|
||||||
autoRemount: int("auto_remount", { mode: "boolean" }).notNull().default(true),
|
autoRemount: int("auto_remount", { mode: "boolean" }).notNull().default(true),
|
||||||
|
agentId: text("agent_id").notNull().default(LOCAL_AGENT_ID),
|
||||||
organizationId: text("organization_id")
|
organizationId: text("organization_id")
|
||||||
.notNull()
|
.notNull()
|
||||||
.references(() => organization.id, { onDelete: "cascade" }),
|
.references(() => organization.id, { onDelete: "cascade" }),
|
||||||
},
|
},
|
||||||
(table) => [
|
(table) => [
|
||||||
unique().on(table.name, table.organizationId),
|
unique().on(table.name, table.organizationId),
|
||||||
|
index("volumes_table_agent_id_idx").on(table.agentId),
|
||||||
uniqueIndex("volumes_table_org_provisioning_id_uidx").on(table.organizationId, table.provisioningId),
|
uniqueIndex("volumes_table_org_provisioning_id_uidx").on(table.organizationId, table.provisioningId),
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
|
|
||||||
18
app/server/jobs/__tests__/cleanup-dangling.test.ts
Normal file
18
app/server/jobs/__tests__/cleanup-dangling.test.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
import { afterEach, expect, test, vi } from "vitest";
|
||||||
|
import { config } from "../../core/config";
|
||||||
|
import { CleanupDanglingMountsJob } from "../cleanup-dangling";
|
||||||
|
import * as mountinfo from "../../utils/mountinfo";
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
config.flags.enableLocalAgent = true;
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("skips controller-local mount inspection when local volume execution is agent-owned", async () => {
|
||||||
|
config.flags.enableLocalAgent = true;
|
||||||
|
const readMountInfo = vi.spyOn(mountinfo, "readMountInfo");
|
||||||
|
|
||||||
|
await new CleanupDanglingMountsJob().run();
|
||||||
|
|
||||||
|
expect(readMountInfo).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
@ -5,14 +5,21 @@ import { volumeService } from "../modules/volumes/volume.service";
|
||||||
import { readMountInfo } from "../utils/mountinfo";
|
import { readMountInfo } from "../utils/mountinfo";
|
||||||
import { getVolumePath } from "../modules/volumes/helpers";
|
import { getVolumePath } from "../modules/volumes/helpers";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { executeUnmount } from "../modules/backends/utils/backend-utils";
|
import { executeUnmount } from "../../../apps/agent/src/volume-host/backends/utils";
|
||||||
import { toMessage } from "../utils/errors";
|
import { toMessage } from "../utils/errors";
|
||||||
import { VOLUME_MOUNT_BASE } from "../core/constants";
|
import { VOLUME_MOUNT_BASE } from "../core/constants";
|
||||||
import { db } from "../db/db";
|
import { db } from "../db/db";
|
||||||
import { withContext } from "../core/request-context";
|
import { withContext } from "../core/request-context";
|
||||||
|
import { config } from "../core/config";
|
||||||
|
import { LOCAL_AGENT_ID } from "../modules/agents/constants";
|
||||||
|
|
||||||
export class CleanupDanglingMountsJob extends Job {
|
export class CleanupDanglingMountsJob extends Job {
|
||||||
async run() {
|
async run() {
|
||||||
|
if (config.flags.enableLocalAgent) {
|
||||||
|
logger.debug("Skipping controller-local dangling mount cleanup because local volume execution is agent-owned.");
|
||||||
|
return { done: true, timestamp: new Date() };
|
||||||
|
}
|
||||||
|
|
||||||
const organizations = await db.query.organization.findMany({});
|
const organizations = await db.query.organization.findMany({});
|
||||||
if (organizations.length === 0) {
|
if (organizations.length === 0) {
|
||||||
logger.warn("No organizations found; skipping dangling mount cleanup to avoid false positives.");
|
logger.warn("No organizations found; skipping dangling mount cleanup to avoid false positives.");
|
||||||
|
|
@ -22,7 +29,7 @@ export class CleanupDanglingMountsJob extends Job {
|
||||||
const allVolumes = [];
|
const allVolumes = [];
|
||||||
for (const org of organizations) {
|
for (const org of organizations) {
|
||||||
const volumes = await withContext({ organizationId: org.id }, async () => volumeService.listVolumes());
|
const volumes = await withContext({ organizationId: org.id }, async () => volumeService.listVolumes());
|
||||||
allVolumes.push(...volumes);
|
allVolumes.push(...volumes.filter((volume) => volume.agentId === LOCAL_AGENT_ID));
|
||||||
}
|
}
|
||||||
|
|
||||||
const allSystemMounts = await readMountInfo();
|
const allSystemMounts = await readMountInfo();
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import { fromAny, fromPartial } from "@total-typescript/shoehorn";
|
||||||
import { Effect } from "effect";
|
import { Effect } from "effect";
|
||||||
import { agentManager, type ProcessWithAgentRuntime } from "../agents-manager";
|
import { agentManager, type ProcessWithAgentRuntime } from "../agents-manager";
|
||||||
import type { AgentManagerRuntime } from "../controller/server";
|
import type { AgentManagerRuntime } from "../controller/server";
|
||||||
import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
|
import type { BackupRunPayload, VolumeCommand } from "@zerobyte/contracts/agent-protocol";
|
||||||
|
|
||||||
const setAgentRuntime = (agentManagerRuntime: Partial<AgentManagerRuntime> | null) => {
|
const setAgentRuntime = (agentManagerRuntime: Partial<AgentManagerRuntime> | null) => {
|
||||||
(process as ProcessWithAgentRuntime).__zerobyteAgentRuntime = {
|
(process as ProcessWithAgentRuntime).__zerobyteAgentRuntime = {
|
||||||
|
|
@ -46,3 +46,32 @@ test("cancelBackup resolves a running backup when the cancel command cannot be d
|
||||||
scheduleId: "schedule-1",
|
scheduleId: "schedule-1",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("runVolumeCommand sends the command to the selected agent", async () => {
|
||||||
|
const runVolumeCommand = vi.fn(() =>
|
||||||
|
Effect.succeed({
|
||||||
|
commandId: "command-1",
|
||||||
|
status: "success" as const,
|
||||||
|
command: { name: "volume.mount" as const, result: { status: "mounted" as const } },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
setAgentRuntime({ runVolumeCommand });
|
||||||
|
|
||||||
|
const command = fromPartial<VolumeCommand>({ name: "volume.mount", volume: { agentId: "agent-1" } });
|
||||||
|
|
||||||
|
await expect(agentManager.runVolumeCommand("agent-1", command)).resolves.toEqual({
|
||||||
|
name: "volume.mount",
|
||||||
|
result: { status: "mounted" },
|
||||||
|
});
|
||||||
|
expect(runVolumeCommand).toHaveBeenCalledWith("agent-1", command);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("runVolumeCommand fails when the selected agent is unavailable", async () => {
|
||||||
|
setAgentRuntime(null);
|
||||||
|
|
||||||
|
const command = fromPartial<VolumeCommand>({ name: "volume.mount", volume: { agentId: "agent-1" } });
|
||||||
|
|
||||||
|
await expect(agentManager.runVolumeCommand("agent-1", command)).rejects.toThrow(
|
||||||
|
"Volume agent agent-1 is not connected",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,23 @@
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
|
import type { BackupRunPayload, VolumeCommand, VolumeCommandResult } from "@zerobyte/contracts/agent-protocol";
|
||||||
import { Effect } from "effect";
|
import { Effect } from "effect";
|
||||||
import { config } from "../../core/config";
|
import { config } from "../../core/config";
|
||||||
import { createAgentManagerRuntime, type AgentManagerEvent } from "./controller/server";
|
import { createAgentManagerRuntime, type AgentManagerEvent } from "./controller/server";
|
||||||
|
import { LOCAL_AGENT_ID } from "./constants";
|
||||||
import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process";
|
import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process";
|
||||||
import type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
|
import {
|
||||||
import { createAgentRuntimeState } from "./helpers/runtime-state";
|
createAgentRuntimeState,
|
||||||
|
type AgentRuntimeState,
|
||||||
|
type BackupExecutionProgress,
|
||||||
|
type BackupExecutionResult,
|
||||||
|
} from "./helpers/runtime-state";
|
||||||
import { getDevAgentRuntimeState } from "./helpers/runtime-state.dev";
|
import { getDevAgentRuntimeState } from "./helpers/runtime-state.dev";
|
||||||
export type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
|
export type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
|
||||||
export type { ProcessWithAgentRuntime } from "./helpers/runtime-state.dev";
|
export type { ProcessWithAgentRuntime } from "./helpers/runtime-state.dev";
|
||||||
|
|
||||||
const productionRuntimeState = createAgentRuntimeState();
|
type ProcessWithProductionAgentRuntime = NodeJS.Process & {
|
||||||
|
__zerobyteProductionAgentRuntime?: AgentRuntimeState;
|
||||||
|
};
|
||||||
|
|
||||||
type AgentRunBackupRequest = {
|
type AgentRunBackupRequest = {
|
||||||
scheduleId: number;
|
scheduleId: number;
|
||||||
|
|
@ -19,7 +26,18 @@ type AgentRunBackupRequest = {
|
||||||
onProgress: (progress: BackupExecutionProgress) => void;
|
onProgress: (progress: BackupExecutionProgress) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
const getAgentRuntimeState = () => (config.__prod__ ? productionRuntimeState : getDevAgentRuntimeState());
|
const getProductionAgentRuntimeState = () => {
|
||||||
|
// Nitro production builds can bundle startup plugins and API handlers into separate chunks.
|
||||||
|
// Keep the live controller on process so both chunks see the same agent sessions.
|
||||||
|
const runtimeProcess = process as ProcessWithProductionAgentRuntime;
|
||||||
|
if (!runtimeProcess.__zerobyteProductionAgentRuntime) {
|
||||||
|
runtimeProcess.__zerobyteProductionAgentRuntime = createAgentRuntimeState();
|
||||||
|
}
|
||||||
|
|
||||||
|
return runtimeProcess.__zerobyteProductionAgentRuntime;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getAgentRuntimeState = () => (config.__prod__ ? getProductionAgentRuntimeState() : getDevAgentRuntimeState());
|
||||||
const getAgentManagerRuntime = () => getAgentRuntimeState().agentManager;
|
const getAgentManagerRuntime = () => getAgentRuntimeState().agentManager;
|
||||||
const getActiveBackupsByScheduleId = () => getAgentRuntimeState().activeBackupsByScheduleId;
|
const getActiveBackupsByScheduleId = () => getAgentRuntimeState().activeBackupsByScheduleId;
|
||||||
const getActiveBackupScheduleIdsByJobId = () => getAgentRuntimeState().activeBackupScheduleIdsByJobId;
|
const getActiveBackupScheduleIdsByJobId = () => getAgentRuntimeState().activeBackupScheduleIdsByJobId;
|
||||||
|
|
@ -266,10 +284,36 @@ export const agentManager = {
|
||||||
cancelBackup: async (agentId: string, scheduleId: number) => {
|
cancelBackup: async (agentId: string, scheduleId: number) => {
|
||||||
return requestBackupCancellation(agentId, scheduleId);
|
return requestBackupCancellation(agentId, scheduleId);
|
||||||
},
|
},
|
||||||
|
runVolumeCommand: async (agentId: string, command: VolumeCommand): Promise<VolumeCommandResult> => {
|
||||||
|
const runtime = getAgentManagerRuntime();
|
||||||
|
if (!runtime) {
|
||||||
|
throw new Error(`Volume agent ${agentId} is not connected`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await Effect.runPromise(runtime.runVolumeCommand(agentId, command));
|
||||||
|
if (!response) {
|
||||||
|
throw new Error(`Failed to send volume command ${command.name} to agent ${agentId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.status === "error") {
|
||||||
|
throw new Error(response.error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.command;
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
export const startLocalAgent = async () => {
|
export const startLocalAgent = async () => {
|
||||||
await spawnLocalAgentProcess(getAgentRuntimeState());
|
const runtime = getAgentRuntimeState();
|
||||||
|
await spawnLocalAgentProcess(runtime);
|
||||||
|
|
||||||
|
if (!runtime.agentManager) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await runtime.agentManager.waitForAgentReady(LOCAL_AGENT_ID))) {
|
||||||
|
throw new Error("Local agent did not become ready before startup");
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// fallow-ignore-next-line unused-export
|
// fallow-ignore-next-line unused-export
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,13 @@
|
||||||
import { Data, Effect, Exit, Fiber, Scope } from "effect";
|
import { Data, Effect, Exit, Fiber, Scope } from "effect";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { toMessage } from "@zerobyte/core/utils";
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
import type { AgentMessage, BackupCancelPayload, BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
|
import type {
|
||||||
|
AgentMessage,
|
||||||
|
BackupCancelPayload,
|
||||||
|
BackupRunPayload,
|
||||||
|
VolumeCommand,
|
||||||
|
VolumeCommandResponsePayload,
|
||||||
|
} from "@zerobyte/contracts/agent-protocol";
|
||||||
import {
|
import {
|
||||||
createControllerAgentSession,
|
createControllerAgentSession,
|
||||||
type AgentConnectionData,
|
type AgentConnectionData,
|
||||||
|
|
@ -40,6 +46,7 @@ class StopAgentManagerServerError extends Data.TaggedError("StopAgentManagerServ
|
||||||
export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) => void) {
|
export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) => void) {
|
||||||
let sessions = new Map<string, ControllerAgentSessionHandle>();
|
let sessions = new Map<string, ControllerAgentSessionHandle>();
|
||||||
let runtimeScope: Scope.CloseableScope | null = null;
|
let runtimeScope: Scope.CloseableScope | null = null;
|
||||||
|
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
const closeSession = (sessionHandle: ControllerAgentSessionHandle) =>
|
const closeSession = (sessionHandle: ControllerAgentSessionHandle) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
|
|
@ -73,6 +80,11 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
|
||||||
const getSessionHandle = (agentId: string) => sessions.get(agentId);
|
const getSessionHandle = (agentId: string) => sessions.get(agentId);
|
||||||
const getSession = (agentId: string) => getSessionHandle(agentId)?.session;
|
const getSession = (agentId: string) => getSessionHandle(agentId)?.session;
|
||||||
|
|
||||||
|
const isAgentReady = (agentId: string) => {
|
||||||
|
const session = getSession(agentId);
|
||||||
|
return !!session && Effect.runSync(session.isReady());
|
||||||
|
};
|
||||||
|
|
||||||
const handleSessionEvent = (params: { agentId: string; agentName: string; sessionId: string }) => {
|
const handleSessionEvent = (params: { agentId: string; agentName: string; sessionId: string }) => {
|
||||||
const { agentId, agentName } = params;
|
const { agentId, agentName } = params;
|
||||||
|
|
||||||
|
|
@ -284,6 +296,19 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
|
||||||
|
|
||||||
return {
|
return {
|
||||||
start,
|
start,
|
||||||
|
waitForAgentReady: async (agentId: string, timeoutMs = 10_000) => {
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
if (isAgentReady(agentId)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
await sleep(50);
|
||||||
|
}
|
||||||
|
|
||||||
|
return isAgentReady(agentId);
|
||||||
|
},
|
||||||
sendBackup: (agentId: string, payload: BackupRunPayload) =>
|
sendBackup: (agentId: string, payload: BackupRunPayload) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const session = getSession(agentId);
|
const session = getSession(agentId);
|
||||||
|
|
@ -319,10 +344,30 @@ export function createAgentManagerRuntime(onEvent: (event: AgentManagerEvent) =>
|
||||||
logger.warn(`Cannot cancel backup command. Agent ${agentId} is no longer accepting commands.`);
|
logger.warn(`Cannot cancel backup command. Agent ${agentId} is no longer accepting commands.`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`Sent backup cancel for command ${payload.jobId} to agent ${agentId}`);
|
logger.info(`Sent backup cancel for command ${payload.jobId} to agent ${agentId}`);
|
||||||
return true;
|
return true;
|
||||||
}),
|
}),
|
||||||
|
runVolumeCommand: (
|
||||||
|
agentId: string,
|
||||||
|
command: VolumeCommand,
|
||||||
|
): Effect.Effect<VolumeCommandResponsePayload | null, Error> =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const session = getSession(agentId);
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
yield* logger.effect.warn(`Cannot send volume command ${command.name}. Agent ${agentId} is not connected.`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(yield* session.isReady())) {
|
||||||
|
yield* logger.effect.warn(`Cannot send volume command ${command.name}. Agent ${agentId} is not ready.`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = yield* session.runVolumeCommand(command);
|
||||||
|
yield* logger.effect.info(`Completed volume command ${command.name} on agent ${agentId}`);
|
||||||
|
return result;
|
||||||
|
}),
|
||||||
stop,
|
stop,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,8 @@ import {
|
||||||
type BackupCancelPayload,
|
type BackupCancelPayload,
|
||||||
type BackupRunPayload,
|
type BackupRunPayload,
|
||||||
type ControllerWireMessage,
|
type ControllerWireMessage,
|
||||||
|
type VolumeCommand,
|
||||||
|
type VolumeCommandResponsePayload,
|
||||||
} from "@zerobyte/contracts/agent-protocol";
|
} from "@zerobyte/contracts/agent-protocol";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { toMessage } from "@zerobyte/core/utils";
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
|
@ -27,13 +29,24 @@ type SessionState = {
|
||||||
lastPongAt: number | null;
|
lastPongAt: number | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type ControllerAgentSessionEvent = AgentMessage | { type: "agent.disconnected" };
|
type PendingVolumeCommand = {
|
||||||
|
resolve: (payload: VolumeCommandResponsePayload) => void;
|
||||||
|
reject: (error: Error) => void;
|
||||||
|
timeout: ReturnType<typeof setTimeout>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ControllerAgentSessionEvent =
|
||||||
|
| Exclude<AgentMessage, { type: "volume.commandResult" }>
|
||||||
|
| {
|
||||||
|
type: "agent.disconnected";
|
||||||
|
};
|
||||||
|
|
||||||
export type ControllerAgentSession = {
|
export type ControllerAgentSession = {
|
||||||
readonly connectionId: string;
|
readonly connectionId: string;
|
||||||
handleMessage: (data: string) => Effect.Effect<void>;
|
handleMessage: (data: string) => Effect.Effect<void>;
|
||||||
sendBackup: (payload: BackupRunPayload) => Effect.Effect<boolean>;
|
sendBackup: (payload: BackupRunPayload) => Effect.Effect<boolean>;
|
||||||
sendBackupCancel: (payload: BackupCancelPayload) => Effect.Effect<boolean>;
|
sendBackupCancel: (payload: BackupCancelPayload) => Effect.Effect<boolean>;
|
||||||
|
runVolumeCommand: (command: VolumeCommand) => Effect.Effect<VolumeCommandResponsePayload, Error>;
|
||||||
isReady: () => Effect.Effect<boolean>;
|
isReady: () => Effect.Effect<boolean>;
|
||||||
run: Effect.Effect<void, never, Scope.Scope>;
|
run: Effect.Effect<void, never, Scope.Scope>;
|
||||||
};
|
};
|
||||||
|
|
@ -45,6 +58,7 @@ export const createControllerAgentSession = (
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
let isClosed = false;
|
let isClosed = false;
|
||||||
const outboundQueue = yield* Queue.bounded<ControllerWireMessage>(64);
|
const outboundQueue = yield* Queue.bounded<ControllerWireMessage>(64);
|
||||||
|
const pendingVolumeCommands = new Map<string, PendingVolumeCommand>();
|
||||||
const state = yield* Ref.make<SessionState>({
|
const state = yield* Ref.make<SessionState>({
|
||||||
isReady: false,
|
isReady: false,
|
||||||
lastSeenAt: null,
|
lastSeenAt: null,
|
||||||
|
|
@ -63,9 +77,18 @@ export const createControllerAgentSession = (
|
||||||
|
|
||||||
const updateState = (update: (current: SessionState) => SessionState) => Ref.update(state, update);
|
const updateState = (update: (current: SessionState) => SessionState) => Ref.update(state, update);
|
||||||
|
|
||||||
|
const rejectPendingVolumeCommands = () => {
|
||||||
|
for (const [commandId, pending] of pendingVolumeCommands) {
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
pending.reject(new Error(`Agent session closed before volume command ${commandId} completed`));
|
||||||
|
}
|
||||||
|
pendingVolumeCommands.clear();
|
||||||
|
};
|
||||||
|
|
||||||
const releaseSession = Effect.gen(function* () {
|
const releaseSession = Effect.gen(function* () {
|
||||||
const disconnectedAt = Date.now();
|
const disconnectedAt = Date.now();
|
||||||
yield* updateState((current) => ({ ...current, isReady: false, lastSeenAt: disconnectedAt }));
|
yield* updateState((current) => ({ ...current, isReady: false, lastSeenAt: disconnectedAt }));
|
||||||
|
yield* Effect.sync(rejectPendingVolumeCommands);
|
||||||
yield* onEvent({ type: "agent.disconnected" });
|
yield* onEvent({ type: "agent.disconnected" });
|
||||||
|
|
||||||
yield* Queue.shutdown(outboundQueue);
|
yield* Queue.shutdown(outboundQueue);
|
||||||
|
|
@ -129,20 +152,43 @@ export const createControllerAgentSession = (
|
||||||
return yield* Effect.never;
|
return yield* Effect.never;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const handleVolumeCommandResult = (payload: VolumeCommandResponsePayload) => {
|
||||||
|
const pending = pendingVolumeCommands.get(payload.commandId);
|
||||||
|
if (!pending) {
|
||||||
|
logger.warn(`Received response for unknown volume command ${payload.commandId}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
pendingVolumeCommands.delete(payload.commandId);
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
pending.resolve(payload);
|
||||||
|
};
|
||||||
|
|
||||||
const handleAgentMessage = (message: AgentMessage) =>
|
const handleAgentMessage = (message: AgentMessage) =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
if (message.type === "agent.ready") {
|
switch (message.type) {
|
||||||
const readyAt = Date.now();
|
case "agent.ready": {
|
||||||
yield* updateState((current) => ({ ...current, isReady: true, lastSeenAt: readyAt }));
|
const readyAt = Date.now();
|
||||||
yield* logger.effect.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
|
yield* updateState((current) => ({ ...current, isReady: true, lastSeenAt: readyAt }));
|
||||||
|
yield* logger.effect.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
|
||||||
|
yield* onEvent(message);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "heartbeat.pong": {
|
||||||
|
const seenAt = Date.now();
|
||||||
|
yield* updateState((current) => ({ ...current, lastSeenAt: seenAt, lastPongAt: message.payload.sentAt }));
|
||||||
|
yield* onEvent(message);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "volume.commandResult": {
|
||||||
|
yield* Effect.sync(() => handleVolumeCommandResult(message.payload));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default: {
|
||||||
|
yield* onEvent(message);
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (message.type === "heartbeat.pong") {
|
|
||||||
const seenAt = Date.now();
|
|
||||||
yield* updateState((current) => ({ ...current, lastSeenAt: seenAt, lastPongAt: message.payload.sentAt }));
|
|
||||||
}
|
|
||||||
|
|
||||||
yield* onEvent(message);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
@ -166,6 +212,37 @@ export const createControllerAgentSession = (
|
||||||
},
|
},
|
||||||
sendBackup: (payload) => offerOutbound(createControllerMessage("backup.run", payload)),
|
sendBackup: (payload) => offerOutbound(createControllerMessage("backup.run", payload)),
|
||||||
sendBackupCancel: (payload) => offerOutbound(createControllerMessage("backup.cancel", payload)),
|
sendBackupCancel: (payload) => offerOutbound(createControllerMessage("backup.cancel", payload)),
|
||||||
|
runVolumeCommand: (command) => {
|
||||||
|
return Effect.tryPromise({
|
||||||
|
try: async () => {
|
||||||
|
const commandId = Bun.randomUUIDv7();
|
||||||
|
const response = new Promise<VolumeCommandResponsePayload>((resolve, reject) => {
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
pendingVolumeCommands.delete(commandId);
|
||||||
|
reject(new Error(`Volume command ${command.name} timed out`));
|
||||||
|
}, 60_000);
|
||||||
|
|
||||||
|
pendingVolumeCommands.set(commandId, { resolve, reject, timeout });
|
||||||
|
});
|
||||||
|
|
||||||
|
const queued = await Effect.runPromise(
|
||||||
|
offerOutbound(createControllerMessage("volume.command", { commandId, command })),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!queued) {
|
||||||
|
const pending = pendingVolumeCommands.get(commandId);
|
||||||
|
if (pending) {
|
||||||
|
clearTimeout(pending.timeout);
|
||||||
|
pendingVolumeCommands.delete(commandId);
|
||||||
|
}
|
||||||
|
throw new Error(`Failed to queue volume command ${command.name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response;
|
||||||
|
},
|
||||||
|
catch: (error) => (error instanceof Error ? error : new Error(toMessage(error))),
|
||||||
|
});
|
||||||
|
},
|
||||||
isReady: () => Ref.get(state).pipe(Effect.map((current) => current.isReady)),
|
isReady: () => Ref.get(state).pipe(Effect.map((current) => current.isReady)),
|
||||||
run,
|
run,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
import type { BackendStatus } from "~/schemas/volumes";
|
|
||||||
import type { Volume } from "../../db/schema";
|
|
||||||
import { getVolumePath } from "../volumes/helpers";
|
|
||||||
import { makeDirectoryBackend } from "./directory/directory-backend";
|
|
||||||
import { makeNfsBackend } from "./nfs/nfs-backend";
|
|
||||||
import { makeRcloneBackend } from "./rclone/rclone-backend";
|
|
||||||
import { makeSmbBackend } from "./smb/smb-backend";
|
|
||||||
import { makeWebdavBackend } from "./webdav/webdav-backend";
|
|
||||||
import { makeSftpBackend } from "./sftp/sftp-backend";
|
|
||||||
|
|
||||||
type OperationResult = {
|
|
||||||
error?: string;
|
|
||||||
status: BackendStatus;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type VolumeBackend = {
|
|
||||||
mount: () => Promise<OperationResult>;
|
|
||||||
unmount: () => Promise<OperationResult>;
|
|
||||||
checkHealth: () => Promise<OperationResult>;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const createVolumeBackend = (volume: Volume, mountPath = getVolumePath(volume)): VolumeBackend => {
|
|
||||||
switch (volume.config.backend) {
|
|
||||||
case "nfs": {
|
|
||||||
return makeNfsBackend(volume.config, mountPath);
|
|
||||||
}
|
|
||||||
case "smb": {
|
|
||||||
return makeSmbBackend(volume.config, mountPath);
|
|
||||||
}
|
|
||||||
case "directory": {
|
|
||||||
return makeDirectoryBackend(volume.config, mountPath);
|
|
||||||
}
|
|
||||||
case "webdav": {
|
|
||||||
return makeWebdavBackend(volume.config, mountPath);
|
|
||||||
}
|
|
||||||
case "rclone": {
|
|
||||||
return makeRcloneBackend(volume.config, mountPath);
|
|
||||||
}
|
|
||||||
case "sftp": {
|
|
||||||
return makeSftpBackend(volume.config, mountPath);
|
|
||||||
}
|
|
||||||
default: {
|
|
||||||
throw new Error("Unsupported backend");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
@ -1,135 +0,0 @@
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
|
|
||||||
import { OPERATION_TIMEOUT } from "../../../core/constants";
|
|
||||||
import { toMessage } from "../../../utils/errors";
|
|
||||||
import { logger } from "@zerobyte/core/node";
|
|
||||||
import { getMountForPath } from "../../../utils/mountinfo";
|
|
||||||
import { withTimeout } from "../../../utils/timeout";
|
|
||||||
import type { VolumeBackend } from "../backend";
|
|
||||||
import { assertMounted, executeMount, executeUnmount } from "../utils/backend-utils";
|
|
||||||
|
|
||||||
const mount = async (config: BackendConfig, path: string) => {
|
|
||||||
logger.debug(`Mounting volume ${path}...`);
|
|
||||||
|
|
||||||
if (config.backend !== "nfs") {
|
|
||||||
logger.error("Provided config is not for NFS backend");
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: "Provided config is not for NFS backend",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("NFS mounting is only supported on Linux hosts.");
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: "NFS mounting is only supported on Linux hosts.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const { status } = await checkHealth(path);
|
|
||||||
if (status === "mounted") {
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status === "error") {
|
|
||||||
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
|
|
||||||
await unmount(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
await fs.mkdir(path, { recursive: true });
|
|
||||||
|
|
||||||
const source = `${config.server}:${config.exportPath}`;
|
|
||||||
const options = [`vers=${config.version}`, `port=${config.port}`];
|
|
||||||
if (config.version === "3") {
|
|
||||||
options.push("nolock");
|
|
||||||
}
|
|
||||||
if (config.readOnly) {
|
|
||||||
options.push("ro");
|
|
||||||
}
|
|
||||||
const args = ["-t", "nfs", "-o", options.join(","), source, path];
|
|
||||||
|
|
||||||
logger.debug(`Mounting volume ${path}...`);
|
|
||||||
logger.info(`Executing mount: mount ${args.join(" ")}`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await executeMount(args);
|
|
||||||
} catch (error) {
|
|
||||||
logger.warn(`Initial NFS mount failed, retrying with -i flag: ${toMessage(error)}`);
|
|
||||||
// Fallback with -i flag if the first mount fails using the mount helper
|
|
||||||
await executeMount(["-i", ...args]);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`NFS volume at ${path} mounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS mount");
|
|
||||||
} catch (err) {
|
|
||||||
logger.error("Error mounting NFS volume", { error: toMessage(err) });
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(err) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const unmount = async (path: string) => {
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("NFS unmounting is only supported on Linux hosts.");
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: "NFS unmounting is only supported on Linux hosts.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
const mount = await getMountForPath(path);
|
|
||||||
if (!mount || mount.mountPoint !== path) {
|
|
||||||
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
await executeUnmount(path);
|
|
||||||
|
|
||||||
await fs.rmdir(path).catch(() => {});
|
|
||||||
|
|
||||||
logger.info(`NFS volume at ${path} unmounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS unmount");
|
|
||||||
} catch (err) {
|
|
||||||
logger.error("Error unmounting NFS volume", {
|
|
||||||
path,
|
|
||||||
error: toMessage(err),
|
|
||||||
});
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(err) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const checkHealth = async (path: string) => {
|
|
||||||
const run = async () => {
|
|
||||||
await assertMounted(path, (fstype) => fstype.startsWith("nfs"));
|
|
||||||
|
|
||||||
logger.debug(`NFS volume at ${path} is healthy and mounted.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check");
|
|
||||||
} catch (error) {
|
|
||||||
const message = toMessage(error);
|
|
||||||
if (message !== "Volume is not mounted") {
|
|
||||||
logger.error("NFS volume health check failed:", message);
|
|
||||||
}
|
|
||||||
return { status: BACKEND_STATUS.error, error: message };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const makeNfsBackend = (config: BackendConfig, path: string): VolumeBackend => ({
|
|
||||||
mount: () => mount(config, path),
|
|
||||||
unmount: () => unmount(path),
|
|
||||||
checkHealth: () => checkHealth(path),
|
|
||||||
});
|
|
||||||
|
|
@ -1,131 +0,0 @@
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import { OPERATION_TIMEOUT, RCLONE_CONFIG_FILE } from "../../../core/constants";
|
|
||||||
import { toMessage } from "../../../utils/errors";
|
|
||||||
import { logger } from "@zerobyte/core/node";
|
|
||||||
import { getMountForPath } from "../../../utils/mountinfo";
|
|
||||||
import { withTimeout } from "../../../utils/timeout";
|
|
||||||
import type { VolumeBackend } from "../backend";
|
|
||||||
import { assertMounted, executeUnmount } from "../utils/backend-utils";
|
|
||||||
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
|
|
||||||
import { safeExec } from "@zerobyte/core/node";
|
|
||||||
import { config as zbConfig } from "~/server/core/config";
|
|
||||||
|
|
||||||
const mount = async (config: BackendConfig, path: string) => {
|
|
||||||
logger.debug(`Mounting rclone volume ${path}...`);
|
|
||||||
|
|
||||||
if (config.backend !== "rclone") {
|
|
||||||
logger.error("Provided config is not for rclone backend");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "Provided config is not for rclone backend" };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("Rclone mounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "Rclone mounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const { status } = await checkHealth(path);
|
|
||||||
if (status === "mounted") {
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status === "error") {
|
|
||||||
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
|
|
||||||
await unmount(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
await fs.mkdir(path, { recursive: true });
|
|
||||||
|
|
||||||
const remotePath = `${config.remote}:${config.path}`;
|
|
||||||
const args = ["mount", remotePath, path, "--daemon"];
|
|
||||||
|
|
||||||
if (config.readOnly) {
|
|
||||||
args.push("--read-only");
|
|
||||||
}
|
|
||||||
|
|
||||||
args.push("--vfs-cache-mode", "writes");
|
|
||||||
args.push("--allow-non-empty");
|
|
||||||
args.push("--allow-other");
|
|
||||||
|
|
||||||
logger.debug(`Mounting rclone volume ${path}...`);
|
|
||||||
logger.info(`Executing rclone: rclone ${args.join(" ")}`);
|
|
||||||
|
|
||||||
const result = await safeExec({
|
|
||||||
command: "rclone",
|
|
||||||
args,
|
|
||||||
env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
|
|
||||||
timeout: zbConfig.serverIdleTimeout * 1000,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (result.exitCode !== 0) {
|
|
||||||
const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error";
|
|
||||||
throw new Error(`Failed to mount rclone volume: ${errorMsg}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`Rclone volume at ${path} mounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), zbConfig.serverIdleTimeout * 1000, "Rclone mount");
|
|
||||||
} catch (error) {
|
|
||||||
const errorMsg = toMessage(error);
|
|
||||||
|
|
||||||
logger.error("Error mounting rclone volume", { error: errorMsg });
|
|
||||||
return { status: BACKEND_STATUS.error, error: errorMsg };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const unmount = async (path: string) => {
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("Rclone unmounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "Rclone unmounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
const mount = await getMountForPath(path);
|
|
||||||
if (!mount || mount.mountPoint !== path) {
|
|
||||||
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
await executeUnmount(path);
|
|
||||||
await fs.rmdir(path).catch(() => {});
|
|
||||||
|
|
||||||
logger.info(`Rclone volume at ${path} unmounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone unmount");
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error unmounting rclone volume", { path, error: toMessage(error) });
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const checkHealth = async (path: string) => {
|
|
||||||
const run = async () => {
|
|
||||||
await assertMounted(path, (fstype) => fstype.includes("rclone"));
|
|
||||||
|
|
||||||
logger.debug(`Rclone volume at ${path} is healthy and mounted.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone health check");
|
|
||||||
} catch (error) {
|
|
||||||
const message = toMessage(error);
|
|
||||||
if (message !== "Volume is not mounted") {
|
|
||||||
logger.error("Rclone volume health check failed:", message);
|
|
||||||
}
|
|
||||||
return { status: BACKEND_STATUS.error, error: message };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const makeRcloneBackend = (config: BackendConfig, path: string): VolumeBackend => ({
|
|
||||||
mount: () => mount(config, path),
|
|
||||||
unmount: () => unmount(path),
|
|
||||||
checkHealth: () => checkHealth(path),
|
|
||||||
});
|
|
||||||
|
|
@ -1,212 +0,0 @@
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import * as path from "node:path";
|
|
||||||
import { spawn } from "node:child_process";
|
|
||||||
import { OPERATION_TIMEOUT, SSH_KEYS_DIR } from "../../../core/constants";
|
|
||||||
import { cryptoUtils } from "../../../utils/crypto";
|
|
||||||
import { toMessage } from "../../../utils/errors";
|
|
||||||
import { logger, FILE_MODES, writeFileWithMode } from "@zerobyte/core/node";
|
|
||||||
import { getMountForPath } from "../../../utils/mountinfo";
|
|
||||||
import { withTimeout } from "../../../utils/timeout";
|
|
||||||
import type { VolumeBackend } from "../backend";
|
|
||||||
import { executeUnmount } from "../utils/backend-utils";
|
|
||||||
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
|
|
||||||
|
|
||||||
const getPrivateKeyPath = (mountPath: string) => {
|
|
||||||
const name = path.basename(mountPath);
|
|
||||||
return path.join(SSH_KEYS_DIR, `${name}.key`);
|
|
||||||
};
|
|
||||||
|
|
||||||
const getKnownHostsPath = (mountPath: string) => {
|
|
||||||
const name = path.basename(mountPath);
|
|
||||||
return path.join(SSH_KEYS_DIR, `${name}.known_hosts`);
|
|
||||||
};
|
|
||||||
|
|
||||||
const mount = async (config: BackendConfig, mountPath: string) => {
|
|
||||||
logger.debug(`Mounting SFTP volume ${mountPath}...`);
|
|
||||||
|
|
||||||
if (config.backend !== "sftp") {
|
|
||||||
logger.error("Provided config is not for SFTP backend");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "Provided config is not for SFTP backend" };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("SFTP mounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "SFTP mounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const { status } = await checkHealth(mountPath);
|
|
||||||
if (status === "mounted") {
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status === "error") {
|
|
||||||
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
|
|
||||||
await unmount(mountPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
await fs.mkdir(mountPath, { recursive: true });
|
|
||||||
await fs.mkdir(SSH_KEYS_DIR, { recursive: true });
|
|
||||||
|
|
||||||
const { uid, gid } = os.userInfo();
|
|
||||||
const options = [
|
|
||||||
"reconnect",
|
|
||||||
"ServerAliveInterval=15",
|
|
||||||
"ServerAliveCountMax=3",
|
|
||||||
"allow_other",
|
|
||||||
`uid=${uid}`,
|
|
||||||
`gid=${gid}`,
|
|
||||||
];
|
|
||||||
|
|
||||||
if (config.skipHostKeyCheck) {
|
|
||||||
options.push("StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null");
|
|
||||||
} else if (config.knownHosts) {
|
|
||||||
const knownHostsPath = getKnownHostsPath(mountPath);
|
|
||||||
await writeFileWithMode(knownHostsPath, config.knownHosts, FILE_MODES.ownerReadWrite);
|
|
||||||
options.push(`UserKnownHostsFile=${knownHostsPath}`, "StrictHostKeyChecking=yes");
|
|
||||||
} else {
|
|
||||||
options.push("StrictHostKeyChecking=yes");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.readOnly) {
|
|
||||||
options.push("ro");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.port) {
|
|
||||||
options.push(`port=${config.port}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const keyPath = getPrivateKeyPath(mountPath);
|
|
||||||
if (config.privateKey) {
|
|
||||||
const decryptedKey = await cryptoUtils.resolveSecret(config.privateKey);
|
|
||||||
let normalizedKey = decryptedKey.replace(/\r\n/g, "\n");
|
|
||||||
if (!normalizedKey.endsWith("\n")) {
|
|
||||||
normalizedKey += "\n";
|
|
||||||
}
|
|
||||||
await writeFileWithMode(keyPath, normalizedKey, FILE_MODES.ownerReadWrite);
|
|
||||||
options.push(`IdentityFile=${keyPath}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const source = `${config.username}@${config.host}:${config.path || ""}`;
|
|
||||||
const args = [source, mountPath, "-o", options.join(",")];
|
|
||||||
|
|
||||||
logger.debug(`Mounting SFTP volume ${mountPath}...`);
|
|
||||||
|
|
||||||
const runSshfs = async (mountArgs: string[], password?: string) => {
|
|
||||||
return new Promise<void>((resolve, reject) => {
|
|
||||||
const child = spawn("sshfs", mountArgs, { stdio: ["pipe", "pipe", "pipe"] });
|
|
||||||
let stdout = "";
|
|
||||||
let stderr = "";
|
|
||||||
|
|
||||||
child.stdout.setEncoding("utf8");
|
|
||||||
child.stderr.setEncoding("utf8");
|
|
||||||
|
|
||||||
child.stdout.on("data", (data) => {
|
|
||||||
stdout += data;
|
|
||||||
});
|
|
||||||
|
|
||||||
child.stderr.on("data", (data) => {
|
|
||||||
stderr += data;
|
|
||||||
});
|
|
||||||
|
|
||||||
child.on("error", (error) => {
|
|
||||||
reject(new Error(`Failed to start sshfs: ${error.message}`));
|
|
||||||
});
|
|
||||||
|
|
||||||
child.on("close", (code) => {
|
|
||||||
if (code === 0) {
|
|
||||||
resolve();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const errorMsg = stderr.trim() || stdout.trim() || "Unknown error";
|
|
||||||
reject(new Error(`Failed to mount SFTP volume: ${errorMsg}`));
|
|
||||||
});
|
|
||||||
|
|
||||||
if (password) {
|
|
||||||
child.stdin.write(password);
|
|
||||||
}
|
|
||||||
child.stdin.end();
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
if (config.password) {
|
|
||||||
const password = await cryptoUtils.resolveSecret(config.password);
|
|
||||||
args.push("-o", "password_stdin");
|
|
||||||
logger.info(`Executing sshfs: sshfs ${args.join(" ")}`);
|
|
||||||
await runSshfs(args, password);
|
|
||||||
} else {
|
|
||||||
logger.info(`Executing sshfs: sshfs ${args.join(" ")}`);
|
|
||||||
await runSshfs(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`SFTP volume at ${mountPath} mounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT * 2, "SFTP mount");
|
|
||||||
} catch (error) {
|
|
||||||
const errorMsg = toMessage(error);
|
|
||||||
logger.error("Error mounting SFTP volume", { error: errorMsg });
|
|
||||||
return { status: BACKEND_STATUS.error, error: errorMsg };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const unmount = async (mountPath: string) => {
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("SFTP unmounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "SFTP unmounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
const mount = await getMountForPath(mountPath);
|
|
||||||
if (!mount || mount.mountPoint !== mountPath) {
|
|
||||||
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
|
|
||||||
} else {
|
|
||||||
await executeUnmount(mountPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
const keyPath = getPrivateKeyPath(mountPath);
|
|
||||||
await fs.unlink(keyPath).catch(() => {});
|
|
||||||
|
|
||||||
const knownHostsPath = getKnownHostsPath(mountPath);
|
|
||||||
await fs.unlink(knownHostsPath).catch(() => {});
|
|
||||||
|
|
||||||
await fs.rmdir(mountPath).catch(() => {});
|
|
||||||
|
|
||||||
logger.info(`SFTP volume at ${mountPath} unmounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "SFTP unmount");
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error unmounting SFTP volume", { mountPath, error: toMessage(error) });
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const checkHealth = async (mountPath: string) => {
|
|
||||||
const mount = await getMountForPath(mountPath);
|
|
||||||
|
|
||||||
if (!mount || mount.mountPoint !== mountPath) {
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (mount.fstype !== "fuse.sshfs") {
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: `Invalid filesystem type: ${mount.fstype} (expected fuse.sshfs)`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
export const makeSftpBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
|
|
||||||
mount: () => mount(config, mountPath),
|
|
||||||
unmount: () => unmount(mountPath),
|
|
||||||
checkHealth: () => checkHealth(mountPath),
|
|
||||||
});
|
|
||||||
|
|
@ -1,141 +0,0 @@
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import { OPERATION_TIMEOUT } from "../../../core/constants";
|
|
||||||
import { cryptoUtils } from "../../../utils/crypto";
|
|
||||||
import { toMessage } from "../../../utils/errors";
|
|
||||||
import { logger } from "@zerobyte/core/node";
|
|
||||||
import { getMountForPath } from "../../../utils/mountinfo";
|
|
||||||
import { withTimeout } from "../../../utils/timeout";
|
|
||||||
import type { VolumeBackend } from "../backend";
|
|
||||||
import { assertMounted, executeMount, executeUnmount } from "../utils/backend-utils";
|
|
||||||
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
|
|
||||||
|
|
||||||
const mount = async (config: BackendConfig, path: string) => {
|
|
||||||
logger.debug(`Mounting SMB volume ${path}...`);
|
|
||||||
|
|
||||||
if (config.backend !== "smb") {
|
|
||||||
logger.error("Provided config is not for SMB backend");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "Provided config is not for SMB backend" };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("SMB mounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "SMB mounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const { status } = await checkHealth(path);
|
|
||||||
if (status === "mounted") {
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status === "error") {
|
|
||||||
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
|
|
||||||
await unmount(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
await fs.mkdir(path, { recursive: true });
|
|
||||||
|
|
||||||
const source = `//${config.server}/${config.share}`;
|
|
||||||
const { uid, gid } = os.userInfo();
|
|
||||||
|
|
||||||
const options = [`port=${config.port}`, `uid=${uid}`, `gid=${gid}`, "iocharset=utf8"];
|
|
||||||
|
|
||||||
if (config.guest) {
|
|
||||||
options.push("username=guest", "password=");
|
|
||||||
} else {
|
|
||||||
const password = await cryptoUtils.resolveSecret(config.password ?? "");
|
|
||||||
const safePassword = password.replace(/\\/g, "\\\\").replace(/,/g, "\\,");
|
|
||||||
|
|
||||||
options.push(`username=${config.username ?? "user"}`, `password=${safePassword}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.domain) {
|
|
||||||
options.push(`domain=${config.domain}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.vers && config.vers !== "auto") {
|
|
||||||
options.push(`vers=${config.vers}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config.readOnly) {
|
|
||||||
options.push("ro");
|
|
||||||
}
|
|
||||||
|
|
||||||
const args = ["-t", "cifs", "-o", options.join(","), source, path];
|
|
||||||
|
|
||||||
logger.debug(`Mounting SMB volume ${path}...`);
|
|
||||||
logger.info(`Executing SMB mount for ${source} at ${path}`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await executeMount(args);
|
|
||||||
} catch (error) {
|
|
||||||
logger.error(`SMB mount failed: ${toMessage(error)}`);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`SMB volume at ${path} mounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB mount");
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error mounting SMB volume", { error: toMessage(error) });
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const unmount = async (path: string) => {
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("SMB unmounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "SMB unmounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
const mount = await getMountForPath(path);
|
|
||||||
if (!mount || mount.mountPoint !== path) {
|
|
||||||
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
await executeUnmount(path);
|
|
||||||
|
|
||||||
await fs.rmdir(path).catch(() => {});
|
|
||||||
|
|
||||||
logger.info(`SMB volume at ${path} unmounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB unmount");
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error unmounting SMB volume", { path, error: toMessage(error) });
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const checkHealth = async (path: string) => {
|
|
||||||
const run = async () => {
|
|
||||||
await assertMounted(path, (fstype) => fstype === "cifs");
|
|
||||||
|
|
||||||
logger.debug(`SMB volume at ${path} is healthy and mounted.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB health check");
|
|
||||||
} catch (error) {
|
|
||||||
const message = toMessage(error);
|
|
||||||
if (message !== "Volume is not mounted") {
|
|
||||||
logger.error("SMB volume health check failed:", message);
|
|
||||||
}
|
|
||||||
return { status: BACKEND_STATUS.error, error: message };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const makeSmbBackend = (config: BackendConfig, path: string): VolumeBackend => ({
|
|
||||||
mount: () => mount(config, path),
|
|
||||||
unmount: () => unmount(path),
|
|
||||||
checkHealth: () => checkHealth(path),
|
|
||||||
});
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
|
||||||
|
|
||||||
vi.mock("node:fs/promises", async (importOriginal) => {
|
|
||||||
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
|
||||||
|
|
||||||
return {
|
|
||||||
...actual,
|
|
||||||
access: vi.fn(actual.access),
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
vi.mock("../../../../utils/mountinfo", async (importOriginal) => {
|
|
||||||
const actual = await importOriginal<typeof import("../../../../utils/mountinfo")>();
|
|
||||||
|
|
||||||
return {
|
|
||||||
...actual,
|
|
||||||
getMountForPath: vi.fn(actual.getMountForPath),
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as mountinfo from "../../../../utils/mountinfo";
|
|
||||||
import { assertMounted } from "../backend-utils";
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
vi.restoreAllMocks();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("assertMountedFilesystem", () => {
|
|
||||||
test("throws when the path is not accessible", async () => {
|
|
||||||
vi.mocked(fs.access).mockRejectedValueOnce(new Error("missing"));
|
|
||||||
|
|
||||||
await expect(assertMounted("/tmp/volume", (fstype) => fstype.startsWith("nfs"))).rejects.toThrow(
|
|
||||||
"Volume is not mounted",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("throws when the mount filesystem does not match", async () => {
|
|
||||||
vi.mocked(fs.access).mockResolvedValueOnce(undefined);
|
|
||||||
vi.mocked(mountinfo.getMountForPath).mockResolvedValueOnce({
|
|
||||||
mountPoint: "/tmp/volume",
|
|
||||||
fstype: "cifs",
|
|
||||||
});
|
|
||||||
|
|
||||||
await expect(assertMounted("/tmp/volume", (fstype) => fstype.startsWith("nfs"))).rejects.toThrow(
|
|
||||||
"Path /tmp/volume is not mounted as correct fstype (found cifs).",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("accepts a matching mounted filesystem", async () => {
|
|
||||||
vi.mocked(fs.access).mockResolvedValueOnce(undefined);
|
|
||||||
vi.mocked(mountinfo.getMountForPath).mockResolvedValueOnce({
|
|
||||||
mountPoint: "/tmp/volume",
|
|
||||||
fstype: "nfs4",
|
|
||||||
});
|
|
||||||
|
|
||||||
await expect(assertMounted("/tmp/volume", (fstype) => fstype.startsWith("nfs"))).resolves.toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,159 +0,0 @@
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import { OPERATION_TIMEOUT } from "../../../core/constants";
|
|
||||||
import { cryptoUtils } from "../../../utils/crypto";
|
|
||||||
import { toMessage } from "../../../utils/errors";
|
|
||||||
import { logger } from "@zerobyte/core/node";
|
|
||||||
import { getMountForPath } from "../../../utils/mountinfo";
|
|
||||||
import { withTimeout } from "../../../utils/timeout";
|
|
||||||
import type { VolumeBackend } from "../backend";
|
|
||||||
import { assertMounted, executeMount, executeUnmount } from "../utils/backend-utils";
|
|
||||||
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
|
|
||||||
|
|
||||||
const mount = async (config: BackendConfig, path: string) => {
|
|
||||||
logger.debug(`Mounting WebDAV volume ${path}...`);
|
|
||||||
|
|
||||||
if (config.backend !== "webdav") {
|
|
||||||
logger.error("Provided config is not for WebDAV backend");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "Provided config is not for WebDAV backend" };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("WebDAV mounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "WebDAV mounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const { status } = await checkHealth(path);
|
|
||||||
if (status === "mounted") {
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status === "error") {
|
|
||||||
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
|
|
||||||
await unmount(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
await fs.mkdir(path, { recursive: true }).catch((err) => {
|
|
||||||
logger.warn(`Failed to create directory ${path}: ${err.message}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
const protocol = config.ssl ? "https" : "http";
|
|
||||||
const defaultPort = config.ssl ? 443 : 80;
|
|
||||||
const port = config.port !== defaultPort ? `:${config.port}` : "";
|
|
||||||
const source = `${protocol}://${config.server}${port}${config.path}`;
|
|
||||||
|
|
||||||
const { uid, gid } = os.userInfo();
|
|
||||||
const options = config.readOnly
|
|
||||||
? [`uid=${uid}`, `gid=${gid}`, "file_mode=0444", "dir_mode=0555", "ro"]
|
|
||||||
: [`uid=${uid}`, `gid=${gid}`, "file_mode=0664", "dir_mode=0775"];
|
|
||||||
|
|
||||||
if (config.username && config.password) {
|
|
||||||
const password = await cryptoUtils.resolveSecret(config.password);
|
|
||||||
const secretsFile = "/etc/davfs2/secrets";
|
|
||||||
const entry = [source, config.username, password].map((value) => value.replace(/[\r\n\t\s]+/g, " ")).join(" ");
|
|
||||||
const secretsContent = `${entry}\n`;
|
|
||||||
await fs.appendFile(secretsFile, secretsContent, { mode: 0o600 });
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(`Mounting WebDAV volume ${path}...`);
|
|
||||||
|
|
||||||
const args = ["-t", "davfs", "-o", options.join(","), source, path];
|
|
||||||
|
|
||||||
try {
|
|
||||||
await executeMount(args);
|
|
||||||
} catch (error) {
|
|
||||||
logger.warn(`Initial WebDAV mount failed, retrying with -i flag: ${toMessage(error)}`);
|
|
||||||
// Fallback with -i flag if the first mount fails using the mount helper
|
|
||||||
await executeMount(["-i", ...args]);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`WebDAV volume at ${path} mounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV mount");
|
|
||||||
} catch (error) {
|
|
||||||
const errorMsg = toMessage(error);
|
|
||||||
|
|
||||||
if (errorMsg.includes("already mounted")) {
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.error("Error mounting WebDAV volume", { error: errorMsg });
|
|
||||||
|
|
||||||
if (errorMsg.includes("option") && errorMsg.includes("requires argument")) {
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: "Invalid mount options. Please check your WebDAV server configuration.",
|
|
||||||
};
|
|
||||||
} else if (errorMsg.includes("connection refused") || errorMsg.includes("Connection refused")) {
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: "Cannot connect to WebDAV server. Please check the server address and port.",
|
|
||||||
};
|
|
||||||
} else if (errorMsg.includes("unauthorized") || errorMsg.includes("Unauthorized")) {
|
|
||||||
return {
|
|
||||||
status: BACKEND_STATUS.error,
|
|
||||||
error: "Authentication failed. Please check your username and password.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { status: BACKEND_STATUS.error, error: errorMsg };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const unmount = async (path: string) => {
|
|
||||||
if (os.platform() !== "linux") {
|
|
||||||
logger.error("WebDAV unmounting is only supported on Linux hosts.");
|
|
||||||
return { status: BACKEND_STATUS.error, error: "WebDAV unmounting is only supported on Linux hosts." };
|
|
||||||
}
|
|
||||||
|
|
||||||
const run = async () => {
|
|
||||||
const mount = await getMountForPath(path);
|
|
||||||
if (!mount || mount.mountPoint !== path) {
|
|
||||||
logger.debug(`Path ${path} is not a mount point. Skipping unmount.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
}
|
|
||||||
|
|
||||||
await executeUnmount(path);
|
|
||||||
|
|
||||||
await fs.rmdir(path).catch(() => {});
|
|
||||||
|
|
||||||
logger.info(`WebDAV volume at ${path} unmounted successfully.`);
|
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV unmount");
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error unmounting WebDAV volume", { path, error: toMessage(error) });
|
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const checkHealth = async (path: string) => {
|
|
||||||
const run = async () => {
|
|
||||||
await assertMounted(path, (fstype) => fstype === "fuse" || fstype === "davfs");
|
|
||||||
|
|
||||||
logger.debug(`WebDAV volume at ${path} is healthy and mounted.`);
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV health check");
|
|
||||||
} catch (error) {
|
|
||||||
const message = toMessage(error);
|
|
||||||
if (message !== "Volume is not mounted") {
|
|
||||||
logger.error("WebDAV volume health check failed:", message);
|
|
||||||
}
|
|
||||||
return { status: BACKEND_STATUS.error, error: message };
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const makeWebdavBackend = (config: BackendConfig, path: string): VolumeBackend => ({
|
|
||||||
mount: () => mount(config, path),
|
|
||||||
unmount: () => unmount(path),
|
|
||||||
checkHealth: () => checkHealth(path),
|
|
||||||
});
|
|
||||||
|
|
@ -421,6 +421,50 @@ describe("backup execution - validation failures", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("backup execution - routing", () => {
|
||||||
|
test("fails local repository backups on non-local volume agents", async () => {
|
||||||
|
const { runBackupMock } = setup();
|
||||||
|
const volume = await createTestVolume({ agentId: "agent-remote" });
|
||||||
|
const repository = await createTestRepository();
|
||||||
|
const schedule = await createTestBackupSchedule({
|
||||||
|
volumeId: volume.id,
|
||||||
|
repositoryId: repository.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
await backupsService.executeBackup(schedule.id);
|
||||||
|
|
||||||
|
const updatedSchedule = await getScheduleByIdOrShortId(schedule.id);
|
||||||
|
expect(updatedSchedule.lastBackupStatus).toBe("error");
|
||||||
|
expect(updatedSchedule.lastBackupError).toBe(
|
||||||
|
`Local repository "${repository.name}" can only be used with the local agent`,
|
||||||
|
);
|
||||||
|
expect(runBackupMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("routes remote repository backups through the owning volume agent", async () => {
|
||||||
|
const { runBackupMock } = setup();
|
||||||
|
const volume = await createTestVolume({ agentId: "agent-remote" });
|
||||||
|
const repository = await createTestRepository({
|
||||||
|
type: "s3",
|
||||||
|
config: {
|
||||||
|
backend: "s3",
|
||||||
|
endpoint: "https://s3.amazonaws.com",
|
||||||
|
bucket: "bucket-name",
|
||||||
|
accessKeyId: "access-key",
|
||||||
|
secretAccessKey: "secret-key",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const schedule = await createTestBackupSchedule({
|
||||||
|
volumeId: volume.id,
|
||||||
|
repositoryId: repository.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
await backupsService.executeBackup(schedule.id);
|
||||||
|
|
||||||
|
expect(runBackupMock).toHaveBeenCalledWith("agent-remote", expect.objectContaining({ scheduleId: schedule.id }));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("stop backup", () => {
|
describe("stop backup", () => {
|
||||||
test("should keep restic warning details when backup completes with read errors", async () => {
|
test("should keep restic warning details when backup completes with read errors", async () => {
|
||||||
const { resticBackupMock } = setup();
|
const { resticBackupMock } = setup();
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import { getVolumePath } from "../volumes/helpers";
|
||||||
import { decryptRepositoryConfig } from "../repositories/repository-config-secrets";
|
import { decryptRepositoryConfig } from "../repositories/repository-config-secrets";
|
||||||
import { createBackupOptions } from "./backup.helpers";
|
import { createBackupOptions } from "./backup.helpers";
|
||||||
import { toErrorDetails } from "../../utils/errors";
|
import { toErrorDetails } from "../../utils/errors";
|
||||||
|
import { BadRequestError } from "http-errors-enhanced";
|
||||||
|
|
||||||
const FUSE_VOLUME_BACKENDS = new Set<Volume["type"]>(["rclone", "sftp", "webdav"]);
|
const FUSE_VOLUME_BACKENDS = new Set<Volume["type"]>(["rclone", "sftp", "webdav"]);
|
||||||
const IGNORE_INODE_FLAG = "--ignore-inode";
|
const IGNORE_INODE_FLAG = "--ignore-inode";
|
||||||
|
|
@ -23,7 +24,17 @@ type BackupExecutionRequest = {
|
||||||
onProgress: (progress: BackupExecutionProgress) => void;
|
onProgress: (progress: BackupExecutionProgress) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
const activeControllersByScheduleId = new Map<number, AbortController>();
|
export type { BackupExecutionResult } from "../agents/agents-manager";
|
||||||
|
|
||||||
|
const activeControllersByScheduleId = new Map<number, { abortController: AbortController; agentId: string | null }>();
|
||||||
|
|
||||||
|
const getBackupExecutionAgentId = (volume: Volume, repository: Repository) => {
|
||||||
|
if (repository.type === "local" && volume.agentId !== LOCAL_AGENT_ID) {
|
||||||
|
throw new BadRequestError(`Local repository "${repository.name}" can only be used with the local agent`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return volume.agentId;
|
||||||
|
};
|
||||||
|
|
||||||
const createBackupRunPayload = async ({
|
const createBackupRunPayload = async ({
|
||||||
jobId,
|
jobId,
|
||||||
|
|
@ -93,17 +104,17 @@ const executeBackupWithoutAgent = async (
|
||||||
export const backupExecutor = {
|
export const backupExecutor = {
|
||||||
track: (scheduleId: number) => {
|
track: (scheduleId: number) => {
|
||||||
const abortController = new AbortController();
|
const abortController = new AbortController();
|
||||||
activeControllersByScheduleId.set(scheduleId, abortController);
|
activeControllersByScheduleId.set(scheduleId, { abortController, agentId: null });
|
||||||
return abortController;
|
return abortController;
|
||||||
},
|
},
|
||||||
untrack: (scheduleId: number, abortController: AbortController) => {
|
untrack: (scheduleId: number, abortController: AbortController) => {
|
||||||
if (activeControllersByScheduleId.get(scheduleId) === abortController) {
|
if (activeControllersByScheduleId.get(scheduleId)?.abortController === abortController) {
|
||||||
activeControllersByScheduleId.delete(scheduleId);
|
activeControllersByScheduleId.delete(scheduleId);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
execute: async (request: Omit<BackupExecutionRequest, "jobId">) => {
|
execute: async (request: Omit<BackupExecutionRequest, "jobId">) => {
|
||||||
const trackedAbortController = activeControllersByScheduleId.get(request.scheduleId);
|
const trackedExecution = activeControllersByScheduleId.get(request.scheduleId);
|
||||||
if (!trackedAbortController || trackedAbortController.signal !== request.signal) {
|
if (!trackedExecution || trackedExecution.abortController.signal !== request.signal) {
|
||||||
throw new Error(`Backup execution for schedule ${request.scheduleId} was not tracked`);
|
throw new Error(`Backup execution for schedule ${request.scheduleId} was not tracked`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -119,27 +130,38 @@ export const backupExecutor = {
|
||||||
throw request.signal.reason || new Error("Operation aborted");
|
throw request.signal.reason || new Error("Operation aborted");
|
||||||
}
|
}
|
||||||
|
|
||||||
const executionResult = await agentManager.runBackup(LOCAL_AGENT_ID, {
|
const executionAgentId = getBackupExecutionAgentId(request.volume, request.repository);
|
||||||
|
trackedExecution.agentId = executionAgentId;
|
||||||
|
|
||||||
|
const executionResult = await agentManager.runBackup(executionAgentId, {
|
||||||
scheduleId: request.scheduleId,
|
scheduleId: request.scheduleId,
|
||||||
payload,
|
payload,
|
||||||
signal: request.signal,
|
signal: request.signal,
|
||||||
onProgress: request.onProgress,
|
onProgress: request.onProgress,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (executionResult.status === "unavailable" && !config.flags.enableLocalAgent) {
|
if (
|
||||||
|
executionResult.status === "unavailable" &&
|
||||||
|
executionAgentId === LOCAL_AGENT_ID &&
|
||||||
|
!config.flags.enableLocalAgent
|
||||||
|
) {
|
||||||
return executeBackupWithoutAgent(payload, request);
|
return executeBackupWithoutAgent(payload, request);
|
||||||
}
|
}
|
||||||
|
|
||||||
return executionResult;
|
return executionResult;
|
||||||
},
|
},
|
||||||
cancel: async (scheduleId: number) => {
|
cancel: async (scheduleId: number) => {
|
||||||
const abortController = activeControllersByScheduleId.get(scheduleId);
|
const trackedExecution = activeControllersByScheduleId.get(scheduleId);
|
||||||
if (!abortController) {
|
if (!trackedExecution) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
abortController.abort();
|
trackedExecution.abortController.abort();
|
||||||
await agentManager.cancelBackup(LOCAL_AGENT_ID, scheduleId);
|
if (!trackedExecution.agentId) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
await agentManager.cancelBackup(trackedExecution.agentId, scheduleId);
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,6 @@ import { copyToMirrors, runForget, syncSnapshotsToMirror } from "./helpers/backu
|
||||||
import { restic } from "../../core/restic";
|
import { restic } from "../../core/restic";
|
||||||
import { mirrorQueries } from "./backups.queries";
|
import { mirrorQueries } from "./backups.queries";
|
||||||
import { toMessage } from "../../utils/errors";
|
import { toMessage } from "../../utils/errors";
|
||||||
|
|
||||||
const listSchedules = async () => {
|
const listSchedules = async () => {
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
const schedules = await db.query.backupSchedulesTable.findMany({
|
const schedules = await db.query.backupSchedulesTable.findMany({
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||||
import { Scheduler } from "../../../core/scheduler";
|
import { Scheduler } from "../../../core/scheduler";
|
||||||
import * as backendModule from "../../backends/backend";
|
|
||||||
import type { VolumeBackend } from "../../backends/backend";
|
|
||||||
import * as bootstrapModule from "../bootstrap";
|
import * as bootstrapModule from "../bootstrap";
|
||||||
|
import { agentManager } from "../../agents/agents-manager";
|
||||||
import { createTestVolume } from "~/test/helpers/volume";
|
import { createTestVolume } from "~/test/helpers/volume";
|
||||||
|
import { config } from "~/server/core/config";
|
||||||
|
import { db } from "~/server/db/db";
|
||||||
|
|
||||||
const loadShutdownModule = async () => {
|
const loadShutdownModule = async () => {
|
||||||
const moduleUrl = new URL("../shutdown.ts", import.meta.url);
|
const moduleUrl = new URL("../shutdown.ts", import.meta.url);
|
||||||
|
|
@ -12,11 +13,12 @@ const loadShutdownModule = async () => {
|
||||||
};
|
};
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
config.flags.enableLocalAgent = true;
|
||||||
vi.restoreAllMocks();
|
vi.restoreAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("shutdown", () => {
|
describe("shutdown", () => {
|
||||||
test("stops the agent runtime before unmounting mounted volumes", async () => {
|
test("does not unmount agent-owned volumes during controller shutdown", async () => {
|
||||||
const events: string[] = [];
|
const events: string[] = [];
|
||||||
const stopScheduler = vi.fn(async () => {
|
const stopScheduler = vi.fn(async () => {
|
||||||
events.push("scheduler.stop");
|
events.push("scheduler.stop");
|
||||||
|
|
@ -24,35 +26,51 @@ describe("shutdown", () => {
|
||||||
const stopApplicationRuntime = vi.fn(async () => {
|
const stopApplicationRuntime = vi.fn(async () => {
|
||||||
events.push("agents.stop");
|
events.push("agents.stop");
|
||||||
});
|
});
|
||||||
const unmountVolume = vi.fn(async () => {
|
const runVolumeCommand = vi.spyOn(agentManager, "runVolumeCommand");
|
||||||
events.push("backend.unmount");
|
|
||||||
return { status: "unmounted" as const };
|
|
||||||
});
|
|
||||||
|
|
||||||
await createTestVolume({
|
const volume = await createTestVolume({
|
||||||
name: "Shutdown test volume",
|
name: "Shutdown test volume",
|
||||||
config: {
|
config: {
|
||||||
backend: "directory",
|
backend: "directory",
|
||||||
path: "/Applications",
|
path: "/Applications",
|
||||||
},
|
},
|
||||||
status: "mounted",
|
status: "mounted",
|
||||||
|
agentId: "agent-1",
|
||||||
});
|
});
|
||||||
|
|
||||||
vi.spyOn(Scheduler, "stop").mockImplementation(stopScheduler);
|
vi.spyOn(Scheduler, "stop").mockImplementation(stopScheduler);
|
||||||
vi.spyOn(bootstrapModule, "stopApplicationRuntime").mockImplementation(stopApplicationRuntime);
|
vi.spyOn(bootstrapModule, "stopApplicationRuntime").mockImplementation(stopApplicationRuntime);
|
||||||
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(
|
|
||||||
() =>
|
|
||||||
({
|
|
||||||
mount: async () => ({ status: "mounted" as const }),
|
|
||||||
unmount: unmountVolume,
|
|
||||||
checkHealth: async () => ({ status: "mounted" as const }),
|
|
||||||
}) satisfies VolumeBackend,
|
|
||||||
);
|
|
||||||
|
|
||||||
const { shutdown } = await loadShutdownModule();
|
const { shutdown } = await loadShutdownModule();
|
||||||
|
|
||||||
await shutdown();
|
await shutdown();
|
||||||
|
|
||||||
expect(events).toEqual(["scheduler.stop", "agents.stop", "backend.unmount"]);
|
expect(events).toEqual(["scheduler.stop", "agents.stop"]);
|
||||||
|
expect(runVolumeCommand).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("keeps legacy controller-local fallback unmount on shutdown", async () => {
|
||||||
|
config.flags.enableLocalAgent = false;
|
||||||
|
const events: string[] = [];
|
||||||
|
vi.spyOn(Scheduler, "stop").mockImplementation(async () => {
|
||||||
|
events.push("scheduler.stop");
|
||||||
|
});
|
||||||
|
vi.spyOn(bootstrapModule, "stopApplicationRuntime").mockImplementation(async () => {
|
||||||
|
events.push("agents.stop");
|
||||||
|
});
|
||||||
|
|
||||||
|
const volume = await createTestVolume({
|
||||||
|
name: "Fallback shutdown test volume",
|
||||||
|
config: { backend: "directory", path: "/Applications" },
|
||||||
|
status: "mounted",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { shutdown } = await loadShutdownModule();
|
||||||
|
|
||||||
|
await shutdown();
|
||||||
|
|
||||||
|
const updated = await db.query.volumesTable.findFirst({ where: { id: volume.id } });
|
||||||
|
expect(events).toEqual(["scheduler.stop", "agents.stop"]);
|
||||||
|
expect(updated?.status).toBe("unmounted");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,29 @@
|
||||||
import { Scheduler } from "../../core/scheduler";
|
import { Scheduler } from "../../core/scheduler";
|
||||||
import { db } from "../../db/db";
|
import { db } from "../../db/db";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { createVolumeBackend } from "../backends/backend";
|
|
||||||
import { stopApplicationRuntime } from "./bootstrap";
|
import { stopApplicationRuntime } from "./bootstrap";
|
||||||
|
import { withContext } from "../../core/request-context";
|
||||||
|
import { volumeService } from "../volumes/volume.service";
|
||||||
|
import { toMessage } from "../../utils/errors";
|
||||||
|
import { config } from "../../core/config";
|
||||||
|
import { LOCAL_AGENT_ID } from "../agents/constants";
|
||||||
|
|
||||||
export const shutdown = async () => {
|
export const shutdown = async () => {
|
||||||
await Scheduler.stop();
|
await Scheduler.stop();
|
||||||
await stopApplicationRuntime();
|
|
||||||
|
|
||||||
const volumes = await db.query.volumesTable.findMany({
|
if (!config.flags.enableLocalAgent) {
|
||||||
where: { status: "mounted" },
|
const volumes = await db.query.volumesTable.findMany({
|
||||||
});
|
where: { AND: [{ status: "mounted" }, { agentId: LOCAL_AGENT_ID }] },
|
||||||
|
});
|
||||||
|
|
||||||
for (const volume of volumes) {
|
for (const volume of volumes) {
|
||||||
const backend = createVolumeBackend(volume);
|
const { status, error } = await withContext({ organizationId: volume.organizationId }, () =>
|
||||||
const { status, error } = await backend.unmount();
|
volumeService.unmountVolume(volume.shortId),
|
||||||
|
).catch((error) => ({ status: "error" as const, error: toMessage(error) }));
|
||||||
|
|
||||||
logger.info(`Volume ${volume.name} unmount status: ${status}${error ? `, error: ${error}` : ""}`);
|
logger.info(`Volume ${volume.name} unmount status: ${status}${error ? `, error: ${error}` : ""}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await stopApplicationRuntime();
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ import { backupsService } from "../backups/backups.service";
|
||||||
import { config } from "~/server/core/config";
|
import { config } from "~/server/core/config";
|
||||||
import { syncProvisionedResources } from "../provisioning/provisioning";
|
import { syncProvisionedResources } from "../provisioning/provisioning";
|
||||||
import { toMessage } from "~/server/utils/errors";
|
import { toMessage } from "~/server/utils/errors";
|
||||||
|
import { LOCAL_AGENT_ID } from "../agents/constants";
|
||||||
|
|
||||||
const ensureLatestConfigurationSchema = async () => {
|
const ensureLatestConfigurationSchema = async () => {
|
||||||
const volumes = await db.query.volumesTable.findMany({});
|
const volumes = await db.query.volumesTable.findMany({});
|
||||||
|
|
@ -71,23 +72,30 @@ export const startup = async () => {
|
||||||
logger.warn(`Removed ${deletedSchedules} orphaned backup schedule(s) during startup`);
|
logger.warn(`Removed ${deletedSchedules} orphaned backup schedule(s) during startup`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const volumes = await db.query.volumesTable.findMany({
|
if (!config.flags.enableLocalAgent) {
|
||||||
where: {
|
const volumes = await db.query.volumesTable.findMany({
|
||||||
OR: [
|
where: {
|
||||||
{ status: "mounted" },
|
AND: [
|
||||||
{
|
{ agentId: LOCAL_AGENT_ID },
|
||||||
AND: [{ autoRemount: true }, { status: "error" }],
|
{
|
||||||
},
|
OR: [
|
||||||
],
|
{ status: "mounted" },
|
||||||
},
|
{
|
||||||
});
|
AND: [{ autoRemount: true }, { status: "error" }],
|
||||||
|
},
|
||||||
for (const volume of volumes) {
|
],
|
||||||
await withContext({ organizationId: volume.organizationId }, async () => {
|
},
|
||||||
await volumeService.mountVolume(volume.shortId).catch((err) => {
|
],
|
||||||
logger.error(`Error auto-remounting volume ${volume.name} on startup: ${err.message}`);
|
},
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
for (const volume of volumes) {
|
||||||
|
await withContext({ organizationId: volume.organizationId }, async () => {
|
||||||
|
await volumeService.mountVolume(volume.shortId).catch((err) => {
|
||||||
|
logger.error(`Error auto-remounting volume ${volume.name} on startup: ${err.message}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await db
|
await db
|
||||||
|
|
@ -102,7 +110,9 @@ export const startup = async () => {
|
||||||
logger.error(`Failed to update stuck backup schedules on startup: ${err.message}`);
|
logger.error(`Failed to update stuck backup schedules on startup: ${err.message}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
Scheduler.build(CleanupDanglingMountsJob).schedule("0 * * * *");
|
if (!config.flags.enableLocalAgent) {
|
||||||
|
Scheduler.build(CleanupDanglingMountsJob).schedule("0 * * * *");
|
||||||
|
}
|
||||||
Scheduler.build(VolumeHealthCheckJob).schedule("*/30 * * * *");
|
Scheduler.build(VolumeHealthCheckJob).schedule("*/30 * * * *");
|
||||||
Scheduler.build(RepositoryHealthCheckJob).schedule("50 12 * * *");
|
Scheduler.build(RepositoryHealthCheckJob).schedule("50 12 * * *");
|
||||||
Scheduler.build(BackupExecutionJob).schedule("* * * * *");
|
Scheduler.build(BackupExecutionJob).schedule("* * * * *");
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ import { config as appConfig } from "~/server/core/config";
|
||||||
import { restic } from "~/server/core/restic";
|
import { restic } from "~/server/core/restic";
|
||||||
import { db } from "~/server/db/db";
|
import { db } from "~/server/db/db";
|
||||||
import { repositoriesTable, volumesTable } from "~/server/db/schema";
|
import { repositoriesTable, volumesTable } from "~/server/db/schema";
|
||||||
|
import { LOCAL_AGENT_ID } from "~/server/modules/agents/constants";
|
||||||
import { mapRepositoryConfigSecrets } from "~/server/modules/repositories/repository-config-secrets";
|
import { mapRepositoryConfigSecrets } from "~/server/modules/repositories/repository-config-secrets";
|
||||||
import { mapVolumeConfigSecrets } from "~/server/modules/volumes/volume-config-secrets";
|
import { mapVolumeConfigSecrets } from "~/server/modules/volumes/volume-config-secrets";
|
||||||
import { BACKEND_TYPES, volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
import { BACKEND_TYPES, volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
||||||
|
|
@ -154,7 +155,6 @@ const syncProvisionedRepositories = async (repositories: ProvisionedRepository[]
|
||||||
|
|
||||||
const existing = existingRepositories.find((r) => r.provisioningId === provisioningId);
|
const existing = existingRepositories.find((r) => r.provisioningId === provisioningId);
|
||||||
const encryptedConfig = await encryptProvisionedRepositoryConfig(repository.config);
|
const encryptedConfig = await encryptProvisionedRepositoryConfig(repository.config);
|
||||||
|
|
||||||
if (!existing) {
|
if (!existing) {
|
||||||
const id = Bun.randomUUIDv7();
|
const id = Bun.randomUUIDv7();
|
||||||
|
|
||||||
|
|
@ -228,6 +228,7 @@ const syncProvisionedVolumes = async (volumes: ProvisionedVolume[]) => {
|
||||||
type: volume.backend,
|
type: volume.backend,
|
||||||
config: await encryptProvisionedVolumeConfig(volume.config),
|
config: await encryptProvisionedVolumeConfig(volume.config),
|
||||||
autoRemount: volume.autoRemount,
|
autoRemount: volume.autoRemount,
|
||||||
|
agentId: LOCAL_AGENT_ID,
|
||||||
status: volume.autoRemount ? "mounted" : "unmounted",
|
status: volume.autoRemount ? "mounted" : "unmounted",
|
||||||
organizationId: volume.organizationId,
|
organizationId: volume.organizationId,
|
||||||
});
|
});
|
||||||
|
|
@ -239,6 +240,7 @@ const syncProvisionedVolumes = async (volumes: ProvisionedVolume[]) => {
|
||||||
type: volume.backend,
|
type: volume.backend,
|
||||||
config: await encryptProvisionedVolumeConfig(volume.config),
|
config: await encryptProvisionedVolumeConfig(volume.config),
|
||||||
autoRemount: volume.autoRemount,
|
autoRemount: volume.autoRemount,
|
||||||
|
agentId: LOCAL_AGENT_ID,
|
||||||
organizationId: volume.organizationId,
|
organizationId: volume.organizationId,
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,6 @@ const createRepository = async (name: string, config: RepositoryConfig, compress
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
const id = Bun.randomUUIDv7();
|
const id = Bun.randomUUIDv7();
|
||||||
const shortId = generateShortId();
|
const shortId = generateShortId();
|
||||||
|
|
||||||
if (config.backend === "local" && !config.isExistingRepository) {
|
if (config.backend === "local" && !config.isExistingRepository) {
|
||||||
config.path = `${config.path}/${shortId}`;
|
config.path = `${config.path}/${shortId}`;
|
||||||
}
|
}
|
||||||
|
|
@ -717,7 +716,6 @@ const updateRepository = async (shortId: ShortId, updates: UpdateRepositoryBody)
|
||||||
const decryptedExisting = await decryptRepositoryConfig(existingConfig);
|
const decryptedExisting = await decryptRepositoryConfig(existingConfig);
|
||||||
const configChanged = updates.config && JSON.stringify(decryptedExisting) !== JSON.stringify(parsedConfig);
|
const configChanged = updates.config && JSON.stringify(decryptedExisting) !== JSON.stringify(parsedConfig);
|
||||||
const encryptedConfig = updates.config ? await encryptRepositoryConfig(parsedConfig) : existingConfig;
|
const encryptedConfig = updates.config ? await encryptRepositoryConfig(parsedConfig) : existingConfig;
|
||||||
|
|
||||||
const updatedAt = Date.now();
|
const updatedAt = Date.now();
|
||||||
const updatePayload = {
|
const updatePayload = {
|
||||||
name: newName,
|
name: newName,
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,26 @@
|
||||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||||
|
const agentManagerMock = vi.hoisted(() => ({
|
||||||
|
runVolumeCommand: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../../agents/agents-manager", () => ({
|
||||||
|
agentManager: agentManagerMock,
|
||||||
|
}));
|
||||||
|
|
||||||
import { volumeService } from "../volume.service";
|
import { volumeService } from "../volume.service";
|
||||||
import { db } from "~/server/db/db";
|
import { db } from "~/server/db/db";
|
||||||
import { volumesTable } from "~/server/db/schema";
|
import { volumesTable } from "~/server/db/schema";
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import os from "node:os";
|
|
||||||
import path from "node:path";
|
|
||||||
import { createTestSession } from "~/test/helpers/auth";
|
import { createTestSession } from "~/test/helpers/auth";
|
||||||
import { withContext } from "~/server/core/request-context";
|
import { withContext } from "~/server/core/request-context";
|
||||||
import { asShortId } from "~/server/utils/branded";
|
import { asShortId } from "~/server/utils/branded";
|
||||||
import { createTestVolume } from "~/test/helpers/volume";
|
import { createTestVolume } from "~/test/helpers/volume";
|
||||||
import * as backendModule from "../../backends/backend";
|
import { config } from "~/server/core/config";
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
config.flags.enableLocalAgent = false;
|
||||||
vi.restoreAllMocks();
|
vi.restoreAllMocks();
|
||||||
|
agentManagerMock.runVolumeCommand.mockReset();
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("volumeService.getVolume", () => {
|
describe("volumeService.getVolume", () => {
|
||||||
|
|
@ -98,13 +105,7 @@ describe("volumeService.getVolume", () => {
|
||||||
describe("volumeService.listFiles security", () => {
|
describe("volumeService.listFiles security", () => {
|
||||||
test("should reject traversal outside the volume root in listFiles", async () => {
|
test("should reject traversal outside the volume root in listFiles", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-vol-svc-"));
|
agentManagerMock.runVolumeCommand.mockRejectedValue(new Error("Invalid path"));
|
||||||
const volumePath = path.join(tempRoot, "vol");
|
|
||||||
const secretPath = path.join(tempRoot, "volume-secret");
|
|
||||||
|
|
||||||
await fs.mkdir(volumePath, { recursive: true });
|
|
||||||
await fs.mkdir(secretPath, { recursive: true });
|
|
||||||
await fs.writeFile(path.join(secretPath, "secret.txt"), "top secret", "utf-8");
|
|
||||||
|
|
||||||
const [volume] = await db
|
const [volume] = await db
|
||||||
.insert(volumesTable)
|
.insert(volumesTable)
|
||||||
|
|
@ -113,44 +114,40 @@ describe("volumeService.listFiles security", () => {
|
||||||
name: `test-vol-${randomUUID().slice(0, 8)}`,
|
name: `test-vol-${randomUUID().slice(0, 8)}`,
|
||||||
type: "directory",
|
type: "directory",
|
||||||
status: "mounted",
|
status: "mounted",
|
||||||
config: { backend: "directory", path: volumePath },
|
config: { backend: "directory", path: "/tmp/volume" },
|
||||||
autoRemount: true,
|
autoRemount: true,
|
||||||
organizationId,
|
organizationId,
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
try {
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
await withContext({ organizationId, userId: user.id }, async () => {
|
await expect(volumeService.listFiles(volume.shortId, "../volume-secret")).rejects.toThrow("Invalid path");
|
||||||
const traversalPath = `../${path.basename(secretPath)}`;
|
});
|
||||||
|
|
||||||
await expect(volumeService.listFiles(volume.shortId, traversalPath)).rejects.toThrow("Invalid path");
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
await fs.rm(tempRoot, { recursive: true, force: true });
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("volumeService.mountVolume", () => {
|
describe("volumeService.mountVolume", () => {
|
||||||
test("unmounts any existing mount before mounting", async () => {
|
test("routes unmount and mount to the owning agent before updating state", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const volume = await createTestVolume({ organizationId, status: "mounted" });
|
const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
|
||||||
const unmount = vi.fn().mockResolvedValue({ status: "unmounted" });
|
agentManagerMock.runVolumeCommand
|
||||||
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
|
.mockResolvedValueOnce({ name: "volume.unmount", result: { status: "unmounted" } })
|
||||||
|
.mockResolvedValueOnce({ name: "volume.mount", result: { status: "mounted" } });
|
||||||
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({
|
|
||||||
mount,
|
|
||||||
unmount,
|
|
||||||
checkHealth: vi.fn().mockResolvedValue({ status: "mounted" }),
|
|
||||||
}));
|
|
||||||
|
|
||||||
await withContext({ organizationId, userId: user.id }, async () => {
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
const result = await volumeService.mountVolume(volume.shortId);
|
const result = await volumeService.mountVolume(volume.shortId);
|
||||||
|
|
||||||
expect(result.status).toBe("mounted");
|
expect(result.status).toBe("mounted");
|
||||||
expect(unmount).toHaveBeenCalledOnce();
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenNthCalledWith(
|
||||||
expect(mount).toHaveBeenCalledOnce();
|
1,
|
||||||
expect(unmount.mock.invocationCallOrder[0]).toBeLessThan(mount.mock.invocationCallOrder[0]);
|
volume.agentId,
|
||||||
|
expect.objectContaining({ name: "volume.unmount", volume: expect.objectContaining({ id: volume.id }) }),
|
||||||
|
);
|
||||||
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenNthCalledWith(
|
||||||
|
2,
|
||||||
|
volume.agentId,
|
||||||
|
expect.objectContaining({ name: "volume.mount", volume: expect.objectContaining({ id: volume.id }) }),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
@ -158,15 +155,8 @@ describe("volumeService.mountVolume", () => {
|
||||||
describe("volumeService.ensureHealthyVolume", () => {
|
describe("volumeService.ensureHealthyVolume", () => {
|
||||||
test("returns ready when the mounted volume passes its health check", async () => {
|
test("returns ready when the mounted volume passes its health check", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const volume = await createTestVolume({ organizationId, status: "mounted" });
|
const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
|
||||||
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
|
agentManagerMock.runVolumeCommand.mockResolvedValue({ name: "volume.checkHealth", result: { status: "mounted" } });
|
||||||
const checkHealth = vi.fn().mockResolvedValue({ status: "mounted" });
|
|
||||||
|
|
||||||
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({
|
|
||||||
mount,
|
|
||||||
unmount: vi.fn().mockResolvedValue({ status: "unmounted" }),
|
|
||||||
checkHealth,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await withContext({ organizationId, userId: user.id }, async () => {
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
||||||
|
|
@ -176,22 +166,21 @@ describe("volumeService.ensureHealthyVolume", () => {
|
||||||
volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }),
|
volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }),
|
||||||
remounted: false,
|
remounted: false,
|
||||||
});
|
});
|
||||||
expect(checkHealth).toHaveBeenCalledOnce();
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledOnce();
|
||||||
expect(mount).not.toHaveBeenCalled();
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
|
||||||
|
volume.agentId,
|
||||||
|
expect.objectContaining({ name: "volume.checkHealth", volume: expect.objectContaining({ id: volume.id }) }),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("auto-remounts when the mounted volume fails its health check", async () => {
|
test("auto-remounts when the mounted volume fails its health check", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: true });
|
const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: true, agentId: "agent-1" });
|
||||||
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
|
agentManagerMock.runVolumeCommand
|
||||||
const checkHealth = vi.fn().mockResolvedValue({ status: "error", error: "stale mount" });
|
.mockResolvedValueOnce({ name: "volume.checkHealth", result: { status: "error", error: "stale mount" } })
|
||||||
|
.mockResolvedValueOnce({ name: "volume.unmount", result: { status: "unmounted" } })
|
||||||
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({
|
.mockResolvedValueOnce({ name: "volume.mount", result: { status: "mounted" } });
|
||||||
mount,
|
|
||||||
unmount: vi.fn().mockResolvedValue({ status: "unmounted" }),
|
|
||||||
checkHealth,
|
|
||||||
}));
|
|
||||||
|
|
||||||
await withContext({ organizationId, userId: user.id }, async () => {
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
||||||
|
|
@ -201,8 +190,7 @@ describe("volumeService.ensureHealthyVolume", () => {
|
||||||
volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }),
|
volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }),
|
||||||
remounted: true,
|
remounted: true,
|
||||||
});
|
});
|
||||||
expect(checkHealth).toHaveBeenCalledOnce();
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledTimes(3);
|
||||||
expect(mount).toHaveBeenCalledOnce();
|
|
||||||
|
|
||||||
const updatedVolume = await db.query.volumesTable.findFirst({ where: { id: volume.id } });
|
const updatedVolume = await db.query.volumesTable.findFirst({ where: { id: volume.id } });
|
||||||
expect(updatedVolume?.status).toBe("mounted");
|
expect(updatedVolume?.status).toBe("mounted");
|
||||||
|
|
@ -212,15 +200,16 @@ describe("volumeService.ensureHealthyVolume", () => {
|
||||||
|
|
||||||
test("returns not ready when the health check fails and auto-remount is disabled", async () => {
|
test("returns not ready when the health check fails and auto-remount is disabled", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: false });
|
const volume = await createTestVolume({
|
||||||
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
|
organizationId,
|
||||||
const checkHealth = vi.fn().mockResolvedValue({ status: "error", error: "stale mount" });
|
status: "mounted",
|
||||||
|
autoRemount: false,
|
||||||
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({
|
agentId: "agent-1",
|
||||||
mount,
|
});
|
||||||
unmount: vi.fn().mockResolvedValue({ status: "unmounted" }),
|
agentManagerMock.runVolumeCommand.mockResolvedValue({
|
||||||
checkHealth,
|
name: "volume.checkHealth",
|
||||||
}));
|
result: { status: "error", error: "stale mount" },
|
||||||
|
});
|
||||||
|
|
||||||
await withContext({ organizationId, userId: user.id }, async () => {
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
||||||
|
|
@ -230,54 +219,17 @@ describe("volumeService.ensureHealthyVolume", () => {
|
||||||
volume: expect.objectContaining({ id: volume.id, status: "error", lastError: "stale mount" }),
|
volume: expect.objectContaining({ id: volume.id, status: "error", lastError: "stale mount" }),
|
||||||
reason: "stale mount",
|
reason: "stale mount",
|
||||||
});
|
});
|
||||||
expect(checkHealth).toHaveBeenCalledOnce();
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledOnce();
|
||||||
expect(mount).not.toHaveBeenCalled();
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("volumeService.testConnection", () => {
|
describe("volumeService.testConnection", () => {
|
||||||
test("uses an isolated temp mount path for backend test connections", async () => {
|
test("routes test connections to the local agent", async () => {
|
||||||
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
|
config.flags.enableLocalAgent = true;
|
||||||
const unmount = vi.fn().mockResolvedValue({ status: "unmounted" });
|
agentManagerMock.runVolumeCommand.mockResolvedValue({
|
||||||
const createVolumeBackendSpy = vi.spyOn(backendModule, "createVolumeBackend").mockReturnValue({
|
name: "volume.testConnection",
|
||||||
mount,
|
result: { success: true, message: "Connection successful" },
|
||||||
unmount,
|
|
||||||
checkHealth: vi.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
await volumeService.testConnection({
|
|
||||||
backend: "nfs",
|
|
||||||
server: "127.0.0.1",
|
|
||||||
exportPath: "/exports/test",
|
|
||||||
version: "4",
|
|
||||||
port: 2049,
|
|
||||||
readOnly: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(createVolumeBackendSpy).toHaveBeenCalledOnce();
|
|
||||||
const [, mountPath] = createVolumeBackendSpy.mock.calls[0];
|
|
||||||
expect(mountPath).toEqual(expect.stringContaining(`${path.sep}zerobyte-test-`));
|
|
||||||
await expect(fs.access(mountPath as string)).rejects.toThrow();
|
|
||||||
expect(mount).toHaveBeenCalledOnce();
|
|
||||||
expect(unmount).toHaveBeenCalledOnce();
|
|
||||||
});
|
|
||||||
|
|
||||||
test("does not fail when backend unmount already removed the temp mount path", async () => {
|
|
||||||
const mount = vi.fn().mockResolvedValue({ status: "mounted" });
|
|
||||||
let mountPath: string | undefined;
|
|
||||||
const unmount = vi.fn().mockImplementation(async () => {
|
|
||||||
await fs.rm(mountPath!, { recursive: true, force: true });
|
|
||||||
return { status: "unmounted" };
|
|
||||||
});
|
|
||||||
|
|
||||||
vi.spyOn(backendModule, "createVolumeBackend").mockImplementation((_volume, tempPath) => {
|
|
||||||
mountPath = tempPath;
|
|
||||||
return {
|
|
||||||
mount,
|
|
||||||
unmount,
|
|
||||||
checkHealth: vi.fn(),
|
|
||||||
};
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
|
|
@ -294,9 +246,9 @@ describe("volumeService.testConnection", () => {
|
||||||
message: "Connection successful",
|
message: "Connection successful",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(mountPath).toEqual(expect.stringContaining(`${path.sep}zerobyte-test-`));
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
|
||||||
await expect(fs.access(mountPath as string)).rejects.toThrow();
|
"local",
|
||||||
expect(mount).toHaveBeenCalledOnce();
|
expect.objectContaining({ name: "volume.testConnection" }),
|
||||||
expect(unmount).toHaveBeenCalledOnce();
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -32,3 +32,7 @@ export const mapVolumeConfigSecrets = async (
|
||||||
export const encryptVolumeConfig = async (config: BackendConfig): Promise<BackendConfig> => {
|
export const encryptVolumeConfig = async (config: BackendConfig): Promise<BackendConfig> => {
|
||||||
return await mapVolumeConfigSecrets(config, cryptoUtils.sealSecret);
|
return await mapVolumeConfigSecrets(config, cryptoUtils.sealSecret);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const decryptVolumeConfig = async (config: BackendConfig): Promise<BackendConfig> => {
|
||||||
|
return await mapVolumeConfigSecrets(config, cryptoUtils.resolveSecret);
|
||||||
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,37 +1,39 @@
|
||||||
import * as fs from "node:fs/promises";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import * as path from "node:path";
|
|
||||||
import { and, eq } from "drizzle-orm";
|
import { and, eq } from "drizzle-orm";
|
||||||
import { BadRequestError, InternalServerError, NotFoundError } from "http-errors-enhanced";
|
import { BadRequestError, InternalServerError, NotFoundError } from "http-errors-enhanced";
|
||||||
import { db } from "../../db/db";
|
import { db } from "../../db/db";
|
||||||
import { volumesTable } from "../../db/schema";
|
import { volumesTable } from "../../db/schema";
|
||||||
import { toMessage } from "../../utils/errors";
|
import { toMessage } from "../../utils/errors";
|
||||||
import { generateShortId } from "../../utils/id";
|
import { generateShortId } from "../../utils/id";
|
||||||
import { getStatFs, type StatFs } from "../../utils/mountinfo";
|
import type { StatFs } from "../../utils/mountinfo";
|
||||||
import { withTimeout } from "../../utils/timeout";
|
import { withTimeout } from "../../utils/timeout";
|
||||||
import { createVolumeBackend } from "../backends/backend";
|
import { config } from "../../core/config";
|
||||||
|
import { LOCAL_AGENT_ID } from "../agents/constants";
|
||||||
|
import { agentManager } from "../agents/agents-manager";
|
||||||
import type { UpdateVolumeBody } from "./volume.dto";
|
import type { UpdateVolumeBody } from "./volume.dto";
|
||||||
import { getVolumePath } from "./helpers";
|
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { serverEvents } from "../../core/events";
|
import { serverEvents } from "../../core/events";
|
||||||
import type { Volume } from "../../db/schema";
|
import type { Volume } from "../../db/schema";
|
||||||
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes";
|
||||||
import { getOrganizationId } from "~/server/core/request-context";
|
import { getOrganizationId } from "~/server/core/request-context";
|
||||||
import { isNodeJSErrnoException } from "~/server/utils/fs";
|
import { type ShortId } from "~/server/utils/branded";
|
||||||
import { asShortId, type ShortId } from "~/server/utils/branded";
|
import { decryptVolumeConfig, encryptVolumeConfig } from "./volume-config-secrets";
|
||||||
import { encryptVolumeConfig } from "./volume-config-secrets";
|
import type { VolumeCommand, VolumeCommandResult } from "@zerobyte/contracts/agent-protocol";
|
||||||
|
import {
|
||||||
|
createVolumeBackend,
|
||||||
|
getStatFs,
|
||||||
|
getVolumePath,
|
||||||
|
type AgentVolume,
|
||||||
|
type BackendConfig as HostBackendConfig,
|
||||||
|
} from "../../../../apps/agent/src/volume-host";
|
||||||
|
import {
|
||||||
|
browseFilesystem as browseHostFilesystem,
|
||||||
|
listVolumeFiles,
|
||||||
|
testVolumeConnection,
|
||||||
|
} from "../../../../apps/agent/src/volume-host/operations";
|
||||||
|
|
||||||
type EnsureHealthyVolumeResult =
|
type EnsureHealthyVolumeResult =
|
||||||
| {
|
| { ready: true; volume: Volume; remounted: boolean }
|
||||||
ready: true;
|
| { ready: false; volume: Volume; reason: string };
|
||||||
volume: Volume;
|
|
||||||
remounted: boolean;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
ready: false;
|
|
||||||
volume: Volume;
|
|
||||||
reason: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
const listVolumes = async () => {
|
const listVolumes = async () => {
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
|
|
@ -52,6 +54,68 @@ const findVolume = async (shortId: ShortId) => {
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const runVolumeCommand = async <TCommand extends VolumeCommand>(agentId: string, command: TCommand) => {
|
||||||
|
const result = await agentManager.runVolumeCommand(agentId, command);
|
||||||
|
if (result.name !== command.name) {
|
||||||
|
throw new InternalServerError(`Unexpected agent response for ${command.name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result as Extract<VolumeCommandResult, { name: TCommand["name"] }>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const volumeForAgent = async (volume: Volume): Promise<Volume> => ({
|
||||||
|
...volume,
|
||||||
|
config: await decryptVolumeConfig(volume.config),
|
||||||
|
});
|
||||||
|
|
||||||
|
const volumeForHost = async (volume: Volume): Promise<AgentVolume> => ({
|
||||||
|
...volume,
|
||||||
|
shortId: volume.shortId,
|
||||||
|
config: (await decryptVolumeConfig(volume.config)) as HostBackendConfig,
|
||||||
|
provisioningId: volume.provisioningId ?? null,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Transitional fallback: older controller-only installs do not run the supervised local agent.
|
||||||
|
// Keep all controller-local host execution behind this predicate so the fallback is easy to delete
|
||||||
|
// once volume operations always go through the local agent.
|
||||||
|
const shouldRunViaAgent = (volume: Volume) => volume.agentId !== LOCAL_AGENT_ID || config.flags.enableLocalAgent;
|
||||||
|
|
||||||
|
const shouldUseControllerLocalVolumeFallback = (volume: Volume) => !shouldRunViaAgent(volume);
|
||||||
|
|
||||||
|
const runVolumeBackendCommand = async (
|
||||||
|
volume: Volume,
|
||||||
|
name: "volume.mount" | "volume.unmount" | "volume.checkHealth",
|
||||||
|
) => {
|
||||||
|
if (shouldUseControllerLocalVolumeFallback(volume)) {
|
||||||
|
const backend = createVolumeBackend(await volumeForHost(volume));
|
||||||
|
switch (name) {
|
||||||
|
case "volume.mount":
|
||||||
|
return backend.mount();
|
||||||
|
case "volume.unmount":
|
||||||
|
return backend.unmount();
|
||||||
|
case "volume.checkHealth":
|
||||||
|
return backend.checkHealth();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const command = await runVolumeCommand(volume.agentId, {
|
||||||
|
name,
|
||||||
|
volume: await volumeForAgent(volume),
|
||||||
|
});
|
||||||
|
return command.result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const mapAgentFileError = (error: unknown) => {
|
||||||
|
const message = toMessage(error);
|
||||||
|
if (message === "Invalid path") {
|
||||||
|
throw new BadRequestError("Invalid path");
|
||||||
|
}
|
||||||
|
if (message === "Directory not found") {
|
||||||
|
throw new NotFoundError("Directory not found");
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
};
|
||||||
|
|
||||||
const createVolume = async (name: string, backendConfig: BackendConfig) => {
|
const createVolume = async (name: string, backendConfig: BackendConfig) => {
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
const trimmedName = name.trim();
|
const trimmedName = name.trim();
|
||||||
|
|
@ -70,6 +134,7 @@ const createVolume = async (name: string, backendConfig: BackendConfig) => {
|
||||||
name: trimmedName,
|
name: trimmedName,
|
||||||
config: encryptedConfig,
|
config: encryptedConfig,
|
||||||
type: backendConfig.backend,
|
type: backendConfig.backend,
|
||||||
|
agentId: LOCAL_AGENT_ID,
|
||||||
organizationId,
|
organizationId,
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
|
|
@ -78,8 +143,7 @@ const createVolume = async (name: string, backendConfig: BackendConfig) => {
|
||||||
throw new InternalServerError("Failed to create volume");
|
throw new InternalServerError("Failed to create volume");
|
||||||
}
|
}
|
||||||
|
|
||||||
const backend = createVolumeBackend(created);
|
const { error, status } = await runVolumeBackendCommand(created, "volume.mount");
|
||||||
const { error, status } = await backend.mount();
|
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.update(volumesTable)
|
.update(volumesTable)
|
||||||
|
|
@ -97,8 +161,7 @@ const deleteVolume = async (shortId: ShortId) => {
|
||||||
throw new NotFoundError("Volume not found");
|
throw new NotFoundError("Volume not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
const backend = createVolumeBackend(volume);
|
await runVolumeBackendCommand(volume, "volume.unmount");
|
||||||
await backend.unmount();
|
|
||||||
await db
|
await db
|
||||||
.delete(volumesTable)
|
.delete(volumesTable)
|
||||||
.where(and(eq(volumesTable.id, volume.id), eq(volumesTable.organizationId, organizationId)));
|
.where(and(eq(volumesTable.id, volume.id), eq(volumesTable.organizationId, organizationId)));
|
||||||
|
|
@ -112,9 +175,8 @@ const mountVolume = async (shortId: ShortId) => {
|
||||||
throw new NotFoundError("Volume not found");
|
throw new NotFoundError("Volume not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
const backend = createVolumeBackend(volume);
|
await runVolumeBackendCommand(volume, "volume.unmount");
|
||||||
await backend.unmount();
|
const { error, status } = await runVolumeBackendCommand(volume, "volume.mount");
|
||||||
const { error, status } = await backend.mount();
|
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.update(volumesTable)
|
.update(volumesTable)
|
||||||
|
|
@ -136,8 +198,7 @@ const unmountVolume = async (shortId: ShortId) => {
|
||||||
throw new NotFoundError("Volume not found");
|
throw new NotFoundError("Volume not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
const backend = createVolumeBackend(volume);
|
const { status, error } = await runVolumeBackendCommand(volume, "volume.unmount");
|
||||||
const { status, error } = await backend.unmount();
|
|
||||||
|
|
||||||
await db
|
await db
|
||||||
.update(volumesTable)
|
.update(volumesTable)
|
||||||
|
|
@ -160,7 +221,15 @@ const getVolume = async (shortId: ShortId) => {
|
||||||
|
|
||||||
let statfs: Partial<StatFs> = {};
|
let statfs: Partial<StatFs> = {};
|
||||||
if (volume.status === "mounted") {
|
if (volume.status === "mounted") {
|
||||||
statfs = await withTimeout(getStatFs(getVolumePath(volume)), 1000, "getStatFs").catch((error) => {
|
statfs = await withTimeout(
|
||||||
|
!shouldUseControllerLocalVolumeFallback(volume)
|
||||||
|
? runVolumeCommand(volume.agentId, { name: "volume.statfs", volume: await volumeForAgent(volume) }).then(
|
||||||
|
(command) => command.result,
|
||||||
|
)
|
||||||
|
: volumeForHost(volume).then((hostVolume) => getStatFs(getVolumePath(hostVolume))),
|
||||||
|
1000,
|
||||||
|
"volume.statfs",
|
||||||
|
).catch((error) => {
|
||||||
logger.warn(`Failed to get statfs for volume ${volume.name}: ${toMessage(error)}`);
|
logger.warn(`Failed to get statfs for volume ${volume.name}: ${toMessage(error)}`);
|
||||||
return {};
|
return {};
|
||||||
});
|
});
|
||||||
|
|
@ -188,8 +257,7 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => {
|
||||||
|
|
||||||
if (configChanged) {
|
if (configChanged) {
|
||||||
logger.debug("Unmounting existing volume before applying new config");
|
logger.debug("Unmounting existing volume before applying new config");
|
||||||
const backend = createVolumeBackend(existing);
|
await runVolumeBackendCommand(existing, "volume.unmount");
|
||||||
await backend.unmount();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const newConfigResult = volumeConfigSchema.safeParse(volumeData.config || existing.config);
|
const newConfigResult = volumeConfigSchema.safeParse(volumeData.config || existing.config);
|
||||||
|
|
@ -217,8 +285,7 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (configChanged) {
|
if (configChanged) {
|
||||||
const backend = createVolumeBackend(updated);
|
const { error, status } = await runVolumeBackendCommand(updated, "volume.mount");
|
||||||
const { error, status } = await backend.mount();
|
|
||||||
await db
|
await db
|
||||||
.update(volumesTable)
|
.update(volumesTable)
|
||||||
.set({ status, lastError: error ?? null, lastHealthCheck: Date.now() })
|
.set({ status, lastError: error ?? null, lastHealthCheck: Date.now() })
|
||||||
|
|
@ -231,38 +298,12 @@ const updateVolume = async (shortId: ShortId, volumeData: UpdateVolumeBody) => {
|
||||||
};
|
};
|
||||||
|
|
||||||
const testConnection = async (backendConfig: BackendConfig) => {
|
const testConnection = async (backendConfig: BackendConfig) => {
|
||||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-test-"));
|
if (!config.flags.enableLocalAgent) {
|
||||||
try {
|
return await testVolumeConnection(backendConfig as HostBackendConfig);
|
||||||
const encryptedConfig = await encryptVolumeConfig(backendConfig);
|
|
||||||
|
|
||||||
const mockVolume = {
|
|
||||||
id: 0,
|
|
||||||
shortId: asShortId("test"),
|
|
||||||
name: "test-connection",
|
|
||||||
config: encryptedConfig,
|
|
||||||
createdAt: Date.now(),
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
lastHealthCheck: Date.now(),
|
|
||||||
type: encryptedConfig.backend,
|
|
||||||
status: "unmounted" as const,
|
|
||||||
lastError: null,
|
|
||||||
provisioningId: null,
|
|
||||||
autoRemount: true,
|
|
||||||
organizationId: "test-org",
|
|
||||||
};
|
|
||||||
|
|
||||||
const backend = createVolumeBackend(mockVolume, tempDir);
|
|
||||||
const { error } = await backend.mount();
|
|
||||||
|
|
||||||
await backend.unmount();
|
|
||||||
|
|
||||||
return {
|
|
||||||
success: !error,
|
|
||||||
message: error ? toMessage(error) : "Connection successful",
|
|
||||||
};
|
|
||||||
} finally {
|
|
||||||
await fs.rm(tempDir, { recursive: true, force: true });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const command = await runVolumeCommand(LOCAL_AGENT_ID, { name: "volume.testConnection", backendConfig });
|
||||||
|
return command.result;
|
||||||
};
|
};
|
||||||
|
|
||||||
const checkHealth = async (shortId: ShortId) => {
|
const checkHealth = async (shortId: ShortId) => {
|
||||||
|
|
@ -273,8 +314,7 @@ const checkHealth = async (shortId: ShortId) => {
|
||||||
throw new NotFoundError("Volume not found");
|
throw new NotFoundError("Volume not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
const backend = createVolumeBackend(volume);
|
const { error, status } = await runVolumeBackendCommand(volume, "volume.checkHealth");
|
||||||
const { error, status } = await backend.checkHealth();
|
|
||||||
|
|
||||||
if (status !== volume.status) {
|
if (status !== volume.status) {
|
||||||
serverEvents.emit("volume:status_changed", { organizationId, volumeName: volume.name, status });
|
serverEvents.emit("volume:status_changed", { organizationId, volumeName: volume.name, status });
|
||||||
|
|
@ -342,7 +382,6 @@ const ensureHealthyVolume = async (shortId: ShortId): Promise<EnsureHealthyVolum
|
||||||
};
|
};
|
||||||
|
|
||||||
const DEFAULT_PAGE_SIZE = 500;
|
const DEFAULT_PAGE_SIZE = 500;
|
||||||
const MAX_PAGE_SIZE = 500;
|
|
||||||
|
|
||||||
const listFiles = async (shortId: ShortId, subPath?: string, offset: number = 0, limit: number = DEFAULT_PAGE_SIZE) => {
|
const listFiles = async (shortId: ShortId, subPath?: string, offset: number = 0, limit: number = DEFAULT_PAGE_SIZE) => {
|
||||||
const volume = await findVolume(shortId);
|
const volume = await findVolume(shortId);
|
||||||
|
|
@ -355,110 +394,33 @@ const listFiles = async (shortId: ShortId, subPath?: string, offset: number = 0,
|
||||||
throw new InternalServerError("Volume is not mounted");
|
throw new InternalServerError("Volume is not mounted");
|
||||||
}
|
}
|
||||||
|
|
||||||
const volumePath = getVolumePath(volume);
|
|
||||||
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
|
|
||||||
const normalizedPath = path.normalize(requestedPath);
|
|
||||||
const relative = path.relative(volumePath, normalizedPath);
|
|
||||||
|
|
||||||
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
||||||
throw new BadRequestError("Invalid path");
|
|
||||||
}
|
|
||||||
|
|
||||||
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
|
|
||||||
const startOffset = Math.max(offset, 0);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
|
if (shouldUseControllerLocalVolumeFallback(volume)) {
|
||||||
|
return await listVolumeFiles(await volumeForHost(volume), subPath, offset, limit);
|
||||||
dirents.sort((a, b) => {
|
|
||||||
const aIsDir = a.isDirectory();
|
|
||||||
const bIsDir = b.isDirectory();
|
|
||||||
|
|
||||||
if (aIsDir === bIsDir) {
|
|
||||||
return a.name.localeCompare(b.name);
|
|
||||||
}
|
|
||||||
return aIsDir ? -1 : 1;
|
|
||||||
});
|
|
||||||
|
|
||||||
const total = dirents.length;
|
|
||||||
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
|
|
||||||
|
|
||||||
const entries = (
|
|
||||||
await Promise.all(
|
|
||||||
paginatedDirents.map(async (dirent) => {
|
|
||||||
const fullPath = path.join(normalizedPath, dirent.name);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const stats = await fs.stat(fullPath);
|
|
||||||
const relativePath = path.relative(volumePath, fullPath);
|
|
||||||
|
|
||||||
return {
|
|
||||||
name: dirent.name,
|
|
||||||
path: `/${relativePath}`,
|
|
||||||
type: dirent.isDirectory() ? ("directory" as const) : ("file" as const),
|
|
||||||
size: dirent.isFile() ? stats.size : undefined,
|
|
||||||
modifiedAt: stats.mtimeMs,
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
).filter((e) => e !== null);
|
|
||||||
|
|
||||||
return {
|
|
||||||
files: entries,
|
|
||||||
path: subPath || "/",
|
|
||||||
offset: startOffset,
|
|
||||||
limit: pageSize,
|
|
||||||
total,
|
|
||||||
hasMore: startOffset + pageSize < total,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
if (isNodeJSErrnoException(error) && error.code === "ENOENT") {
|
|
||||||
throw new NotFoundError("Directory not found");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const command = await runVolumeCommand(volume.agentId, {
|
||||||
|
name: "volume.listFiles",
|
||||||
|
volume: await volumeForAgent(volume),
|
||||||
|
subPath,
|
||||||
|
offset,
|
||||||
|
limit,
|
||||||
|
});
|
||||||
|
return command.result;
|
||||||
|
} catch (error) {
|
||||||
|
mapAgentFileError(error);
|
||||||
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
|
throw new InternalServerError(`Failed to list files: ${toMessage(error)}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const browseFilesystem = async (browsePath: string) => {
|
const browseFilesystem = async (browsePath: string) => {
|
||||||
const normalizedPath = path.normalize(browsePath);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const entries = await fs.readdir(normalizedPath, { withFileTypes: true });
|
if (!config.flags.enableLocalAgent) {
|
||||||
|
return await browseHostFilesystem(browsePath);
|
||||||
|
}
|
||||||
|
|
||||||
const directories = await Promise.all(
|
const command = await runVolumeCommand(LOCAL_AGENT_ID, { name: "filesystem.browse", path: browsePath });
|
||||||
entries
|
return command.result;
|
||||||
.filter((entry) => entry.isDirectory())
|
|
||||||
.map(async (entry) => {
|
|
||||||
const fullPath = path.join(normalizedPath, entry.name);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const stats = await fs.stat(fullPath);
|
|
||||||
return {
|
|
||||||
name: entry.name,
|
|
||||||
path: fullPath,
|
|
||||||
type: "directory" as const,
|
|
||||||
size: undefined,
|
|
||||||
modifiedAt: stats.mtimeMs,
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
return {
|
|
||||||
name: entry.name,
|
|
||||||
path: fullPath,
|
|
||||||
type: "directory" as const,
|
|
||||||
size: undefined,
|
|
||||||
modifiedAt: undefined,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
return {
|
|
||||||
directories: directories.sort((a, b) => a.name.localeCompare(b.name)),
|
|
||||||
path: normalizedPath,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new InternalServerError(`Failed to browse filesystem: ${toMessage(error)}`);
|
throw new InternalServerError(`Failed to browse filesystem: ${toMessage(error)}`);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import { db } from "~/server/db/db";
|
import { db } from "~/server/db/db";
|
||||||
import { faker } from "@faker-js/faker";
|
import { faker } from "@faker-js/faker";
|
||||||
import { volumesTable, type VolumeInsert } from "~/server/db/schema";
|
import { volumesTable, type VolumeInsert } from "~/server/db/schema";
|
||||||
|
import { LOCAL_AGENT_ID } from "~/server/modules/agents/constants";
|
||||||
import { ensureTestOrganization, TEST_ORG_ID } from "./organization";
|
import { ensureTestOrganization, TEST_ORG_ID } from "./organization";
|
||||||
import { generateShortId } from "~/server/utils/id";
|
import { generateShortId } from "~/server/utils/id";
|
||||||
|
|
||||||
|
|
@ -17,6 +18,7 @@ export const createTestVolume = async (overrides: Partial<VolumeInsert> = {}) =>
|
||||||
autoRemount: true,
|
autoRemount: true,
|
||||||
shortId: generateShortId(),
|
shortId: generateShortId(),
|
||||||
type: "directory",
|
type: "directory",
|
||||||
|
agentId: LOCAL_AGENT_ID,
|
||||||
organizationId: TEST_ORG_ID,
|
organizationId: TEST_ORG_ID,
|
||||||
...overrides,
|
...overrides,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import { handleBackupCancelCommand } from "./backup-cancel";
|
||||||
import { handleBackupRunCommand } from "./backup-run";
|
import { handleBackupRunCommand } from "./backup-run";
|
||||||
import type { ControllerCommandContext } from "../context";
|
import type { ControllerCommandContext } from "../context";
|
||||||
import { handleHeartbeatPingCommand } from "./heartbeat-ping";
|
import { handleHeartbeatPingCommand } from "./heartbeat-ping";
|
||||||
|
import { handleVolumeCommand } from "./volume";
|
||||||
|
|
||||||
export const handleControllerCommand = (context: ControllerCommandContext, message: ControllerMessage) => {
|
export const handleControllerCommand = (context: ControllerCommandContext, message: ControllerMessage) => {
|
||||||
switch (message.type) {
|
switch (message.type) {
|
||||||
|
|
@ -12,6 +13,9 @@ export const handleControllerCommand = (context: ControllerCommandContext, messa
|
||||||
case "backup.cancel": {
|
case "backup.cancel": {
|
||||||
return handleBackupCancelCommand(context, message.payload);
|
return handleBackupCancelCommand(context, message.payload);
|
||||||
}
|
}
|
||||||
|
case "volume.command": {
|
||||||
|
return handleVolumeCommand(context, message.payload);
|
||||||
|
}
|
||||||
case "heartbeat.ping": {
|
case "heartbeat.ping": {
|
||||||
return handleHeartbeatPingCommand(context, message.payload);
|
return handleHeartbeatPingCommand(context, message.payload);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
127
apps/agent/src/commands/volume.test.ts
Normal file
127
apps/agent/src/commands/volume.test.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
import { afterEach, expect, test, vi } from "vitest";
|
||||||
|
import { Effect } from "effect";
|
||||||
|
import { fromPartial } from "@total-typescript/shoehorn";
|
||||||
|
import {
|
||||||
|
parseAgentMessage,
|
||||||
|
type AgentWireMessage,
|
||||||
|
type VolumeCommandPayload,
|
||||||
|
} from "@zerobyte/contracts/agent-protocol";
|
||||||
|
import type { ControllerCommandContext } from "../context";
|
||||||
|
|
||||||
|
const volumeHostMock = vi.hoisted(() => ({
|
||||||
|
createVolumeBackend: vi.fn(),
|
||||||
|
getStatFs: vi.fn(),
|
||||||
|
getVolumePath: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const operationsMock = vi.hoisted(() => ({
|
||||||
|
browseFilesystem: vi.fn(),
|
||||||
|
listVolumeFiles: vi.fn(),
|
||||||
|
testVolumeConnection: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../volume-host", () => volumeHostMock);
|
||||||
|
vi.mock("../volume-host/operations", () => operationsMock);
|
||||||
|
|
||||||
|
import { handleVolumeCommand } from "./volume";
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
volumeHostMock.createVolumeBackend.mockReset();
|
||||||
|
volumeHostMock.getStatFs.mockReset();
|
||||||
|
volumeHostMock.getVolumePath.mockReset();
|
||||||
|
operationsMock.browseFilesystem.mockReset();
|
||||||
|
operationsMock.listVolumeFiles.mockReset();
|
||||||
|
operationsMock.testVolumeConnection.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
|
const runVolumeCommand = async (payload: VolumeCommandPayload) => {
|
||||||
|
const outboundMessages: string[] = [];
|
||||||
|
const context = fromPartial<ControllerCommandContext>({
|
||||||
|
offerOutbound: (message: AgentWireMessage) =>
|
||||||
|
Effect.sync(() => {
|
||||||
|
outboundMessages.push(message);
|
||||||
|
return true;
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
await Effect.runPromise(handleVolumeCommand(context, payload));
|
||||||
|
return outboundMessages.map((message) => parseAgentMessage(message));
|
||||||
|
};
|
||||||
|
|
||||||
|
test("runs backend-backed volume commands on the agent host", async () => {
|
||||||
|
const mount = vi.fn(async () => ({ status: "mounted" as const }));
|
||||||
|
volumeHostMock.createVolumeBackend.mockReturnValue({ mount });
|
||||||
|
|
||||||
|
const messages = await runVolumeCommand(
|
||||||
|
fromPartial<VolumeCommandPayload>({
|
||||||
|
commandId: "command-1",
|
||||||
|
command: {
|
||||||
|
name: "volume.mount",
|
||||||
|
volume: { id: 1, config: { backend: "directory", path: "/tmp/source" }, provisioningId: undefined },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(volumeHostMock.createVolumeBackend).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ id: 1, config: { backend: "directory", path: "/tmp/source" }, provisioningId: null }),
|
||||||
|
);
|
||||||
|
expect(mount).toHaveBeenCalledOnce();
|
||||||
|
expect(messages[0]?.success).toBe(true);
|
||||||
|
if (messages[0]?.success && messages[0].data.type === "volume.commandResult") {
|
||||||
|
expect(messages[0].data.payload).toEqual({
|
||||||
|
commandId: "command-1",
|
||||||
|
status: "success",
|
||||||
|
command: { name: "volume.mount", result: { status: "mounted" } },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns command errors without throwing", async () => {
|
||||||
|
operationsMock.browseFilesystem.mockRejectedValue(new Error("permission denied"));
|
||||||
|
|
||||||
|
const messages = await runVolumeCommand({
|
||||||
|
commandId: "command-2",
|
||||||
|
command: { name: "filesystem.browse", path: "/root" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(messages[0]?.success).toBe(true);
|
||||||
|
if (messages[0]?.success && messages[0].data.type === "volume.commandResult") {
|
||||||
|
expect(messages[0].data.payload).toEqual({
|
||||||
|
commandId: "command-2",
|
||||||
|
status: "error",
|
||||||
|
error: "permission denied",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("routes file listing commands to host operations", async () => {
|
||||||
|
operationsMock.listVolumeFiles.mockResolvedValue({
|
||||||
|
files: [],
|
||||||
|
path: "/logs",
|
||||||
|
offset: 0,
|
||||||
|
limit: 10,
|
||||||
|
total: 0,
|
||||||
|
hasMore: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
await runVolumeCommand(
|
||||||
|
fromPartial<VolumeCommandPayload>({
|
||||||
|
commandId: "command-3",
|
||||||
|
command: {
|
||||||
|
name: "volume.listFiles",
|
||||||
|
volume: { id: 1, config: { backend: "directory", path: "/tmp/source" } },
|
||||||
|
subPath: "/logs",
|
||||||
|
offset: 0,
|
||||||
|
limit: 10,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(operationsMock.listVolumeFiles).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ id: 1, config: { backend: "directory", path: "/tmp/source" } }),
|
||||||
|
"/logs",
|
||||||
|
0,
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
});
|
||||||
76
apps/agent/src/commands/volume.ts
Normal file
76
apps/agent/src/commands/volume.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
import { Effect } from "effect";
|
||||||
|
import {
|
||||||
|
createAgentMessage,
|
||||||
|
type VolumeCommand,
|
||||||
|
type VolumeCommandPayload,
|
||||||
|
type VolumeCommandResult,
|
||||||
|
} from "@zerobyte/contracts/agent-protocol";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { createVolumeBackend, getStatFs, getVolumePath, type AgentVolume, type BackendConfig } from "../volume-host";
|
||||||
|
import { browseFilesystem, listVolumeFiles, testVolumeConnection } from "../volume-host/operations";
|
||||||
|
import type { ControllerCommandContext } from "../context";
|
||||||
|
|
||||||
|
type VolumeBackedCommand = Extract<VolumeCommand, { volume: unknown }>;
|
||||||
|
|
||||||
|
const asVolume = (volume: VolumeBackedCommand["volume"]): AgentVolume => ({
|
||||||
|
...volume,
|
||||||
|
config: volume.config as BackendConfig,
|
||||||
|
provisioningId: volume.provisioningId ?? null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const runBackendOperation = async (
|
||||||
|
command: Extract<VolumeCommand, { volume: unknown }>,
|
||||||
|
operation: "mount" | "unmount" | "checkHealth",
|
||||||
|
) => {
|
||||||
|
const backend = createVolumeBackend(asVolume(command.volume));
|
||||||
|
return backend[operation]();
|
||||||
|
};
|
||||||
|
|
||||||
|
const executeVolumeCommand = async (command: VolumeCommand): Promise<VolumeCommandResult> => {
|
||||||
|
switch (command.name) {
|
||||||
|
case "volume.mount":
|
||||||
|
return { name: command.name, result: await runBackendOperation(command, "mount") };
|
||||||
|
case "volume.unmount":
|
||||||
|
return { name: command.name, result: await runBackendOperation(command, "unmount") };
|
||||||
|
case "volume.checkHealth":
|
||||||
|
return { name: command.name, result: await runBackendOperation(command, "checkHealth") };
|
||||||
|
case "volume.statfs":
|
||||||
|
return { name: command.name, result: await getStatFs(getVolumePath(asVolume(command.volume))) };
|
||||||
|
case "volume.listFiles":
|
||||||
|
return {
|
||||||
|
name: command.name,
|
||||||
|
result: await listVolumeFiles(asVolume(command.volume), command.subPath, command.offset, command.limit),
|
||||||
|
};
|
||||||
|
case "volume.testConnection":
|
||||||
|
return { name: command.name, result: await testVolumeConnection(command.backendConfig as BackendConfig) };
|
||||||
|
case "filesystem.browse":
|
||||||
|
return { name: command.name, result: await browseFilesystem(command.path) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const handleVolumeCommand = (context: ControllerCommandContext, payload: VolumeCommandPayload) => {
|
||||||
|
return Effect.promise(async () => {
|
||||||
|
try {
|
||||||
|
const command = await executeVolumeCommand(payload.command);
|
||||||
|
await Effect.runPromise(
|
||||||
|
context.offerOutbound(
|
||||||
|
createAgentMessage("volume.commandResult", {
|
||||||
|
commandId: payload.commandId,
|
||||||
|
status: "success",
|
||||||
|
command,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
await Effect.runPromise(
|
||||||
|
context.offerOutbound(
|
||||||
|
createAgentMessage("volume.commandResult", {
|
||||||
|
commandId: payload.commandId,
|
||||||
|
status: "error",
|
||||||
|
error: toMessage(error),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
@ -1,14 +1,32 @@
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import { createControllerSession, type ControllerSession } from "./controller-session";
|
import { createControllerSession, type ControllerSession } from "./controller-session";
|
||||||
|
import { cleanupDanglingVolumeMountDirectories } from "./volume-host/cleanup";
|
||||||
|
|
||||||
const controllerUrl = process.env.ZEROBYTE_CONTROLLER_URL;
|
const controllerUrl = process.env.ZEROBYTE_CONTROLLER_URL;
|
||||||
const agentToken = process.env.ZEROBYTE_AGENT_TOKEN;
|
const agentToken = process.env.ZEROBYTE_AGENT_TOKEN;
|
||||||
const RECONNECT_DELAY_MS = 1000;
|
const RECONNECT_DELAY_MS = 1000;
|
||||||
|
const VOLUME_CLEANUP_INTERVAL_MS = 60 * 60 * 1000;
|
||||||
|
|
||||||
export class Agent {
|
export class Agent {
|
||||||
private ws: WebSocket | null = null;
|
private ws: WebSocket | null = null;
|
||||||
private controllerSession: ControllerSession | null = null;
|
private controllerSession: ControllerSession | null = null;
|
||||||
private reconnectTimeout: ReturnType<typeof setTimeout> | null = null;
|
private reconnectTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
private volumeCleanupInterval: ReturnType<typeof setInterval> | null = null;
|
||||||
|
|
||||||
|
private runVolumeCleanup() {
|
||||||
|
void cleanupDanglingVolumeMountDirectories().catch((error) => {
|
||||||
|
logger.warn(`Agent volume cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private startVolumeCleanup() {
|
||||||
|
if (this.volumeCleanupInterval) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.runVolumeCleanup();
|
||||||
|
this.volumeCleanupInterval = setInterval(() => this.runVolumeCleanup(), VOLUME_CLEANUP_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
private scheduleReconnect() {
|
private scheduleReconnect() {
|
||||||
if (this.reconnectTimeout) {
|
if (this.reconnectTimeout) {
|
||||||
|
|
@ -22,6 +40,8 @@ export class Agent {
|
||||||
}
|
}
|
||||||
|
|
||||||
connect() {
|
connect() {
|
||||||
|
this.startVolumeCleanup();
|
||||||
|
|
||||||
if (this.reconnectTimeout) {
|
if (this.reconnectTimeout) {
|
||||||
clearTimeout(this.reconnectTimeout);
|
clearTimeout(this.reconnectTimeout);
|
||||||
this.reconnectTimeout = null;
|
this.reconnectTimeout = null;
|
||||||
|
|
|
||||||
49
apps/agent/src/volume-host/__tests__/cleanup.test.ts
Normal file
49
apps/agent/src/volume-host/__tests__/cleanup.test.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { afterEach, expect, test, vi } from "vitest";
|
||||||
|
|
||||||
|
let tempRoot: string | undefined;
|
||||||
|
let mockMountPoints: string[] = [];
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
vi.doUnmock("./constants");
|
||||||
|
vi.doUnmock("./fs");
|
||||||
|
vi.resetModules();
|
||||||
|
mockMountPoints = [];
|
||||||
|
if (tempRoot) {
|
||||||
|
await fs.rm(tempRoot, { recursive: true, force: true });
|
||||||
|
tempRoot = undefined;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const loadCleanup = async () => {
|
||||||
|
vi.resetModules();
|
||||||
|
tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-agent-cleanup-"));
|
||||||
|
vi.doMock("./constants", () => ({ VOLUME_MOUNT_BASE: tempRoot }));
|
||||||
|
vi.doMock("./fs", () => ({
|
||||||
|
readMountInfo: async () => mockMountPoints.map((mountPoint) => ({ mountPoint, fstype: "fuse.sshfs" })),
|
||||||
|
}));
|
||||||
|
|
||||||
|
return import("../cleanup");
|
||||||
|
};
|
||||||
|
|
||||||
|
test("removes stale volume directories that are not mounted on the agent host", async () => {
|
||||||
|
const { cleanupDanglingVolumeMountDirectories } = await loadCleanup();
|
||||||
|
await fs.mkdir(path.join(tempRoot!, "stale-volume", "_data"), { recursive: true });
|
||||||
|
|
||||||
|
await cleanupDanglingVolumeMountDirectories();
|
||||||
|
|
||||||
|
await expect(fs.access(path.join(tempRoot!, "stale-volume"))).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("keeps volume directories that are still mounted on the agent host", async () => {
|
||||||
|
const { cleanupDanglingVolumeMountDirectories } = await loadCleanup();
|
||||||
|
const localVolumeDir = path.join(tempRoot!, "mounted-volume");
|
||||||
|
mockMountPoints = [path.join(localVolumeDir, "_data")];
|
||||||
|
await fs.mkdir(path.join(localVolumeDir, "_data"), { recursive: true });
|
||||||
|
|
||||||
|
await cleanupDanglingVolumeMountDirectories();
|
||||||
|
|
||||||
|
await expect(fs.access(localVolumeDir)).resolves.toBeNull();
|
||||||
|
});
|
||||||
66
apps/agent/src/volume-host/__tests__/operations.test.ts
Normal file
66
apps/agent/src/volume-host/__tests__/operations.test.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { afterEach, expect, test } from "vitest";
|
||||||
|
import { listVolumeFiles } from "../operations";
|
||||||
|
import type { AgentVolume } from "../types";
|
||||||
|
|
||||||
|
let tempRoot: string | undefined;
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (tempRoot) {
|
||||||
|
await fs.rm(tempRoot, { recursive: true, force: true });
|
||||||
|
tempRoot = undefined;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const createDirectoryVolume = async (): Promise<AgentVolume> => {
|
||||||
|
tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-volume-ops-"));
|
||||||
|
return {
|
||||||
|
id: 1,
|
||||||
|
shortId: "volume-1",
|
||||||
|
name: "Test volume",
|
||||||
|
config: { backend: "directory", path: tempRoot },
|
||||||
|
createdAt: Date.now(),
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
lastHealthCheck: Date.now(),
|
||||||
|
type: "directory",
|
||||||
|
status: "mounted",
|
||||||
|
lastError: null,
|
||||||
|
provisioningId: null,
|
||||||
|
autoRemount: true,
|
||||||
|
agentId: "local",
|
||||||
|
organizationId: "org-1",
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
test("listVolumeFiles returns sorted paginated entries inside the volume", async () => {
|
||||||
|
const volume = await createDirectoryVolume();
|
||||||
|
await fs.mkdir(path.join(tempRoot!, "z-dir"));
|
||||||
|
await fs.mkdir(path.join(tempRoot!, "a-dir"));
|
||||||
|
await fs.writeFile(path.join(tempRoot!, "b-file.txt"), "hello");
|
||||||
|
|
||||||
|
const result = await listVolumeFiles(volume, undefined, 1, 2);
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
path: "/",
|
||||||
|
offset: 1,
|
||||||
|
limit: 2,
|
||||||
|
total: 3,
|
||||||
|
hasMore: false,
|
||||||
|
});
|
||||||
|
expect(result.files.map((entry) => entry.name)).toEqual(["z-dir", "b-file.txt"]);
|
||||||
|
expect(result.files[1]).toMatchObject({ path: "/b-file.txt", type: "file", size: 5 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("listVolumeFiles rejects traversal outside the volume", async () => {
|
||||||
|
const volume = await createDirectoryVolume();
|
||||||
|
|
||||||
|
await expect(listVolumeFiles(volume, "../outside", 0, 10)).rejects.toThrow("Invalid path");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("listVolumeFiles reports missing directories consistently", async () => {
|
||||||
|
const volume = await createDirectoryVolume();
|
||||||
|
|
||||||
|
await expect(listVolumeFiles(volume, "missing", 0, 10)).rejects.toThrow("Directory not found");
|
||||||
|
});
|
||||||
|
|
@ -1,12 +1,11 @@
|
||||||
import * as fs from "node:fs/promises";
|
import * as fs from "node:fs/promises";
|
||||||
import { toMessage } from "../../../utils/errors";
|
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import type { VolumeBackend } from "../backend";
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
|
import type { BackendConfig, VolumeBackend } from "../types";
|
||||||
|
|
||||||
const mount = async (config: BackendConfig, _volumePath: string) => {
|
const mount = async (config: BackendConfig, volumePath: string) => {
|
||||||
if (config.backend !== "directory") {
|
if (config.backend !== "directory") {
|
||||||
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
|
return { status: "error" as const, error: "Invalid backend type" };
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("Mounting directory volume from:", config.path);
|
logger.info("Mounting directory volume from:", config.path);
|
||||||
|
|
@ -16,33 +15,33 @@ const mount = async (config: BackendConfig, _volumePath: string) => {
|
||||||
const stats = await fs.stat(config.path);
|
const stats = await fs.stat(config.path);
|
||||||
|
|
||||||
if (!stats.isDirectory()) {
|
if (!stats.isDirectory()) {
|
||||||
return { status: BACKEND_STATUS.error, error: "Path is not a directory" };
|
return { status: "error" as const, error: "Path is not a directory" };
|
||||||
}
|
}
|
||||||
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
return { status: "mounted" as const };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Failed to mount directory volume:", error);
|
logger.error("Failed to mount directory volume:", error);
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const unmount = async () => {
|
const unmount = async () => {
|
||||||
logger.info("Cannot unmount directory volume.");
|
logger.info("Cannot unmount directory volume.");
|
||||||
return { status: BACKEND_STATUS.unmounted };
|
return { status: "unmounted" as const };
|
||||||
};
|
};
|
||||||
|
|
||||||
const checkHealth = async (config: BackendConfig) => {
|
const checkHealth = async (config: BackendConfig) => {
|
||||||
if (config.backend !== "directory") {
|
if (config.backend !== "directory") {
|
||||||
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
|
return { status: "error" as const, error: "Invalid backend type" };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fs.access(config.path);
|
await fs.access(config.path);
|
||||||
|
|
||||||
return { status: BACKEND_STATUS.mounted };
|
return { status: "mounted" as const };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Directory health check failed:", error);
|
logger.error("Directory health check failed:", error);
|
||||||
return { status: BACKEND_STATUS.error, error: toMessage(error) };
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
112
apps/agent/src/volume-host/backends/nfs.ts
Normal file
112
apps/agent/src/volume-host/backends/nfs.ts
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import { logger } from "@zerobyte/core/node";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { OPERATION_TIMEOUT } from "../constants";
|
||||||
|
import { withTimeout } from "../timeout";
|
||||||
|
import { getMountForPath } from "../fs";
|
||||||
|
import type { BackendConfig, VolumeBackend } from "../types";
|
||||||
|
import { assertMounted, executeMount, executeUnmount } from "./utils";
|
||||||
|
|
||||||
|
const checkHealth = async (mountPath: string) => {
|
||||||
|
const run = async () => {
|
||||||
|
await assertMounted(mountPath, (fstype) => fstype.startsWith("nfs"));
|
||||||
|
|
||||||
|
logger.debug(`NFS volume at ${mountPath} is healthy and mounted.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check");
|
||||||
|
} catch (error) {
|
||||||
|
const message = toMessage(error);
|
||||||
|
if (message !== "Volume is not mounted") {
|
||||||
|
logger.error("NFS volume health check failed:", message);
|
||||||
|
}
|
||||||
|
return { status: "error" as const, error: message };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const unmount = async (mountPath: string) => {
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("NFS unmounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "NFS unmounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) {
|
||||||
|
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeUnmount(mountPath);
|
||||||
|
await fs.rmdir(mountPath).catch(() => {});
|
||||||
|
|
||||||
|
logger.info(`NFS volume at ${mountPath} unmounted successfully.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS unmount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error unmounting NFS volume", { mountPath, error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
|
logger.debug(`Mounting volume ${mountPath}...`);
|
||||||
|
|
||||||
|
if (config.backend !== "nfs") {
|
||||||
|
logger.error("Provided config is not for NFS backend");
|
||||||
|
return { status: "error" as const, error: "Provided config is not for NFS backend" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("NFS mounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "NFS mounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { status } = await checkHealth(mountPath);
|
||||||
|
if (status === "mounted") return { status: "mounted" as const };
|
||||||
|
|
||||||
|
if (status === "error") {
|
||||||
|
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
|
||||||
|
await unmount(mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
await fs.mkdir(mountPath, { recursive: true });
|
||||||
|
const options = [`vers=${config.version}`, `port=${config.port}`];
|
||||||
|
if (config.version === "3") options.push("nolock");
|
||||||
|
if (config.readOnly) options.push("ro");
|
||||||
|
const args = ["-t", "nfs", "-o", options.join(","), `${config.server}:${config.exportPath}`, mountPath];
|
||||||
|
|
||||||
|
logger.debug(`Mounting volume ${mountPath}...`);
|
||||||
|
logger.info(`Executing mount: mount ${args.join(" ")}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await executeMount(args);
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn(`Initial NFS mount failed, retrying with -i flag: ${toMessage(error)}`);
|
||||||
|
await executeMount(["-i", ...args]);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`NFS volume at ${mountPath} mounted successfully.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS mount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error mounting NFS volume", { error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const makeNfsBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
|
||||||
|
mount: () => mount(config, mountPath),
|
||||||
|
unmount: () => unmount(mountPath),
|
||||||
|
checkHealth: () => checkHealth(mountPath),
|
||||||
|
});
|
||||||
124
apps/agent/src/volume-host/backends/rclone.ts
Normal file
124
apps/agent/src/volume-host/backends/rclone.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import { logger, safeExec } from "@zerobyte/core/node";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { OPERATION_TIMEOUT, RCLONE_CONFIG_FILE, RCLONE_TIMEOUT } from "../constants";
|
||||||
|
import { withTimeout } from "../timeout";
|
||||||
|
import { getMountForPath } from "../fs";
|
||||||
|
import type { BackendConfig, VolumeBackend } from "../types";
|
||||||
|
import { assertMounted, executeUnmount } from "./utils";
|
||||||
|
|
||||||
|
const checkHealth = async (mountPath: string) => {
|
||||||
|
const run = async () => {
|
||||||
|
await assertMounted(mountPath, (fstype) => fstype.includes("rclone"));
|
||||||
|
|
||||||
|
logger.debug(`Rclone volume at ${mountPath} is healthy and mounted.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone health check");
|
||||||
|
} catch (error) {
|
||||||
|
const message = toMessage(error);
|
||||||
|
if (message !== "Volume is not mounted") {
|
||||||
|
logger.error("Rclone volume health check failed:", message);
|
||||||
|
}
|
||||||
|
return { status: "error" as const, error: message };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const unmount = async (mountPath: string) => {
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("Rclone unmounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "Rclone unmounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) {
|
||||||
|
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeUnmount(mountPath);
|
||||||
|
await fs.rmdir(mountPath).catch(() => {});
|
||||||
|
|
||||||
|
logger.info(`Rclone volume at ${mountPath} unmounted successfully.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "Rclone unmount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error unmounting rclone volume", { mountPath, error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
|
logger.debug(`Mounting rclone volume ${mountPath}...`);
|
||||||
|
|
||||||
|
if (config.backend !== "rclone") {
|
||||||
|
logger.error("Provided config is not for rclone backend");
|
||||||
|
return { status: "error" as const, error: "Provided config is not for rclone backend" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("Rclone mounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "Rclone mounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { status } = await checkHealth(mountPath);
|
||||||
|
if (status === "mounted") return { status: "mounted" as const };
|
||||||
|
|
||||||
|
if (status === "error") {
|
||||||
|
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
|
||||||
|
await unmount(mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
await fs.mkdir(mountPath, { recursive: true });
|
||||||
|
const args = [
|
||||||
|
"mount",
|
||||||
|
`${config.remote}:${config.path}`,
|
||||||
|
mountPath,
|
||||||
|
"--daemon",
|
||||||
|
"--vfs-cache-mode",
|
||||||
|
"writes",
|
||||||
|
"--allow-non-empty",
|
||||||
|
"--allow-other",
|
||||||
|
];
|
||||||
|
if (config.readOnly) args.push("--read-only");
|
||||||
|
|
||||||
|
logger.debug(`Mounting rclone volume ${mountPath}...`);
|
||||||
|
logger.info(`Executing rclone: rclone ${args.join(" ")}`);
|
||||||
|
|
||||||
|
const result = await safeExec({
|
||||||
|
command: "rclone",
|
||||||
|
args,
|
||||||
|
env: { RCLONE_CONFIG: RCLONE_CONFIG_FILE },
|
||||||
|
timeout: RCLONE_TIMEOUT,
|
||||||
|
});
|
||||||
|
if (result.exitCode !== 0) {
|
||||||
|
const errorMsg = result.stderr.toString() || result.stdout.toString() || "Unknown error";
|
||||||
|
throw new Error(`Failed to mount rclone volume: ${errorMsg}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Rclone volume at ${mountPath} mounted successfully.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), RCLONE_TIMEOUT, "Rclone mount");
|
||||||
|
} catch (error) {
|
||||||
|
const errorMsg = toMessage(error);
|
||||||
|
logger.error("Error mounting rclone volume", { error: errorMsg });
|
||||||
|
return { status: "error" as const, error: errorMsg };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const makeRcloneBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
|
||||||
|
mount: () => mount(config, mountPath),
|
||||||
|
unmount: () => unmount(mountPath),
|
||||||
|
checkHealth: () => checkHealth(mountPath),
|
||||||
|
});
|
||||||
161
apps/agent/src/volume-host/backends/sftp.ts
Normal file
161
apps/agent/src/volume-host/backends/sftp.ts
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { spawn } from "node:child_process";
|
||||||
|
import { FILE_MODES, logger, writeFileWithMode } from "@zerobyte/core/node";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { OPERATION_TIMEOUT, SSH_KEYS_DIR } from "../constants";
|
||||||
|
import { getMountForPath } from "../fs";
|
||||||
|
import { withTimeout } from "../timeout";
|
||||||
|
import type { BackendConfig, VolumeBackend } from "../types";
|
||||||
|
import { executeUnmount } from "./utils";
|
||||||
|
|
||||||
|
const getPrivateKeyPath = (mountPath: string) => path.join(SSH_KEYS_DIR, `${path.basename(mountPath)}.key`);
|
||||||
|
const getKnownHostsPath = (mountPath: string) => path.join(SSH_KEYS_DIR, `${path.basename(mountPath)}.known_hosts`);
|
||||||
|
|
||||||
|
const runSshfs = async (args: string[], password?: string) =>
|
||||||
|
new Promise<void>((resolve, reject) => {
|
||||||
|
const child = spawn("sshfs", args, { stdio: ["pipe", "pipe", "pipe"] });
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
|
||||||
|
child.stdout.on("data", (data) => {
|
||||||
|
stdout += data;
|
||||||
|
});
|
||||||
|
child.stderr.on("data", (data) => {
|
||||||
|
stderr += data;
|
||||||
|
});
|
||||||
|
child.on("error", (error) => {
|
||||||
|
reject(new Error(`Failed to start sshfs: ${error.message}`));
|
||||||
|
});
|
||||||
|
child.on("close", (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const errorMsg = stderr.trim() || stdout.trim() || "Unknown error";
|
||||||
|
reject(new Error(`Failed to mount SFTP volume: ${errorMsg}`));
|
||||||
|
});
|
||||||
|
|
||||||
|
if (password) child.stdin.write(password);
|
||||||
|
child.stdin.end();
|
||||||
|
});
|
||||||
|
|
||||||
|
const checkHealth = async (mountPath: string) => {
|
||||||
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) return { status: "unmounted" as const };
|
||||||
|
if (mount.fstype !== "fuse.sshfs") {
|
||||||
|
return { status: "error" as const, error: `Invalid filesystem type: ${mount.fstype} (expected fuse.sshfs)` };
|
||||||
|
}
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
const unmount = async (mountPath: string) => {
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("SFTP unmounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "SFTP unmounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) {
|
||||||
|
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
|
||||||
|
} else {
|
||||||
|
await executeUnmount(mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.unlink(getPrivateKeyPath(mountPath)).catch(() => {});
|
||||||
|
await fs.unlink(getKnownHostsPath(mountPath)).catch(() => {});
|
||||||
|
await fs.rmdir(mountPath).catch(() => {});
|
||||||
|
|
||||||
|
logger.info(`SFTP volume at ${mountPath} unmounted successfully.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "SFTP unmount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error unmounting SFTP volume", { mountPath, error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
|
logger.debug(`Mounting SFTP volume ${mountPath}...`);
|
||||||
|
|
||||||
|
if (config.backend !== "sftp") {
|
||||||
|
logger.error("Provided config is not for SFTP backend");
|
||||||
|
return { status: "error" as const, error: "Provided config is not for SFTP backend" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("SFTP mounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "SFTP mounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { status } = await checkHealth(mountPath);
|
||||||
|
if (status === "mounted") return { status: "mounted" as const };
|
||||||
|
|
||||||
|
if (status === "error") {
|
||||||
|
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
|
||||||
|
await unmount(mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
await fs.mkdir(mountPath, { recursive: true });
|
||||||
|
await fs.mkdir(SSH_KEYS_DIR, { recursive: true });
|
||||||
|
const { uid, gid } = os.userInfo();
|
||||||
|
const options = [
|
||||||
|
"reconnect",
|
||||||
|
"ServerAliveInterval=15",
|
||||||
|
"ServerAliveCountMax=3",
|
||||||
|
"allow_other",
|
||||||
|
`uid=${uid}`,
|
||||||
|
`gid=${gid}`,
|
||||||
|
];
|
||||||
|
|
||||||
|
if (config.skipHostKeyCheck) {
|
||||||
|
options.push("StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null");
|
||||||
|
} else if (config.knownHosts) {
|
||||||
|
await writeFileWithMode(getKnownHostsPath(mountPath), config.knownHosts, FILE_MODES.ownerReadWrite);
|
||||||
|
options.push(`UserKnownHostsFile=${getKnownHostsPath(mountPath)}`, "StrictHostKeyChecking=yes");
|
||||||
|
} else {
|
||||||
|
options.push("StrictHostKeyChecking=yes");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.readOnly) options.push("ro");
|
||||||
|
if (config.port) options.push(`port=${config.port}`);
|
||||||
|
if (config.privateKey) {
|
||||||
|
let key = config.privateKey.replace(/\r\n/g, "\n");
|
||||||
|
if (!key.endsWith("\n")) key += "\n";
|
||||||
|
await writeFileWithMode(getPrivateKeyPath(mountPath), key, FILE_MODES.ownerReadWrite);
|
||||||
|
options.push(`IdentityFile=${getPrivateKeyPath(mountPath)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const args = [`${config.username}@${config.host}:${config.path || ""}`, mountPath, "-o", options.join(",")];
|
||||||
|
if (config.password) args.push("-o", "password_stdin");
|
||||||
|
|
||||||
|
logger.info(`Executing sshfs: sshfs ${args.join(" ")}`);
|
||||||
|
await runSshfs(args, config.password);
|
||||||
|
|
||||||
|
logger.info(`SFTP volume at ${mountPath} mounted successfully.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT * 2, "SFTP mount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error mounting SFTP volume", { error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const makeSftpBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
|
||||||
|
mount: () => mount(config, mountPath),
|
||||||
|
unmount: () => unmount(mountPath),
|
||||||
|
checkHealth: () => checkHealth(mountPath),
|
||||||
|
});
|
||||||
124
apps/agent/src/volume-host/backends/smb.ts
Normal file
124
apps/agent/src/volume-host/backends/smb.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import { logger } from "@zerobyte/core/node";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { OPERATION_TIMEOUT } from "../constants";
|
||||||
|
import { withTimeout } from "../timeout";
|
||||||
|
import { getMountForPath } from "../fs";
|
||||||
|
import type { BackendConfig, VolumeBackend } from "../types";
|
||||||
|
import { assertMounted, executeMount, executeUnmount } from "./utils";
|
||||||
|
|
||||||
|
const checkHealth = async (mountPath: string) => {
|
||||||
|
const run = async () => {
|
||||||
|
await assertMounted(mountPath, (fstype) => fstype === "cifs");
|
||||||
|
|
||||||
|
logger.debug(`SMB volume at ${mountPath} is healthy and mounted.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB health check");
|
||||||
|
} catch (error) {
|
||||||
|
const message = toMessage(error);
|
||||||
|
if (message !== "Volume is not mounted") {
|
||||||
|
logger.error("SMB volume health check failed:", message);
|
||||||
|
}
|
||||||
|
return { status: "error" as const, error: message };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const unmount = async (mountPath: string) => {
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("SMB unmounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "SMB unmounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) {
|
||||||
|
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeUnmount(mountPath);
|
||||||
|
await fs.rmdir(mountPath).catch(() => {});
|
||||||
|
|
||||||
|
logger.info(`SMB volume at ${mountPath} unmounted successfully.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB unmount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error unmounting SMB volume", { mountPath, error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
|
logger.debug(`Mounting SMB volume ${mountPath}...`);
|
||||||
|
|
||||||
|
if (config.backend !== "smb") {
|
||||||
|
logger.error("Provided config is not for SMB backend");
|
||||||
|
return { status: "error" as const, error: "Provided config is not for SMB backend" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("SMB mounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "SMB mounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { status } = await checkHealth(mountPath);
|
||||||
|
if (status === "mounted") return { status: "mounted" as const };
|
||||||
|
|
||||||
|
if (status === "error") {
|
||||||
|
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
|
||||||
|
await unmount(mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
await fs.mkdir(mountPath, { recursive: true });
|
||||||
|
const { uid, gid } = os.userInfo();
|
||||||
|
const options = [`port=${config.port}`, `uid=${uid}`, `gid=${gid}`, "iocharset=utf8"];
|
||||||
|
|
||||||
|
if (config.guest) {
|
||||||
|
options.push("username=guest", "password=");
|
||||||
|
} else {
|
||||||
|
const safePassword = (config.password ?? "").replace(/\\/g, "\\\\").replace(/,/g, "\\,");
|
||||||
|
options.push(`username=${config.username ?? "user"}`, `password=${safePassword}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.domain) options.push(`domain=${config.domain}`);
|
||||||
|
if (config.vers && config.vers !== "auto") options.push(`vers=${config.vers}`);
|
||||||
|
if (config.readOnly) options.push("ro");
|
||||||
|
|
||||||
|
const source = `//${config.server}/${config.share}`;
|
||||||
|
const args = ["-t", "cifs", "-o", options.join(","), source, mountPath];
|
||||||
|
|
||||||
|
logger.debug(`Mounting SMB volume ${mountPath}...`);
|
||||||
|
logger.info(`Executing SMB mount for ${source} at ${mountPath}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await executeMount(args);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(`SMB mount failed: ${toMessage(error)}`);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`SMB volume at ${mountPath} mounted successfully.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "SMB mount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error mounting SMB volume", { error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const makeSmbBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
|
||||||
|
mount: () => mount(config, mountPath),
|
||||||
|
unmount: () => unmount(mountPath),
|
||||||
|
checkHealth: () => checkHealth(mountPath),
|
||||||
|
});
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
import * as fs from "node:fs/promises";
|
import * as fs from "node:fs/promises";
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger, safeExec } from "@zerobyte/core/node";
|
||||||
import { safeExec } from "@zerobyte/core/node";
|
import { getMountForPath } from "../fs";
|
||||||
import { getMountForPath } from "../../../utils/mountinfo";
|
|
||||||
|
|
||||||
export const executeMount = async (args: string[]): Promise<void> => {
|
export const executeMount = async (args: string[]): Promise<void> => {
|
||||||
const shouldBeVerbose = process.env.LOG_LEVEL === "debug" || process.env.NODE_ENV !== "production";
|
const shouldBeVerbose = process.env.LOG_LEVEL === "debug" || process.env.NODE_ENV !== "production";
|
||||||
|
|
@ -10,7 +9,6 @@ export const executeMount = async (args: string[]): Promise<void> => {
|
||||||
|
|
||||||
logger.debug(`Executing mount ${effectiveArgs.join(" ")}`);
|
logger.debug(`Executing mount ${effectiveArgs.join(" ")}`);
|
||||||
const result = await safeExec({ command: "mount", args: effectiveArgs, timeout: 10000 });
|
const result = await safeExec({ command: "mount", args: effectiveArgs, timeout: 10000 });
|
||||||
|
|
||||||
const stdout = result.stdout.toString().trim();
|
const stdout = result.stdout.toString().trim();
|
||||||
const stderr = result.stderr.toString().trim();
|
const stderr = result.stderr.toString().trim();
|
||||||
|
|
||||||
|
|
@ -26,37 +24,29 @@ export const executeMount = async (args: string[]): Promise<void> => {
|
||||||
throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr || stdout || "unknown error"}`);
|
throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr || stdout || "unknown error"}`);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const executeUnmount = async (path: string): Promise<void> => {
|
export const executeUnmount = async (mountPath: string): Promise<void> => {
|
||||||
let stderr: string | undefined;
|
logger.debug(`Executing umount -l ${mountPath}`);
|
||||||
|
const result = await safeExec({ command: "umount", args: ["-l", mountPath], timeout: 10000 });
|
||||||
logger.debug(`Executing umount -l ${path}`);
|
const stderr = result.stderr.toString();
|
||||||
const result = await safeExec({ command: "umount", args: ["-l", path], timeout: 10000 });
|
|
||||||
|
|
||||||
stderr = result.stderr.toString();
|
|
||||||
|
|
||||||
if (stderr?.trim()) {
|
|
||||||
logger.warn(stderr.trim());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
if (stderr.trim()) logger.warn(stderr.trim());
|
||||||
if (result.exitCode !== 0) {
|
if (result.exitCode !== 0) {
|
||||||
throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr?.trim()}`);
|
throw new Error(`Mount command failed with exit code ${result.exitCode}: ${stderr.trim()}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const assertMounted = async (path: string, isExpectedFilesystem: (fstype: string) => boolean) => {
|
export const assertMounted = async (mountPath: string, isExpectedFilesystem: (fstype: string) => boolean) => {
|
||||||
try {
|
try {
|
||||||
await fs.access(path);
|
await fs.access(mountPath);
|
||||||
} catch {
|
} catch {
|
||||||
throw new Error("Volume is not mounted");
|
throw new Error("Volume is not mounted");
|
||||||
}
|
}
|
||||||
|
|
||||||
const mount = await getMountForPath(path);
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) {
|
||||||
if (!mount || mount.mountPoint !== path) {
|
|
||||||
throw new Error("Volume is not mounted");
|
throw new Error("Volume is not mounted");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isExpectedFilesystem(mount.fstype)) {
|
if (!isExpectedFilesystem(mount.fstype)) {
|
||||||
throw new Error(`Path ${path} is not mounted as correct fstype (found ${mount.fstype}).`);
|
throw new Error(`Path ${mountPath} is not mounted as correct fstype (found ${mount.fstype}).`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
144
apps/agent/src/volume-host/backends/webdav.ts
Normal file
144
apps/agent/src/volume-host/backends/webdav.ts
Normal file
|
|
@ -0,0 +1,144 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import { logger } from "@zerobyte/core/node";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { OPERATION_TIMEOUT } from "../constants";
|
||||||
|
import { withTimeout } from "../timeout";
|
||||||
|
import { getMountForPath } from "../fs";
|
||||||
|
import type { BackendConfig, VolumeBackend } from "../types";
|
||||||
|
import { assertMounted, executeMount, executeUnmount } from "./utils";
|
||||||
|
|
||||||
|
const checkHealth = async (mountPath: string) => {
|
||||||
|
const run = async () => {
|
||||||
|
await assertMounted(mountPath, (fstype) => fstype === "fuse" || fstype === "davfs");
|
||||||
|
|
||||||
|
logger.debug(`WebDAV volume at ${mountPath} is healthy and mounted.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV health check");
|
||||||
|
} catch (error) {
|
||||||
|
const message = toMessage(error);
|
||||||
|
if (message !== "Volume is not mounted") {
|
||||||
|
logger.error("WebDAV volume health check failed:", message);
|
||||||
|
}
|
||||||
|
return { status: "error" as const, error: message };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const unmount = async (mountPath: string) => {
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("WebDAV unmounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "WebDAV unmounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
const mount = await getMountForPath(mountPath);
|
||||||
|
if (!mount || mount.mountPoint !== mountPath) {
|
||||||
|
logger.debug(`Path ${mountPath} is not a mount point. Skipping unmount.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeUnmount(mountPath);
|
||||||
|
await fs.rmdir(mountPath).catch(() => {});
|
||||||
|
|
||||||
|
logger.info(`WebDAV volume at ${mountPath} unmounted successfully.`);
|
||||||
|
return { status: "unmounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV unmount");
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error unmounting WebDAV volume", { mountPath, error: toMessage(error) });
|
||||||
|
return { status: "error" as const, error: toMessage(error) };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const mount = async (config: BackendConfig, mountPath: string) => {
|
||||||
|
logger.debug(`Mounting WebDAV volume ${mountPath}...`);
|
||||||
|
|
||||||
|
if (config.backend !== "webdav") {
|
||||||
|
logger.error("Provided config is not for WebDAV backend");
|
||||||
|
return { status: "error" as const, error: "Provided config is not for WebDAV backend" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (os.platform() !== "linux") {
|
||||||
|
logger.error("WebDAV mounting is only supported on Linux hosts.");
|
||||||
|
return { status: "error" as const, error: "WebDAV mounting is only supported on Linux hosts." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { status } = await checkHealth(mountPath);
|
||||||
|
if (status === "mounted") return { status: "mounted" as const };
|
||||||
|
|
||||||
|
if (status === "error") {
|
||||||
|
logger.debug(`Trying to unmount any existing mounts at ${mountPath} before mounting...`);
|
||||||
|
await unmount(mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
await fs.mkdir(mountPath, { recursive: true }).catch((error) => {
|
||||||
|
logger.warn(`Failed to create directory ${mountPath}: ${toMessage(error)}`);
|
||||||
|
});
|
||||||
|
const protocol = config.ssl ? "https" : "http";
|
||||||
|
const defaultPort = config.ssl ? 443 : 80;
|
||||||
|
const source = `${protocol}://${config.server}${config.port !== defaultPort ? `:${config.port}` : ""}${config.path}`;
|
||||||
|
const { uid, gid } = os.userInfo();
|
||||||
|
const options = config.readOnly
|
||||||
|
? [`uid=${uid}`, `gid=${gid}`, "file_mode=0444", "dir_mode=0555", "ro"]
|
||||||
|
: [`uid=${uid}`, `gid=${gid}`, "file_mode=0664", "dir_mode=0775"];
|
||||||
|
|
||||||
|
if (config.username && config.password) {
|
||||||
|
const entry = [source, config.username, config.password]
|
||||||
|
.map((value) => value.replace(/[\r\n\t\s]+/g, " "))
|
||||||
|
.join(" ");
|
||||||
|
await fs.appendFile("/etc/davfs2/secrets", `${entry}\n`, { mode: 0o600 });
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(`Mounting WebDAV volume ${mountPath}...`);
|
||||||
|
|
||||||
|
const args = ["-t", "davfs", "-o", options.join(","), source, mountPath];
|
||||||
|
try {
|
||||||
|
await executeMount(args);
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn(`Initial WebDAV mount failed, retrying with -i flag: ${toMessage(error)}`);
|
||||||
|
await executeMount(["-i", ...args]);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`WebDAV volume at ${mountPath} mounted successfully.`);
|
||||||
|
return { status: "mounted" as const };
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await withTimeout(run(), OPERATION_TIMEOUT, "WebDAV mount");
|
||||||
|
} catch (error) {
|
||||||
|
const message = toMessage(error);
|
||||||
|
if (message.includes("already mounted")) return { status: "mounted" as const };
|
||||||
|
|
||||||
|
logger.error("Error mounting WebDAV volume", { error: message });
|
||||||
|
|
||||||
|
if (message.includes("option") && message.includes("requires argument")) {
|
||||||
|
return {
|
||||||
|
status: "error" as const,
|
||||||
|
error: "Invalid mount options. Please check your WebDAV server configuration.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (message.includes("connection refused") || message.includes("Connection refused")) {
|
||||||
|
return {
|
||||||
|
status: "error" as const,
|
||||||
|
error: "Cannot connect to WebDAV server. Please check the server address and port.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (message.includes("unauthorized") || message.includes("Unauthorized")) {
|
||||||
|
return { status: "error" as const, error: "Authentication failed. Please check your username and password." };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { status: "error" as const, error: message };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const makeWebdavBackend = (config: BackendConfig, mountPath: string): VolumeBackend => ({
|
||||||
|
mount: () => mount(config, mountPath),
|
||||||
|
unmount: () => unmount(mountPath),
|
||||||
|
checkHealth: () => checkHealth(mountPath),
|
||||||
|
});
|
||||||
28
apps/agent/src/volume-host/cleanup.ts
Normal file
28
apps/agent/src/volume-host/cleanup.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { logger } from "@zerobyte/core/node";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { VOLUME_MOUNT_BASE } from "./constants";
|
||||||
|
import { readMountInfo } from "./fs";
|
||||||
|
|
||||||
|
export const cleanupDanglingVolumeMountDirectories = async () => {
|
||||||
|
const mounts = await readMountInfo().catch((error) => {
|
||||||
|
logger.warn(`Failed to read mount info for volume cleanup: ${toMessage(error)}`);
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
const mountedPaths = new Set(mounts.map((mount) => mount.mountPoint));
|
||||||
|
const volumeDirs = await fs.readdir(VOLUME_MOUNT_BASE).catch(() => []);
|
||||||
|
|
||||||
|
for (const dir of volumeDirs) {
|
||||||
|
const mountPath = path.join(VOLUME_MOUNT_BASE, dir, "_data");
|
||||||
|
if (mountedPaths.has(mountPath)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullPath = path.join(VOLUME_MOUNT_BASE, dir);
|
||||||
|
logger.info(`Removing stale volume mount directory at ${fullPath}`);
|
||||||
|
await fs.rm(fullPath, { recursive: true, force: true }).catch((error) => {
|
||||||
|
logger.warn(`Failed to remove stale volume mount directory ${fullPath}: ${toMessage(error)}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
9
apps/agent/src/volume-host/constants.ts
Normal file
9
apps/agent/src/volume-host/constants.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
import * as path from "node:path";
|
||||||
|
|
||||||
|
export const OPERATION_TIMEOUT = 5000;
|
||||||
|
export const VOLUME_MOUNT_BASE = process.env.ZEROBYTE_VOLUMES_DIR || "/var/lib/zerobyte/volumes";
|
||||||
|
export const SSH_KEYS_DIR = "/var/lib/zerobyte/ssh";
|
||||||
|
export const RCLONE_CONFIG_DIR = process.env.RCLONE_CONFIG_DIR || "/root/.config/rclone";
|
||||||
|
export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf");
|
||||||
|
const serverIdleTimeout = Number(process.env.SERVER_IDLE_TIMEOUT ?? 60);
|
||||||
|
export const RCLONE_TIMEOUT = (Number.isFinite(serverIdleTimeout) ? serverIdleTimeout : 60) * 1000;
|
||||||
66
apps/agent/src/volume-host/fs.ts
Normal file
66
apps/agent/src/volume-host/fs.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import { isPathWithin } from "@zerobyte/core/utils";
|
||||||
|
|
||||||
|
type MountInfo = {
|
||||||
|
mountPoint: string;
|
||||||
|
fstype: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const unescapeMount = (value: string) =>
|
||||||
|
value.replace(/\\([0-7]{3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8)));
|
||||||
|
|
||||||
|
export const readMountInfo = async (): Promise<MountInfo[]> => {
|
||||||
|
const text = await fs.readFile("/proc/self/mountinfo", "utf-8");
|
||||||
|
const result: MountInfo[] = [];
|
||||||
|
|
||||||
|
for (const line of text.split("\n")) {
|
||||||
|
if (!line) continue;
|
||||||
|
const sep = line.indexOf(" - ");
|
||||||
|
if (sep === -1) continue;
|
||||||
|
|
||||||
|
const left = line.slice(0, sep).split(" ");
|
||||||
|
const right = line.slice(sep + 3).split(" ");
|
||||||
|
const mpRaw = left[4];
|
||||||
|
const fstype = right[0];
|
||||||
|
|
||||||
|
if (!mpRaw || !fstype) continue;
|
||||||
|
result.push({ mountPoint: unescapeMount(mpRaw), fstype });
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getMountForPath = async (targetPath: string): Promise<MountInfo | undefined> => {
|
||||||
|
const mounts = await readMountInfo();
|
||||||
|
let best: MountInfo | undefined;
|
||||||
|
|
||||||
|
for (const mount of mounts) {
|
||||||
|
if (!isPathWithin(mount.mountPoint, targetPath)) continue;
|
||||||
|
if (!best || mount.mountPoint.length > best.mountPoint.length) {
|
||||||
|
best = mount;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return best;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getStatFs = async (mountPoint: string) => {
|
||||||
|
const stat = await fs.statfs(mountPoint, { bigint: true });
|
||||||
|
const unit = stat.bsize > 0n ? stat.bsize : 1n;
|
||||||
|
const blocks = stat.blocks > 0n ? stat.blocks : 0n;
|
||||||
|
let bfree = stat.bfree > 0n ? stat.bfree : 0n;
|
||||||
|
if (bfree > blocks) bfree = blocks;
|
||||||
|
const bavail = stat.bavail > 0n ? stat.bavail : 0n;
|
||||||
|
const max = BigInt(Number.MAX_SAFE_INTEGER);
|
||||||
|
const toNumber = (value: bigint) => (value > max ? Number.MAX_SAFE_INTEGER : Number(value));
|
||||||
|
|
||||||
|
return {
|
||||||
|
total: toNumber(blocks * unit),
|
||||||
|
used: toNumber((blocks - bfree) * unit),
|
||||||
|
free: toNumber(bavail * unit),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isNodeJSErrnoException = (error: unknown): error is NodeJS.ErrnoException => {
|
||||||
|
return error instanceof Error && "code" in error;
|
||||||
|
};
|
||||||
31
apps/agent/src/volume-host/index.ts
Normal file
31
apps/agent/src/volume-host/index.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
import { makeDirectoryBackend } from "./backends/directory";
|
||||||
|
import { makeNfsBackend } from "./backends/nfs";
|
||||||
|
import { makeRcloneBackend } from "./backends/rclone";
|
||||||
|
import { makeSftpBackend } from "./backends/sftp";
|
||||||
|
import { makeSmbBackend } from "./backends/smb";
|
||||||
|
import { makeWebdavBackend } from "./backends/webdav";
|
||||||
|
import { getVolumePath } from "./paths";
|
||||||
|
import type { AgentVolume, VolumeBackend } from "./types";
|
||||||
|
|
||||||
|
export { getStatFs, isNodeJSErrnoException } from "./fs";
|
||||||
|
export { getVolumePath } from "./paths";
|
||||||
|
export type { AgentVolume, BackendConfig, VolumeBackend } from "./types";
|
||||||
|
|
||||||
|
export const createVolumeBackend = (volume: AgentVolume, mountPath = getVolumePath(volume)): VolumeBackend => {
|
||||||
|
switch (volume.config.backend) {
|
||||||
|
case "directory":
|
||||||
|
return makeDirectoryBackend(volume.config, mountPath);
|
||||||
|
case "nfs":
|
||||||
|
return makeNfsBackend(volume.config, mountPath);
|
||||||
|
case "smb":
|
||||||
|
return makeSmbBackend(volume.config, mountPath);
|
||||||
|
case "webdav":
|
||||||
|
return makeWebdavBackend(volume.config, mountPath);
|
||||||
|
case "rclone":
|
||||||
|
return makeRcloneBackend(volume.config, mountPath);
|
||||||
|
case "sftp":
|
||||||
|
return makeSftpBackend(volume.config, mountPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error("Unsupported backend");
|
||||||
|
};
|
||||||
156
apps/agent/src/volume-host/operations.ts
Normal file
156
apps/agent/src/volume-host/operations.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import { toMessage } from "@zerobyte/core/utils";
|
||||||
|
import { createVolumeBackend, getVolumePath, isNodeJSErrnoException } from ".";
|
||||||
|
import type { AgentVolume, BackendConfig } from "./types";
|
||||||
|
|
||||||
|
const DEFAULT_PAGE_SIZE = 500;
|
||||||
|
const MAX_PAGE_SIZE = 500;
|
||||||
|
|
||||||
|
export const listVolumeFiles = async (
|
||||||
|
volume: AgentVolume,
|
||||||
|
subPath?: string,
|
||||||
|
offset: number = 0,
|
||||||
|
limit: number = DEFAULT_PAGE_SIZE,
|
||||||
|
) => {
|
||||||
|
const volumePath = getVolumePath(volume);
|
||||||
|
const requestedPath = subPath ? path.join(volumePath, subPath) : volumePath;
|
||||||
|
const normalizedPath = path.normalize(requestedPath);
|
||||||
|
const relative = path.relative(volumePath, normalizedPath);
|
||||||
|
|
||||||
|
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
||||||
|
throw new Error("Invalid path");
|
||||||
|
}
|
||||||
|
|
||||||
|
const pageSize = Math.min(Math.max(limit, 1), MAX_PAGE_SIZE);
|
||||||
|
const startOffset = Math.max(offset, 0);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const dirents = await fs.readdir(normalizedPath, { withFileTypes: true });
|
||||||
|
|
||||||
|
dirents.sort((a, b) => {
|
||||||
|
const aIsDir = a.isDirectory();
|
||||||
|
const bIsDir = b.isDirectory();
|
||||||
|
|
||||||
|
if (aIsDir === bIsDir) {
|
||||||
|
return a.name.localeCompare(b.name);
|
||||||
|
}
|
||||||
|
return aIsDir ? -1 : 1;
|
||||||
|
});
|
||||||
|
|
||||||
|
const total = dirents.length;
|
||||||
|
const paginatedDirents = dirents.slice(startOffset, startOffset + pageSize);
|
||||||
|
|
||||||
|
const entries = (
|
||||||
|
await Promise.all(
|
||||||
|
paginatedDirents.map(async (dirent) => {
|
||||||
|
const fullPath = path.join(normalizedPath, dirent.name);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const stats = await fs.stat(fullPath);
|
||||||
|
const relativePath = path.relative(volumePath, fullPath);
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: dirent.name,
|
||||||
|
path: `/${relativePath}`,
|
||||||
|
type: dirent.isDirectory() ? ("directory" as const) : ("file" as const),
|
||||||
|
size: dirent.isFile() ? stats.size : undefined,
|
||||||
|
modifiedAt: stats.mtimeMs,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
).filter((entry) => entry !== null);
|
||||||
|
|
||||||
|
return {
|
||||||
|
files: entries,
|
||||||
|
path: subPath || "/",
|
||||||
|
offset: startOffset,
|
||||||
|
limit: pageSize,
|
||||||
|
total,
|
||||||
|
hasMore: startOffset + pageSize < total,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
if (isNodeJSErrnoException(error) && error.code === "ENOENT") {
|
||||||
|
throw new Error("Directory not found");
|
||||||
|
}
|
||||||
|
if (toMessage(error) === "Invalid path") {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
throw new Error(`Failed to list files: ${toMessage(error)}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const browseFilesystem = async (browsePath: string) => {
|
||||||
|
const normalizedPath = path.normalize(browsePath);
|
||||||
|
const entries = await fs.readdir(normalizedPath, { withFileTypes: true });
|
||||||
|
|
||||||
|
const directories = await Promise.all(
|
||||||
|
entries
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.map(async (entry) => {
|
||||||
|
const fullPath = path.join(normalizedPath, entry.name);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const stats = await fs.stat(fullPath);
|
||||||
|
return {
|
||||||
|
name: entry.name,
|
||||||
|
path: fullPath,
|
||||||
|
type: "directory" as const,
|
||||||
|
size: undefined,
|
||||||
|
modifiedAt: stats.mtimeMs,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
name: entry.name,
|
||||||
|
path: fullPath,
|
||||||
|
type: "directory" as const,
|
||||||
|
size: undefined,
|
||||||
|
modifiedAt: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
directories: directories.sort((a, b) => a.name.localeCompare(b.name)),
|
||||||
|
path: normalizedPath,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const testVolumeConnection = async (backendConfig: BackendConfig) => {
|
||||||
|
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zerobyte-test-"));
|
||||||
|
try {
|
||||||
|
const mockVolume: AgentVolume = {
|
||||||
|
id: 0,
|
||||||
|
shortId: "test",
|
||||||
|
name: "test-connection",
|
||||||
|
config: backendConfig,
|
||||||
|
createdAt: Date.now(),
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
lastHealthCheck: Date.now(),
|
||||||
|
type: backendConfig.backend,
|
||||||
|
status: "unmounted",
|
||||||
|
lastError: null,
|
||||||
|
provisioningId: null,
|
||||||
|
autoRemount: true,
|
||||||
|
agentId: "local",
|
||||||
|
organizationId: "test-org",
|
||||||
|
};
|
||||||
|
|
||||||
|
const backend = createVolumeBackend(mockVolume, tempDir);
|
||||||
|
const { error } = await backend.mount();
|
||||||
|
|
||||||
|
await backend.unmount();
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: !error,
|
||||||
|
message: error ? toMessage(error) : "Connection successful",
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
await fs.rm(tempDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
};
|
||||||
10
apps/agent/src/volume-host/paths.ts
Normal file
10
apps/agent/src/volume-host/paths.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
import { VOLUME_MOUNT_BASE } from "./constants";
|
||||||
|
import type { AgentVolume } from "./types";
|
||||||
|
|
||||||
|
export const getVolumePath = (volume: AgentVolume) => {
|
||||||
|
if (volume.config.backend === "directory") {
|
||||||
|
return volume.config.path;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${VOLUME_MOUNT_BASE}/${volume.shortId}/_data`;
|
||||||
|
};
|
||||||
21
apps/agent/src/volume-host/timeout.ts
Normal file
21
apps/agent/src/volume-host/timeout.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
class TimeoutError extends Error {
|
||||||
|
code = "ETIMEOUT";
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "TimeoutError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const withTimeout = async <T>(promise: Promise<T>, ms: number, label: string): Promise<T> => {
|
||||||
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
try {
|
||||||
|
return await Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise<never>((_, reject) => {
|
||||||
|
timeout = setTimeout(() => reject(new TimeoutError(`${label} timed out after ${ms}ms`)), ms);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
if (timeout) clearTimeout(timeout);
|
||||||
|
}
|
||||||
|
};
|
||||||
67
apps/agent/src/volume-host/types.ts
Normal file
67
apps/agent/src/volume-host/types.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
export type BackendStatus = "mounted" | "unmounted" | "error";
|
||||||
|
|
||||||
|
type BaseConfig = { backend: string; readOnly?: boolean };
|
||||||
|
|
||||||
|
export type BackendConfig =
|
||||||
|
| (BaseConfig & { backend: "directory"; path: string })
|
||||||
|
| (BaseConfig & { backend: "nfs"; server: string; exportPath: string; port: number; version: "3" | "4" | "4.1" })
|
||||||
|
| (BaseConfig & {
|
||||||
|
backend: "smb";
|
||||||
|
server: string;
|
||||||
|
share: string;
|
||||||
|
username?: string;
|
||||||
|
password?: string;
|
||||||
|
guest?: boolean;
|
||||||
|
vers?: "1.0" | "2.0" | "2.1" | "3.0" | "auto";
|
||||||
|
domain?: string;
|
||||||
|
port: number;
|
||||||
|
})
|
||||||
|
| (BaseConfig & {
|
||||||
|
backend: "webdav";
|
||||||
|
server: string;
|
||||||
|
path: string;
|
||||||
|
username?: string;
|
||||||
|
password?: string;
|
||||||
|
port: number;
|
||||||
|
ssl?: boolean;
|
||||||
|
})
|
||||||
|
| (BaseConfig & { backend: "rclone"; remote: string; path: string })
|
||||||
|
| (BaseConfig & {
|
||||||
|
backend: "sftp";
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
username: string;
|
||||||
|
password?: string;
|
||||||
|
privateKey?: string;
|
||||||
|
path: string;
|
||||||
|
skipHostKeyCheck?: boolean;
|
||||||
|
knownHosts?: string;
|
||||||
|
});
|
||||||
|
|
||||||
|
export type AgentVolume = {
|
||||||
|
id: number;
|
||||||
|
shortId: string;
|
||||||
|
name: string;
|
||||||
|
config: BackendConfig;
|
||||||
|
createdAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
lastHealthCheck: number;
|
||||||
|
type: string;
|
||||||
|
status: BackendStatus;
|
||||||
|
lastError: string | null;
|
||||||
|
provisioningId?: string | null;
|
||||||
|
autoRemount: boolean;
|
||||||
|
agentId: string;
|
||||||
|
organizationId: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type OperationResult = {
|
||||||
|
status: BackendStatus;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type VolumeBackend = {
|
||||||
|
mount: () => Promise<OperationResult>;
|
||||||
|
unmount: () => Promise<OperationResult>;
|
||||||
|
checkHealth: () => Promise<OperationResult>;
|
||||||
|
};
|
||||||
|
|
@ -51,6 +51,111 @@ const backupCancelSchema = z.object({
|
||||||
payload: z.object({ jobId: z.string(), scheduleId: z.string() }),
|
payload: z.object({ jobId: z.string(), scheduleId: z.string() }),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const backendStatusSchema = z.enum(["mounted", "unmounted", "error"]);
|
||||||
|
|
||||||
|
const volumeSchema = z.object({
|
||||||
|
id: z.number(),
|
||||||
|
shortId: z.string(),
|
||||||
|
name: z.string(),
|
||||||
|
path: z.string().nullable().optional(),
|
||||||
|
config: z.record(z.string(), z.unknown()).and(z.object({ backend: z.string() })),
|
||||||
|
createdAt: z.number(),
|
||||||
|
updatedAt: z.number(),
|
||||||
|
lastHealthCheck: z.number(),
|
||||||
|
type: z.string(),
|
||||||
|
status: backendStatusSchema,
|
||||||
|
lastError: z.string().nullable(),
|
||||||
|
provisioningId: z.string().nullable().optional(),
|
||||||
|
autoRemount: z.boolean(),
|
||||||
|
agentId: z.string(),
|
||||||
|
organizationId: z.string(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const volumeOperationResultSchema = z.object({
|
||||||
|
status: backendStatusSchema,
|
||||||
|
error: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const statfsSchema = z.object({
|
||||||
|
total: z.number().optional(),
|
||||||
|
used: z.number().optional(),
|
||||||
|
free: z.number().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const fileEntrySchema = z.object({
|
||||||
|
name: z.string(),
|
||||||
|
path: z.string(),
|
||||||
|
type: z.enum(["directory", "file"]),
|
||||||
|
size: z.number().optional(),
|
||||||
|
modifiedAt: z.number().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const directoryEntrySchema = z.object({
|
||||||
|
name: z.string(),
|
||||||
|
path: z.string(),
|
||||||
|
type: z.literal("directory"),
|
||||||
|
size: z.undefined().optional(),
|
||||||
|
modifiedAt: z.number().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const volumeCommandSchema = z.discriminatedUnion("name", [
|
||||||
|
z.object({ name: z.literal("volume.mount"), volume: volumeSchema }),
|
||||||
|
z.object({ name: z.literal("volume.unmount"), volume: volumeSchema }),
|
||||||
|
z.object({ name: z.literal("volume.checkHealth"), volume: volumeSchema }),
|
||||||
|
z.object({ name: z.literal("volume.statfs"), volume: volumeSchema }),
|
||||||
|
z.object({
|
||||||
|
name: z.literal("volume.listFiles"),
|
||||||
|
volume: volumeSchema,
|
||||||
|
subPath: z.string().optional(),
|
||||||
|
offset: z.number(),
|
||||||
|
limit: z.number(),
|
||||||
|
}),
|
||||||
|
z.object({ name: z.literal("volume.testConnection"), backendConfig: z.record(z.string(), z.unknown()) }),
|
||||||
|
z.object({ name: z.literal("filesystem.browse"), path: z.string() }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const volumeCommandRequestSchema = z.object({
|
||||||
|
type: z.literal("volume.command"),
|
||||||
|
payload: z.object({
|
||||||
|
commandId: z.string(),
|
||||||
|
command: volumeCommandSchema,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const volumeCommandResultSchema = z.discriminatedUnion("name", [
|
||||||
|
z.object({ name: z.literal("volume.mount"), result: volumeOperationResultSchema }),
|
||||||
|
z.object({ name: z.literal("volume.unmount"), result: volumeOperationResultSchema }),
|
||||||
|
z.object({ name: z.literal("volume.checkHealth"), result: volumeOperationResultSchema }),
|
||||||
|
z.object({ name: z.literal("volume.statfs"), result: statfsSchema }),
|
||||||
|
z.object({
|
||||||
|
name: z.literal("volume.listFiles"),
|
||||||
|
result: z.object({
|
||||||
|
files: z.array(fileEntrySchema),
|
||||||
|
path: z.string(),
|
||||||
|
offset: z.number(),
|
||||||
|
limit: z.number(),
|
||||||
|
total: z.number(),
|
||||||
|
hasMore: z.boolean(),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
z.object({
|
||||||
|
name: z.literal("volume.testConnection"),
|
||||||
|
result: z.object({ success: z.boolean(), message: z.string() }),
|
||||||
|
}),
|
||||||
|
z.object({
|
||||||
|
name: z.literal("filesystem.browse"),
|
||||||
|
result: z.object({ directories: z.array(directoryEntrySchema), path: z.string() }),
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const volumeCommandResponseSchema = z.object({
|
||||||
|
type: z.literal("volume.commandResult"),
|
||||||
|
payload: z.discriminatedUnion("status", [
|
||||||
|
z.object({ commandId: z.string(), status: z.literal("success"), command: volumeCommandResultSchema }),
|
||||||
|
z.object({ commandId: z.string(), status: z.literal("error"), error: z.string() }),
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
|
||||||
const heartbeatPingSchema = z.object({
|
const heartbeatPingSchema = z.object({
|
||||||
type: z.literal("heartbeat.ping"),
|
type: z.literal("heartbeat.ping"),
|
||||||
payload: z.object({ sentAt: z.number() }),
|
payload: z.object({ sentAt: z.number() }),
|
||||||
|
|
@ -113,6 +218,7 @@ const heartbeatPongSchema = z.object({
|
||||||
const controllerMessageSchema = z.discriminatedUnion("type", [
|
const controllerMessageSchema = z.discriminatedUnion("type", [
|
||||||
backupRunSchema,
|
backupRunSchema,
|
||||||
backupCancelSchema,
|
backupCancelSchema,
|
||||||
|
volumeCommandRequestSchema,
|
||||||
heartbeatPingSchema,
|
heartbeatPingSchema,
|
||||||
]);
|
]);
|
||||||
const agentMessageSchema = z.discriminatedUnion("type", [
|
const agentMessageSchema = z.discriminatedUnion("type", [
|
||||||
|
|
@ -122,6 +228,7 @@ const agentMessageSchema = z.discriminatedUnion("type", [
|
||||||
backupCompletedSchema,
|
backupCompletedSchema,
|
||||||
backupFailedSchema,
|
backupFailedSchema,
|
||||||
backupCancelledSchema,
|
backupCancelledSchema,
|
||||||
|
volumeCommandResponseSchema,
|
||||||
heartbeatPongSchema,
|
heartbeatPongSchema,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
@ -132,6 +239,10 @@ export type BackupProgressPayload = z.infer<typeof backupProgressSchema>["payloa
|
||||||
export type BackupCompletedPayload = z.infer<typeof backupCompletedSchema>["payload"];
|
export type BackupCompletedPayload = z.infer<typeof backupCompletedSchema>["payload"];
|
||||||
export type BackupFailedPayload = z.infer<typeof backupFailedSchema>["payload"];
|
export type BackupFailedPayload = z.infer<typeof backupFailedSchema>["payload"];
|
||||||
export type BackupCancelledPayload = z.infer<typeof backupCancelledSchema>["payload"];
|
export type BackupCancelledPayload = z.infer<typeof backupCancelledSchema>["payload"];
|
||||||
|
export type VolumeCommandPayload = z.infer<typeof volumeCommandRequestSchema>["payload"];
|
||||||
|
export type VolumeCommand = z.infer<typeof volumeCommandSchema>;
|
||||||
|
export type VolumeCommandResult = z.infer<typeof volumeCommandResultSchema>;
|
||||||
|
export type VolumeCommandResponsePayload = z.infer<typeof volumeCommandResponseSchema>["payload"];
|
||||||
export type ControllerMessage = z.infer<typeof controllerMessageSchema>;
|
export type ControllerMessage = z.infer<typeof controllerMessageSchema>;
|
||||||
export type AgentMessage = z.infer<typeof agentMessageSchema>;
|
export type AgentMessage = z.infer<typeof agentMessageSchema>;
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue