fix: mark agent offline only if the session was removed properly
This commit is contained in:
parent
4e17daf6fe
commit
9bb5d2c916
5 changed files with 36 additions and 40 deletions
|
|
@ -1,7 +1,7 @@
|
||||||
import { logger } from "@zerobyte/core/node";
|
import { logger } from "@zerobyte/core/node";
|
||||||
import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
|
import type { BackupRunPayload } from "@zerobyte/contracts/agent-protocol";
|
||||||
import { config } from "../../core/config";
|
import { config } from "../../core/config";
|
||||||
import type { AgentBackupEventHandlers } from "./controller/server";
|
import { createAgentManagerRuntime, type AgentBackupEventHandlers } from "./controller/server";
|
||||||
import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process";
|
import { spawnLocalAgentProcess, stopLocalAgentProcess } from "./local/process";
|
||||||
import type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
|
import type { BackupExecutionProgress, BackupExecutionResult } from "./helpers/runtime-state";
|
||||||
import { createAgentRuntimeState } from "./helpers/runtime-state";
|
import { createAgentRuntimeState } from "./helpers/runtime-state";
|
||||||
|
|
@ -155,7 +155,6 @@ export const startAgentController = async () => {
|
||||||
runtime.agentManager = null;
|
runtime.agentManager = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { createAgentManagerRuntime } = await import("./controller/server");
|
|
||||||
const nextAgentManager = createAgentManagerRuntime();
|
const nextAgentManager = createAgentManagerRuntime();
|
||||||
nextAgentManager.setBackupEventHandlers(backupEventHandlers);
|
nextAgentManager.setBackupEventHandlers(backupEventHandlers);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -197,10 +197,11 @@ export function createAgentManagerRuntime() {
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
close: (ws) => {
|
close: (ws) => {
|
||||||
removeSession(ws.data.agentId, ws.data.id);
|
if (removeSession(ws.data.agentId, ws.data.id)) {
|
||||||
void agentsService.markAgentOffline(ws.data.agentId).catch((error) => {
|
void agentsService.markAgentOffline(ws.data.agentId).catch((error) => {
|
||||||
logger.error(`Failed to mark agent ${ws.data.agentId} as offline: ${toMessage(error)}`);
|
logger.error(`Failed to mark agent ${ws.data.agentId} as offline: ${toMessage(error)}`);
|
||||||
});
|
});
|
||||||
|
}
|
||||||
logger.info(`Agent "${ws.data.agentName}" (${ws.data.agentId}) disconnected`);
|
logger.info(`Agent "${ws.data.agentName}" (${ws.data.agentId}) disconnected`);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
@ -214,11 +215,9 @@ export function createAgentManagerRuntime() {
|
||||||
catch: (error) => new StopAgentManagerServerError({ cause: error }),
|
catch: (error) => new StopAgentManagerServerError({ cause: error }),
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
Effect.catchAll((error) =>
|
Effect.catchAll((error) => {
|
||||||
Effect.sync(() => {
|
return logger.effect.error(`Failed to stop Agent Manager server: ${toMessage(error.cause)}`);
|
||||||
logger.error(`Failed to stop Agent Manager server: ${toMessage(error.cause)}`);
|
}),
|
||||||
}),
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -233,6 +232,7 @@ export function createAgentManagerRuntime() {
|
||||||
await Effect.runPromise(Scope.close(scope, Exit.succeed(undefined)));
|
await Effect.runPromise(Scope.close(scope, Exit.succeed(undefined)));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// TODO: Move the effect boundary up
|
||||||
const start = async () => {
|
const start = async () => {
|
||||||
if (runtimeScope) {
|
if (runtimeScope) {
|
||||||
await stop();
|
await stop();
|
||||||
|
|
|
||||||
|
|
@ -136,16 +136,13 @@ export const createControllerAgentSession = (
|
||||||
yield* Effect.addFinalizer(() => closeSession());
|
yield* Effect.addFinalizer(() => closeSession());
|
||||||
|
|
||||||
const handleSendFailure = (reason: string) => {
|
const handleSendFailure = (reason: string) => {
|
||||||
logger.error(
|
return Effect.gen(function* () {
|
||||||
`Closing session for agent ${socket.data.agentId} on ${socket.data.id} after an outbound websocket send failed: ${reason}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
socket.close();
|
|
||||||
|
|
||||||
void Effect.runPromise(closeSession()).catch((error) => {
|
|
||||||
logger.error(
|
logger.error(
|
||||||
`Failed to close session for agent ${socket.data.agentId} on ${socket.data.id}: ${toMessage(error)}`,
|
`Closing session for agent ${socket.data.agentId} on ${socket.data.id} after an outbound websocket send failed: ${reason}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
yield* Effect.sync(() => socket.close());
|
||||||
|
yield* closeSession();
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -154,17 +151,16 @@ export const createControllerAgentSession = (
|
||||||
Effect.forever(
|
Effect.forever(
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const message = yield* Queue.take(outboundQueue);
|
const message = yield* Queue.take(outboundQueue);
|
||||||
yield* Effect.sync(() => {
|
|
||||||
try {
|
const sendResult = yield* Effect.try({
|
||||||
const sendResult = socket.send(message);
|
try: () => socket.send(message),
|
||||||
if (sendResult === 0) {
|
catch: (error) => toMessage(error),
|
||||||
handleSendFailure("connection issue");
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
handleSendFailure(toMessage(error));
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}),
|
|
||||||
|
if (sendResult === 0) {
|
||||||
|
yield* handleSendFailure("connection issue");
|
||||||
|
}
|
||||||
|
}).pipe(Effect.catchAll((reason) => handleSendFailure(reason))),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -200,9 +196,7 @@ export const createControllerAgentSession = (
|
||||||
at: readyAt,
|
at: readyAt,
|
||||||
});
|
});
|
||||||
|
|
||||||
yield* Effect.sync(() => {
|
yield* logger.effect.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
|
||||||
logger.info(`Agent "${socket.data.agentName}" (${socket.data.agentId}) is ready`);
|
|
||||||
});
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "backup.started": {
|
case "backup.started": {
|
||||||
|
|
@ -210,7 +204,7 @@ export const createControllerAgentSession = (
|
||||||
scheduleId: message.payload.scheduleId,
|
scheduleId: message.payload.scheduleId,
|
||||||
state: "active",
|
state: "active",
|
||||||
});
|
});
|
||||||
logger.info(
|
yield* logger.effect.info(
|
||||||
`Backup ${message.payload.jobId} started on agent ${socket.data.agentId} for schedule ${message.payload.scheduleId}`,
|
`Backup ${message.payload.jobId} started on agent ${socket.data.agentId} for schedule ${message.payload.scheduleId}`,
|
||||||
);
|
);
|
||||||
yield* handlers.onBackupStarted(message.payload);
|
yield* handlers.onBackupStarted(message.payload);
|
||||||
|
|
@ -258,16 +252,12 @@ export const createControllerAgentSession = (
|
||||||
const parsed = parseAgentMessage(data);
|
const parsed = parseAgentMessage(data);
|
||||||
|
|
||||||
if (parsed === null) {
|
if (parsed === null) {
|
||||||
yield* Effect.sync(() => {
|
yield* logger.effect.warn(`Invalid JSON from agent ${socket.data.agentId}`);
|
||||||
logger.warn(`Invalid JSON from agent ${socket.data.agentId}`);
|
|
||||||
});
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!parsed.success) {
|
if (!parsed.success) {
|
||||||
yield* Effect.sync(() => {
|
yield* logger.effect.warn(`Invalid agent message from ${socket.data.agentId}: ${parsed.error.message}`);
|
||||||
logger.warn(`Invalid agent message from ${socket.data.agentId}: ${parsed.error.message}`);
|
|
||||||
});
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ export const handleBackupRunCommand = (context: ControllerCommandContext, payloa
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info(`Starting backup ${payload.jobId} for schedule ${payload.scheduleId}`);
|
yield* logger.effect.info(`Starting backup ${payload.jobId} for schedule ${payload.scheduleId}`);
|
||||||
const abortController = new AbortController();
|
const abortController = new AbortController();
|
||||||
yield* context.setRunningJob(payload.jobId, { scheduleId: payload.scheduleId, abortController });
|
yield* context.setRunningJob(payload.jobId, { scheduleId: payload.scheduleId, abortController });
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { format } from "date-fns";
|
||||||
import { createConsola, type ConsolaReporter } from "consola";
|
import { createConsola, type ConsolaReporter } from "consola";
|
||||||
import { formatWithOptions } from "node:util";
|
import { formatWithOptions } from "node:util";
|
||||||
import { sanitizeSensitiveData } from "../utils/sanitize";
|
import { sanitizeSensitiveData } from "../utils/sanitize";
|
||||||
|
import { Effect } from "effect";
|
||||||
|
|
||||||
type LogLevel = "debug" | "info" | "warn" | "error";
|
type LogLevel = "debug" | "info" | "warn" | "error";
|
||||||
|
|
||||||
|
|
@ -102,4 +103,10 @@ export const logger = {
|
||||||
info: (...messages: unknown[]) => consola.info(formatMessages(messages).join(" ")),
|
info: (...messages: unknown[]) => consola.info(formatMessages(messages).join(" ")),
|
||||||
warn: (...messages: unknown[]) => consola.warn(formatMessages(messages).join(" ")),
|
warn: (...messages: unknown[]) => consola.warn(formatMessages(messages).join(" ")),
|
||||||
error: (...messages: unknown[]) => consola.error(formatMessages(messages).join(" ")),
|
error: (...messages: unknown[]) => consola.error(formatMessages(messages).join(" ")),
|
||||||
|
effect: {
|
||||||
|
debug: (...messages: unknown[]) => Effect.sync(() => consola.debug(formatMessages(messages).join(" "))),
|
||||||
|
info: (...messages: unknown[]) => Effect.sync(() => consola.info(formatMessages(messages).join(" "))),
|
||||||
|
warn: (...messages: unknown[]) => Effect.sync(() => consola.warn(formatMessages(messages).join(" "))),
|
||||||
|
error: (...messages: unknown[]) => Effect.sync(() => consola.error(formatMessages(messages).join(" "))),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue