fix(shutdown): keep the current volume status after shutdown cleanup
This commit is contained in:
parent
6bcb0bef71
commit
23b00a1c20
4 changed files with 50 additions and 13 deletions
|
|
@ -58,7 +58,7 @@ describe("shutdown", () => {
|
||||||
expect(updated!.status).toBe("mounted");
|
expect(updated!.status).toBe("mounted");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps legacy controller-local fallback unmount on shutdown", async () => {
|
test("keeps mounted status while running the legacy controller-local fallback unmount on shutdown", async () => {
|
||||||
config.flags.enableLocalAgent = false;
|
config.flags.enableLocalAgent = false;
|
||||||
const events: string[] = [];
|
const events: string[] = [];
|
||||||
vi.spyOn(Scheduler, "stop").mockImplementation(async () => {
|
vi.spyOn(Scheduler, "stop").mockImplementation(async () => {
|
||||||
|
|
@ -85,6 +85,6 @@ describe("shutdown", () => {
|
||||||
expect(events).toEqual(["scheduler.stop", "agents.stop"]);
|
expect(events).toEqual(["scheduler.stop", "agents.stop"]);
|
||||||
expect(runVolumeCommand).not.toHaveBeenCalled();
|
expect(runVolumeCommand).not.toHaveBeenCalled();
|
||||||
expect(updated).toBeDefined();
|
expect(updated).toBeDefined();
|
||||||
expect(updated!.status).toBe("unmounted");
|
expect(updated!.status).toBe("mounted");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ export const shutdown = async () => {
|
||||||
|
|
||||||
for (const volume of volumes) {
|
for (const volume of volumes) {
|
||||||
const { status, error } = await withContext({ organizationId: volume.organizationId }, () =>
|
const { status, error } = await withContext({ organizationId: volume.organizationId }, () =>
|
||||||
volumeService.unmountVolume(volume.shortId),
|
volumeService.unmountVolume(volume.shortId, { persistStatus: false }),
|
||||||
).catch((error) => ({ status: "error" as const, error: toMessage(error) }));
|
).catch((error) => ({ status: "error" as const, error: toMessage(error) }));
|
||||||
|
|
||||||
logger.info(`Volume ${volume.name} unmount status: ${status}${error ? `, error: ${error}` : ""}`);
|
logger.info(`Volume ${volume.name} unmount status: ${status}${error ? `, error: ${error}` : ""}`);
|
||||||
|
|
|
||||||
|
|
@ -152,11 +152,38 @@ describe("volumeService.mountVolume", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("volumeService.unmountVolume", () => {
|
||||||
|
test("persists the unmounted status for normal unmount requests", async () => {
|
||||||
|
const { organizationId, user } = await createTestSession();
|
||||||
|
const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
|
||||||
|
agentManagerMock.runVolumeCommand.mockResolvedValueOnce({
|
||||||
|
name: "volume.unmount",
|
||||||
|
result: { status: "unmounted" },
|
||||||
|
});
|
||||||
|
|
||||||
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
|
const result = await volumeService.unmountVolume(volume.shortId);
|
||||||
|
|
||||||
|
expect(result.status).toBe("unmounted");
|
||||||
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
|
||||||
|
volume.agentId,
|
||||||
|
expect.objectContaining({ name: "volume.unmount", volume: expect.objectContaining({ id: volume.id }) }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const updatedVolume = await db.query.volumesTable.findFirst({ where: { id: volume.id } });
|
||||||
|
expect(updatedVolume?.status).toBe("unmounted");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("volumeService.ensureHealthyVolume", () => {
|
describe("volumeService.ensureHealthyVolume", () => {
|
||||||
test("returns ready when the mounted volume passes its health check", async () => {
|
test("returns ready when the mounted volume passes its health check", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
|
const volume = await createTestVolume({ organizationId, status: "mounted", agentId: "agent-1" });
|
||||||
agentManagerMock.runVolumeCommand.mockResolvedValue({ name: "volume.checkHealth", result: { status: "mounted" } });
|
agentManagerMock.runVolumeCommand.mockResolvedValue({
|
||||||
|
name: "volume.checkHealth",
|
||||||
|
result: { status: "mounted" },
|
||||||
|
});
|
||||||
|
|
||||||
await withContext({ organizationId, userId: user.id }, async () => {
|
await withContext({ organizationId, userId: user.id }, async () => {
|
||||||
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
const result = await volumeService.ensureHealthyVolume(volume.shortId);
|
||||||
|
|
@ -169,14 +196,22 @@ describe("volumeService.ensureHealthyVolume", () => {
|
||||||
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledOnce();
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledOnce();
|
||||||
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
|
expect(agentManagerMock.runVolumeCommand).toHaveBeenCalledWith(
|
||||||
volume.agentId,
|
volume.agentId,
|
||||||
expect.objectContaining({ name: "volume.checkHealth", volume: expect.objectContaining({ id: volume.id }) }),
|
expect.objectContaining({
|
||||||
|
name: "volume.checkHealth",
|
||||||
|
volume: expect.objectContaining({ id: volume.id }),
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("auto-remounts when the mounted volume fails its health check", async () => {
|
test("auto-remounts when the mounted volume fails its health check", async () => {
|
||||||
const { organizationId, user } = await createTestSession();
|
const { organizationId, user } = await createTestSession();
|
||||||
const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: true, agentId: "agent-1" });
|
const volume = await createTestVolume({
|
||||||
|
organizationId,
|
||||||
|
status: "mounted",
|
||||||
|
autoRemount: true,
|
||||||
|
agentId: "agent-1",
|
||||||
|
});
|
||||||
agentManagerMock.runVolumeCommand
|
agentManagerMock.runVolumeCommand
|
||||||
.mockResolvedValueOnce({ name: "volume.checkHealth", result: { status: "error", error: "stale mount" } })
|
.mockResolvedValueOnce({ name: "volume.checkHealth", result: { status: "error", error: "stale mount" } })
|
||||||
.mockResolvedValueOnce({ name: "volume.unmount", result: { status: "unmounted" } })
|
.mockResolvedValueOnce({ name: "volume.unmount", result: { status: "unmounted" } })
|
||||||
|
|
|
||||||
|
|
@ -174,7 +174,7 @@ const mountVolume = async (shortId: ShortId) => {
|
||||||
return { error, status };
|
return { error, status };
|
||||||
};
|
};
|
||||||
|
|
||||||
const unmountVolume = async (shortId: ShortId) => {
|
const unmountVolume = async (shortId: ShortId, options?: { persistStatus?: boolean }) => {
|
||||||
const organizationId = getOrganizationId();
|
const organizationId = getOrganizationId();
|
||||||
const volume = await findVolume(shortId);
|
const volume = await findVolume(shortId);
|
||||||
|
|
||||||
|
|
@ -184,13 +184,15 @@ const unmountVolume = async (shortId: ShortId) => {
|
||||||
|
|
||||||
const { status, error } = await runVolumeBackendCommand(volume, "volume.unmount");
|
const { status, error } = await runVolumeBackendCommand(volume, "volume.unmount");
|
||||||
|
|
||||||
await db
|
if (options?.persistStatus !== false) {
|
||||||
.update(volumesTable)
|
await db
|
||||||
.set({ status })
|
.update(volumesTable)
|
||||||
.where(and(eq(volumesTable.id, volume.id), eq(volumesTable.organizationId, organizationId)));
|
.set({ status })
|
||||||
|
.where(and(eq(volumesTable.id, volume.id), eq(volumesTable.organizationId, organizationId)));
|
||||||
|
|
||||||
if (status === "unmounted") {
|
if (status === "unmounted") {
|
||||||
serverEvents.emit("volume:unmounted", { organizationId, volumeName: volume.name });
|
serverEvents.emit("volume:unmounted", { organizationId, volumeName: volume.name });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { error, status };
|
return { error, status };
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue