pediatric-ai-scribe-v3/test/vision-tool.test.js

// A text-only chat model can be lent a pair of eyes: it calls look_at_image, a
// vision model describes the attachment, and the chat model answers from the
// description. The attachment never reaches the model that cannot read it.
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const visionTool = require('../src/utils/visionTool');

const read = file => fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
const IMAGES = [{ mimeType: 'image/png', dataBase64: 'aW1n' }];
const call = (question) => ({
  id: 'call_1', type: 'function',
  function: { name: 'look_at_image', arguments: JSON.stringify({ question }) }
});

function recorder(replies) {
  const calls = [];
  const queue = replies.slice();
  return {
    calls,
    callAI: async (messages, options) => {
      calls.push({ messages, options });
      return queue.length > 1 ? queue.shift() : queue[0];
    }
  };
}

test('the tool describes itself as eyesight the model does not have', () => {
  const fn = visionTool.tools[0].function;
  assert.equal(fn.name, 'look_at_image');
  assert.match(fn.description, /You cannot see images yourself/);
  assert.equal(fn.parameters.required[0], 'question');
});

test('a call routes the image to the vision model, never to the chat model', async () => {
  const r = recorder([{ content: 'A red maculopapular rash over the trunk.' }, { content: 'That pattern fits...' }]);
  const out = await visionTool.dispatch({ content: '', toolCalls: [call('describe the rash')] }, {
    images: IMAGES, visionModel: 'seeing-model', callAI: r.callAI,
    messages: [{ role: 'user', content: 'what is this rash' }], options: { model: 'blind-model' }
  });

  assert.equal(r.calls.length, 2);
  // First hop: the vision model, with the picture.
  assert.equal(r.calls[0].options.model, 'seeing-model');
  assert.equal(r.calls[0].options.images, IMAGES);
  assert.match(r.calls[0].messages[0].content, /describe the rash/);
  // Second hop: the chat model, with words and no picture.
  assert.equal(r.calls[1].options.model, 'blind-model');
  assert.equal(r.calls[1].options.images, undefined, 'the blind model never receives the image');
  assert.equal(r.calls[1].options.toolChoice, 'none', 'it has its answer; it must now use it');
  const toolMessage = r.calls[1].messages.find(m => m.role === 'tool');
  assert.match(toolMessage.content, /maculopapular/);

  assert.equal(out.visionHandled, true);
  assert.equal(out.content, 'That pattern fits...');
});

test('a question that presumes the wrong thing is corrected, not answered', () => {
  // The question is written by a model that has not seen the image, from what
  // the user said — so "describe this rash" can arrive about a photograph of a
  // drug chart. Answering the presumption would send that back as fact.
  const prompt = visionTool.describePrompt('what is the distribution of this rash');
  assert.match(prompt, /may assume something the image does not show/);
  assert.match(prompt, /If the question presumes something absent/);
  assert.match(prompt, /describe what the image does contain/);
});

test('the seeing model is told to report, not to diagnose', () => {
  // It has a picture and no conversation, no sources and no system prompt. An
  // opinion from it would carry weight it has not earned.
  const prompt = visionTool.describePrompt('what is this');
  assert.match(prompt, /Report only what is observable/);
  assert.match(prompt, /Do not diagnose/);
  assert.match(prompt, /say plainly when something is unclear/);
});

test('a reply with no tool call passes straight through', async () => {
  const r = recorder([{ content: 'unused' }]);
  const ai = { content: 'A direct answer.', toolCalls: [] };
  assert.equal(await visionTool.dispatch(ai, { images: IMAGES, visionModel: 'm', callAI: r.callAI }), ai);
  assert.equal(r.calls.length, 0);
});

test('another tool’s call is left for that tool', async () => {
  const r = recorder([{ content: 'unused' }]);
  const ai = { content: '', toolCalls: [{ id: 'x', type: 'function', function: { name: 'generate_image', arguments: '{}' } }] };
  assert.equal(await visionTool.dispatch(ai, { images: IMAGES, visionModel: 'm', callAI: r.callAI }), ai);
  assert.equal(r.calls.length, 0);
});

test('calling it with nothing attached, or no vision model, says so plainly', async () => {
  const r = recorder([{ content: 'x' }]);
  await assert.rejects(
    () => visionTool.dispatch({ content: '', toolCalls: [call('what is this')] },
      { images: [], visionModel: 'm', callAI: r.callAI }),
    err => /no image attached/i.test(err.message) && err.statusCode === 400);

  await assert.rejects(
    () => visionTool.dispatch({ content: '', toolCalls: [call('what is this')] },
      { images: IMAGES, visionModel: '', callAI: r.callAI }),
    err => /No vision model is configured/.test(err.message) && err.statusCode === 503);
});

test('a vision model that says nothing is an error, not an empty description', async () => {
  const r = recorder([{ content: '   ' }]);
  await assert.rejects(
    () => visionTool.dispatch({ content: '', toolCalls: [call('describe')] },
      { images: IMAGES, visionModel: 'seeing', callAI: r.callAI, messages: [] }),
    err => /no description/i.test(err.message));
});

test('the route withholds the attachment only from a model the gateway calls blind', () => {
  const route = read('src/routes/clinicalAssistant.js');
  assert.match(route, /function assistantToolset\(prepared\)/);
  // Delegation is opt-in on an explicit false, never on an unknown.
  assert.match(route, /supportsVision\([\s\S]{0,80}\) === false/);
  assert.match(route, /tools: imageTool\.tools\.concat\(visionTool\.tools\), images: undefined/);
  // Both call sites build the toolset, and both completion paths reuse the same
  // decision — a finalize that re-attached the withheld image would undo it.
  assert.equal((route.match(/var toolset = assistantToolset\(prepared\);/g) || []).length, 2);
  assert.equal((route.match(/images: assistantToolset\(prepared\)\.images/g) || []).length, 2);
});

test('vision capability is asked of the gateway and cached, never guessed from the id', () => {
  const util = read('src/utils/modelVision.js');
  assert.match(util, /supports_vision === 'boolean' \? info\.supports_vision : null/);
  assert.match(util, /never inferred from the model id/);
  assert.match(util, /CACHE_MS/);
  assert.doesNotMatch(util, /\/gemini|\/gpt-4|test\(modelId\)/, 'no name matching');
});