pediatric-ai-scribe-v3/src/utils/visionTool.js

// ============================================================
// VISION TOOL
// ============================================================
// Lets a text-only model ask a model that can see.
//
// The same shape as the image tool, and for the same reason: the model is handed
// a capability and decides when to use it. Here the capability is eyesight. When
// someone attaches a photograph and the configured chat model cannot accept
// image input, the attachment used to be either refused by the provider or
// silently dropped — the person got an answer about a picture nobody had looked
// at, which is worse than a refusal.
//
// The attachment never reaches the chat model. It goes to the vision model,
// whose description comes back as a tool result, and the chat model answers from
// that. So the clinical reasoning stays with the model an administrator chose
// for it, and only the seeing is delegated.
var MAX_QUESTION = 500;
var MAX_DESCRIPTION_TOKENS = 900;

var tools = [{
  type: 'function',
  function: {
    name: 'look_at_image',
    description: 'Look at the image the user attached to this conversation. ' +
      'You cannot see images yourself; this asks a model that can and returns ' +
      'what it sees in words. Call it whenever the user\'s message refers to an ' +
      'attached image, and ask for exactly what you need to answer — a general ' +
      'description, or a specific question such as the reading on a device, the ' +
      'distribution of a rash, or the text on a label.',
    parameters: {
      type: 'object',
      additionalProperties: false,
      properties: {
        question: {
          type: 'string',
          minLength: 1,
          maxLength: MAX_QUESTION,
          description: 'What you need to know about the image.'
        }
      },
      required: ['question']
    }
  }
}];

function failure(statusCode, message) {
  var error = new Error(message);
  error.statusCode = statusCode;
  return error;
}

// What the seeing model is told. Deliberately narrow: it reports, it does not
// diagnose. The clinical judgement belongs to the chat model, which has the
// conversation, the retrieved sources and the system prompt; this one has a
// picture and no context, and an opinion from it would carry undeserved weight.
function describePrompt(question) {
  return 'Describe what is visible in the attached image so that a clinician who ' +
    'cannot see it can reason about it. Report only what is observable — colours, ' +
    'distribution, morphology, text, readings, scale. Do not diagnose, do not ' +
    'speculate about causes, and say plainly when something is unclear or cut off ' +
    'rather than guessing.\n\n' +
    // The question is written by a model that has not seen the image, from what
    // the user said. So it can presume something that is not there — "describe
    // this rash" about a photograph of a drug chart. Answering the presumption
    // rather than the picture is the failure that matters, because the answer
    // then travels back as fact.
    'The question below was written by someone who cannot see the image, so it ' +
    'may assume something the image does not show. Describe what is actually ' +
    'there. If the question presumes something absent — a finding, a body part, ' +
    'a kind of document — say so first, in plain words, and then describe what ' +
    'the image does contain.\n\n' +
    'The question: ' + question;
}

/**
 * Run a look_at_image call and let the model finish its answer.
 * Returns `ai` unchanged when the model did not call the tool.
 */
async function dispatch(ai, opts) {
  opts = opts || {};
  if (!ai || !ai.toolCalls || !ai.toolCalls.length) return ai;
  var call = ai.toolCalls.find(function (c) {
    return c && c.function && c.function.name === 'look_at_image';
  });
  if (!call) return ai;

  if (!Array.isArray(opts.images) || !opts.images.length) {
    throw failure(400, 'There is no image attached to this conversation.');
  }
  if (!opts.visionModel) {
    throw failure(503, 'No vision model is configured, so an attached image cannot be read.');
  }

  var question = '';
  try {
    var args = JSON.parse(String(call.function.arguments || '{}'));
    question = String(args.question || '').slice(0, MAX_QUESTION).trim();
  } catch (e) {
    throw failure(400, 'The image question could not be read.');
  }
  if (!question) question = 'Describe this image.';

  var seen = await opts.callAI(
    [{ role: 'user', content: describePrompt(question) }],
    { model: opts.visionModel, temperature: 0.1, images: opts.images, maxTokens: MAX_DESCRIPTION_TOKENS }
  );
  var description = String((seen && seen.content) || '').trim();
  if (!description) throw failure(502, 'The vision model returned no description of the image.');

  // Handed back as a tool result so the chat model answers in its own voice,
  // with its own sources and system prompt, from a description rather than a
  // picture. toolChoice none: it has what it asked for and must now answer.
  var completed = await opts.callAI(
    (opts.messages || []).concat([
      { role: 'assistant', content: null, tool_calls: [call] },
      { role: 'tool', tool_call_id: call.id, content: JSON.stringify({ description: description }) }
    ]),
    Object.assign({}, opts.options || {}, { tools: tools, toolChoice: 'none' })
  );

  return Object.assign({}, completed, {
    visionHandled: true,
    visionModel: opts.visionModel,
    visionDescription: description
  });
}

module.exports = { tools, dispatch, describePrompt, MAX_QUESTION };