// ============================================================ // VISION TOOL // ============================================================ // Lets a text-only model ask a model that can see. // // The same shape as the image tool, and for the same reason: the model is handed // a capability and decides when to use it. Here the capability is eyesight. When // someone attaches a photograph and the configured chat model cannot accept // image input, the attachment used to be either refused by the provider or // silently dropped — the person got an answer about a picture nobody had looked // at, which is worse than a refusal. // // The attachment never reaches the chat model. It goes to the vision model, // whose description comes back as a tool result, and the chat model answers from // that. So the clinical reasoning stays with the model an administrator chose // for it, and only the seeing is delegated. var MAX_QUESTION = 500; var MAX_DESCRIPTION_TOKENS = 900; var tools = [{ type: 'function', function: { name: 'look_at_image', description: 'Look at the image the user attached to this conversation. ' + 'You cannot see images yourself; this asks a model that can and returns ' + 'what it sees in words. Call it whenever the user\'s message refers to an ' + 'attached image, and ask for exactly what you need to answer — a general ' + 'description, or a specific question such as the reading on a device, the ' + 'distribution of a rash, or the text on a label.', parameters: { type: 'object', additionalProperties: false, properties: { question: { type: 'string', minLength: 1, maxLength: MAX_QUESTION, description: 'What you need to know about the image.' } }, required: ['question'] } } }]; function failure(statusCode, message) { var error = new Error(message); error.statusCode = statusCode; return error; } // What the seeing model is told. Deliberately narrow: it reports, it does not // diagnose. The clinical judgement belongs to the chat model, which has the // conversation, the retrieved sources and the system prompt; this one has a // picture and no context, and an opinion from it would carry undeserved weight. function describePrompt(question) { return 'Describe what is visible in the attached image so that a clinician who ' + 'cannot see it can reason about it. Report only what is observable — colours, ' + 'distribution, morphology, text, readings, scale. Do not diagnose, do not ' + 'speculate about causes, and say plainly when something is unclear or cut off ' + 'rather than guessing.\n\n' + // The question is written by a model that has not seen the image, from what // the user said. So it can presume something that is not there — "describe // this rash" about a photograph of a drug chart. Answering the presumption // rather than the picture is the failure that matters, because the answer // then travels back as fact. 'The question below was written by someone who cannot see the image, so it ' + 'may assume something the image does not show. Describe what is actually ' + 'there. If the question presumes something absent — a finding, a body part, ' + 'a kind of document — say so first, in plain words, and then describe what ' + 'the image does contain.\n\n' + 'The question: ' + question; } /** * Run a look_at_image call and let the model finish its answer. * Returns `ai` unchanged when the model did not call the tool. */ async function dispatch(ai, opts) { opts = opts || {}; if (!ai || !ai.toolCalls || !ai.toolCalls.length) return ai; var call = ai.toolCalls.find(function (c) { return c && c.function && c.function.name === 'look_at_image'; }); if (!call) return ai; if (!Array.isArray(opts.images) || !opts.images.length) { throw failure(400, 'There is no image attached to this conversation.'); } if (!opts.visionModel) { throw failure(503, 'No vision model is configured, so an attached image cannot be read.'); } var question = ''; try { var args = JSON.parse(String(call.function.arguments || '{}')); question = String(args.question || '').slice(0, MAX_QUESTION).trim(); } catch (e) { throw failure(400, 'The image question could not be read.'); } if (!question) question = 'Describe this image.'; var seen = await opts.callAI( [{ role: 'user', content: describePrompt(question) }], { model: opts.visionModel, temperature: 0.1, images: opts.images, maxTokens: MAX_DESCRIPTION_TOKENS } ); var description = String((seen && seen.content) || '').trim(); if (!description) throw failure(502, 'The vision model returned no description of the image.'); // Handed back as a tool result so the chat model answers in its own voice, // with its own sources and system prompt, from a description rather than a // picture. toolChoice none: it has what it asked for and must now answer. var completed = await opts.callAI( (opts.messages || []).concat([ { role: 'assistant', content: null, tool_calls: [call] }, { role: 'tool', tool_call_id: call.id, content: JSON.stringify({ description: description }) } ]), Object.assign({}, opts.options || {}, { tools: tools, toolChoice: 'none' }) ); return Object.assign({}, completed, { visionHandled: true, visionModel: opts.visionModel, visionDescription: description }); } module.exports = { tools, dispatch, describePrompt, MAX_QUESTION };