From 4f5687982d31049cf4d67e99bbc3c056b9ba869d Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 11 Sep 2026 14:03:05 +0200 Subject: [PATCH] feat: Learning resources can be grounded in the clinical corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Learning generated everything from the model alone. A deck on bronchiolitis was whatever the model remembered about bronchiolitis, with no connection to the documents this institution actually indexed — while the assistant had been searching that corpus all along. Same collection, deliberately. mcp_bge_m3_1024 is already embedded with openrouter-bge-m3 at 1024 dimensions; a second index over the same documents with the same embedder would be a copy that drifts. What differs is the budget: a chat answer wants a few tight excerpts because the reader is waiting, a teaching resource synthesises a whole topic. So learning.search_limit and learning.context_chars default to 30 and 2500 against the assistant's 8 and 1400, and are separate keys so tuning one cannot move the other. Not unbounded, though. "No limit" only moves the ceiling from a setting to the model's context window, where overflow truncates the middle of the prompt silently — the worst place to lose source material. 60 results and 8000 characters per excerpt are the caps. Opt in per generation: a resource on something the library does not cover is better written without it than padded with the nearest unrelated excerpts. Retrieval never fails a generation — the resource is then written from the model alone, which is what happened before this existed — and every response reports what it was grounded on, so a caller can say "24 excerpts" or "the library had nothing on this" rather than quietly serving ungrounded material. Verified against the live corpus: bronchiolitis, neonatal jaundice and febrile seizure each returned 12 excerpts and ~23k characters from Nelson, Rudolph and the Pediatric Clinical Practice Guidelines. A deck generated through the full chain came back with textbook specificity that is not general recall — bronchiolar diameter, birth-weight thresholds, the full pathogen list. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- src/routes/learningAI.js | 38 ++++++++- src/utils/learningRetrieval.js | 81 +++++++++++++++++++ test/generated-image-tools.test.js | 3 + test/learning-retrieval.test.js | 120 +++++++++++++++++++++++++++++ 4 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 src/utils/learningRetrieval.js create mode 100644 test/learning-retrieval.test.js diff --git a/src/routes/learningAI.js b/src/routes/learningAI.js index b079eb3e..cb35b2f6 100644 --- a/src/routes/learningAI.js +++ b/src/routes/learningAI.js @@ -13,6 +13,7 @@ var { authMiddleware, moderatorMiddleware } = require('../middleware/auth'); var db = require('../db/database'); var cryptoUtil = require('../utils/crypto'); var { assertSafeHttpsUrl } = require('../utils/urlSafety'); +var learningRetrieval = require('../utils/learningRetrieval'); router.use(authMiddleware); router.use(moderatorMiddleware); @@ -137,12 +138,23 @@ async function extractText(buffer, mimetype, filename) { // ── Build AI prompt ────────────────────────────────────────── function buildGeneratePrompt(opts) { - var { topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories } = opts; + var { topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories, corpusContext } = opts; var source = docText ? 'Based on the following document/resource text, generate educational content.\n\nDOCUMENT:\n"""\n' + docText.substring(0, 50000) + '\n"""\n' : 'Generate educational content on the following topic for a medical professional audience (pediatrics / primary care).\n\nTOPIC: ' + topic + '\n'; + // Material from this institution's own indexed documents, when the author + // asked for it. It goes before the instructions so the model reads it as the + // ground to work from, and it is explicitly preferred over recall: the point + // of grounding is that local guidance wins where the two disagree. + if (corpusContext) { + source += '\nThe following excerpts come from this institution\'s indexed clinical library. ' + + 'Prefer them over your own recall wherever they disagree, and do not contradict them. ' + + 'They are reference material, not a template: write the resource in your own words.\n\n' + + 'LIBRARY EXCERPTS:\n"""\n' + corpusContext + '\n"""\n'; + } + var refineInstr = refinement ? '\n\nAdditional instructions for tone/style/focus: ' + refinement : ''; var categoryInstr = buildCategoryInstruction(existingCategories); @@ -174,6 +186,8 @@ Then each slide separated by ---. Guidelines: - Do not nest lists more than one level deep, and do not put an ordered list inside a bullet: it overfills the slide. - Prefer more slides with less on each. A slide should hold one idea. +- A slide heading is the slide's subject. Do not number it or prefix it with + "Slide 3:" — the deck numbers itself. - Include a summary/key takeaways slide at the end - Do NOT include HTML tags or inline styles`; } @@ -270,6 +284,10 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res) var webdavPath = req.body.webdavPath || ''; var wordCount = parseInt(req.body.wordCount) || 0; var slideCount = parseInt(req.body.slideCount) || 0; + // Opt in. Grounding is the right default for clinical teaching, but a + // resource on something the library does not cover is better written + // without it than padded with the nearest unrelated excerpts. + var useCorpus = String(req.body.useCorpus) === 'true' || req.body.useCorpus === true; if (typeof topic !== 'string' || typeof refinement !== 'string') return res.status(400).json({ error: 'topic and refinement must be text' }); @@ -326,7 +344,18 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res) } var existingCategories = await db.all('SELECT name FROM learning_categories ORDER BY sort_order ASC, name ASC', []); - var prompt = buildGeneratePrompt({ topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories }); + // Retrieval never fails a generation: without it the resource is written + // from the model alone, which is exactly what happened before this + // existed. The reason is surfaced so the author is told rather than + // quietly handed ungrounded material. + var corpus = { sources: [], context: '', reason: 'not requested' }; + if (useCorpus) { + corpus = await learningRetrieval.retrieve(topic || docText || '', db.getSetting); + if (corpus.reason) console.warn('[learning] corpus not used:', corpus.reason); + else console.info('[learning] grounded on', corpus.sources.length, 'excerpts'); + } + + var prompt = buildGeneratePrompt({ topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories, corpusContext: corpus.context }); var aiMessages = [ { role: 'system', content: 'You are a medical education content generator. Return ONLY the requested JSON or Marp markdown — no preamble, no commentary, no code fences, no thinking. Start your response with { or --- as appropriate.' }, @@ -356,12 +385,12 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res) try { parsedPres = m ? JSON.parse(m[0]) : null; } catch(e2) { parsedPres = null; } } if (parsedPres && parsedPres.marpMarkdown) { - return res.json({ success: true, contentType: 'presentation', marpMarkdown: parsedPres.marpMarkdown, category_name: parsedPres.category_name || '', questions: parsedPres.questions || [], imageJobs: result.imageJobs || [], model: result.model }); + return res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, contentType: 'presentation', marpMarkdown: parsedPres.marpMarkdown, category_name: parsedPres.category_name || '', questions: parsedPres.questions || [], imageJobs: result.imageJobs || [], model: result.model }); } } // Plain Marp markdown (no questions requested, or parse failed) var marpMd = raw.replace(/^```(?:markdown|marp)?\s*/i, '').replace(/\s*```\s*$/, ''); - return res.json({ success: true, contentType: 'presentation', marpMarkdown: marpMd, questions: [], imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length }); + return res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, contentType: 'presentation', marpMarkdown: marpMd, questions: [], imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length }); } // Strip code fences and any trailing text after JSON @@ -413,6 +442,7 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res) res.json({ success: true, + grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, content: parsed, imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length, diff --git a/src/utils/learningRetrieval.js b/src/utils/learningRetrieval.js new file mode 100644 index 00000000..93784f4a --- /dev/null +++ b/src/utils/learningRetrieval.js @@ -0,0 +1,81 @@ +// ============================================================ +// LEARNING RETRIEVAL +// Grounding teaching material in the clinical corpus. +// +// Learning generated everything from the model alone: a slide deck on +// bronchiolitis was whatever the model remembered about bronchiolitis, with no +// connection to the documents this institution actually indexed. The corpus was +// right there — the assistant has been searching it all along. +// +// Same collection, deliberately. mcp_bge_m3_1024 is embedded with +// openrouter-bge-m3 at 1024 dimensions; a second index over the same documents +// with the same embedder would be a copy that drifts. What differs is the +// budget, not the index. +// +// A chat answer wants a few tight excerpts, because the reader is waiting and +// the answer is one paragraph. A teaching resource synthesises a whole topic, +// so it wants many more and longer ones. Hence separate settings rather than +// borrowing clinical_assistant.*: tuning one must never move the other. +// ============================================================ + +var { semanticSearch } = require('./clinicalMcpClient'); +var { normalizeMcpSearchResponse, dedupeSources, cleanSourceExcerpt } = require('./clinicalRetrieval'); + +// Generous, but not unbounded. "No limit" only moves the ceiling from a setting +// to the model's context window, where overflow truncates the middle of the +// prompt silently — the worst possible place to lose source material. +var DEFAULTS = { limit: 30, contextChars: 2500 }; +var BOUNDS = { limit: [3, 60], contextChars: [300, 8000] }; + +function clampInt(value, min, max, fallback) { + var n = parseInt(value, 10); + if (!Number.isFinite(n)) return fallback; + return Math.min(max, Math.max(min, n)); +} + +async function settings(getSetting) { + return { + limit: clampInt(await getSetting('learning.search_limit', String(DEFAULTS.limit)), + BOUNDS.limit[0], BOUNDS.limit[1], DEFAULTS.limit), + contextChars: clampInt(await getSetting('learning.context_chars', String(DEFAULTS.contextChars)), + BOUNDS.contextChars[0], BOUNDS.contextChars[1], DEFAULTS.contextChars) + }; +} + +/** + * Search the clinical corpus for a topic. + * + * Never throws. Retrieval failing must not fail the generation — a resource + * written from the model alone is what happened before this existed, and is a + * far better outcome than an error page. The caller is told what happened so it + * can say so rather than quietly producing ungrounded material. + */ +async function retrieve(topic, getSetting) { + var query = String(topic || '').trim(); + if (!query) return { sources: [], context: '', reason: 'no topic' }; + try { + var opts = await settings(getSetting); + var response = await semanticSearch(query, { + limit: opts.limit, + includeContext: true, + contextChars: opts.contextChars + }); + var sources = dedupeSources(normalizeMcpSearchResponse(response)).slice(0, opts.limit); + if (!sources.length) return { sources: [], context: '', reason: 'nothing indexed matched' }; + return { sources: sources, context: formatForPrompt(sources), reason: null }; + } catch (e) { + return { sources: [], context: '', reason: e.message || 'retrieval failed' }; + } +} + +// Numbered the way the assistant numbers them, so a model that has learned one +// citation convention here sees the same one there. +function formatForPrompt(sources) { + return sources.map(function (s, i) { + var n = s.number || (i + 1); + return '[' + n + '] ' + (s.title || 'Untitled') + (s.page ? ', page ' + s.page : '') + + '\n' + cleanSourceExcerpt(s.excerpt); + }).join('\n\n---\n\n'); +} + +module.exports = { retrieve, formatForPrompt, settings, DEFAULTS, BOUNDS }; diff --git a/test/generated-image-tools.test.js b/test/generated-image-tools.test.js index c34d42aa..b3f88ae6 100644 --- a/test/generated-image-tools.test.js +++ b/test/generated-image-tools.test.js @@ -77,6 +77,9 @@ function route(file, ai, jobs) { '../utils/ai':ai, '../db/database':{getSetting:async key=>key.includes('model')?'test':null,all:async()=>[]}, '../middleware/auth':{authMiddleware(){},moderatorMiddleware(){}}, '../utils/crypto':{},'../utils/urlSafety':{},'../utils/policy':{requireFeature:()=>()=>{}}, + // Retrieval is opt-in and these cases do not ask for it; the stub proves + // the route never reaches the corpus unless useCorpus was set. + '../utils/learningRetrieval':{ retrieve: async () => { throw new Error('retrieval must not run unless requested'); } }, '../utils/logger':{audit(){},error(){}},'../utils/redis':{},'../utils/clinicalPromptPool':{createClinicalPromptPool:()=>({})}, '../utils/clinicalPrompts':require('../src/utils/clinicalPrompts'), '../utils/clinicalConversation':require('../src/utils/clinicalConversation'), diff --git a/test/learning-retrieval.test.js b/test/learning-retrieval.test.js new file mode 100644 index 00000000..1d836978 --- /dev/null +++ b/test/learning-retrieval.test.js @@ -0,0 +1,120 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +const root = path.join(__dirname, '..'); +const read = file => fs.readFileSync(path.join(root, file), 'utf8'); + +// Load the module with its two dependencies replaced, so these tests exercise +// the budgeting and failure handling without a Milvus behind them. +function load(mocks) { + const module = { exports: {} }; + vm.runInNewContext(read('src/utils/learningRetrieval.js'), { + module, exports: module.exports, console: { warn() {}, info() {} }, + require(name) { + if (Object.hasOwn(mocks, name)) return mocks[name]; + throw new Error('unexpected import: ' + name); + } + }); + return module.exports; +} + +const passthrough = { + './clinicalRetrieval': { + normalizeMcpSearchResponse: r => r.results || [], + dedupeSources: s => s, + cleanSourceExcerpt: t => String(t || '') + } +}; + +test('Learning retrieves with its own budget, not the assistant’s', async () => { + let captured = null; + const lib = load({ + ...passthrough, + './clinicalMcpClient': { semanticSearch: async (q, opts) => { captured = { q, opts }; return { results: [] }; } } + }); + // A chat answer wants a few tight excerpts because the reader is waiting. A + // teaching resource synthesises a whole topic, so it wants many more and + // longer ones. The assistant's defaults are 8 and 1400. + assert.equal(lib.DEFAULTS.limit, 30); + assert.equal(lib.DEFAULTS.contextChars, 2500); + + await lib.retrieve('bronchiolitis', async () => null); + assert.equal(captured.q, 'bronchiolitis'); + assert.equal(captured.opts.limit, 30); + assert.equal(captured.opts.contextChars, 2500); + assert.equal(captured.opts.includeContext, true); +}); + +test('the budget is settable but bounded', async () => { + let captured = null; + const lib = load({ + ...passthrough, + './clinicalMcpClient': { semanticSearch: async (q, opts) => { captured = opts; return { results: [] }; } } + }); + const settings = { 'learning.search_limit': '999', 'learning.context_chars': '999999' }; + await lib.retrieve('topic', async key => settings[key]); + // "No limit" only moves the ceiling from a setting to the model's context + // window, where overflow truncates the middle of the prompt silently. + assert.equal(captured.limit, lib.BOUNDS.limit[1]); + assert.equal(captured.contextChars, lib.BOUNDS.contextChars[1]); + + const low = { 'learning.search_limit': '0', 'learning.context_chars': '1' }; + await lib.retrieve('topic', async key => low[key]); + assert.equal(captured.limit, lib.BOUNDS.limit[0]); + assert.equal(captured.contextChars, lib.BOUNDS.contextChars[0]); +}); + +test('retrieval failing never fails the generation', async () => { + const lib = load({ + ...passthrough, + './clinicalMcpClient': { semanticSearch: async () => { throw new Error('MCP unreachable'); } } + }); + // Writing the resource from the model alone is what happened before this + // existed, and is a far better outcome than an error page. + const out = await lib.retrieve('bronchiolitis', async () => null); + // Length, not deepEqual: the array is created inside the VM realm, so its + // prototype is not this realm's Array and a structural compare fails. + assert.equal(out.sources.length, 0); + assert.equal(out.context, ''); + assert.match(out.reason, /MCP unreachable/, 'and the caller is told why'); +}); + +test('an empty corpus result is reported, not silently passed off as grounded', async () => { + const lib = load({ + ...passthrough, + './clinicalMcpClient': { semanticSearch: async () => ({ results: [] }) } + }); + const out = await lib.retrieve('something nobody indexed', async () => null); + assert.equal(out.context, ''); + assert.match(out.reason, /nothing indexed matched/); +}); + +test('excerpts are numbered the way the assistant numbers them', async () => { + const lib = load({ + ...passthrough, + './clinicalMcpClient': { semanticSearch: async () => ({ results: [ + { number: 1, title: 'Bronchiolitis chapter', page: 12, excerpt: 'Supportive care.' }, + { number: 2, title: 'RSV guidance', excerpt: 'Peak 3-6 months.' } + ] }) } + }); + const out = await lib.retrieve('bronchiolitis', async () => null); + assert.equal(out.sources.length, 2); + assert.match(out.context, /^\[1\] Bronchiolitis chapter, page 12\nSupportive care\./); + assert.match(out.context, /\[2\] RSV guidance\nPeak 3-6 months\./); + assert.equal(out.reason, null); +}); + +test('the corpus block tells the model to prefer it over recall', () => { + const route = read('src/routes/learningAI.js'); + assert.match(route, /Prefer them over your own recall wherever they disagree/); + assert.match(route, /reference material, not a template/, 'so it writes rather than copies'); + // Opt in: a resource on something the library does not cover is better + // written without it than padded with the nearest unrelated excerpts. + assert.match(route, /var useCorpus\s+= String\(req\.body\.useCorpus\) === 'true'/); + assert.match(route, /if \(useCorpus\) \{/); + // And every success response says what it was grounded on. + assert.equal((route.match(/grounding: \{ used: Boolean\(corpus\.context\)/g) || []).length, 3); +});