feat: Learning resources can be grounded in the clinical corpus

Learning generated everything from the model alone. A deck on bronchiolitis was
whatever the model remembered about bronchiolitis, with no connection to the
documents this institution actually indexed — while the assistant had been
searching that corpus all along.

Same collection, deliberately. mcp_bge_m3_1024 is already embedded with
openrouter-bge-m3 at 1024 dimensions; a second index over the same documents
with the same embedder would be a copy that drifts. What differs is the budget:
a chat answer wants a few tight excerpts because the reader is waiting, a
teaching resource synthesises a whole topic. So learning.search_limit and
learning.context_chars default to 30 and 2500 against the assistant's 8 and
1400, and are separate keys so tuning one cannot move the other.

Not unbounded, though. "No limit" only moves the ceiling from a setting to the
model's context window, where overflow truncates the middle of the prompt
silently — the worst place to lose source material. 60 results and 8000
characters per excerpt are the caps.

Opt in per generation: a resource on something the library does not cover is
better written without it than padded with the nearest unrelated excerpts.
Retrieval never fails a generation — the resource is then written from the model
alone, which is what happened before this existed — and every response reports
what it was grounded on, so a caller can say "24 excerpts" or "the library had
nothing on this" rather than quietly serving ungrounded material.

Verified against the live corpus: bronchiolitis, neonatal jaundice and febrile
seizure each returned 12 excerpts and ~23k characters from Nelson, Rudolph and
the Pediatric Clinical Practice Guidelines. A deck generated through the full
chain came back with textbook specificity that is not general recall —
bronchiolar diameter, birth-weight thresholds, the full pathogen list.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU

src/routes/learningAI.js

@@ -13,6 +13,7 @@ var { authMiddleware, moderatorMiddleware } = require('../middleware/auth');
 var db = require('../db/database');
 var cryptoUtil = require('../utils/crypto');
 var { assertSafeHttpsUrl } = require('../utils/urlSafety');
+var learningRetrieval = require('../utils/learningRetrieval');
 
 router.use(authMiddleware);
 router.use(moderatorMiddleware);
@@ -137,12 +138,23 @@ async function extractText(buffer, mimetype, filename) {
 // ── Build AI prompt ──────────────────────────────────────────
 
 function buildGeneratePrompt(opts) {
-  var { topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories } = opts;
+  var { topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories, corpusContext } = opts;
 
   var source = docText
     ? 'Based on the following document/resource text, generate educational content.\n\nDOCUMENT:\n"""\n' + docText.substring(0, 50000) + '\n"""\n'
     : 'Generate educational content on the following topic for a medical professional audience (pediatrics / primary care).\n\nTOPIC: ' + topic + '\n';
 
+  // Material from this institution's own indexed documents, when the author
+  // asked for it. It goes before the instructions so the model reads it as the
+  // ground to work from, and it is explicitly preferred over recall: the point
+  // of grounding is that local guidance wins where the two disagree.
+  if (corpusContext) {
+    source += '\nThe following excerpts come from this institution\'s indexed clinical library. ' +
+      'Prefer them over your own recall wherever they disagree, and do not contradict them. ' +
+      'They are reference material, not a template: write the resource in your own words.\n\n' +
+      'LIBRARY EXCERPTS:\n"""\n' + corpusContext + '\n"""\n';
+  }
+
   var refineInstr = refinement ? '\n\nAdditional instructions for tone/style/focus: ' + refinement : '';
   var categoryInstr = buildCategoryInstruction(existingCategories);
 
@@ -174,6 +186,8 @@ Then each slide separated by ---. Guidelines:
 - Do not nest lists more than one level deep, and do not put an ordered list
   inside a bullet: it overfills the slide.
 - Prefer more slides with less on each. A slide should hold one idea.
+- A slide heading is the slide's subject. Do not number it or prefix it with
+  "Slide 3:" — the deck numbers itself.
 - Include a summary/key takeaways slide at the end
 - Do NOT include HTML tags or inline styles`;
     }
@@ -270,6 +284,10 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res)
     var webdavPath   = req.body.webdavPath || '';
     var wordCount    = parseInt(req.body.wordCount) || 0;
     var slideCount   = parseInt(req.body.slideCount) || 0;
+    // Opt in. Grounding is the right default for clinical teaching, but a
+    // resource on something the library does not cover is better written
+    // without it than padded with the nearest unrelated excerpts.
+    var useCorpus    = String(req.body.useCorpus) === 'true' || req.body.useCorpus === true;
 
     if (typeof topic !== 'string' || typeof refinement !== 'string') return res.status(400).json({ error: 'topic and refinement must be text' });
 
@@ -326,7 +344,18 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res)
     }
 
     var existingCategories = await db.all('SELECT name FROM learning_categories ORDER BY sort_order ASC, name ASC', []);
-    var prompt = buildGeneratePrompt({ topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories });
+    // Retrieval never fails a generation: without it the resource is written
+    // from the model alone, which is exactly what happened before this
+    // existed. The reason is surfaced so the author is told rather than
+    // quietly handed ungrounded material.
+    var corpus = { sources: [], context: '', reason: 'not requested' };
+    if (useCorpus) {
+      corpus = await learningRetrieval.retrieve(topic || docText || '', db.getSetting);
+      if (corpus.reason) console.warn('[learning] corpus not used:', corpus.reason);
+      else console.info('[learning] grounded on', corpus.sources.length, 'excerpts');
+    }
+
+    var prompt = buildGeneratePrompt({ topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories, corpusContext: corpus.context });
 
     var aiMessages = [
         { role: 'system', content: 'You are a medical education content generator. Return ONLY the requested JSON or Marp markdown — no preamble, no commentary, no code fences, no thinking. Start your response with { or --- as appropriate.' },
@@ -356,12 +385,12 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res)
           try { parsedPres = m ? JSON.parse(m[0]) : null; } catch(e2) { parsedPres = null; }
         }
         if (parsedPres && parsedPres.marpMarkdown) {
-          return res.json({ success: true, contentType: 'presentation', marpMarkdown: parsedPres.marpMarkdown, category_name: parsedPres.category_name || '', questions: parsedPres.questions || [], imageJobs: result.imageJobs || [], model: result.model });
+          return res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, contentType: 'presentation', marpMarkdown: parsedPres.marpMarkdown, category_name: parsedPres.category_name || '', questions: parsedPres.questions || [], imageJobs: result.imageJobs || [], model: result.model });
         }
       }
       // Plain Marp markdown (no questions requested, or parse failed)
       var marpMd = raw.replace(/^```(?:markdown|marp)?\s*/i, '').replace(/\s*```\s*$/, '');
-      return res.json({ success: true, contentType: 'presentation', marpMarkdown: marpMd, questions: [], imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length });
+      return res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, contentType: 'presentation', marpMarkdown: marpMd, questions: [], imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length });
     }
 
     // Strip code fences and any trailing text after JSON
@@ -413,6 +442,7 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res)
 
     res.json({
       success: true,
+      grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null },
       content: parsed,
       imageJobs: result.imageJobs || [], model: result.model,
       docLength: docText.length,

src/utils/learningRetrieval.js

@@ -0,0 +1,81 @@
+// ============================================================
+// LEARNING RETRIEVAL
+// Grounding teaching material in the clinical corpus.
+//
+// Learning generated everything from the model alone: a slide deck on
+// bronchiolitis was whatever the model remembered about bronchiolitis, with no
+// connection to the documents this institution actually indexed. The corpus was
+// right there — the assistant has been searching it all along.
+//
+// Same collection, deliberately. mcp_bge_m3_1024 is embedded with
+// openrouter-bge-m3 at 1024 dimensions; a second index over the same documents
+// with the same embedder would be a copy that drifts. What differs is the
+// budget, not the index.
+//
+// A chat answer wants a few tight excerpts, because the reader is waiting and
+// the answer is one paragraph. A teaching resource synthesises a whole topic,
+// so it wants many more and longer ones. Hence separate settings rather than
+// borrowing clinical_assistant.*: tuning one must never move the other.
+// ============================================================
+
+var { semanticSearch } = require('./clinicalMcpClient');
+var { normalizeMcpSearchResponse, dedupeSources, cleanSourceExcerpt } = require('./clinicalRetrieval');
+
+// Generous, but not unbounded. "No limit" only moves the ceiling from a setting
+// to the model's context window, where overflow truncates the middle of the
+// prompt silently — the worst possible place to lose source material.
+var DEFAULTS = { limit: 30, contextChars: 2500 };
+var BOUNDS = { limit: [3, 60], contextChars: [300, 8000] };
+
+function clampInt(value, min, max, fallback) {
+  var n = parseInt(value, 10);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(max, Math.max(min, n));
+}
+
+async function settings(getSetting) {
+  return {
+    limit: clampInt(await getSetting('learning.search_limit', String(DEFAULTS.limit)),
+      BOUNDS.limit[0], BOUNDS.limit[1], DEFAULTS.limit),
+    contextChars: clampInt(await getSetting('learning.context_chars', String(DEFAULTS.contextChars)),
+      BOUNDS.contextChars[0], BOUNDS.contextChars[1], DEFAULTS.contextChars)
+  };
+}
+
+/**
+ * Search the clinical corpus for a topic.
+ *
+ * Never throws. Retrieval failing must not fail the generation — a resource
+ * written from the model alone is what happened before this existed, and is a
+ * far better outcome than an error page. The caller is told what happened so it
+ * can say so rather than quietly producing ungrounded material.
+ */
+async function retrieve(topic, getSetting) {
+  var query = String(topic || '').trim();
+  if (!query) return { sources: [], context: '', reason: 'no topic' };
+  try {
+    var opts = await settings(getSetting);
+    var response = await semanticSearch(query, {
+      limit: opts.limit,
+      includeContext: true,
+      contextChars: opts.contextChars
+    });
+    var sources = dedupeSources(normalizeMcpSearchResponse(response)).slice(0, opts.limit);
+    if (!sources.length) return { sources: [], context: '', reason: 'nothing indexed matched' };
+    return { sources: sources, context: formatForPrompt(sources), reason: null };
+  } catch (e) {
+    return { sources: [], context: '', reason: e.message || 'retrieval failed' };
+  }
+}
+
+// Numbered the way the assistant numbers them, so a model that has learned one
+// citation convention here sees the same one there.
+function formatForPrompt(sources) {
+  return sources.map(function (s, i) {
+    var n = s.number || (i + 1);
+    return '[' + n + '] ' + (s.title || 'Untitled') + (s.page ? ', page ' + s.page : '') +
+      '\n' + cleanSourceExcerpt(s.excerpt);
+  }).join('\n\n---\n\n');
+}
+
+module.exports = { retrieve, formatForPrompt, settings, DEFAULTS, BOUNDS };

test/generated-image-tools.test.js

@@ -77,6 +77,9 @@ function route(file, ai, jobs) {
     '../utils/ai':ai, '../db/database':{getSetting:async key=>key.includes('model')?'test':null,all:async()=>[]},
     '../middleware/auth':{authMiddleware(){},moderatorMiddleware(){}},
     '../utils/crypto':{},'../utils/urlSafety':{},'../utils/policy':{requireFeature:()=>()=>{}},
+    // Retrieval is opt-in and these cases do not ask for it; the stub proves
+    // the route never reaches the corpus unless useCorpus was set.
+    '../utils/learningRetrieval':{ retrieve: async () => { throw new Error('retrieval must not run unless requested'); } },
     '../utils/logger':{audit(){},error(){}},'../utils/redis':{},'../utils/clinicalPromptPool':{createClinicalPromptPool:()=>({})},
     '../utils/clinicalPrompts':require('../src/utils/clinicalPrompts'),
     '../utils/clinicalConversation':require('../src/utils/clinicalConversation'),

test/learning-retrieval.test.js

@@ -0,0 +1,120 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const vm = require('node:vm');
+
+const root = path.join(__dirname, '..');
+const read = file => fs.readFileSync(path.join(root, file), 'utf8');
+
+// Load the module with its two dependencies replaced, so these tests exercise
+// the budgeting and failure handling without a Milvus behind them.
+function load(mocks) {
+  const module = { exports: {} };
+  vm.runInNewContext(read('src/utils/learningRetrieval.js'), {
+    module, exports: module.exports, console: { warn() {}, info() {} },
+    require(name) {
+      if (Object.hasOwn(mocks, name)) return mocks[name];
+      throw new Error('unexpected import: ' + name);
+    }
+  });
+  return module.exports;
+}
+
+const passthrough = {
+  './clinicalRetrieval': {
+    normalizeMcpSearchResponse: r => r.results || [],
+    dedupeSources: s => s,
+    cleanSourceExcerpt: t => String(t || '')
+  }
+};
+
+test('Learning retrieves with its own budget, not the assistant’s', async () => {
+  let captured = null;
+  const lib = load({
+    ...passthrough,
+    './clinicalMcpClient': { semanticSearch: async (q, opts) => { captured = { q, opts }; return { results: [] }; } }
+  });
+  // A chat answer wants a few tight excerpts because the reader is waiting. A
+  // teaching resource synthesises a whole topic, so it wants many more and
+  // longer ones. The assistant's defaults are 8 and 1400.
+  assert.equal(lib.DEFAULTS.limit, 30);
+  assert.equal(lib.DEFAULTS.contextChars, 2500);
+
+  await lib.retrieve('bronchiolitis', async () => null);
+  assert.equal(captured.q, 'bronchiolitis');
+  assert.equal(captured.opts.limit, 30);
+  assert.equal(captured.opts.contextChars, 2500);
+  assert.equal(captured.opts.includeContext, true);
+});
+
+test('the budget is settable but bounded', async () => {
+  let captured = null;
+  const lib = load({
+    ...passthrough,
+    './clinicalMcpClient': { semanticSearch: async (q, opts) => { captured = opts; return { results: [] }; } }
+  });
+  const settings = { 'learning.search_limit': '999', 'learning.context_chars': '999999' };
+  await lib.retrieve('topic', async key => settings[key]);
+  // "No limit" only moves the ceiling from a setting to the model's context
+  // window, where overflow truncates the middle of the prompt silently.
+  assert.equal(captured.limit, lib.BOUNDS.limit[1]);
+  assert.equal(captured.contextChars, lib.BOUNDS.contextChars[1]);
+
+  const low = { 'learning.search_limit': '0', 'learning.context_chars': '1' };
+  await lib.retrieve('topic', async key => low[key]);
+  assert.equal(captured.limit, lib.BOUNDS.limit[0]);
+  assert.equal(captured.contextChars, lib.BOUNDS.contextChars[0]);
+});
+
+test('retrieval failing never fails the generation', async () => {
+  const lib = load({
+    ...passthrough,
+    './clinicalMcpClient': { semanticSearch: async () => { throw new Error('MCP unreachable'); } }
+  });
+  // Writing the resource from the model alone is what happened before this
+  // existed, and is a far better outcome than an error page.
+  const out = await lib.retrieve('bronchiolitis', async () => null);
+  // Length, not deepEqual: the array is created inside the VM realm, so its
+  // prototype is not this realm's Array and a structural compare fails.
+  assert.equal(out.sources.length, 0);
+  assert.equal(out.context, '');
+  assert.match(out.reason, /MCP unreachable/, 'and the caller is told why');
+});
+
+test('an empty corpus result is reported, not silently passed off as grounded', async () => {
+  const lib = load({
+    ...passthrough,
+    './clinicalMcpClient': { semanticSearch: async () => ({ results: [] }) }
+  });
+  const out = await lib.retrieve('something nobody indexed', async () => null);
+  assert.equal(out.context, '');
+  assert.match(out.reason, /nothing indexed matched/);
+});
+
+test('excerpts are numbered the way the assistant numbers them', async () => {
+  const lib = load({
+    ...passthrough,
+    './clinicalMcpClient': { semanticSearch: async () => ({ results: [
+      { number: 1, title: 'Bronchiolitis chapter', page: 12, excerpt: 'Supportive care.' },
+      { number: 2, title: 'RSV guidance', excerpt: 'Peak 3-6 months.' }
+    ] }) }
+  });
+  const out = await lib.retrieve('bronchiolitis', async () => null);
+  assert.equal(out.sources.length, 2);
+  assert.match(out.context, /^\[1\] Bronchiolitis chapter, page 12\nSupportive care\./);
+  assert.match(out.context, /\[2\] RSV guidance\nPeak 3-6 months\./);
+  assert.equal(out.reason, null);
+});
+
+test('the corpus block tells the model to prefer it over recall', () => {
+  const route = read('src/routes/learningAI.js');
+  assert.match(route, /Prefer them over your own recall wherever they disagree/);
+  assert.match(route, /reference material, not a template/, 'so it writes rather than copies');
+  // Opt in: a resource on something the library does not cover is better
+  // written without it than padded with the nearest unrelated excerpts.
+  assert.match(route, /var useCorpus\s+= String\(req\.body\.useCorpus\) === 'true'/);
+  assert.match(route, /if \(useCorpus\) \{/);
+  // And every success response says what it was grounded on.
+  assert.equal((route.match(/grounding: \{ used: Boolean\(corpus\.context\)/g) || []).length, 3);
+});