pediatric-ai-scribe-v3/test/learning-retrieval.test.js

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');

// Load the module with its two dependencies replaced, so these tests exercise
// the budgeting and failure handling without a Milvus behind them.
function load(mocks) {
  const module = { exports: {} };
  vm.runInNewContext(read('src/utils/learningRetrieval.js'), {
    module, exports: module.exports, console: { warn() {}, info() {} },
    require(name) {
      if (Object.hasOwn(mocks, name)) return mocks[name];
      throw new Error('unexpected import: ' + name);
    }
  });
  return module.exports;
}

const passthrough = {
  './clinicalRetrieval': {
    normalizeMcpSearchResponse: r => r.results || [],
    dedupeSources: s => s,
    cleanSourceExcerpt: t => String(t || '')
  }
};

test('Learning retrieves with its own budget, not the assistant’s', async () => {
  let captured = null;
  const lib = load({
    ...passthrough,
    './clinicalMcpClient': { semanticSearch: async (q, opts) => { captured = { q, opts }; return { results: [] }; } }
  });
  // A chat answer wants a few tight excerpts because the reader is waiting. A
  // teaching resource synthesises a whole topic, so it wants many more and
  // longer ones. The assistant's defaults are 8 and 1400.
  assert.equal(lib.DEFAULTS.limit, 30);
  assert.equal(lib.DEFAULTS.contextChars, 2500);

  await lib.retrieve('bronchiolitis', async () => null);
  assert.equal(captured.q, 'bronchiolitis');
  assert.equal(captured.opts.limit, 30);
  assert.equal(captured.opts.contextChars, 2500);
  assert.equal(captured.opts.includeContext, true);
});

test('the budget is settable but bounded', async () => {
  let captured = null;
  const lib = load({
    ...passthrough,
    './clinicalMcpClient': { semanticSearch: async (q, opts) => { captured = opts; return { results: [] }; } }
  });
  const settings = { 'learning.search_limit': '999', 'learning.context_chars': '999999' };
  await lib.retrieve('topic', async key => settings[key]);
  // "No limit" only moves the ceiling from a setting to the model's context
  // window, where overflow truncates the middle of the prompt silently.
  assert.equal(captured.limit, lib.BOUNDS.limit[1]);
  assert.equal(captured.contextChars, lib.BOUNDS.contextChars[1]);

  const low = { 'learning.search_limit': '0', 'learning.context_chars': '1' };
  await lib.retrieve('topic', async key => low[key]);
  assert.equal(captured.limit, lib.BOUNDS.limit[0]);
  assert.equal(captured.contextChars, lib.BOUNDS.contextChars[0]);
});

test('retrieval failing never fails the generation', async () => {
  const lib = load({
    ...passthrough,
    './clinicalMcpClient': { semanticSearch: async () => { throw new Error('MCP unreachable'); } }
  });
  // Writing the resource from the model alone is what happened before this
  // existed, and is a far better outcome than an error page.
  const out = await lib.retrieve('bronchiolitis', async () => null);
  // Length, not deepEqual: the array is created inside the VM realm, so its
  // prototype is not this realm's Array and a structural compare fails.
  assert.equal(out.sources.length, 0);
  assert.equal(out.context, '');
  assert.match(out.reason, /MCP unreachable/, 'and the caller is told why');
});

test('an empty corpus result is reported, not silently passed off as grounded', async () => {
  const lib = load({
    ...passthrough,
    './clinicalMcpClient': { semanticSearch: async () => ({ results: [] }) }
  });
  const out = await lib.retrieve('something nobody indexed', async () => null);
  assert.equal(out.context, '');
  assert.match(out.reason, /nothing indexed matched/);
});

test('excerpts are numbered the way the assistant numbers them', async () => {
  const lib = load({
    ...passthrough,
    './clinicalMcpClient': { semanticSearch: async () => ({ results: [
      { number: 1, title: 'Bronchiolitis chapter', page: 12, excerpt: 'Supportive care.' },
      { number: 2, title: 'RSV guidance', excerpt: 'Peak 3-6 months.' }
    ] }) }
  });
  const out = await lib.retrieve('bronchiolitis', async () => null);
  assert.equal(out.sources.length, 2);
  assert.match(out.context, /^\[1\] Bronchiolitis chapter, page 12\nSupportive care\./);
  assert.match(out.context, /\[2\] RSV guidance\nPeak 3-6 months\./);
  assert.equal(out.reason, null);
});

test('the corpus block tells the model to prefer it over recall', () => {
  const route = read('src/routes/learningAI.js');
  assert.match(route, /Prefer them over your own recall wherever they disagree/);
  assert.match(route, /reference material, not a template/, 'so it writes rather than copies');
  // Provenance a clinician can check, on the deck itself rather than only in a
  // log. Restricted to excerpts actually used, and explicitly not invented.
  // References belong at the end, never in the body: a slide carrying [1]
  // markers is unreadable from the back of a room, and an article reads as
  // prose, not as a paper.
  assert.match(route, /Do NOT cite in the body: no \[1\] markers, no bracketed numbers/);
  assert.match(route, /In a presentation this is the final slide, titled/);
  assert.match(route, /'References\. Do not invent references/);
  assert.match(route, /Do not invent references, and do not list an excerpt you did not use/);
  // Opt in: a resource on something the library does not cover is better
  // written without it than padded with the nearest unrelated excerpts.
  assert.match(route, /var useCorpus\s+= String\(req\.body\.useCorpus\) === 'true'/);
  assert.match(route, /if \(useCorpus\) \{/);
  // And every success response says what it was grounded on.
  assert.equal((route.match(/grounding: \{ used: Boolean\(corpus\.context\)/g) || []).length, 3);
});

test('the Learning screen can ask for grounding and says what it got', () => {
  const html = read('public/components/cms.html');
  const js = read('public/js/learningHub.js');

  // Opt-out in the UI rather than opt-in: for clinical teaching the library is
  // nearly always the right source, so forgetting the checkbox should give the
  // grounded version, not the ungrounded one.
  assert.match(html, /<input type="checkbox" id="lh-ai-use-corpus" checked>/);
  assert.match(js, /formData\.append\('useCorpus', corpusBox && corpusBox\.checked === false \? 'false' : 'true'\)/);

  // The checkbox explains when to turn it OFF, which is the non-obvious case.
  assert.match(html, /Turn this off for a topic the\s*\n?\s*library does not cover/);

  // And the result is reported. Ungrounded material presented as grounded is
  // the failure worth preventing, so the wording must never imply the library
  // was used when it was not.
  assert.match(js, /'Written from ' \+ g\.count \+ ' library excerpt'/);
  assert.match(js, /'Not grounded' \+ \(g\.reason \? ' \\u2014 ' \+ g\.reason : ''\) \+ '\. Written from the model alone\.'/);
  assert.match(html, /id="lh-ai-grounding-result"/);
});