pediatric-ai-scribe-v3/test/my-resources-refine.test.js

// Behavioural cover for modifying a resource. The rest of my-resources.test.js
// reads the source; these run the handler, because the three faults they pin
// were all invisible to source reading — the code said the right thing and did
// the wrong one.
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const read = file => fs.readFileSync(path.join(__dirname, '..', file), 'utf8');

// Normalised, because that is the shape the route stores. Comparing a
// hand-written deck with a parsed reply would compare normalisation, not content.
const deckBuild = require('../src/utils/deckBuild');
const DECK = Object.assign(
  { title: 'Croup', subtitle: 'Teaching deck', date: '2026-01-01' },
  deckBuild.parse(JSON.stringify({
    slides: [
      { type: 'title', title: 'Croup' },
      { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor'] }
    ]
  }))
);

function router(t, overrides = {}) {
  const module = { exports: {} };
  const aiCalls = [];
  const replies = overrides.replies ? overrides.replies.slice() : null;
  const reviewCalls = [];
  const updates = [];
  const row = Object.assign({
    id: 5, user_id: 7, title: 'Croup', kind: 'presentation', topic: 'croup',
    markdown: '# Croup\n\n- Barking cough\n- Stridor\n', deck: JSON.stringify(DECK)
  }, overrides.row || {});
  const quiet = { log() {}, error() {}, warn() {}, info() {} };

  const mocks = {
    express: require('express'),
    os: require('os'),
    path: require('path'),
    'fs/promises': require('fs/promises'),
    '../db/database': {
      get: async (sql, params) => {
        if (/^UPDATE user_resources/.test(sql)) {
          updates.push({ sql, params });
          return { id: row.id, title: row.title, updated_at: '2026-01-02T00:00:00Z' };
        }
        return row;
      },
      all: async () => [],
      run: async () => ({}),
      getSetting: async key => {
        if (key === 'clinical_assistant.image_model') return 'synthetic-image';
        if (key === 'my_resources.review_model') return overrides.visionModel || '';
        return '';
      }
    },
    '../middleware/auth': { authMiddleware: (req, res, next) => next() },
    '../utils/ai': {
      callAI: async (messages, options) => {
        aiCalls.push({ messages, options });
        // A scripted sequence when the test needs the model to answer
        // differently on each attempt; the last reply repeats if it runs out.
        const content = replies ? (replies.length > 1 ? replies.shift() : replies[0]) : overrides.reply;
        return { content: content, model: 'synthetic' };
      },
      discoverModels: async () => []
    },
    '../utils/deckBuild': deckBuild,
    '../utils/deckSchema': require('../src/utils/deckSchema'),
    '../utils/deckReview': {
      slideImages: async () => (overrides.slideImages || []),
      review: async (deck) => { reviewCalls.push(deck); return { deck: deck, reviewed: true, reason: 'ok' }; },
      MAX_SLIDES: 20
    },
    '../utils/documentExport': { FORMATS: { pptx: { mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation' } }, GOTENBERG: 'http://gotenberg:3000', isSupported: () => false, filename: () => 'x', mimeFor: () => '', render: async () => ({}), renderDeck: async () => Buffer.from('pptx') },
    '../utils/generatedImages': { service: () => ({ get: async () => null, asset: async () => null }), workflows: ['my_resources'] },
    '../utils/learningRetrieval': { retrieve: async () => ({ context: '', sources: [], reason: null }) },
    '../utils/logger': quiet,
    '../utils/nextcloudFiles': { send: async () => '/PediatricScribe/2026-01-01/x.pptx' },
    '../utils/metrics': { resourceRefines: { inc() {} }, resourceVocabularyGaps: { inc() {} } },
    '../utils/pubmedSearch': { isAvailable: async () => false, search: async () => ({ results: [] }), formatForPrompt: () => '' },
    '../utils/webSearch': { isAvailable: async () => false, search: async () => ({ results: [] }), formatForPrompt: () => '' },
    '../utils/resourceImages': { tools: [{ name: 'draw' }], dispatch: async x => x, guidance: () => 'GUIDANCE.', requestedCount: () => 0, MAX_IMAGES: 6 }
  };

  vm.runInNewContext(read('src/routes/myResources.js'), {
    module, exports: module.exports, console: quiet, Buffer, JSON, Date, Math,
    process: { env: {} }, setTimeout() {},
    require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; }
  });

  async function request(method, routePath, body) {
    const layer = module.exports.stack.find(l => l.route && l.route.path === routePath && l.route.methods[method]);
    assert.ok(layer, 'route missing: ' + method + ' ' + routePath);
    const stack = layer.route.stack;
    const handler = stack[stack.length - 1].handle;
    const res = {
      statusCode: 200, body: null,
      status(code) { this.statusCode = code; return this; },
      json(payload) { this.body = payload; return this; },
      setHeader() {}, send() { return this; }
    };
    await handler({ body, params: { id: '5' }, query: {}, user: { id: 7 } }, res);
    return res;
  }
  return { request, aiCalls, updates, reviewCalls };
}

test('modifying a deck asks for a deck, even when illustration is on', async () => {
  // The fault: existingDeck was declared below the branch that read it, so var
  // hoisting made it undefined there. Illustration on a deck therefore appended
  // the markdown instruction — "Returning the markdown is still required" —
  // to a prompt whose body asked for deck JSON. The model was told to produce
  // two different artifacts at once and the reply parsed as neither, so every
  // modification of a presentation with illustration ticked failed with 502.
  const r = router(null, {
    reply: JSON.stringify({ slides: [DECK.slides[0], { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor', 'Hoarse voice'] }] })
  });
  const res = await r.request('post', '/my-resources/:id/refine', {
    instructions: 'add a third feature', withImages: 'true'
  });

  assert.equal(res.statusCode, 200, 'a deck modification with images on must not 502');
  const prompt = r.aiCalls[0].messages[0].content;
  assert.match(prompt, /Revise the following slide deck/);
  assert.match(prompt, /Add "image_prompt" to the slides/, 'the deck illustration instruction');
  assert.doesNotMatch(prompt, /Returning the markdown is still required/,
    'the markdown instruction must never reach a deck prompt');
  assert.doesNotMatch(prompt, /An illustration tool is available/,
    'a deck declares its figures; it is not given the tool');
  assert.equal(r.aiCalls[0].options.tools, undefined, 'and no tool schema is attached');
});

test('a modification the model returned unchanged says so instead of claiming success', async () => {
  // Answering "Applied" for a reply identical to the original sent people off
  // to download the same file and conclude the feature was broken. It was
  // logged server-side, where the person who could reword the instruction
  // could not see it.
  const r = router(null, { reply: JSON.stringify({ slides: DECK.slides }) });
  const res = await r.request('post', '/my-resources/:id/refine', { instructions: 'make it better' });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.unchanged, true, 'the caller is told the deck came back identical');
});

test('a modification that did change the deck reports itself as changed', async () => {
  const r = router(null, {
    reply: JSON.stringify({ slides: [DECK.slides[0], { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor', 'Hoarse voice'] }] })
  });
  const res = await r.request('post', '/my-resources/:id/refine', { instructions: 'add a third feature' });

  assert.equal(res.body.unchanged, false);
  assert.equal(r.updates.length, 1, 'and the row is written');
  assert.match(r.updates[0].sql, /updated_at = NOW\(\)/);
});

test('a presentation with no deck still modifies, through the markdown path', async () => {
  const r = router(null, { row: { deck: null }, reply: '# Croup\n\n- Barking cough\n- Stridor\n- Hoarse voice\n' });
  const res = await r.request('post', '/my-resources/:id/refine', {
    instructions: 'add a third feature', withImages: 'true'
  });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.unchanged, false);
  const prompt = r.aiCalls[0].messages[0].content;
  assert.match(prompt, /Revise the following Pandoc markdown/);
  assert.match(prompt, /An illustration tool is available/, 'markdown has nowhere to declare a figure');
});

test('the library says which presentations carry a deck, and the row shows when it was modified', () => {
  // Both halves of "modify does nothing": the list read created_at, so a
  // modification that did apply left the visible timestamp untouched, and
  // nothing distinguished a real deck from a flat one.
  const route = read('src/routes/myResources.js');
  assert.match(route, /AS has_deck/);

  const ui = read('public/js/myResources.js');
  assert.match(ui, /row\.updated_at \? new Date\(row\.updated_at\)/);
  assert.match(ui, /edited \? 'modified '/);
  assert.match(ui, /has_deck === false \? ' · plain text, no slide layout' : ''/);
  assert.match(ui, /if \(data\.unchanged\) \{/);
});

// ── Generating a deck ──────────────────────────────────────────────────────

const GOOD_DECK = JSON.stringify({
  slides: [
    { type: 'title', title: 'Croup' },
    { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor'] }
  ]
});

test('a deck the model fumbles once is asked for a second time, not abandoned', async () => {
  // Falling straight back to markdown after one unlucky reply produced a
  // materially worse artifact: plain slides inferred from markdown instead of
  // the layouts the model chose. Measured on the stored library, this happened
  // once in eight generations.
  const r = router(null, { replies: ['Sorry, I cannot do that.', GOOD_DECK] });
  const res = await r.request('post', '/my-resources/generate', {
    topic: 'croup', kind: 'presentation'
  });

  assert.equal(res.statusCode, 200);
  assert.equal(r.aiCalls.length, 2, 'the deck is asked for twice before giving up');
  assert.match(r.aiCalls[1].messages[0].content, /teaching presentation/,
    'the retry is the same deck prompt, not the weaker markdown one');
  assert.equal(res.body.deckFallback, null, 'and the retry succeeded, so nothing fell back');
});

test('a deck that fails twice falls back to markdown and says why', async () => {
  const r = router(null, { replies: ['Sorry, I cannot help with that.', 'I am unable to comply.', '# Croup\n\n- Barking cough\n'] });
  const res = await r.request('post', '/my-resources/generate', {
    topic: 'croup', kind: 'presentation'
  });

  assert.equal(res.statusCode, 200);
  assert.equal(r.aiCalls.length, 3, 'two deck attempts, then markdown');
  assert.equal(res.body.deckFallback, 'the reply was not a deck',
    'the caller is told it came out plain, and why');
});

test('a truncated deck reply is named as truncated, not as the wrong shape', async () => {
  // The three causes want different fixes — a smaller deck, a different model,
  // a reworded topic — so the message distinguishes them. Truncation is only
  // claimed for a reply that began as JSON: an apology in prose does not end in
  // "}" either, and naming that "cut short" points at the wrong fix.
  const cut = GOOD_DECK.slice(0, GOOD_DECK.length - 30);
  const r = router(null, { replies: [cut, cut, '# Croup\n'] });
  const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' });

  assert.match(res.body.deckFallback, /cut short at \d+ characters/);
});

test('a deck that parses first time is never asked for twice', async () => {
  const r = router(null, { replies: [GOOD_DECK] });
  const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' });

  assert.equal(res.statusCode, 200);
  assert.equal(r.aiCalls.length, 1, 'the retry costs a call and must only happen on failure');
  assert.equal(res.body.deckFallback, null);
});

// ── Modifying with sight ───────────────────────────────────────────────────

const TWO_PNGS = [
  { mimeType: 'image/png', dataBase64: 'aW1hZ2Ux' },
  { mimeType: 'image/png', dataBase64: 'aW1hZ2Uy' }
];

const EDITED = JSON.stringify({
  slides: [
    { type: 'title', title: 'Croup' },
    { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor', 'Hoarse voice'] }
  ]
});

test('modifying a deck shows the model what the deck currently looks like', async () => {
  // Most of what people ask for while modifying is about the rendered page —
  // "that slide is crowded", "the diagram is in the wrong place" — and none of
  // it is answerable from the JSON alone.
  const r = router(null, { visionModel: 'seeing-model', slideImages: TWO_PNGS, replies: [EDITED] });
  const res = await r.request('post', '/my-resources/:id/refine', {
    instructions: 'slide 2 looks crowded, split it'
  });

  assert.equal(res.statusCode, 200);
  assert.equal(r.aiCalls[0].options.images, TWO_PNGS, 'the rendered slides are attached');
  assert.equal(r.aiCalls[0].options.model, 'seeing-model', 'and the model that can see does the editing');
  assert.match(r.aiCalls[0].messages[0].content, /You can see the deck as it renders now: 2 images/);
  assert.match(r.aiCalls[0].messages[0].content, /first image is slide index 0/,
    'the images must be anchored to the JSON or a slide reference means nothing');
  assert.equal(res.body.saw, 2);
});

test('the edited deck is rendered again and checked', async () => {
  // The edit was made against how the deck looked *before* it. A slide that
  // gained two bullets only overflows once it is rendered again.
  const r = router(null, { visionModel: 'seeing-model', slideImages: TWO_PNGS, replies: [EDITED] });
  const res = await r.request('post', '/my-resources/:id/refine', { instructions: 'add a feature' });

  assert.equal(r.reviewCalls.length, 1, 'the result goes back past the reviewer');
  assert.equal(res.body.review.applied, true);
});

test('with no vision model configured, modify still works and never renders', async () => {
  // Sight is an upgrade, not a dependency. Nothing here may become a new way
  // for a modification to fail.
  const r = router(null, { replies: [EDITED] });
  const res = await r.request('post', '/my-resources/:id/refine', { instructions: 'add a feature' });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.saw, 0);
  assert.equal(r.aiCalls[0].options.images, undefined);
  assert.equal(r.reviewCalls.length, 0, 'and nothing is rendered or reviewed');
});

test('a render that fails falls through to editing blind rather than failing', async () => {
  // Gotenberg down, LibreOffice wedged, a deck too big: none of them may cost
  // the author their modification.
  const r = router(null, { visionModel: 'seeing-model', slideImages: [], replies: [EDITED] });
  const res = await r.request('post', '/my-resources/:id/refine', { instructions: 'add a feature' });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.saw, 0);
  assert.equal(r.aiCalls[0].options.images, undefined);
  assert.notEqual(r.aiCalls[0].options.model, 'seeing-model',
    'no images means no reason to pay for the vision model');
});

test('an echo is still reported as an echo, even when the reviewer moved something', async () => {
  // The reviewer may reposition, which would otherwise make an instruction that
  // achieved nothing look like it had worked.
  const r = router(null, { visionModel: 'seeing-model', slideImages: TWO_PNGS,
                           replies: [JSON.stringify({ slides: DECK.slides })] });
  const res = await r.request('post', '/my-resources/:id/refine', { instructions: 'make it better' });

  assert.equal(res.body.unchanged, true, 'judged on the model edit, before the reviewer ran');
});