From 6e2ad26547d7bf8891112bd2893d1bfa9e11ada2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 14 Sep 2026 18:25:05 +0200 Subject: [PATCH] feat: starter questions are things to learn, not scenarios, and DeepSeek writes them without thinking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pool was cases — "A 6-week-old with 3 days of projectile vomiting… which fluid do you start?" — and the user, reading them on the assistant's front page: "Vague nonsense clinical scenarios. I want concrete things people learn from and pathophysiology if possible, all related to peds. No more clinical scenarios. Use deepseek with its reasoning off." So the brief is inverted. A question asks one definite thing with a definite answer — a mechanism, a distinction between two look-alikes, the number that decides, the reason behind a rule of practice — mechanism first, naming its disease, drug or finding, and staying paediatric. The filter refuses a vignette opener and a bare definition instead of demanding a number; the prompt version moves to 3 so every deployment rebuilds the pool once; the model defaults to DeepSeek flash with thinking disabled, which the assistant already knows how to ask for. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016fZGJNyDvERbMgS2Uc2msP --- src/utils/clinicalPromptPool.js | 37 +++++++++++++-------- test/clinical-assistant-prompt-pool.test.js | 23 +++++++------ 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/utils/clinicalPromptPool.js b/src/utils/clinicalPromptPool.js index 016c2b6b..6a324d2e 100644 --- a/src/utils/clinicalPromptPool.js +++ b/src/utils/clinicalPromptPool.js @@ -251,7 +251,7 @@ function parseJsonObject(text) { // Bumped whenever the generation prompt or the filter changes. A pool made by // an earlier version is served while a new one is built, so nobody sees an // empty screen and nobody has to remember to press Regenerate. -var PROMPT_VERSION = 2; +var PROMPT_VERSION = 3; function createClinicalPromptPool(opts) { opts = opts || {}; @@ -332,9 +332,10 @@ function createClinicalPromptPool(opts) { async function buildCorpusPromptPool() { var taxonomy = taxonomyWithQuotas(target); - // A small non-reasoning model writes the pool: a reasoning model spends its - // budget thinking about a twenty-line list and returns nothing. - var chatModel = await opts.getSetting('clinical_assistant.prompt_model', '') || process.env.CLINICAL_ASSISTANT_PROMPT_MODEL || 'openrouter-gpt-4.1-mini'; + // DeepSeek with thinking off writes the pool (the user's choice): a list + // of questions needs no reasoning budget, and a reasoning model spent its + // budget thinking about a twenty-line list and returned nothing. + var chatModel = await opts.getSetting('clinical_assistant.prompt_model', '') || process.env.CLINICAL_ASSISTANT_PROMPT_MODEL || 'ds-deepseek-v4.1-flash'; var generated = []; var startedAt = Date.now(); // A build that produced nothing used to vanish without a word; now it says @@ -357,11 +358,16 @@ function createClinicalPromptPool(opts) { var ai = await opts.callAI([ { role: 'system', - content: 'Write starter questions for a pediatric clinical assistant, the kind a clinician actually types at 3 a.m. with a patient in front of them. Use only the provided titles/snippets as inspiration; do not answer the questions and do not mention source names. Every question is a case and a decision: it opens with a one-line vignette — an age with a number and unit (e.g. "a 6-week-old", "a 14-year-old"), the setting, and two or three specific findings of which at least one is a number (a vital sign, a lab value, a dose, a duration, a weight) — and then asks ONE thing the clinician must decide now: the next test, the threshold at which to act, a dose with units, admit or discharge, when to repeat, what changes the plan. Never ask a textbook question: no "What is…", "What are the causes/features/signs of…", "How is X managed?", "Which scores are useful…". 18 to 45 words. Return strict JSON only: {"questions":[{"label":"2-5 word label","prompt":"question ending with ?","category":"' + item.category + '","age_band":"' + item.ageBands.join('|') + '","intent":"diagnosis|management|red_flags|differential|counseling|dosing|admission|discharge|review"}]}.' + content: 'Write starter questions for a pediatric clinical assistant. They are what a resident or a paediatrician would ask to learn something concrete about children — not a case, not a scenario, not a vignette. Use only the provided titles/snippets as inspiration; do not answer the questions and do not mention source names. Each question asks one definite thing with a definite answer: a mechanism (why a disease produces a finding, how a drug or a test works, what goes wrong in the physiology), a distinction (what separates two conditions that look alike, and why), a threshold or a number that matters in children (a dose, a cut-off, an age at which something changes), or the reason behind a rule of practice. Prefer pathophysiology wherever the source supports it. Name the disease, drug, organism, test or finding in the question itself, and keep it paediatric: an age group, a childhood disease, or a paediatric dose or value. Never open with a patient ("A 6-week-old with…", "A child presents…"), never invent findings, never ask "What is X?" alone. 10 to 30 words. Return strict JSON only: {"questions":[{"label":"2-5 word label","prompt":"question ending with ?","category":"' + item.category + '","age_band":"' + item.ageBands.join('|') + '","intent":"diagnosis|management|red_flags|differential|counseling|dosing|admission|discharge|review"}]}.' }, - { role: 'user', content: 'Category: ' + item.category + '\nAge bands: ' + item.ageBands.join(', ') + '\nIndexed pediatric source snippets:\n\n' + sourceText + '\n\nCreate 20 starter questions for this category, each a specific case with a specific decision. Vary the setting (emergency department, ward, clinic, neonatal unit, telephone advice), the decision (next test, threshold, dose, fluids, admit/discharge, timing, what if a finding changes), and the age. Example of the depth wanted: "A 6-week-old with 3 days of projectile non-bilious vomiting, weight down 8% from birth and a chloride of 88: which fluid do you start, and what corrects before theatre is safe?"' } + { role: 'user', content: 'Category: ' + item.category + '\nAge bands: ' + item.ageBands.join(', ') + '\nIndexed pediatric source snippets:\n\n' + sourceText + '\n\nCreate 20 starter questions for this category, each a concrete thing to learn, mechanism first. Vary what is asked (why a finding happens, how a treatment works, what distinguishes two conditions, which number decides, why a rule exists) and the age group. Examples of the kind wanted: "Why does pyloric stenosis produce a hypochloraemic metabolic alkalosis, and why is the urine paradoxically acidic?", "How does a ductal-dependent lesion stay hidden until the duct closes, and what does prostaglandin E1 do?", "Why are infants under 3 months given broader empiric cover for fever than older children?" No scenarios.' } ], { model: chatModel || undefined, + // The pool is a list, not a reasoning task: DeepSeek writes it + // without thinking, which is faster and — measured on the assistant + // itself — no less accurate for a grounded task. + reasoningEffort: 'none', + reasoningFormat: 'hidden', temperature: 0.78, // A case is three to four times the length of a heading; the old // ceiling cut the list off mid-JSON and a whole category was lost. @@ -567,17 +573,20 @@ function createClinicalPromptPool(opts) { return taxonomyItem && taxonomyItem.ageBands && taxonomyItem.ageBands[0] || 'school_age'; } - // A starter question is a case and a decision. "What red flags in a child's - // headache history warrant investigation?" is a chapter heading; it survives - // nothing here. A number is what tells a case from a heading — an age, a - // vital sign, a lab, a dose, a duration — so one is required, and the - // openers that only ever introduce a heading are refused outright. - var TEXTBOOK_OPENER = /^(what (is|are)( the)? (definition|cause|causes|feature|features|sign|signs|symptom|symptoms|treatment|management|differential|complication|complications|indication|indications|role|difference)\b|what (is|are) (a |an |the )?[a-z\- ]{1,40}\?$|how (is|are|should) [a-z\- ]{1,60} (managed|treated|evaluated|diagnosed)\?$|which (clinical )?(scores?|tools?|scales?) (is|are) useful\b)/i; + // A starter question is one concrete thing to learn, mechanism first. The + // pool used to be cases — "A 6-week-old with 3 days of vomiting…: which + // fluid?" — and the user found them "vague nonsense clinical scenarios": + // "I want concrete things people learn from and pathophysiology if possible, + // all related to peds. No more clinical scenarios." So a vignette opener is + // refused, and so is the bare definition ("What is X?") that teaches + // nothing; a question must name its subject and ask one definite thing. + var VIGNETTE_OPENER = /^(a|an|the|this|my|our|in a|in an|for a|for an|you (are|have|see)|there is)\s+(\d+[\s-]*(day|week|month|year)s?[\s-]*old|\d+\s*(yo|y\/o|mo|wk)\b|(newborn|neonate|infant|baby|toddler|child|boy|girl|teen|adolescent|patient|term|preterm)\b[^?]{0,80}\b(presents?|presenting|brought|arrives?|is seen|comes? in|admitted|on the ward|in (the )?(ed|clinic|emergency|nicu|picu))\b)/i; + var BARE_DEFINITION = /^what (is|are) (a |an |the )?[a-z\- ]{1,40}\?$/i; function isUsefulQuestion(prompt, item) { if (!prompt || prompt.length < 40 || prompt.length > 320) return false; if (prompt.indexOf('?') === -1) return false; - if (!/\d/.test(prompt)) return false; - if (TEXTBOOK_OPENER.test(prompt)) return false; + if (VIGNETTE_OPENER.test(prompt)) return false; + if (BARE_DEFINITION.test(prompt)) return false; if (!hasPediatricSignal(prompt, item)) return false; return !/\b(source|snippet|textbook|chapter|document|database)\b/i.test(prompt); } diff --git a/test/clinical-assistant-prompt-pool.test.js b/test/clinical-assistant-prompt-pool.test.js index c01b3dee..53f13921 100644 --- a/test/clinical-assistant-prompt-pool.test.js +++ b/test/clinical-assistant-prompt-pool.test.js @@ -24,7 +24,7 @@ test('clinical assistant starter prompts are Redis or indexed-source backed', () assert.match(pool, /CLINICAL_ASSISTANT_PROMPT_POOL_TARGET, 1000/); // Built once, then only from the admin button: no timer unless the environment asks for one. assert.match(pool, /CLINICAL_ASSISTANT_PROMPT_POOL_REFRESH_MS, 0\)/); - assert.match(pool, /process\.env\.CLINICAL_ASSISTANT_PROMPT_MODEL \|\| 'openrouter-gpt-4\.1-mini'/); + assert.match(pool, /process\.env\.CLINICAL_ASSISTANT_PROMPT_MODEL \|\| 'ds-deepseek-v4\.1-flash'/); assert.match(route, /loadStoredPromptPool: loadLatestPromptPoolSnapshot/); assert.match(pool, /PEDIATRIC_TAXONOMY/); assert.match(pool, /taxonomyWithQuotas\(target\)/); @@ -82,18 +82,19 @@ function fakePool(overrides) { }, overrides || {})) }; } -test('a starter question is a case and a decision, not a chapter heading', () => { +test('a starter question is one concrete thing to learn, not a scenario', () => { const { pool } = fakePool(); const ok = q => pool.isUsefulQuestion(q, { category: 'respiratory', intent: 'management', age_band: 'infant' }); - // What the old pool was full of. - assert.equal(ok('What red flags in a child\'s headache history warrant further investigation?'), false, 'no number, no case'); - assert.equal(ok('Which clinical scores are useful for assessing asthma severity in children?'), false, 'a heading'); - assert.equal(ok('How is bronchiolitis managed in infants?'), false, 'a heading'); - assert.equal(ok('What are the causes of neonatal jaundice presenting at 2 days?'), false, 'a heading with a number in it is still a heading'); - // What it should hold. - assert.equal(ok('A 6-week-old with 3 days of projectile non-bilious vomiting, weight down 8% and a chloride of 88: which fluid do you start, and what corrects before theatre is safe?'), true); - assert.equal(ok('A 14-year-old on the ward with asthma, SpO2 91% after two salbutamol nebulisers: at what point do you add magnesium, and what dose?'), true); - assert.equal(ok('In a neonate with bilious emesis and a scaphoid abdomen at 2 days of life, which imaging comes first and what wait is acceptable?'), true); + // What the user refused: "vague nonsense clinical scenarios". + assert.equal(ok('A 6-week-old with 3 days of projectile non-bilious vomiting, weight down 8% and a chloride of 88: which fluid do you start, and what corrects before theatre is safe?'), false, 'a vignette'); + assert.equal(ok('A 14-year-old on the ward with asthma, SpO2 91% after two salbutamol nebulisers: at what point do you add magnesium?'), false, 'a vignette'); + assert.equal(ok('An infant presents with bilious emesis and a scaphoid abdomen: which imaging comes first?'), false, 'a vignette without an age'); + assert.equal(ok('What is bronchiolitis?'), false, 'a bare definition'); + // What it should hold: mechanisms, distinctions, the numbers that matter. + assert.equal(ok('Why does pyloric stenosis produce a hypochloraemic metabolic alkalosis in infants, and why is the urine paradoxically acidic?'), true); + assert.equal(ok('How does a ductal-dependent heart lesion stay silent in a newborn until the duct closes, and what does prostaglandin E1 do?'), true); + assert.equal(ok('Why are infants under 3 months given broader empiric antibiotic cover for fever than older children?'), true); + assert.equal(ok('What separates croup from epiglottitis in a child, and why does one respond to dexamethasone?'), true); }); test('a pool built by an older prompt is served, then rebuilt in the background', async () => {