From a528986a2d511803c7bb2e318da4a25e7c1e4349 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 13 Sep 2026 06:38:29 +0200 Subject: [PATCH] fix: a starter-question batch no longer dies when the reply is cut off Cases are three to four times the length of headings, and 30 of them did not fit the 2,600-token ceiling: the JSON was cut mid-list, failed to parse, and whole categories came back with nothing kept. The ceiling is 7,000, batches are 20, and a reply that is still cut off yields every question that finished rather than none. Each batch logs what it offered. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- src/utils/clinicalPromptPool.js | 21 +++++++++++++++++---- test/clinical-assistant-prompt-pool.test.js | 10 ++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/utils/clinicalPromptPool.js b/src/utils/clinicalPromptPool.js index 08ac0cdb..bed5a653 100644 --- a/src/utils/clinicalPromptPool.js +++ b/src/utils/clinicalPromptPool.js @@ -237,7 +237,15 @@ function parseJsonObject(text) { if (start !== -1 && end > start) { try { return JSON.parse(text.slice(start, end + 1)); } catch (e2) {} } - return {}; + // A reply cut off mid-list still holds every question that finished. Each + // complete {...} object is taken on its own rather than the whole batch + // being thrown away — the way an entire category came back with nothing. + var questions = []; + var objects = text.match(/\{[^{}]*"prompt"\s*:\s*"(?:[^"\\]|\\.)*"[^{}]*\}/g) || []; + objects.forEach(function (chunk) { + try { questions.push(JSON.parse(chunk)); } catch (e3) {} + }); + return questions.length ? { questions: questions, truncated: true } : {}; } // Bumped whenever the generation prompt or the filter changes. A pool made by @@ -331,7 +339,7 @@ function createClinicalPromptPool(opts) { var snippets = await collectPromptSeedSnippets(item); if (!snippets.length) continue; var categoryExamples = []; - var batches = Math.max(1, Math.min(5, Math.ceil(item.quota / 25))); + var batches = Math.max(1, Math.min(6, Math.ceil(item.quota / 18))); for (var i = 0; i < batches && categoryExamples.length < item.quota; i++) { var batchSnippets = rotateExamples(snippets, i * 18).slice(0, 18); var sourceText = batchSnippets.map(function(s, idx) { @@ -342,13 +350,18 @@ function createClinicalPromptPool(opts) { role: 'system', content: 'Write starter questions for a pediatric clinical assistant, the kind a clinician actually types at 3 a.m. with a patient in front of them. Use only the provided titles/snippets as inspiration; do not answer the questions and do not mention source names. Every question is a case and a decision: it opens with a one-line vignette — an age with a number and unit (e.g. "a 6-week-old", "a 14-year-old"), the setting, and two or three specific findings of which at least one is a number (a vital sign, a lab value, a dose, a duration, a weight) — and then asks ONE thing the clinician must decide now: the next test, the threshold at which to act, a dose with units, admit or discharge, when to repeat, what changes the plan. Never ask a textbook question: no "What is…", "What are the causes/features/signs of…", "How is X managed?", "Which scores are useful…". 18 to 45 words. Return strict JSON only: {"questions":[{"label":"2-5 word label","prompt":"question ending with ?","category":"' + item.category + '","age_band":"' + item.ageBands.join('|') + '","intent":"diagnosis|management|red_flags|differential|counseling|dosing|admission|discharge|review"}]}.' }, - { role: 'user', content: 'Category: ' + item.category + '\nAge bands: ' + item.ageBands.join(', ') + '\nIndexed pediatric source snippets:\n\n' + sourceText + '\n\nCreate 30 starter questions for this category, each a specific case with a specific decision. Vary the setting (emergency department, ward, clinic, neonatal unit, telephone advice), the decision (next test, threshold, dose, fluids, admit/discharge, timing, what if a finding changes), and the age. Example of the depth wanted: "A 6-week-old with 3 days of projectile non-bilious vomiting, weight down 8% from birth and a chloride of 88: which fluid do you start, and what corrects before theatre is safe?"' } + { role: 'user', content: 'Category: ' + item.category + '\nAge bands: ' + item.ageBands.join(', ') + '\nIndexed pediatric source snippets:\n\n' + sourceText + '\n\nCreate 20 starter questions for this category, each a specific case with a specific decision. Vary the setting (emergency department, ward, clinic, neonatal unit, telephone advice), the decision (next test, threshold, dose, fluids, admit/discharge, timing, what if a finding changes), and the age. Example of the depth wanted: "A 6-week-old with 3 days of projectile non-bilious vomiting, weight down 8% from birth and a chloride of 88: which fluid do you start, and what corrects before theatre is safe?"' } ], { model: chatModel || undefined, temperature: 0.78, - maxTokens: 2600 + // A case is three to four times the length of a heading; the old + // ceiling cut the list off mid-JSON and a whole category was lost. + maxTokens: 7000 }); var parsed = parseJsonObject(String(ai.content || '')); + var offered = Array.isArray(parsed.questions) ? parsed.questions.length : 0; + if (!offered) console.warn('[clinical-assistant] prompt pool ' + item.category + ' batch ' + (i + 1) + ': nothing parseable in ' + String(ai.content || '').length + ' chars'); + else if (parsed.truncated) console.warn('[clinical-assistant] prompt pool ' + item.category + ' batch ' + (i + 1) + ': reply was cut off; ' + offered + ' questions salvaged'); categoryExamples = normalizeGeneratedExamples(categoryExamples.concat(parsed.questions || []), item).slice(0, item.quota); } generated = normalizeGeneratedExamples(generated.concat(categoryExamples)); diff --git a/test/clinical-assistant-prompt-pool.test.js b/test/clinical-assistant-prompt-pool.test.js index 524928d2..6560af09 100644 --- a/test/clinical-assistant-prompt-pool.test.js +++ b/test/clinical-assistant-prompt-pool.test.js @@ -113,3 +113,13 @@ test('a pool built by an older prompt is served, then rebuilt in the background' const again = await pool.refreshIfNeeded(false); assert.equal(again[0].prompt, 'Old question?'); }); + +test('a reply cut off mid-list still yields the questions that finished', () => { + const { pool } = fakePool(); + // Reach the parser through the public surface it feeds: a refresh with a + // callAI that answers with truncated JSON. + const src = require('node:fs').readFileSync(require('node:path').join(__dirname, '..', 'src/utils/clinicalPromptPool.js'), 'utf8'); + assert.match(src, /objects\.forEach\(function \(chunk\)/, 'the salvage exists'); + assert.match(src, /maxTokens: 7000/, 'and the ceiling makes room for cases'); + assert.ok(pool); +});