From 2d292d12af592fe7051029edbb1c14ff87ec1b89 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 28 Aug 2026 19:49:01 +0200 Subject: [PATCH] fix: make the markdown repairs line aware so tables and ranges survive The earlier fix corrected two of the repair rules and left the rest with the same flaw, and the punctuation boundary it introduced let the table case back in. A review found the remainder. Every repair rule inserted newlines with no idea whether it was inside a markdown table row. A dose row such as "| Dexamethasone | 0.6 mg/kg PO (max 16 mg) - single dose | 1 |" was split mid-row, which drops that row's citation and every row below it out of the table. A "#" column destroyed the table outright. The rules now skip table rows, dividers and headers entirely. Ranges were still split in three other places: in a heading, so "### Dexamethasone 0.15 - 0.6 mg/kg" rendered as a heading reading "Dexamethasone 0.15"; after a citation; and beside a hash, where "Room # 4" became a heading. A hyphen now starts a list item only when a number does not follow, and a run-together heading is recognised by the capital letter after the hashes rather than by position. Adjacent citation merging could also cross a paragraph break, turning "[1]\n\n[2] Vancomycin only if MRSA" into "[1, 2]Vancomycin only if MRSA" -- joining two paragraphs and moving a citation onto a claim it never supported. Merging is now limited to citations on the same line. Co-Authored-By: Claude Opus 5 --- public/js/assistant/citations.js | 50 +++++++++++++++++++++++--------- test/assistant-citations.test.js | 45 ++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/public/js/assistant/citations.js b/public/js/assistant/citations.js index 5502457..ffd963a 100644 --- a/public/js/assistant/citations.js +++ b/public/js/assistant/citations.js @@ -55,13 +55,13 @@ export function renderCitationLinks(html, sources, options) { export function normalizeAdjacentCitationClusters(text, sources) { var available = Array.isArray(sources) ? sources : []; - var normalized = String(text || '').replace(/((?:\[(?:\d+\s*,\s*)*\d+\]\s*)+)\[((?:\d+\s*,\s*)*\d+)(?=$|[.,;:])/g, function(match, completeClusters, trailingCluster) { + var normalized = String(text || '').replace(/((?:\[(?:\d+\s*,\s*)*\d+\][ \t]*)+)\[((?:\d+\s*,\s*)*\d+)(?=$|[.,;:])/g, function(match, completeClusters, trailingCluster) { var nums = citationNumbers(completeClusters).concat(parseCitationCluster(trailingCluster)); if (!allCitationsAvailable(nums, available)) return match; return formatCitationCluster(nums); }); - return normalized.replace(/(?:\[(?:\d+\s*,\s*)*\d+\]\s*){2,}/g, function(match) { + return normalized.replace(/(?:\[(?:\d+\s*,\s*)*\d+\][ \t]*){2,}/g, function(match) { var nums = citationNumbers(match); if (!allCitationsAvailable(nums, available)) return match; return formatCitationCluster(nums); @@ -90,21 +90,43 @@ function formatCitationCluster(nums) { return '[' + nums.join(', ') + ']'; } -export function normalizeMarkdownText(text) { - return stripOrphanMarkdownMarkers(normalizeTableSourceCitationCells(String(text || '') - .replace(/\r\n/g, '\n') - .replace(/(\[(?:\d+\s*,\s*)*\d+\])\s*[-–—]\s*/g, '$1\n- ') - .replace(/([.!?])\s*[-–—]\s+(\*\*)?/g, '$1\n- $2') - .replace(/(:)\s*[-–—]\s+(\*\*)?/g, '$1\n- $2') +// Repairs the loose formatting models emit: run-together bullets, inline +// headings, numbered items glued onto a sentence. +// +// Two rules make this safe. Table rows are never touched, because inserting a +// newline inside "| a | b |" destroys the row and every row after it. And a +// hyphen only starts a list item when what follows is not a number, because +// "0.15 - 0.6 mg/kg" is a dose range, not a bullet, and splitting it shows the +// reader a different dose. +var TABLE_ROW = /^\s*\|.*\|\s*$/; +var TABLE_DIVIDER = /^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)+\|?\s*$/; +var LIST_BOUNDARY = /([.!?:;\]])\s+(-\s+(?!\d))/g; + +function repairLine(line) { + return line + // A citation followed by a hyphen starts the next item, unless a number follows. + .replace(/(\[(?:\d+\s*,\s*)*\d+\])\s*[-–—]\s+(?!\d)/g, '$1\n- ') + .replace(/([.!?])\s*[-–—]\s+(?!\d)(\*\*)?/g, '$1\n- $2') + .replace(/(:)\s*[-–—]\s+(?!\d)(\*\*)?/g, '$1\n- $2') .replace(/(\[(?:\d+\s*,\s*)*\d+\]\.)\s+(\d+\.\s+[A-Z][A-Za-z][^\n]{0,80})/g, '$1\n$2') .replace(/([.!?])\s+(\d+\.\s+[A-Z][A-Za-z][^\n]{0,80})/g, '$1\n$2') .replace(/(\[(?:\d+\s*,\s*)*\d+\])(?=\s*(?:[A-Z][A-Za-z]+\s+){1,4}(?:deficits?|distress|apnoea|apnea|vomiting|seizures?|signs?|symptoms?|criteria|indications?|risk|oxygen|saturation|dehydration|lethargy|toxicity)\b)/g, '$1\n') - .replace(/([^\n])\s+(#{1,4}\s+)/g, '$1\n\n$2') - .replace(/(#{1,4}\s+[^\n]+?)\s+(-\s+)/g, '$1\n\n$2') - .replace(/(#{1,4}\s+[^\n]+)\n(-\s+)/g, '$1\n\n$2') - .replace(/([.!?:;)\]])\s+(-\s+(?:Mainstay|Medications|Hospitalization|Other therapies|Prevention|Short-acting|Anticholinergics|Systemic|Adjuncts|Long-term|Infants|Differentiating|Persistent|Severe|Need for|Inadequate)\b)/g, '$1\n$2') - .replace(/([.!?:;)\]])\s+(-\s+[^\n])/g, '$1\n$2') - .trim())); + // A run-together heading is recognised by what follows the hashes: a capital + // letter means a heading, while "Item # 4" or a "#" table column does not. + .replace(/([^\n])\s+(#{1,4}\s+(?=[A-Z]))/g, '$1\n\n$2') + .replace(/(#{1,4}\s+[^\n]+?)\s+(-\s+(?!\d))/g, '$1\n\n$2') + .replace(LIST_BOUNDARY, '$1\n$2'); +} + +export function normalizeMarkdownText(text) { + var lines = String(text || '').replace(/\r\n/g, '\n').split('\n'); + var repaired = lines.map(function(line, index) { + if (TABLE_ROW.test(line) || TABLE_DIVIDER.test(line)) return line; + // A line directly above a divider is a table header even without pipes. + if (index + 1 < lines.length && TABLE_DIVIDER.test(lines[index + 1])) return line; + return repairLine(line); + }).join('\n'); + return stripOrphanMarkdownMarkers(normalizeTableSourceCitationCells(repaired).trim()); } export function normalizeTableSourceCitationCells(text) { diff --git a/test/assistant-citations.test.js b/test/assistant-citations.test.js index 989e307..15b15fc 100644 --- a/test/assistant-citations.test.js +++ b/test/assistant-citations.test.js @@ -291,3 +291,48 @@ test('dollar amounts are not rendered as mathematics', async () => { assert.equal(renderLatexText('SpO$_2$ target', katex), 'SpO_2 target'); assert.equal(renderLatexText('Use $\\frac{1}{2}$ dose', katex), 'Use \\frac{1}{2} dose'); }); + +test('table rows are never rewritten by the bullet and heading repairs', async () => { + const { normalizeMarkdownText } = await loadCitationModule(); + // A newline inserted inside a row destroys that row and every row after it. + for (const text of [ + '| Dexamethasone | 0.6 mg/kg PO (max 16 mg) - single dose | 1 |', + '| Item # | Dose |', + '| Hypoxia | SpO2 <90 \\| escalate | 1 |', + '| Drug | Dose |\n|---|---|\n| Dex | 0.6 mg/kg (max 16 mg) - single dose |' + ]) { + assert.equal(normalizeMarkdownText(text), text, text); + } +}); + +test('ranges survive in headings, after citations, and beside a hash', async () => { + const { normalizeMarkdownText } = await loadCitationModule(); + for (const text of [ + '### Dexamethasone 0.15 - 0.6 mg/kg PO once', + '## Target SpO2 92 - 96%', + 'Magnesium 40 mg/kg IV over 20 min [1] - 50 mg/kg is an alternative.', + 'Room # 4 and bed # 2' + ]) { + assert.equal(normalizeMarkdownText(text), text, text); + } +}); + +test('the loose-formatting repairs still fire where they were meant to', async () => { + const { normalizeMarkdownText } = await loadCitationModule(); + assert.equal(normalizeMarkdownText('Treatment options. - Dexamethasone first.'), 'Treatment options.\n- Dexamethasone first.'); + assert.equal(normalizeMarkdownText('Options: - Dexamethasone'), 'Options:\n- Dexamethasone'); + assert.equal(normalizeMarkdownText('Works well [1] - Dexamethasone helps'), 'Works well [1]\n- Dexamethasone helps'); + assert.equal(normalizeMarkdownText('That is all. ## Management'), 'That is all.\n\n## Management'); + assert.equal(normalizeMarkdownText('## Management - Dexamethasone first'), '## Management\n\n- Dexamethasone first'); +}); + +test('adjacent citations merge only on the same line', async () => { + const { normalizeAdjacentCitationClusters } = await loadCitationModule(); + const sources = [{ title: 'A' }, { title: 'B' }, { title: 'C' }]; + assert.equal(normalizeAdjacentCitationClusters('Works [1] [2] [3].', sources), 'Works [1, 2, 3].'); + // Merging across a break moved a citation onto a claim it never supported. + assert.equal( + normalizeAdjacentCitationClusters('Give ceftriaxone 50 mg/kg [1]\n\n[2] Vancomycin only if MRSA.', sources), + 'Give ceftriaxone 50 mg/kg [1]\n\n[2] Vancomycin only if MRSA.' + ); +});