. Images render only from
+ * the safe allowlist; anything else falls back to its alt text. Both used to be
+ * done by holding the block out of the text before parsing and splicing HTML
+ * back in afterwards, which is what a renderer rule is for.
+ */
+export function installCitations(md) {
+ if (!md || md.__assistantCitations) return md;
+ md.__assistantCitations = true;
+ // Before 'escape', or \[1\] is consumed as an escaped bracket first.
+ md.inline.ruler.before('escape', 'assistant_cite', citationRule);
+ md.renderer.rules.assistant_cite = renderCitation;
+ var image = md.renderer.rules.image || function (t, i, o, e, self) { return self.renderToken(t, i, o); };
+ md.renderer.rules.image = function (tokens, idx, o, env, self) {
+ if (!safeImageUrl(tokens[idx].attrGet('src'))) return escapeHtml(tokens[idx].content || 'Image');
+ return image(tokens, idx, o, env, self);
+ };
+ // markdown-it writes column alignment as style="text-align:…"; the export,
+ // the stylesheet and a decade of email clients read the align attribute.
+ // Both are emitted so nothing that looks for either is wrong.
+ ['th_open', 'td_open'].forEach(function (name) {
+ var base = md.renderer.rules[name] || function (t, i, o, e, self) { return self.renderToken(t, i, o); };
+ md.renderer.rules[name] = function (tokens, idx, o, env, self) {
+ var style = tokens[idx].attrGet('style') || '';
+ var m = /text-align:\s*(left|right|center)/.exec(style);
+ if (m && !tokens[idx].attrGet('align')) tokens[idx].attrSet('align', m[1]);
+ return base(tokens, idx, o, env, self);
+ };
+ });
+ // Math is not markdown, so the parser needs telling where it is — and once
+ // told, a $ inside a URL, a code span or an HTML attribute is never math,
+ // by construction, which is what a text pre-pass could not promise.
+ md.inline.ruler.before('escape', 'assistant_math', mathInlineRule);
+ md.block.ruler.before('fence', 'assistant_math_block', mathBlockRule, { alt: ['paragraph', 'reference', 'blockquote', 'list'] });
+ md.renderer.rules.assistant_math = function (tokens, idx, o, env) { return renderMath(env, tokens[idx], false); };
+ md.renderer.rules.assistant_math_block = function (tokens, idx, o, env) { return renderMath(env, tokens[idx], true) + '\n'; };
+ md.renderer.rules.fence = function (tokens, idx) {
+ var lang = String(tokens[idx].info || '').trim().toLowerCase();
+ var code = String(tokens[idx].content || '').replace(/\n$/, '');
+ // Percent-encoding keeps Mermaid arrows intact through DOMPurify.
+ if (lang === 'mermaid') return 'Rendering graph...\n';
+ if (lang === 'chart' || lang === 'chartjs') return '\n';
+ return '' + escapeHtml(code) + '
\n';
+ };
+ return md;
+}
+
+// ─── Math as tokens ────────────────────────────────────────────────────
+// $…$ and \(…\) inline; $$…$$ and \[…\] display, on one line or spanning
+// several; \ce{…} and \pu{…} for mhchem. Rendered by KaTeX when the page has
+// it, otherwise escaped and shown as written.
+var MATH_INLINE_RE = /^(?:\$\$([\s\S]+?)\$\$|\\\[([\s\S]+?)\\\]|\$([^$\n]+?)\$|\\\((.+?)\\\)|\\(ce|pu)\{([^{}\n]*)\})/;
+
+function mathInlineRule(state, silent) {
+ var c = state.src.charCodeAt(state.pos);
+ if (c !== 0x24 /* $ */ && c !== 0x5C /* \ */) return false;
+ var m = MATH_INLINE_RE.exec(state.src.slice(state.pos));
+ if (!m) return false;
+ // A lone $ with a space either side is a dollar sign, not an equation.
+ if (m[3] !== undefined && (/^\s/.test(m[3]) || /\s$/.test(m[3]))) return false;
+ if (!silent) {
+ var token = state.push('assistant_math', 'span', 0);
+ var display = m[1] !== undefined || m[2] !== undefined;
+ token.content = m[1] !== undefined ? m[1] : m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3]
+ : m[4] !== undefined ? m[4] : '\\' + m[5] + '{' + m[6] + '}';
+ // Without KaTeX the expression is shown exactly as written — \(a\) stays
+ // \(a\), not re-dressed as $a$ — so raw is kept beside the content.
+ token.meta = { display: display, raw: m[0] };
+ if (display) token.type = 'assistant_math_block';
+ }
+ state.pos += m[0].length;
+ return true;
+}
+
+// $$ or \[ opening a line, closing on the same or a later line.
+function mathBlockRule(state, startLine, endLine, silent) {
+ var start = state.bMarks[startLine] + state.tShift[startLine];
+ var max = state.eMarks[startLine];
+ var first = state.src.slice(start, max);
+ var open = /^\$\$/.test(first) ? '$$' : /^\\\[/.test(first) ? '\\[' : null;
+ if (!open) return false;
+ var close = open === '$$' ? '$$' : '\\]';
+ var rest = first.slice(open.length);
+ var content, next = startLine;
+ var closeAt = rest.indexOf(close);
+ if (closeAt !== -1 && rest.slice(closeAt + close.length).trim() === '') {
+ content = rest.slice(0, closeAt);
+ } else {
+ var lines = [rest];
+ for (next = startLine + 1; next < endLine; next++) {
+ var line = state.src.slice(state.bMarks[next] + state.tShift[next], state.eMarks[next]);
+ var at = line.indexOf(close);
+ if (at !== -1) { lines.push(line.slice(0, at)); break; }
+ lines.push(line);
+ }
+ if (next >= endLine) return false; // never closed: leave it to the paragraph rule
+ content = lines.join('\n');
+ }
+ if (silent) return true;
+ var token = state.push('assistant_math_block', 'div', 0);
+ token.content = content.trim();
+ token.meta = { display: true, raw: state.getLines(startLine, next + 1, 0, false).trim() };
+ token.map = [startLine, next + 1];
+ state.line = next + 1;
+ return true;
+}
+
+function renderMath(env, token, display) {
+ var katex = env && env.assistantOpts && env.assistantOpts.katex;
+ if (!katex) return escapeHtml((token.meta && token.meta.raw) || token.content);
+ return safeKatex(katex, token.content, display);
+}
+
+var defaultRenderer = null;
+function resolveRenderer(opts) {
+ if (opts && opts.markdownIt && typeof opts.markdownIt.render === 'function') return installCitations(opts.markdownIt);
+ if (!defaultRenderer) {
+ // The browser's window is globalThis. A test sandbox keeps window as a
+ // property of its own global, so both are looked at.
+ var g = typeof globalThis !== 'undefined' ? globalThis : {};
+ var factory = typeof g.markdownit === 'function' ? g.markdownit
+ : (g.window && typeof g.window.markdownit === 'function' ? g.window.markdownit : null);
+ // html:true, as marked rendered before it: an answer may carry a or
+ // a
, and the single sanitisation boundary is what makes that safe.
+ if (factory) defaultRenderer = installCitations(factory({ html: true, linkify: true, typographer: true, breaks: true }));
+ }
+ return defaultRenderer;
+}
+
+// The text as the parser should see it: placeholders dropped, math rendered and
+// held, fences and inline code held while the legacy table repair runs, then put
+// back so the parser's own fence rule renders them. Shared by rendering and by
+// ordering, so the two cannot disagree about what is a citation.
+function prepareMarkdown(md, opts, renderer) {
+ // The legacy table repair and the fence hold need a lexer to know where
+ // blocks are. It is the same parser that will render — one, not two.
+ opts = Object.assign({}, opts, { markdownIt: renderer });
var text = String(md || '')
// The model occasionally writes [src]/[source] placeholders instead of
// real citation numbers. Never invent numbers — drop the tokens; the
// sources panel still lists the actual sources.
- .replace(/\[(?:src|source)\]/gi, '');
+ .replace(/\[(?:src|source)\]/gi, '')
+ // A cluster the model ran out of room for: "[1][4][2][3" at the very end.
+ // Only when it directly follows complete brackets, so a lone "[3" is left.
+ .replace(/((?:\[\d+\]\s*)+)\[(\d+)\s*$/, '$1[$2]');
var embedded = textProtection(text, 'html');
- // Literal/embedded content bypasses prose, math and citation rewriting, then
- // rejoins the parsed HTML before the single sanitization boundary.
- text = protectBlocks(text, opts, function(code, lang) {
- lang = String(lang || '').toLowerCase();
- var html;
- // Percent-encoding keeps Mermaid arrows intact through DOMPurify.
- if (lang === 'mermaid') html = 'Rendering graph...';
- else if (lang === 'chart' || lang === 'chartjs') html = '';
- else html = '' + escapeHtml(code) + '
';
- return embedded.hold(html);
- }, true);
+ var fences = textProtection(text, 'fence');
+ text = protectBlocks(text, opts, function (code, lang) { return fences.hold('```' + (lang || '') + '\n' + code + '\n```'); }, true);
var inlineCode = textProtection(text, 'code');
text = text.replace(/(`+)[\s\S]*?\1/g, inlineCode.hold);
- var links = textProtection(text, 'links');
- text = text.replace(new RegExp(
- /!?\[[^\]\n]*\]\([^\n]*?\)|https?:\/\/(?:\\[^\s]|[^\s<>|\\])+|<(?:https?:\/\/|mailto:)[^\s<>]+>|<[^\s<>@]+@[^\s<>@]+>/.source + '|' + HTML_TAG_PATTERN.source,
- 'gi'), function(match) {
- if (match.charAt(0) === '!') {
- var imageLink = match.match(/^!\[([^\]\n]*)\]\(([^\n]*?)\)$/);
- if (imageLink && !safeImageUrl(imageLink[2])) return links.hold(escapeHtml(imageLink[1] || 'Image'), match);
- }
- return links.hold(match);
- });
+ // \[1\] is a citation the model escaped, not display math — but \[ ... \] is
+ // the display-math delimiter, and the math pass runs next. A bracket holding
+ // nothing but numbers is never an equation, so it is unescaped here, before
+ // the math pass can hold it. This is the one place text syntax is genuinely
+ // ambiguous, and the parser cannot resolve it because math is not markdown.
text = text.replace(/\\\[((?:\d+\s*,\s*)*\d+)\\\]/g, '[$1]');
- text = renderLatexText(text, opts.katex, embedded.hold);
var limited = false;
text = normalizeMarkdownText(text, Object.assign({}, opts, {
- onUnrecoverable: function(raw) {
+ onUnrecoverable: function (raw) {
limited = true;
- raw = inlineCode.restore(links.restore(embedded.restore(raw, true)));
+ raw = inlineCode.restore(fences.restore(embedded.restore(raw, true)));
return embedded.hold('' + escapeHtml(raw) + '
', raw);
}
}));
- text = links.restore(normalizeAdjacentCitationClusters(text, sources || []));
+ text = fences.restore(inlineCode.restore(text));
+ return { text: text, embedded: embedded, limited: limited };
+}
- text = inlineCode.restore(text);
- var html;
- if (opts.marked && typeof opts.marked.parse === 'function') {
- html = opts.marked.parse(text, { breaks: true, gfm: true });
- } else if (opts.markdownIt && typeof opts.markdownIt.render === 'function') {
- html = opts.markdownIt.render(text);
- } else {
- html = fallbackMarkdown(text);
- }
-
- html = renderCitationLinks(html, sources || [], opts);
- html = embedded.restore(html);
+export function renderAssistantMarkdown(md, sources, options) {
+ var opts = options || {};
+ var renderer = resolveRenderer(opts);
+ if (!renderer) throw new Error('renderAssistantMarkdown: no markdown-it renderer is available');
+ var prepared = prepareMarkdown(md, opts, renderer);
+ var env = { assistantSources: sources || [], assistantOpts: opts, assistantCiteOrder: [] };
+ var html = renderer.render(prepared.text, env);
+ html = prepared.embedded.restore(html);
html = wrapTables(html);
- if (limited) html += 'Could not safely recover this flattened table. Stored text is shown unchanged; missing cells or rows cannot be recovered.
';
+ if (prepared.limited) html += 'Could not safely recover this flattened table. Stored text is shown unchanged; missing cells or rows may be present.
';
if (opts.notice) html += '' + escapeHtml(opts.notice) + '
';
-
return typeof opts.sanitize === 'function' ? opts.sanitize(html) : html;
}
+/**
+ * Link `[n]` markers in HTML that did not come from our renderer.
+ *
+ * The translation service is handed HTML and returns HTML, with the markers
+ * carried across as plain text. That cannot go through the tokenizer, but the
+ * document has structure of its own: only text nodes are visited, and none
+ * inside code, pre, a or script. An attribute value is not a text node, so it is
+ * never touched — the case that defeated the pattern approach.
+ */
+export function linkCitationsInHtml(html, sources, options) {
+ if (typeof document === 'undefined') return String(html || '');
+ var opts = options || {};
+ var tpl = document.createElement('template');
+ tpl.innerHTML = String(html || '');
+ var walker = document.createTreeWalker(tpl.content, 4 /* NodeFilter.SHOW_TEXT */);
+ var texts = [];
+ for (var node = walker.nextNode(); node; node = walker.nextNode()) {
+ var skip = false;
+ for (var el = node.parentNode; el && el !== tpl.content; el = el.parentNode) {
+ if (/^(code|pre|a|script|style)$/i.test(el.nodeName)) { skip = true; break; }
+ }
+ if (!skip && node.nodeValue.indexOf('[') !== -1) texts.push(node);
+ }
+ var env = { assistantSources: sources || [], assistantOpts: opts, assistantCiteOrder: [] };
+ texts.forEach(function (node) {
+ var parts = node.nodeValue.split(/(\\?\[(?:\d+\s*,\s*)*\d+\\?\])/);
+ if (parts.length === 1) return;
+ var frag = document.createDocumentFragment();
+ parts.forEach(function (part) {
+ var m = /^\\?\[((?:\d+\s*,\s*)*\d+)\\?\]$/.exec(part);
+ var nums = m ? m[1].split(',').map(function (n) { return Number(n.trim()); }) : null;
+ if (!nums || !nums.every(function (n) { return n > 0 && sourceByNumber(env.assistantSources, n); })) {
+ frag.appendChild(document.createTextNode(part));
+ return;
+ }
+ var span = document.createElement('span');
+ span.innerHTML = nums.map(function (n, i) {
+ return renderCitation([{ type: 'assistant_cite', meta: { source: n, display: displayNumber(env, n) } }], 0, {}, env)
+ .replace(/^ /, i ? ' ' : '');
+ }).join('');
+ while (span.firstChild) frag.appendChild(span.firstChild);
+ });
+ node.parentNode.replaceChild(frag, node);
+ });
+ return tpl.innerHTML;
+}
+
export function wrapTables(html) {
return String(html || '')
.replace(/]*)?>/g, '')
@@ -95,144 +343,58 @@ export function wrapTables(html) {
// on the number cannot drift.
function sourceByNumber(sources, n) {
var list = sources || [];
+ // A list ordered by citation carries each source's original number as
+ // sourceNumber and its display position as number. `[7]` in the text names
+ // the original, always.
for (var i = 0; i < list.length; i++) {
- if (list[i] && Number(list[i].number) === Number(n)) return list[i];
+ if (list[i] && list[i].sourceNumber != null && Number(list[i].sourceNumber) === Number(n)) return list[i];
+ }
+ if (list.some(function (s) { return s && s.sourceNumber != null; })) return undefined;
+ for (var j = 0; j < list.length; j++) {
+ if (list[j] && Number(list[j].number) === Number(n)) return list[j];
}
// Sources predating the numbering, or a caller passing a bare list.
return list[n - 1];
}
/**
- * Sources in the order the answer cites them, renumbered to match.
+ * Sources in the order the answer cites them.
*
- * They arrive in retrieval order, which is an order the reader never sees and
- * has no way to follow: an answer whose first citation is [7] opened a list
- * that began at [1], so matching a marker to a source meant hunting. Reference
- * lists in published writing are numbered by first appearance for exactly this
- * reason.
+ * Derived from the same token walk that renders the chips, so the list and the
+ * numbers on screen cannot disagree. The text is returned exactly as given:
+ * `[7]` stays `[7]`, because that number is the identity of the source in the
+ * saved chat, the export and every chip's data-source-number. Only `number` —
+ * what the reader sees — is by first appearance; `sourceNumber` keeps the
+ * original.
*
- * Cited sources come first, renumbered 1..n by first appearance. Anything
- * retrieved and not cited keeps its place after them — it is still evidence of
- * what the search returned, which is what the panel is for, and it is no longer
- * mixed in among the numbers the answer actually used.
- *
- * Returns new objects. Renumbering in place would corrupt a stored answer whose
- * text still holds the original markers.
+ * Cited sources first. Anything retrieved and not cited follows, marked, and
+ * still numbered so the list stays contiguous — it is evidence of what the
+ * search returned, which is what the panel is for.
*/
-// Fenced code, inline code and math, then a citation cluster. Alternation, so
-// a cluster inside one of those regions matches the protecting branch first and
-// is never rewritten — the same trick renderCitationLinks uses on HTML.
-//
-// This is not cosmetic. An answer containing arr[2][1] in a code block would
-// otherwise have its indices rewritten to whatever the source mapping said, and
-// nothing downstream could tell that the code had been altered.
-var CITATION_SCAN = /```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`|\$\$[\s\S]*?\$\$|\$[^$\n]*\$|\[((?:\d+\s*,\s*)*\d+)\]/g;
-
-export function orderSourcesByCitation(text, sources) {
+export function orderSourcesByCitation(text, sources, options) {
var list = Array.isArray(sources) ? sources : [];
- if (!list.length) return { text: String(text || ''), sources: list };
-
- // First appearance wins, and only markers that resolve to a real source
- // count — an invented number must not reserve a position in the list.
- var order = [];
- String(text || '').replace(CITATION_SCAN, function (match, cluster) {
- if (!cluster) return match; // a protected region
- cluster.split(',').forEach(function (part) {
- var n = Number(part.trim());
- if (!Number.isInteger(n) || n < 1) return;
- if (!sourceByNumber(list, n)) return;
- if (order.indexOf(n) === -1) order.push(n);
- });
- return '';
- });
- if (!order.length) return { text: String(text || ''), sources: list };
-
- var renumbered = [];
- var mapping = {};
- order.forEach(function (was, i) {
- var source = sourceByNumber(list, was);
- mapping[was] = i + 1;
- renumbered.push(Object.assign({}, source, { number: i + 1 }));
+ var plain = String(text || '');
+ var renderer = resolveRenderer(options || {});
+ if (!list.length || !renderer) return { text: plain, sources: list };
+ var env = { assistantSources: list, assistantOpts: {}, assistantCiteOrder: [] };
+ renderer.parse(prepareMarkdown(plain, Object.assign({}, options || {}, { katex: null }), renderer).text, env);
+ var order = env.assistantCiteOrder;
+ if (!order.length) return { text: plain, sources: list };
+ var out = order.map(function (n, i) {
+ return Object.assign({}, sourceByNumber(list, n), { number: i + 1, sourceNumber: n });
});
list.forEach(function (source, i) {
- var was = Number(source && source.number) || i + 1;
- if (mapping[was]) return; // already placed
- renumbered.push(Object.assign({}, source, { number: renumbered.length + 1, uncited: true }));
+ var n = Number(source && source.number);
+ if (order.indexOf(n) !== -1) return;
+ // Identity is whatever `number` was — kept raw, never coerced. A legacy
+ // save can carry a non-numeric value here, and inventing a clean one in
+ // its place would change which card a saved chip points at.
+ var identity = source && source.number != null ? source.number : i + 1;
+ out.push(Object.assign({}, source, { number: out.length + 1, sourceNumber: identity, uncited: true }));
});
-
- // Rewrite the markers in one pass. Doing it number by number would renumber
- // something twice — 2 becomes 1, then that 1 becomes whatever 1 maps to.
- var rewritten = String(text || '').replace(CITATION_SCAN, function (match, cluster) {
- if (!cluster) return match; // a protected region
- var nums = cluster.split(',').map(function (p) { return Number(p.trim()); });
- if (nums.some(function (n) { return !mapping[n]; })) return match; // leave anything unresolved alone
- return '[' + nums.map(function (n) { return mapping[n]; }).join(', ') + ']';
- });
- return { text: rewritten, sources: renumbered };
+ return { text: plain, sources: out };
}
-export function renderCitationLinks(html, sources, options) {
- var opts = options || {};
- return String(html || '').replace(new RegExp(
- /<(pre|code|a)\b[^>]*>[\s\S]*?<\/\1>/.source + '|' + HTML_TAG_PATTERN.source + '|' + /\[((?:\d+\s*,\s*)*\d+)\]/.source,
- 'gi'), function (match, tag, cluster) {
- if (!cluster) return match;
- var nums = cluster.split(',').map(function (n) { return Number(n.trim()); }).filter(function (n) { return Number.isInteger(n) && n > 0; });
- if (!nums.length || nums.some(function (n) { return !sourceByNumber(sources, n); })) return match;
- return nums.map(function (n) {
- var source = sourceByNumber(sources, n);
- var title = source ? source.title || source.resource || 'Source' : 'Source';
- var page = source && (source.page || source.page_number || source.pageNumber);
- var label = 'Source ' + n + ': ' + title + (page ? ', page ' + page : '');
- // The number, not "src". Every marker used to read the same, so the only
- // way to tell one citation from another was to hover it — and the list
- // below is numbered, which made the numbering useless to match against.
- // The export has always shown numbers; the screen now agrees with it.
- var text = opts.citationLabel === 'text' ? 'src' : String(n);
- return '' + text + '';
- }).join(' ');
- });
-}
-
-export function normalizeAdjacentCitationClusters(text, sources) {
- var available = Array.isArray(sources) ? sources : [];
- var normalized = String(text || '').replace(/((?:\[(?:\d+\s*,\s*)*\d+\]\s*)+)\[((?:\d+\s*,\s*)*\d+)(?=$|[.,;:])/g, function(match, completeClusters, trailingCluster) {
- var nums = citationNumbers(completeClusters).concat(parseCitationCluster(trailingCluster));
- if (!allCitationsAvailable(nums, available)) return match;
- return formatCitationCluster(nums);
- });
-
- return normalized.replace(/(?:\[(?:\d+\s*,\s*)*\d+\]\s*){2,}/g, function(match) {
- var nums = citationNumbers(match);
- if (!allCitationsAvailable(nums, available)) return match;
- return formatCitationCluster(nums);
- });
-}
-
-function citationNumbers(text) {
- var clusters = String(text || '').match(/\[((?:\d+\s*,\s*)*\d+)\]/g) || [];
- var nums = [];
- clusters.forEach(function(cluster) {
- nums = nums.concat(parseCitationCluster(cluster.slice(1, -1)));
- });
- return nums;
-}
-
-function parseCitationCluster(cluster) {
- return String(cluster || '').split(',').map(function(n) { return Number(n.trim()); }).filter(function(n) { return Number.isInteger(n) && n > 0; });
-}
-
-function allCitationsAvailable(nums, sources) {
- return nums.length > 0 && nums.every(function(n) { return sourceByNumber(sources, n); });
-}
-
-function formatCitationCluster(nums) {
- nums = Array.from(new Set(nums)).sort(function(a, b) { return a - b; });
- return '[' + nums.join(', ') + ']';
-}
-
-// Keep Markdown structure and literal syntax out of prose cleanups. The installed
-// parser identifies complete blocks; the no-parser path only shields pipe lines.
function textProtection(text, kind) {
var prefix = '\uE000' + (kind || 'markdown') + ':';
while (text.indexOf(prefix) !== -1) prefix += '\uE000';
@@ -410,91 +572,10 @@ export function stripOrphanMarkdownMarkers(text) {
.trim();
}
-export function fallbackMarkdown(text) {
- var html = escapeHtml(text)
- .replace(/^### (.*)$/gm, '$1
')
- .replace(/^## (.*)$/gm, '$1
')
- .replace(/^# (.*)$/gm, '$1
')
- .replace(/\*\*(.*?)\*\*/g, '$1')
- .replace(/\*(.*?)\*/g, '$1')
- .replace(/`([^`]+)`/g, '$1');
- html = html.split(/\n{2,}/).map(function (block) {
- if (/^\s*' + lines.filter(Boolean).map(function (l) { return '' + l.replace(/^\s*[-*] /, '') + ' '; }).join('') + '';
- }
- if (lines.every(function (l) { return /^\s*\d+\. /.test(l) || !l.trim(); })) {
- return '' + lines.filter(Boolean).map(function (l) { return '- ' + l.replace(/^\s*\d+\. /, '') + '
'; }).join('') + '
';
- }
- return '' + block.replace(/\n/g, '
') + '
';
- }).join('');
- return html;
-}
-
-function renderMixedList(lines) {
- var html = '';
- var list = [];
- var paragraph = [];
- lines.forEach(function(line) {
- if (/^\s*[-*] /.test(line)) {
- if (paragraph.length) {
- html += '' + paragraph.join('
') + '
';
- paragraph = [];
- }
- list.push(line.replace(/^\s*[-*] /, ''));
- return;
- }
- if (list.length) {
- html += '' + list.map(function(item) { return '- ' + item + '
'; }).join('') + '
';
- list = [];
- }
- if (line.trim()) paragraph.push(line);
- });
- if (paragraph.length) html += '' + paragraph.join('
') + '
';
- if (list.length) html += '' + list.map(function(item) { return '- ' + item + '
'; }).join('') + '
';
- return html;
-}
-
-function isMarkdownTable(lines) {
- return lines.length >= 2 && /^\s*\|.*\|\s*$/.test(lines[0]) && /^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)+\|?\s*$/.test(lines[1]);
-}
-
-function renderFallbackTable(lines) {
- var header = tableCells(lines[0]);
- var rows = lines.slice(2).filter(function(line) { return /^\s*\|.*\|\s*$/.test(line); }).map(tableCells);
- return '' + header.map(function(cell) { return '' + cell + ' '; }).join('') + ' ' +
- rows.map(function(row) { return '' + row.map(function(cell) { return '' + cell + ' '; }).join('') + ' '; }).join('') +
- '
';
-}
-
function tableCells(line) {
return String(line || '').trim().replace(/^\|/, '').replace(/\|$/, '').split(/(?[' + escapeHtml(n) + '] ' + escapeHtml(title) + (page ? ', page ' + escapeHtml(page) : '') + '.';
+ return '[' + escapeHtml(n) + '] ' + escapeHtml(title) + (page ? ', page ' + escapeHtml(page) : '') + '. ';
}).join('');
}
diff --git a/public/js/assistant/sources.js b/public/js/assistant/sources.js
index 46a88974..2d0f3c2a 100644
--- a/public/js/assistant/sources.js
+++ b/public/js/assistant/sources.js
@@ -3,7 +3,11 @@ import { escapeAttr, escapeHtml } from './citations.js';
export function renderSourcesList(sources) {
if (!sources || sources.length === 0) return 'No citations returned.
';
return sources.map(function (s, idx) {
+ // `number` is what the reader sees — by first citation when the list has
+ // been ordered. The card's id is the source's identity, which is what a
+ // chip's href names, so a click lands on the right card whatever the order.
var n = s.number || idx + 1;
+ var identity = s.sourceNumber != null ? s.sourceNumber : n;
var page = s.page || s.page_number || s.pageNumber;
var meta = [];
if (page) meta.push('page ' + page);
@@ -16,7 +20,7 @@ export function renderSourcesList(sources) {
// is also a view of what the search found — but worth distinguishing from
// the numbers the answer actually used.
if (s.uncited) meta.unshift('not cited');
- return '' +
+ return '' +
'[' + escapeHtml(n) + '] ' + escapeHtml(s.title || s.resource || 'Untitled source') + '' +
renderSourceBadges(s) +
'' +
diff --git a/public/js/clinicalAssistant.js b/public/js/clinicalAssistant.js
index a9466a58..27c9e050 100644
--- a/public/js/clinicalAssistant.js
+++ b/public/js/clinicalAssistant.js
@@ -4,7 +4,7 @@
// server can call native MCP directly without routing through mcpo.
// ============================================================
import { EMPTY_PROMPT_SETS } from './assistant/data.js';
-import { escapeAttr, escapeHtml, orderSourcesByCitation, renderAssistantMarkdown, renderCitationLinks, safeImageUrl, wrapTables } from './assistant/citations.js';
+import { escapeAttr, escapeHtml, linkCitationsInHtml, orderSourcesByCitation, renderAssistantMarkdown, safeImageUrl, wrapTables } from './assistant/citations.js';
import { renderSourcesList } from './assistant/sources.js';
import { createAssistantExporter } from './assistant/export.js';
import { createAssistantImageStore } from './assistant/images.js';
@@ -602,14 +602,15 @@ import {
// done while streaming: the order is the order of first citation, and a
// citation that has not arrived yet cannot take its place — numbers would
// shuffle under the reader mid-sentence.
- var ordered = orderSourcesByCitation(
- finalData.answer || finalData.markdown || '',
- finalData.sources || finalData.citations || streamSources);
- lastAnswer = ordered.text;
- lastSources = ordered.sources;
- replaceLoadingMessage(loading, lastAnswer, lastSources, finalData.suggestions || []);
+ // Stored as the server sent them. Ordering by first citation happens at
+ // the render points below and nowhere else, so the text and the saved
+ // sources keep their identity: `[7]` stays `[7]`, and source 7 stays 7.
+ lastAnswer = finalData.answer || finalData.markdown || '';
+ lastSources = finalData.sources || finalData.citations || streamSources;
+ var shown = displaySources(lastAnswer, lastSources);
+ replaceLoadingMessage(loading, lastAnswer, shown, finalData.suggestions || []);
attachImageJobs(loading, messages[messages.length - 1], finalData.imageJobs || []);
- renderSources(lastSources);
+ renderSources(shown);
// Last, not first. setBusy(false) announces assistant-answer-done, and
// firing it before lastAnswer was assigned meant every listener — voice
// mode among them — was handed the *previous* answer. It now fires once the
@@ -1003,7 +1004,6 @@ import {
var opts = options || {};
try {
return renderAssistantMarkdown(md, sources || [], {
- marked: window.marked,
markdownIt: getMarkdownRenderer(),
katex: window.katex,
sanitize: sanitize,
@@ -1022,8 +1022,10 @@ import {
function getMarkdownRenderer() {
if (markdownRenderer) return markdownRenderer;
if (typeof window.markdownit === 'function') {
+ // html:true, as marked rendered before it. An answer may carry a
+ // or a
; the single sanitisation boundary is what makes that safe.
markdownRenderer = window.markdownit({
- html: false,
+ html: true,
linkify: true,
typographer: true,
breaks: true
@@ -1056,6 +1058,16 @@ import {
if (label) label.textContent = count === 1 ? '1 source' : count + ' sources';
}
+ // Sources as the reader should see them: ordered by first citation, with
+ // `number` the display position and `sourceNumber` the identity. Used only
+ // where something is rendered. What is stored — messages, lastSources, the
+ // save payload — keeps the sources exactly as the server sent them, so a
+ // saved chat is the same shape however many times it is opened and saved.
+ function displaySources(text, sources) {
+ if (!Array.isArray(sources) || !sources.length) return sources || [];
+ return orderSourcesByCitation(text, sources, { markdownIt: getMarkdownRenderer() }).sources;
+ }
+
function renderSources(sources) {
var wrap = document.getElementById('assistant-sources');
if (!wrap) return;
@@ -1175,9 +1187,14 @@ import {
var bubble = citation.closest('.assistant-bubble');
if (bubble && Array.isArray(bubble.assistantSources)) {
renderSources(bubble.assistantSources);
+ // data-source-number is the source's identity; the list is ordered by
+ // first citation, so position no longer means anything.
var number = Number(citation.getAttribute('data-source-number'));
- var source = bubble.assistantSources[number - 1];
- if (source) openSourceModal(source, number);
+ var source = bubble.assistantSources.filter(function (s) {
+ return Number(s && (s.sourceNumber || s.number)) === number;
+ })[0] || bubble.assistantSources[number - 1];
+ var shown = Number(citation.getAttribute('data-display-number')) || number;
+ if (source) openSourceModal(source, shown);
}
return; // The native anchor navigates to the matching source in the refreshed panel.
}
@@ -2301,13 +2318,16 @@ import {
// Sanitize what the service returned, THEN turn the surviving [n] markers
// into the usual chips bound to this message's sources, then restore the
// scroll wrapper the simplification removed.
- var html = wrapTables(renderCitationLinks(sanitize(String(translated || '')), sources, {}));
+ // The service returns HTML with the markers as plain text. Linked by
+ // walking the document's text nodes — not the tokenizer, which never
+ // sees this HTML, and not a pattern over the string.
+ var html = wrapTables(linkCitationsInHtml(sanitize(String(translated || '')), sources, {}));
var lost = expected.filter(function(n) { return citationNumbersIn(translated).indexOf(n) === -1; });
if (lost.length) {
// Keep dropped evidence reachable rather than letting it disappear.
html += '' +
escapeHtml('Sources not carried into the translation') + '' +
- renderCitationLinks(lost.map(function(n) { return '[' + n + ']'; }).join(' '), sources, {}) +
+ linkCitationsInHtml(lost.map(function(n) { return '[' + n + ']'; }).join(' '), sources, {}) +
'';
}
bubble.innerHTML = html +
@@ -2680,13 +2700,17 @@ import {
messages.forEach(function (m, index) {
var display = savedMessagePresentation(m);
display.attachments = m.attachments;
- var row = appendMessageNode(m.role, display.answer, m.sources && m.sources.length ? m.sources : lastSources, null, false, display);
+ // Ordered for display here; m.sources itself stays as saved, so the
+ // next save writes back exactly what was read.
+ var turnSources = m.sources && m.sources.length ? m.sources : lastSources;
+ var row = appendMessageNode(m.role, display.answer,
+ m.role === 'assistant' ? displaySources(display.answer, turnSources) : turnSources, null, false, display);
row.dataset.messageIndex = String(index);
attachImageJobs(row, m, m.imageJobs);
});
wrap.scrollTop = wrap.scrollHeight;
}
- renderSources(lastSources);
+ renderSources(displaySources(lastAnswer, lastSources));
exporter.invalidate();
updateConversationBudget();
}
diff --git a/test/assistant-attachment-roundtrip.test.js b/test/assistant-attachment-roundtrip.test.js
index d24896fc..7f1bb9db 100644
--- a/test/assistant-attachment-roundtrip.test.js
+++ b/test/assistant-attachment-roundtrip.test.js
@@ -149,6 +149,7 @@ test('client restores attachments as thumbnails and saves them plus the generate
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const saves = [];
diff --git a/test/assistant-autosave.test.js b/test/assistant-autosave.test.js
index adee1347..208b19df 100644
--- a/test/assistant-autosave.test.js
+++ b/test/assistant-autosave.test.js
@@ -28,6 +28,7 @@ function ui(t, options = {}) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const timers = fakeTimers();
diff --git a/test/assistant-citation-modal.test.js b/test/assistant-citation-modal.test.js
index b31cc3bc..f24d1014 100644
--- a/test/assistant-citation-modal.test.js
+++ b/test/assistant-citation-modal.test.js
@@ -14,6 +14,7 @@ function ui(t) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const toasts = [];
diff --git a/test/assistant-citations.test.js b/test/assistant-citations.test.js
index 3c660ed4..04ff86b4 100644
--- a/test/assistant-citations.test.js
+++ b/test/assistant-citations.test.js
@@ -6,6 +6,10 @@ const MarkdownIt = require('markdown-it');
let modulePromise;
+// The module is real ESM with no window; it finds its parser on globalThis,
+// which is what window is in the browser.
+globalThis.markdownit = require('markdown-it');
+
async function loadCitationModule() {
if (!modulePromise) {
modulePromise = fs.readFile(path.join(__dirname, '..', 'public', 'js', 'assistant', 'citations.js'), 'utf8')
@@ -69,21 +73,32 @@ test('does not merge separate citations across separate claims', async () => {
assert.doesNotMatch(html, /data-source-number="1"[^>]*>\d+<\/a>]*data-source-number="2"/);
});
-test('sorts adjacent citation tokens into one safe Vancouver cluster', async () => {
+test('adjacent citations render as one run of chips, numbered by first appearance', async () => {
+ // They used to be sorted into ascending source order. Nothing is sorted now:
+ // the chip shows the display number, which is first-appearance order, so a
+ // reader sees 1 2 3 4 in either case — and the text underneath is untouched,
+ // which is what keeps a saved answer's markers meaning what they meant.
const { renderAssistantMarkdown } = await loadCitationModule();
const html = renderAssistantMarkdown('Deteriorating course [1][4][2][3].', [
{ title: 'A' }, { title: 'B' }, { title: 'C' }, { title: 'D' }
]);
+ assert.match(html, /data-source-number="1"[^>]*>1<\/a> ]*data-source-number="4"[^>]*>2<\/a> ]*data-source-number="2"[^>]*>3<\/a> ]*data-source-number="3"[^>]*>4<\/a>/);
+ assert.doesNotMatch(html, /\]]*>\d+<\/a> ]*data-source-number="2"[^>]*>\d+<\/a> ]*data-source-number="3"[^>]*>\d+<\/a> ]*data-source-number="4"[^>]*>\d+<\/a>/);
assert.doesNotMatch(html, /\] {
+ // "[1][4][2][3" at the very end is a bracket the model ran out of room for.
+ // Repaired before parsing, so the rule sees a complete cluster; numbered by
+ // first appearance like any other.
const { renderAssistantMarkdown } = await loadCitationModule();
const html = renderAssistantMarkdown('Deteriorating course [1][4][2][3', [
{ title: 'A' }, { title: 'B' }, { title: 'C' }, { title: 'D' }
]);
- assert.match(html, /data-source-number="1"[^>]*>\d+<\/a> ]*data-source-number="2"[^>]*>\d+<\/a> ]*data-source-number="3"[^>]*>\d+<\/a> ]*data-source-number="4"[^>]*>\d+<\/a>/);
+ assert.match(html, /data-source-number="1"[^>]*>1<\/a> ]*data-source-number="4"[^>]*>2<\/a> ]*data-source-number="2"[^>]*>3<\/a> ]*data-source-number="3"[^>]*>4<\/a>/);
+ assert.doesNotMatch(html, /\[3(?!\])/, 'the unterminated bracket is gone');
});
test('does not normalize adjacent citations if any source number is unknown', async () => {
@@ -344,3 +359,13 @@ test('the sources toggle is a boolean the server enforces, with the legacy key h
assert.match(fs.readFileSync(path.join(root, 'public/components/admin.html'), 'utf8'),
/id="assistant-show-sources"/);
});
+
+test('a markdown link or footnote whose text is a bare number stays a link, not a citation', async () => {
+ const { renderAssistantMarkdown } = await loadCitationModule();
+ // The citation rule runs before markdown-it's link rule, so it has to step
+ // aside for "[1](url)" and "[1]: url" or every numbered link would become a chip.
+ const html = renderAssistantMarkdown('See [1](https://example.org/guide) and [2].', sources);
+ assert.match(html, /]+href="https:\/\/example\.org\/guide"[^>]*>1<\/a>/);
+ assert.equal((html.match(/assistant-cite/g) || []).length, 1);
+ assert.match(html, /data-source-number="2"/);
+});
diff --git a/test/assistant-math-mhchem.test.js b/test/assistant-math-mhchem.test.js
index 3897b19b..5494f579 100644
--- a/test/assistant-math-mhchem.test.js
+++ b/test/assistant-math-mhchem.test.js
@@ -2,6 +2,7 @@ const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const path = require('node:path');
+globalThis.markdownit = require('markdown-it'); // the renderer finds its parser here, as the browser's window
let modulePromise;
diff --git a/test/assistant-message-actions.test.js b/test/assistant-message-actions.test.js
index e58f53c3..dd3a1e86 100644
--- a/test/assistant-message-actions.test.js
+++ b/test/assistant-message-actions.test.js
@@ -15,6 +15,7 @@ function ui(t, options = {}) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const copies = [];
diff --git a/test/assistant-saved-tables.test.js b/test/assistant-saved-tables.test.js
index 8c63ec50..f2fb1c1d 100644
--- a/test/assistant-saved-tables.test.js
+++ b/test/assistant-saved-tables.test.js
@@ -21,6 +21,7 @@ function ui(t, parser = marked) {
style.textContent = read('public/css/assistant.css');
window.document.head.appendChild(style);
window.marked = parser;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const saves = [];
@@ -52,10 +53,14 @@ test('raw v2 actual save/load and SSE/fallback/append/export share all table cel
c.fetchAssistantChat = async () => ({ success: true, answer: table, sources });
const loading = c.appendLoadingMessage();
await c.streamAssistantResponse({}, loading);
- // The chip shows its source number now rather than the word "src", so a
- // reader can match a marker to the numbered list under the answer.
- assert.deepEqual(rows(bubble(app)), [['Alpha', '1.25', 'A - B', '2'], ['Beta', '2-4', 'unchanged', '1']]);
- assert.match(bubble(app).querySelector('.assistant-cite').title, /Source 2: Synthetic B, page 19/);
+ // The chip shows its display number — order of first appearance — not the
+ // stored source number. Alpha's source 2 is cited first, so it reads 1.
+ // The stored text still says 2; data-source-number still says 2.
+ assert.deepEqual(rows(bubble(app)), [['Alpha', '1.25', 'A - B', '1'], ['Beta', '2-4', 'unchanged', '2']]);
+ // The title names the number the reader sees on the chip — display order —
+ // so hovering "1" says "Source 1". Identity is on data-source-number.
+ assert.match(bubble(app).querySelector('.assistant-cite').title, /Source 1: Synthetic B, page 19/);
+ assert.equal(bubble(app).querySelector('.assistant-cite').getAttribute('data-source-number'), '2');
assert.equal(bubble(app).querySelectorAll('td')[1].getAttribute('align'), 'right');
assert.equal(app.window.getComputedStyle(bubble(app).querySelectorAll('td')[1]).textAlign, 'right');
assert.equal(bubble(app).querySelector('.assistant-table-scroll').getAttribute('tabindex'), '0');
@@ -67,7 +72,7 @@ test('raw v2 actual save/load and SSE/fallback/append/export share all table cel
assert.equal(rows(bubble(app)).length, 2);
c.exportAnswerPdf();
const exported = app.document.querySelector('#assistant-export-modal');
- assert.deepEqual(rows(exported), [['Alpha', '1.25', 'A - B', '2'], ['Beta', '2-4', 'unchanged', '1']]);
+ assert.deepEqual(rows(exported), [['Alpha', '1.25', 'A - B', '1'], ['Beta', '2-4', 'unchanged', '2']]);
for (const cite of exported.querySelectorAll('.assistant-cite')) {
assert.ok(exported.querySelector(cite.getAttribute('href')), 'export citation resolves to its own reference');
}
@@ -112,7 +117,7 @@ test('code, math, escaped pipes and URLs cannot become lists or steal source-col
assert.equal(rows(bubble(app)).length, 2);
assert.equal(rows(bubble(app))[0][1], 'A - B [1] and $x - y$ and \\(a - b\\)');
assert.equal(rows(bubble(app))[1][1], 'a|b and URL');
- assert.equal(rows(bubble(app))[1][2], '1');
+ assert.equal(rows(bubble(app))[1][2], '2', 'row B cites source 1, cited second → reads 2');
assert.equal(bubble(app).querySelector('code .assistant-cite'), null);
assert.equal(bubble(app).querySelector('a[href^="https:"]').getAttribute('href'), 'https://example.test/a-b?q=1%7C2');
for (const literal of ['`' + collapse(table) + '`', '~~~md\n' + collapse(table) + '\n~~~', ' ' + collapse(table), '$$' + collapse(table) + '$$', '\\[' + collapse(table) + '\\]', 'https://example.test/' + collapse(table).replace(/ /g, '%20')]) {
@@ -179,7 +184,7 @@ test('legacy recovery refuses missing boundaries; supports independent lines and
reopen(app, collapse(escaped), 1);
assert.equal(rows(bubble(app)).length, 2);
assert.equal(rows(bubble(app))[0][1], 'a|b and x|y and $x - y$');
- assert.equal(rows(bubble(app))[0][2], '2');
+ assert.equal(rows(bubble(app))[0][2], '1', 'row A cites source 2, cited first → reads 1');
reopen(app, collapse(table) + '\n\nCaption two\n\n' + collapse(table), 1);
assert.equal(bubble(app).querySelectorAll('table').length, 2);
for (const raw of ['| A | B | --- | --- | x | y |', '| A | B | | --- | --- | | x | y', '| A | B | | --- | --- | | x | y | trailing prose']) {
@@ -200,7 +205,9 @@ test('math/code literals and sentinel-shaped input survive postprocessing withou
assert.equal(bubble(app).querySelector('code').textContent, 'a|b [2][1] $notmath$');
assert.equal(bubble(app).querySelector('code .katex, .katex .assistant-cite'), null);
assert.deepEqual(expressions, ['x \\mid y [1]', 'a - b']);
- assert.equal(rows(bubble(app))[1][2], '1');
+ // Alpha's source 2 is cited first (its [1] is inside math, held, not a
+ // citation), so Beta's [1] is the second citation and reads 2.
+ assert.equal(rows(bubble(app))[1][2], '2');
for (const raw of ['$$\n' + table + '\n$$', '~~~md\n' + table + '\n[1][2] $code$\n~~~', '\uE000html:0\uE001 and `\uE000markdown:0\uE001 [2][1]`']) {
reopen(app, raw);
assert.equal(bubble(app).querySelectorAll('table, .assistant-cite').length, 0);
@@ -411,11 +418,9 @@ test('durable jobs preserve legacy provenance, provisional/clicked turn sources
await c.performAutosave(); const saved = app.saves.at(-1);
assert.equal(saved.messages[0].content, raw); assert.equal(saved.messages[0].retainedAnswer, retained); assert.equal(saved.messages[0].legacyClipped, true);
assert.deepEqual(saved.messages[0].sources, sources);
- // Sources are reordered into citation order on the final render, and anything
- // retrieved but never cited is marked so the panel can say so. This answer
- // cites only the first, so the second is carried along as "not cited".
- assert.deepEqual(saved.sources,
- [secondSources[0], Object.assign({}, secondSources[1], { uncited: true })]);
+ // Saved exactly as the server sent them. Ordering by citation is a display
+ // matter and never reaches the stored sources or the save payload.
+ assert.deepEqual(saved.sources, secondSources);
assert.deepEqual(saved.messages[0].imageJobs, [{ jobId: id }]); assert.equal(saved.messages.at(-1).content, second);
c.restoreSavedChat(saved); await tick();
const before = JSON.stringify(c.messages);
diff --git a/test/assistant-streaming-blocks.test.js b/test/assistant-streaming-blocks.test.js
index 20f69e2b..9c0c4b07 100644
--- a/test/assistant-streaming-blocks.test.js
+++ b/test/assistant-streaming-blocks.test.js
@@ -23,6 +23,7 @@ function ui(t) {
window.eval(read('public/js/accountBoundary.js'));
window.AccountBoundary.enter({ id: 'synthetic-streaming-owner' }, true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const context = {
diff --git a/test/assistant-translate.test.js b/test/assistant-translate.test.js
index cfceefc8..b53ee2fa 100644
--- a/test/assistant-translate.test.js
+++ b/test/assistant-translate.test.js
@@ -126,6 +126,7 @@ function client(t, options = {}) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const calls = [];
diff --git a/test/assistant-user-markdown.test.js b/test/assistant-user-markdown.test.js
index 7dbae8b1..b8e737b2 100644
--- a/test/assistant-user-markdown.test.js
+++ b/test/assistant-user-markdown.test.js
@@ -17,6 +17,7 @@ function ui(t) {
style.textContent = read('public/css/assistant.css');
window.document.head.appendChild(style);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const context = { window, document: window.document, console, URL, Blob, TextDecoder, AbortController,
diff --git a/test/assistant-voice.test.js b/test/assistant-voice.test.js
index cd8cb822..870686a8 100644
--- a/test/assistant-voice.test.js
+++ b/test/assistant-voice.test.js
@@ -15,6 +15,7 @@ function ui(t, options = {}) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-voice-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const transcriptions = [];
diff --git a/test/assistant-workspace-layout.test.js b/test/assistant-workspace-layout.test.js
index ad4cbf67..547840aa 100644
--- a/test/assistant-workspace-layout.test.js
+++ b/test/assistant-workspace-layout.test.js
@@ -20,6 +20,7 @@ function workspace(t) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const fetched = [];
diff --git a/test/citation-ordering.test.js b/test/citation-ordering.test.js
index 66a8c377..20667e66 100644
--- a/test/citation-ordering.test.js
+++ b/test/citation-ordering.test.js
@@ -1,142 +1,87 @@
// Sources arrived in retrieval order — an order the reader never sees and
-// cannot follow. An answer whose first citation was [7] opened a list that
-// began at [1], so matching a marker to a source meant hunting for it.
+// cannot follow: an answer whose first citation was [7] opened a list that
+// began at [1]. Reference lists are numbered by first appearance for exactly
+// this reason.
//
-// Borrowed from the quiz app, where validating citations and ordering them fall
-// out of the same pass: it collects the sources the answer actually used into an
-// insertion-ordered map, so the list comes back in first-citation order for
-// free. Reference lists in published writing are numbered by first appearance
-// for the same reason.
+// Ordering is derived from the same token walk that renders the chips, so the
+// list and the numbers on screen cannot disagree. The text is never rewritten:
+// `[7]` stays `[7]`, because that number is the source's identity in the saved
+// chat, the export and every chip. Two earlier versions rewrote the text and
+// both were wrong — one hit array indexing in code, the other an HTML
+// attribute — and a third would have hit the next literal context.
const test = require('node:test');
const assert = require('node:assert/strict');
-const fs = require('node:fs');
-const path = require('node:path');
-const vm = require('node:vm');
+const MarkdownIt = require('markdown-it');
+globalThis.markdownit = MarkdownIt;
-function load() {
- const src = fs.readFileSync(path.join(__dirname, '..', 'public/js/assistant/citations.js'), 'utf8')
- .replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, '')
- .replace(/^export /gm, '');
- const ctx = { window: {}, document: undefined, console };
- vm.createContext(ctx);
- vm.runInContext(src + '\nthis.orderSourcesByCitation = orderSourcesByCitation;', ctx);
- return ctx;
+let mod;
+async function load() {
+ if (!mod) {
+ const fs = require('node:fs'); const path = require('node:path');
+ const src = fs.readFileSync(path.join(__dirname, '..', 'public/js/assistant/citations.js'), 'utf8');
+ mod = await import('data:text/javascript;charset=utf-8,' + encodeURIComponent(src));
+ }
+ return mod;
}
+const four = [{ number: 1, title: 'Alpha' }, { number: 2, title: 'Beta' }, { number: 3, title: 'Gamma' }, { number: 4, title: 'Delta' }];
+const titles = out => Array.from(out.sources, s => s.title);
-const four = [
- { number: 1, title: 'Alpha' }, { number: 2, title: 'Beta' },
- { number: 3, title: 'Gamma' }, { number: 4, title: 'Delta' }
-];
-
-test('sources come back in the order the answer cites them', () => {
- const { orderSourcesByCitation } = load();
+test('sources come back in the order the answer cites them', async () => {
+ const { orderSourcesByCitation } = await load();
const out = orderSourcesByCitation('Third first [3]. Then the first [1].', four);
- assert.deepEqual(Array.from(out.sources.slice(0, 2), s => s.title), ['Gamma', 'Alpha']);
- assert.equal(out.text, 'Third first [1]. Then the first [2].');
+ assert.deepEqual(titles(out).slice(0, 2), ['Gamma', 'Alpha']);
+ assert.deepEqual(Array.from(out.sources.slice(0, 2), s => [s.number, s.sourceNumber]), [[1, 3], [2, 1]]);
});
-test('a cluster is renumbered as a whole, in its own order', () => {
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('Both of these [4, 2] agree.', four);
- assert.equal(out.text, 'Both of these [1, 2] agree.');
- assert.deepEqual(Array.from(out.sources.slice(0, 2), s => s.title), ['Delta', 'Beta']);
+test('the text is returned exactly as given', async () => {
+ const { orderSourcesByCitation } = await load();
+ const text = 'Third first [3]. Then the first [1].';
+ assert.equal(orderSourcesByCitation(text, four).text, text);
});
-test('renumbering happens in one pass, so nothing is renumbered twice', () => {
- // Rewriting number by number turns 2 into 1, then that 1 into whatever 1
- // maps to. The whole text is rewritten once instead.
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('[2] then [1] then [2] again.', four);
- assert.equal(out.text, '[1] then [2] then [1] again.');
+test('a source cited twice keeps its first position', async () => {
+ const { orderSourcesByCitation } = await load();
+ assert.deepEqual(titles(orderSourcesByCitation('[3] ... [1] ... [3] again.', four)).slice(0, 2), ['Gamma', 'Alpha']);
});
-test('a source cited twice keeps its first position', () => {
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('[3] ... [1] ... [3] again.', four);
- assert.deepEqual(Array.from(out.sources.slice(0, 2), s => s.title), ['Gamma', 'Alpha']);
-});
-
-test('retrieved but uncited sources follow, marked and still numbered', () => {
- // The panel is also a view of what the search returned, so they stay — just
- // no longer mixed in among the numbers the answer used.
- const { orderSourcesByCitation } = load();
+test('retrieved but uncited sources follow, marked and still numbered', async () => {
+ const { orderSourcesByCitation } = await load();
const out = orderSourcesByCitation('Only this one [2].', four);
assert.equal(out.sources[0].title, 'Beta');
- assert.equal(out.sources[0].uncited, undefined);
assert.equal(out.sources.length, 4, 'nothing is dropped');
assert.deepEqual(Array.from(out.sources.slice(1), s => s.uncited), [true, true, true]);
assert.deepEqual(Array.from(out.sources, s => s.number), [1, 2, 3, 4], 'numbering stays contiguous');
});
-test('an invented citation reserves no place and is left alone', () => {
- // It is not turned into a link either; the audit records it. What matters
- // here is that it cannot push a real source down the list.
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('Invented [9]. Real [2].', four);
- assert.equal(out.sources[0].title, 'Beta');
- assert.match(out.text, /Invented \[9\]/, 'the unresolved marker is untouched');
- assert.match(out.text, /Real \[1\]/);
+test('an invented citation reserves no place', async () => {
+ const { orderSourcesByCitation } = await load();
+ assert.equal(orderSourcesByCitation('Invented [9]. Real [2].', four).sources[0].title, 'Beta');
});
-test('a cluster containing an invented number is left whole', () => {
- // Renumbering half of it would silently change which source the good half
- // points at.
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('Mixed [2, 9].', four);
- assert.equal(out.text, 'Mixed [2, 9].');
+test('a bracket inside code, math or an HTML attribute is not a citation', async () => {
+ // By construction, not by pattern: the rule only runs where the parser runs
+ // inline rules, and those are places it never does.
+ const { orderSourcesByCitation } = await load();
+ const text = '```\nx = arr[3][1]\n```\n\n`m[4]` and $f[3]$ and t\n\nReal [2].';
+ const out = orderSourcesByCitation(text, four);
+ assert.equal(out.sources[0].title, 'Beta', 'the first real citation is first');
+ assert.equal(out.sources[0].sourceNumber, 2);
});
-test('an answer that cites nothing is returned untouched', () => {
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('No citations here.', four);
- assert.equal(out.text, 'No citations here.');
- assert.deepEqual(Array.from(out.sources), four);
+test('the rendered chips agree with the ordered list', async () => {
+ const { orderSourcesByCitation, renderAssistantMarkdown } = await load();
+ const text = 'See [4], then [2], then [4] again.';
+ const ordered = orderSourcesByCitation(text, four).sources;
+ const html = renderAssistantMarkdown(text, ordered);
+ // [4] is display 1, [2] is display 2 — in the chip text and in the list.
+ assert.match(html, /data-source-number="4"[^>]*data-display-number="1"[^>]*>1<\/a>/);
+ assert.match(html, /data-source-number="2"[^>]*data-display-number="2"[^>]*>2<\/a>/);
+ assert.equal(ordered[0].sourceNumber, 4); assert.equal(ordered[0].number, 1);
});
-test('the originals are not mutated, so a stored answer stays readable', () => {
- // Its text still holds the original markers; renumbering in place would make
- // the two disagree.
- const { orderSourcesByCitation } = load();
+test('the originals are not mutated', async () => {
+ const { orderSourcesByCitation } = await load();
const before = JSON.parse(JSON.stringify(four));
orderSourcesByCitation('[3] [1]', four);
assert.deepEqual(four, before);
});
-
-test('sources with no number field fall back to position', () => {
- const { orderSourcesByCitation } = load();
- const bare = [{ title: 'One' }, { title: 'Two' }, { title: 'Three' }];
- const out = orderSourcesByCitation('Cite the third [3].', bare);
- assert.equal(out.sources[0].title, 'Three');
- assert.equal(out.text, 'Cite the third [1].');
-});
-
-// ---- what must never be touched ---------------------------------------------
-
-test('a citation-shaped index inside code is not a citation', () => {
- // arr[2][1] is array indexing. Renumbering it would alter the code and
- // nothing downstream could tell.
- const { orderSourcesByCitation } = load();
- const text = 'Use [3] here.\n\n```js\nconst x = arr[2][1];\n```\n\nAnd `m[1][2]` inline.';
- const out = orderSourcesByCitation(text, four);
- assert.match(out.text, /Use \[1\] here/);
- assert.match(out.text, /arr\[2\]\[1\]/, 'the fenced code was rewritten');
- assert.match(out.text, /`m\[1\]\[2\]`/, 'the inline code was rewritten');
-});
-
-test('a bracket inside math is not a citation', () => {
- const { orderSourcesByCitation } = load();
- const text = 'See [2].\n\n$$ f[1] = x $$ and $g[3]$.';
- const out = orderSourcesByCitation(text, four);
- assert.match(out.text, /See \[1\]/);
- assert.match(out.text, /\$\$ f\[1\] = x \$\$/);
- assert.match(out.text, /\$g\[3\]\$/);
-});
-
-test('a bracket inside code reserves no place in the order either', () => {
- // If [1] inside a code block counted as a citation, the first real citation
- // would come out numbered 2.
- const { orderSourcesByCitation } = load();
- const out = orderSourcesByCitation('```\nx[1]\n```\n\nReal [4].', four);
- assert.equal(out.sources[0].title, 'Delta');
- assert.match(out.text, /Real \[1\]/);
-});
diff --git a/test/clinical-conversation.test.js b/test/clinical-conversation.test.js
index 5914c458..b3773705 100644
--- a/test/clinical-conversation.test.js
+++ b/test/clinical-conversation.test.js
@@ -171,6 +171,8 @@ function browserUI(options = {}) {
});
const calls = { stream: [], save: [] };
const escapeHtml = text => String(text).replace(/&/g, '&').replace(//g, '>');
+ // The real renderer, when a test injects it, finds its parser on the window.
+ dom.window.markdownit = require('markdown-it');
const context = {
window: dom.window, document: dom.window.document, navigator: dom.window.navigator,
console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require("node:crypto").webcrypto,
@@ -287,7 +289,11 @@ test('real save/reopen and UI renderer contain malicious legacy numbers without
const cards = ui.document.querySelectorAll('.assistant-source');
assert.equal(cards.length, 2);
assert.equal(cards[1].id, 'assistant-source-' + malicious);
- assert.equal(cards[1].querySelector('strong').textContent, '[' + malicious + '] Legacy markup');
+ // The visible number is the card's position — this source is uncited, so it
+ // follows the one cited source and reads [2]. Identity lives in the id and in
+ // every chip's data-source-number, which is what the assertions around this
+ // one check; the legacy value is never coerced or invented.
+ assert.equal(cards[1].querySelector('strong').textContent, '[2] Legacy markup');
assert.equal(ui.document.querySelector('img, script, [onerror], [data-injected]'), null);
assert.equal(ui.document.querySelector('.assistant-cite').getAttribute('href'), '#assistant-source-1');
assert.equal(ui.document.getElementById('assistant-source-1'), cards[0]);
diff --git a/test/clinical-release-integration.test.js b/test/clinical-release-integration.test.js
index 3ea816d8..abbd0a1f 100644
--- a/test/clinical-release-integration.test.js
+++ b/test/clinical-release-integration.test.js
@@ -69,6 +69,7 @@ test('native admin and assistant modules retain budget, table/source identity an
window.showToast = values.showToast;
window.confirm = () => true;
window.marked = require('marked').marked;
+ window.markdownit = require('markdown-it');
window.matchMedia = () => ({ matches: true }); // Exercise the real inline export, without printing/downloading.
t.after(async () => {
await new Promise(resolve => setTimeout(resolve, 2000)); // let autosave + late fetch chains settle against this mock
diff --git a/test/generated-images-ui.test.js b/test/generated-images-ui.test.js
index b2d519f9..ae62aa85 100644
--- a/test/generated-images-ui.test.js
+++ b/test/generated-images-ui.test.js
@@ -105,7 +105,7 @@ test('legacy HTTP/data and Share-only fallbacks keep the ORIGINAL owner across c
test('selected sidebar B survives A-done -> B-queued -> save/reopen/export, without removing A from its conversation turn', async () => {
const b='22345678-1234-1234-1234-123456789abc';let status='pending',saved;
const ui=client();ui.dom.window.document.body.innerHTML=fs.readFileSync('public/components/assistant.html','utf8');
- Object.assign(ui.context,{escapeAttr:String,escapeHtml:String,EMPTY_PROMPT_SETS:[],renderAssistantMarkdown:text=>''+text+'
',renderSourcesList:()=>'',
+ Object.assign(ui.context,{escapeAttr:String,escapeHtml:String,EMPTY_PROMPT_SETS:[],orderSourcesByCitation: (text, sources) => ({ text: text, sources: sources || [] }), renderAssistantMarkdown:text=>''+text+'
',renderSourcesList:()=>'',
createAssistantExporter:()=>({invalidate(){}}),createAssistantImageStore:()=>({renderGeneratedImage:url=>'
',clear(){}}),
startAssistantImageJob:async()=>({success:true,jobId:b,status:'pending'}),
saveAssistantChat:async payload=>{saved=JSON.parse(JSON.stringify(payload));return {success:true};},fetchSavedAssistantChats:async()=>({success:true,chats:[]})});
diff --git a/test/patient-takehome.test.js b/test/patient-takehome.test.js
index f9e8e6df..3cf7c576 100644
--- a/test/patient-takehome.test.js
+++ b/test/patient-takehome.test.js
@@ -185,6 +185,7 @@ function client(t, options = {}) {
window.eval(read('public/js/accountBoundary.js'));
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-takehome-owner' }, true), true);
window.marked = marked;
+ window.markdownit = require('markdown-it');
window.DOMPurify = require('dompurify')(window);
window.matchMedia = () => ({ matches: true });
const calls = [];