refactor: citations are a markdown-it token, and the numbers you see are display order
The old renderer rewrote the text: it found "[n]" with regexes, renumbered them, and swapped the result back in — which broke inside `arr[2][1]`, inside HTML attributes, and whenever two turns disagreed about what "[3]" meant. It also had a fallback markdown renderer of its own for when the rewrite produced something markdown-it would not parse. Now "[n]" is an inline rule registered on the same markdown-it instance that renders everything else. The parser decides what is prose and what is code, a link, or a URL, so the rule never sees "[1]" inside a code span, and it steps aside for "[1](url)". Math is two more rules on the same parser instead of a regex pre-pass, so "$" inside a URL is no longer math. Identity vs display: the stored "[n]" and each card's id are the source's identity (sourceNumber) and are never rewritten. The number a reader sees is the order of first appearance, computed at render time from the token stream (orderSourcesByCitation), so "one, then seven" cannot happen and a saved chat re-opens pointing at the same cards it was saved with. Stored messages and sources are untouched; export and the modal resolve by identity. Translated HTML gets the same links through a TreeWalker over text nodes (linkCitationsInHtml) rather than a regex over markup. Deleted: renderCitationLinks, normalizeAdjacentCitationClusters, the fallback renderer (fallbackMarkdown/renderMixedList/renderFallbackTable), renderLatexText, CITATION_SCAN. Tests that asserted rewritten text now assert token output; harnesses that render for real are given a parser. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU
This commit is contained in:
Daniel
parent
e376f69502
commit
8a6a4df121
21 changed files with 498 additions and 392 deletions
|
|
@ -20,68 +20,316 @@ export function safeImageUrl(src) {
|
|||
}
|
||||
}
|
||||
|
||||
export function renderAssistantMarkdown(md, sources, options) {
|
||||
var opts = options || {};
|
||||
// ─── Citations as tokens ───────────────────────────────────────────────
|
||||
//
|
||||
// A citation is recognised where markdown-it runs inline rules — in prose —
|
||||
// and nowhere else. A bracket inside a code span, a fence, an HTML tag, a link
|
||||
// destination or a held math expression never reaches this rule, because the
|
||||
// tokenizer never runs inline rules there. That is the whole design: the parser
|
||||
// already knows where prose is, and nothing here re-derives it with patterns.
|
||||
//
|
||||
// What it replaced was five hundred lines that did exactly that re-deriving:
|
||||
// hold code, hold links, hold HTML, hold math, rewrite the text, parse, then
|
||||
// regex the HTML for what survived. Every literal context was another branch,
|
||||
// and each one found was another that had been missed. The last two found were
|
||||
// HTML attributes and array indexing in code.
|
||||
//
|
||||
// The stored text is never rewritten. `[7]` in the answer stays `[7]`, and it
|
||||
// is the identity of that source everywhere — the chip's data-source-number,
|
||||
// the card's id, a saved chat, an export. What changes is only what the reader
|
||||
// sees: chips are numbered in order of first appearance, and the sources list
|
||||
// is ordered the same way, both derived from the same walk of the same tokens.
|
||||
|
||||
var CITE_RE = /^\\?\[((?:\d+\s*,\s*)*\d+)\\?\]/;
|
||||
|
||||
function citationRule(state, silent) {
|
||||
var src = state.src, pos = state.pos, c = src.charCodeAt(pos);
|
||||
if (c !== 0x5B && !(c === 0x5C && src.charCodeAt(pos + 1) === 0x5B)) return false;
|
||||
var m = CITE_RE.exec(src.slice(pos, pos + 64));
|
||||
if (!m) return false;
|
||||
// [3](url) is a link and [3]: a definition. [3][2] is two citations —
|
||||
// reference links with all-digit labels do not occur in model output, and
|
||||
// adjacent clusters do, constantly.
|
||||
var after = src.charAt(pos + m[0].length);
|
||||
if (after === '(' || after === ':') return false;
|
||||
var env = state.env || {};
|
||||
var sources = env.assistantSources || [];
|
||||
var nums = m[1].split(',').map(function (n) { return Number(n.trim()); });
|
||||
// Every number must name a retrieved source. An invented one is not a
|
||||
// citation: it stays as the text the model wrote, visibly unlinked, and the
|
||||
// server's citation audit records it. Nothing here guesses.
|
||||
if (!nums.every(function (n) { return n > 0 && sourceByNumber(sources, n); })) return false;
|
||||
if (!silent) {
|
||||
nums.forEach(function (n) {
|
||||
var token = state.push('assistant_cite', 'a', 0);
|
||||
token.meta = { source: n, display: displayNumber(env, n) };
|
||||
});
|
||||
}
|
||||
state.pos += m[0].length;
|
||||
return true;
|
||||
}
|
||||
|
||||
// First appearance wins and never moves: a later citation only appends, so a
|
||||
// number assigned while streaming is the number it keeps.
|
||||
function displayNumber(env, n) {
|
||||
var order = env.assistantCiteOrder || (env.assistantCiteOrder = []);
|
||||
var i = order.indexOf(n);
|
||||
if (i === -1) { order.push(n); i = order.length - 1; }
|
||||
return i + 1;
|
||||
}
|
||||
|
||||
function renderCitation(tokens, idx, o, env) {
|
||||
var meta = tokens[idx].meta;
|
||||
var opts = (env && env.assistantOpts) || {};
|
||||
var source = sourceByNumber((env && env.assistantSources) || [], meta.source);
|
||||
// A list ordered by citation already says what the reader should see, and
|
||||
// the panel shows the same number. Otherwise, first appearance in this text.
|
||||
var display = source && source.sourceNumber != null ? source.number : meta.display;
|
||||
var title = source ? source.title || source.resource || 'Source' : 'Source';
|
||||
var page = source && (source.page || source.page_number || source.pageNumber);
|
||||
var label = 'Source ' + display + ': ' + title + (page ? ', page ' + page : '');
|
||||
var text = opts.citationLabel === 'text' ? 'src' : String(display);
|
||||
// Chips in one cluster, or back to back, are separated by a space rather
|
||||
// than run together.
|
||||
var gap = idx > 0 && tokens[idx - 1].type === 'assistant_cite' ? ' ' : '';
|
||||
return gap + '<a class="assistant-cite" href="#' + escapeAttr(opts.citationTargetPrefix || 'assistant-source-') + meta.source +
|
||||
'" data-source-number="' + meta.source + '" data-display-number="' + display +
|
||||
'" title="' + escapeHtml(label) + '" aria-label="' + escapeAttr(label) + '">' + text + '</a>';
|
||||
}
|
||||
|
||||
/**
|
||||
* Teach a markdown-it instance the assistant's three renderer rules. Idempotent,
|
||||
* so the same instance can be handed in on every render.
|
||||
*
|
||||
* Fences render mermaid and chart blocks to the placeholders the page then
|
||||
* draws, and everything else to a plain <pre><code>. Images render only from
|
||||
* the safe allowlist; anything else falls back to its alt text. Both used to be
|
||||
* done by holding the block out of the text before parsing and splicing HTML
|
||||
* back in afterwards, which is what a renderer rule is for.
|
||||
*/
|
||||
export function installCitations(md) {
|
||||
if (!md || md.__assistantCitations) return md;
|
||||
md.__assistantCitations = true;
|
||||
// Before 'escape', or \[1\] is consumed as an escaped bracket first.
|
||||
md.inline.ruler.before('escape', 'assistant_cite', citationRule);
|
||||
md.renderer.rules.assistant_cite = renderCitation;
|
||||
var image = md.renderer.rules.image || function (t, i, o, e, self) { return self.renderToken(t, i, o); };
|
||||
md.renderer.rules.image = function (tokens, idx, o, env, self) {
|
||||
if (!safeImageUrl(tokens[idx].attrGet('src'))) return escapeHtml(tokens[idx].content || 'Image');
|
||||
return image(tokens, idx, o, env, self);
|
||||
};
|
||||
// markdown-it writes column alignment as style="text-align:…"; the export,
|
||||
// the stylesheet and a decade of email clients read the align attribute.
|
||||
// Both are emitted so nothing that looks for either is wrong.
|
||||
['th_open', 'td_open'].forEach(function (name) {
|
||||
var base = md.renderer.rules[name] || function (t, i, o, e, self) { return self.renderToken(t, i, o); };
|
||||
md.renderer.rules[name] = function (tokens, idx, o, env, self) {
|
||||
var style = tokens[idx].attrGet('style') || '';
|
||||
var m = /text-align:\s*(left|right|center)/.exec(style);
|
||||
if (m && !tokens[idx].attrGet('align')) tokens[idx].attrSet('align', m[1]);
|
||||
return base(tokens, idx, o, env, self);
|
||||
};
|
||||
});
|
||||
// Math is not markdown, so the parser needs telling where it is — and once
|
||||
// told, a $ inside a URL, a code span or an HTML attribute is never math,
|
||||
// by construction, which is what a text pre-pass could not promise.
|
||||
md.inline.ruler.before('escape', 'assistant_math', mathInlineRule);
|
||||
md.block.ruler.before('fence', 'assistant_math_block', mathBlockRule, { alt: ['paragraph', 'reference', 'blockquote', 'list'] });
|
||||
md.renderer.rules.assistant_math = function (tokens, idx, o, env) { return renderMath(env, tokens[idx], false); };
|
||||
md.renderer.rules.assistant_math_block = function (tokens, idx, o, env) { return renderMath(env, tokens[idx], true) + '\n'; };
|
||||
md.renderer.rules.fence = function (tokens, idx) {
|
||||
var lang = String(tokens[idx].info || '').trim().toLowerCase();
|
||||
var code = String(tokens[idx].content || '').replace(/\n$/, '');
|
||||
// Percent-encoding keeps Mermaid arrows intact through DOMPurify.
|
||||
if (lang === 'mermaid') return '<div class="assistant-mermaid" data-mermaid="' + escapeAttr(encodeURIComponent(code)) + '">Rendering graph...</div>\n';
|
||||
if (lang === 'chart' || lang === 'chartjs') return '<canvas class="assistant-chart" data-chart="' + escapeAttr(code) + '"></canvas>\n';
|
||||
return '<pre><code>' + escapeHtml(code) + '</code></pre>\n';
|
||||
};
|
||||
return md;
|
||||
}
|
||||
|
||||
// ─── Math as tokens ────────────────────────────────────────────────────
|
||||
// $…$ and \(…\) inline; $$…$$ and \[…\] display, on one line or spanning
|
||||
// several; \ce{…} and \pu{…} for mhchem. Rendered by KaTeX when the page has
|
||||
// it, otherwise escaped and shown as written.
|
||||
var MATH_INLINE_RE = /^(?:\$\$([\s\S]+?)\$\$|\\\[([\s\S]+?)\\\]|\$([^$\n]+?)\$|\\\((.+?)\\\)|\\(ce|pu)\{([^{}\n]*)\})/;
|
||||
|
||||
function mathInlineRule(state, silent) {
|
||||
var c = state.src.charCodeAt(state.pos);
|
||||
if (c !== 0x24 /* $ */ && c !== 0x5C /* \ */) return false;
|
||||
var m = MATH_INLINE_RE.exec(state.src.slice(state.pos));
|
||||
if (!m) return false;
|
||||
// A lone $ with a space either side is a dollar sign, not an equation.
|
||||
if (m[3] !== undefined && (/^\s/.test(m[3]) || /\s$/.test(m[3]))) return false;
|
||||
if (!silent) {
|
||||
var token = state.push('assistant_math', 'span', 0);
|
||||
var display = m[1] !== undefined || m[2] !== undefined;
|
||||
token.content = m[1] !== undefined ? m[1] : m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3]
|
||||
: m[4] !== undefined ? m[4] : '\\' + m[5] + '{' + m[6] + '}';
|
||||
// Without KaTeX the expression is shown exactly as written — \(a\) stays
|
||||
// \(a\), not re-dressed as $a$ — so raw is kept beside the content.
|
||||
token.meta = { display: display, raw: m[0] };
|
||||
if (display) token.type = 'assistant_math_block';
|
||||
}
|
||||
state.pos += m[0].length;
|
||||
return true;
|
||||
}
|
||||
|
||||
// $$ or \[ opening a line, closing on the same or a later line.
|
||||
function mathBlockRule(state, startLine, endLine, silent) {
|
||||
var start = state.bMarks[startLine] + state.tShift[startLine];
|
||||
var max = state.eMarks[startLine];
|
||||
var first = state.src.slice(start, max);
|
||||
var open = /^\$\$/.test(first) ? '$$' : /^\\\[/.test(first) ? '\\[' : null;
|
||||
if (!open) return false;
|
||||
var close = open === '$$' ? '$$' : '\\]';
|
||||
var rest = first.slice(open.length);
|
||||
var content, next = startLine;
|
||||
var closeAt = rest.indexOf(close);
|
||||
if (closeAt !== -1 && rest.slice(closeAt + close.length).trim() === '') {
|
||||
content = rest.slice(0, closeAt);
|
||||
} else {
|
||||
var lines = [rest];
|
||||
for (next = startLine + 1; next < endLine; next++) {
|
||||
var line = state.src.slice(state.bMarks[next] + state.tShift[next], state.eMarks[next]);
|
||||
var at = line.indexOf(close);
|
||||
if (at !== -1) { lines.push(line.slice(0, at)); break; }
|
||||
lines.push(line);
|
||||
}
|
||||
if (next >= endLine) return false; // never closed: leave it to the paragraph rule
|
||||
content = lines.join('\n');
|
||||
}
|
||||
if (silent) return true;
|
||||
var token = state.push('assistant_math_block', 'div', 0);
|
||||
token.content = content.trim();
|
||||
token.meta = { display: true, raw: state.getLines(startLine, next + 1, 0, false).trim() };
|
||||
token.map = [startLine, next + 1];
|
||||
state.line = next + 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
function renderMath(env, token, display) {
|
||||
var katex = env && env.assistantOpts && env.assistantOpts.katex;
|
||||
if (!katex) return escapeHtml((token.meta && token.meta.raw) || token.content);
|
||||
return safeKatex(katex, token.content, display);
|
||||
}
|
||||
|
||||
var defaultRenderer = null;
|
||||
function resolveRenderer(opts) {
|
||||
if (opts && opts.markdownIt && typeof opts.markdownIt.render === 'function') return installCitations(opts.markdownIt);
|
||||
if (!defaultRenderer) {
|
||||
// The browser's window is globalThis. A test sandbox keeps window as a
|
||||
// property of its own global, so both are looked at.
|
||||
var g = typeof globalThis !== 'undefined' ? globalThis : {};
|
||||
var factory = typeof g.markdownit === 'function' ? g.markdownit
|
||||
: (g.window && typeof g.window.markdownit === 'function' ? g.window.markdownit : null);
|
||||
// html:true, as marked rendered before it: an answer may carry a <span> or
|
||||
// a <br>, and the single sanitisation boundary is what makes that safe.
|
||||
if (factory) defaultRenderer = installCitations(factory({ html: true, linkify: true, typographer: true, breaks: true }));
|
||||
}
|
||||
return defaultRenderer;
|
||||
}
|
||||
|
||||
// The text as the parser should see it: placeholders dropped, math rendered and
|
||||
// held, fences and inline code held while the legacy table repair runs, then put
|
||||
// back so the parser's own fence rule renders them. Shared by rendering and by
|
||||
// ordering, so the two cannot disagree about what is a citation.
|
||||
function prepareMarkdown(md, opts, renderer) {
|
||||
// The legacy table repair and the fence hold need a lexer to know where
|
||||
// blocks are. It is the same parser that will render — one, not two.
|
||||
opts = Object.assign({}, opts, { markdownIt: renderer });
|
||||
var text = String(md || '')
|
||||
// The model occasionally writes [src]/[source] placeholders instead of
|
||||
// real citation numbers. Never invent numbers — drop the tokens; the
|
||||
// sources panel still lists the actual sources.
|
||||
.replace(/\[(?:src|source)\]/gi, '');
|
||||
.replace(/\[(?:src|source)\]/gi, '')
|
||||
// A cluster the model ran out of room for: "[1][4][2][3" at the very end.
|
||||
// Only when it directly follows complete brackets, so a lone "[3" is left.
|
||||
.replace(/((?:\[\d+\]\s*)+)\[(\d+)\s*$/, '$1[$2]');
|
||||
var embedded = textProtection(text, 'html');
|
||||
// Literal/embedded content bypasses prose, math and citation rewriting, then
|
||||
// rejoins the parsed HTML before the single sanitization boundary.
|
||||
text = protectBlocks(text, opts, function(code, lang) {
|
||||
lang = String(lang || '').toLowerCase();
|
||||
var html;
|
||||
// Percent-encoding keeps Mermaid arrows intact through DOMPurify.
|
||||
if (lang === 'mermaid') html = '<div class="assistant-mermaid" data-mermaid="' + escapeAttr(encodeURIComponent(code)) + '">Rendering graph...</div>';
|
||||
else if (lang === 'chart' || lang === 'chartjs') html = '<canvas class="assistant-chart" data-chart="' + escapeAttr(code) + '"></canvas>';
|
||||
else html = '<pre><code>' + escapeHtml(code) + '</code></pre>';
|
||||
return embedded.hold(html);
|
||||
}, true);
|
||||
var fences = textProtection(text, 'fence');
|
||||
text = protectBlocks(text, opts, function (code, lang) { return fences.hold('```' + (lang || '') + '\n' + code + '\n```'); }, true);
|
||||
var inlineCode = textProtection(text, 'code');
|
||||
text = text.replace(/(`+)[\s\S]*?\1/g, inlineCode.hold);
|
||||
var links = textProtection(text, 'links');
|
||||
text = text.replace(new RegExp(
|
||||
/!?\[[^\]\n]*\]\([^\n]*?\)|https?:\/\/(?:\\[^\s]|[^\s<>|\\])+|<(?:https?:\/\/|mailto:)[^\s<>]+>|<[^\s<>@]+@[^\s<>@]+>/.source + '|' + HTML_TAG_PATTERN.source,
|
||||
'gi'), function(match) {
|
||||
if (match.charAt(0) === '!') {
|
||||
var imageLink = match.match(/^!\[([^\]\n]*)\]\(([^\n]*?)\)$/);
|
||||
if (imageLink && !safeImageUrl(imageLink[2])) return links.hold(escapeHtml(imageLink[1] || 'Image'), match);
|
||||
}
|
||||
return links.hold(match);
|
||||
});
|
||||
// \[1\] is a citation the model escaped, not display math — but \[ ... \] is
|
||||
// the display-math delimiter, and the math pass runs next. A bracket holding
|
||||
// nothing but numbers is never an equation, so it is unescaped here, before
|
||||
// the math pass can hold it. This is the one place text syntax is genuinely
|
||||
// ambiguous, and the parser cannot resolve it because math is not markdown.
|
||||
text = text.replace(/\\\[((?:\d+\s*,\s*)*\d+)\\\]/g, '[$1]');
|
||||
text = renderLatexText(text, opts.katex, embedded.hold);
|
||||
var limited = false;
|
||||
text = normalizeMarkdownText(text, Object.assign({}, opts, {
|
||||
onUnrecoverable: function(raw) {
|
||||
onUnrecoverable: function (raw) {
|
||||
limited = true;
|
||||
raw = inlineCode.restore(links.restore(embedded.restore(raw, true)));
|
||||
raw = inlineCode.restore(fences.restore(embedded.restore(raw, true)));
|
||||
return embedded.hold('<pre>' + escapeHtml(raw) + '</pre>', raw);
|
||||
}
|
||||
}));
|
||||
text = links.restore(normalizeAdjacentCitationClusters(text, sources || []));
|
||||
text = fences.restore(inlineCode.restore(text));
|
||||
return { text: text, embedded: embedded, limited: limited };
|
||||
}
|
||||
|
||||
text = inlineCode.restore(text);
|
||||
var html;
|
||||
if (opts.marked && typeof opts.marked.parse === 'function') {
|
||||
html = opts.marked.parse(text, { breaks: true, gfm: true });
|
||||
} else if (opts.markdownIt && typeof opts.markdownIt.render === 'function') {
|
||||
html = opts.markdownIt.render(text);
|
||||
} else {
|
||||
html = fallbackMarkdown(text);
|
||||
}
|
||||
|
||||
html = renderCitationLinks(html, sources || [], opts);
|
||||
html = embedded.restore(html);
|
||||
export function renderAssistantMarkdown(md, sources, options) {
|
||||
var opts = options || {};
|
||||
var renderer = resolveRenderer(opts);
|
||||
if (!renderer) throw new Error('renderAssistantMarkdown: no markdown-it renderer is available');
|
||||
var prepared = prepareMarkdown(md, opts, renderer);
|
||||
var env = { assistantSources: sources || [], assistantOpts: opts, assistantCiteOrder: [] };
|
||||
var html = renderer.render(prepared.text, env);
|
||||
html = prepared.embedded.restore(html);
|
||||
html = wrapTables(html);
|
||||
if (limited) html += '<p class="assistant-render-notice" role="note">Could not safely recover this flattened table. Stored text is shown unchanged; missing cells or rows cannot be recovered.</p>';
|
||||
if (prepared.limited) html += '<p class="assistant-render-notice" role="note">Could not safely recover this flattened table. Stored text is shown unchanged; missing cells or rows may be present.</p>';
|
||||
if (opts.notice) html += '<p class="assistant-render-notice" role="note">' + escapeHtml(opts.notice) + '</p>';
|
||||
|
||||
return typeof opts.sanitize === 'function' ? opts.sanitize(html) : html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Link `[n]` markers in HTML that did not come from our renderer.
|
||||
*
|
||||
* The translation service is handed HTML and returns HTML, with the markers
|
||||
* carried across as plain text. That cannot go through the tokenizer, but the
|
||||
* document has structure of its own: only text nodes are visited, and none
|
||||
* inside code, pre, a or script. An attribute value is not a text node, so it is
|
||||
* never touched — the case that defeated the pattern approach.
|
||||
*/
|
||||
export function linkCitationsInHtml(html, sources, options) {
|
||||
if (typeof document === 'undefined') return String(html || '');
|
||||
var opts = options || {};
|
||||
var tpl = document.createElement('template');
|
||||
tpl.innerHTML = String(html || '');
|
||||
var walker = document.createTreeWalker(tpl.content, 4 /* NodeFilter.SHOW_TEXT */);
|
||||
var texts = [];
|
||||
for (var node = walker.nextNode(); node; node = walker.nextNode()) {
|
||||
var skip = false;
|
||||
for (var el = node.parentNode; el && el !== tpl.content; el = el.parentNode) {
|
||||
if (/^(code|pre|a|script|style)$/i.test(el.nodeName)) { skip = true; break; }
|
||||
}
|
||||
if (!skip && node.nodeValue.indexOf('[') !== -1) texts.push(node);
|
||||
}
|
||||
var env = { assistantSources: sources || [], assistantOpts: opts, assistantCiteOrder: [] };
|
||||
texts.forEach(function (node) {
|
||||
var parts = node.nodeValue.split(/(\\?\[(?:\d+\s*,\s*)*\d+\\?\])/);
|
||||
if (parts.length === 1) return;
|
||||
var frag = document.createDocumentFragment();
|
||||
parts.forEach(function (part) {
|
||||
var m = /^\\?\[((?:\d+\s*,\s*)*\d+)\\?\]$/.exec(part);
|
||||
var nums = m ? m[1].split(',').map(function (n) { return Number(n.trim()); }) : null;
|
||||
if (!nums || !nums.every(function (n) { return n > 0 && sourceByNumber(env.assistantSources, n); })) {
|
||||
frag.appendChild(document.createTextNode(part));
|
||||
return;
|
||||
}
|
||||
var span = document.createElement('span');
|
||||
span.innerHTML = nums.map(function (n, i) {
|
||||
return renderCitation([{ type: 'assistant_cite', meta: { source: n, display: displayNumber(env, n) } }], 0, {}, env)
|
||||
.replace(/^ /, i ? ' ' : '');
|
||||
}).join('');
|
||||
while (span.firstChild) frag.appendChild(span.firstChild);
|
||||
});
|
||||
node.parentNode.replaceChild(frag, node);
|
||||
});
|
||||
return tpl.innerHTML;
|
||||
}
|
||||
|
||||
export function wrapTables(html) {
|
||||
return String(html || '')
|
||||
.replace(/<table(\s[^>]*)?>/g, '<div class="assistant-table-scroll" tabindex="0" role="region" aria-label="Scrollable table"><table$1>')
|
||||
|
|
@ -95,144 +343,58 @@ export function wrapTables(html) {
|
|||
// on the number cannot drift.
|
||||
function sourceByNumber(sources, n) {
|
||||
var list = sources || [];
|
||||
// A list ordered by citation carries each source's original number as
|
||||
// sourceNumber and its display position as number. `[7]` in the text names
|
||||
// the original, always.
|
||||
for (var i = 0; i < list.length; i++) {
|
||||
if (list[i] && Number(list[i].number) === Number(n)) return list[i];
|
||||
if (list[i] && list[i].sourceNumber != null && Number(list[i].sourceNumber) === Number(n)) return list[i];
|
||||
}
|
||||
if (list.some(function (s) { return s && s.sourceNumber != null; })) return undefined;
|
||||
for (var j = 0; j < list.length; j++) {
|
||||
if (list[j] && Number(list[j].number) === Number(n)) return list[j];
|
||||
}
|
||||
// Sources predating the numbering, or a caller passing a bare list.
|
||||
return list[n - 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sources in the order the answer cites them, renumbered to match.
|
||||
* Sources in the order the answer cites them.
|
||||
*
|
||||
* They arrive in retrieval order, which is an order the reader never sees and
|
||||
* has no way to follow: an answer whose first citation is [7] opened a list
|
||||
* that began at [1], so matching a marker to a source meant hunting. Reference
|
||||
* lists in published writing are numbered by first appearance for exactly this
|
||||
* reason.
|
||||
* Derived from the same token walk that renders the chips, so the list and the
|
||||
* numbers on screen cannot disagree. The text is returned exactly as given:
|
||||
* `[7]` stays `[7]`, because that number is the identity of the source in the
|
||||
* saved chat, the export and every chip's data-source-number. Only `number` —
|
||||
* what the reader sees — is by first appearance; `sourceNumber` keeps the
|
||||
* original.
|
||||
*
|
||||
* Cited sources come first, renumbered 1..n by first appearance. Anything
|
||||
* retrieved and not cited keeps its place after them — it is still evidence of
|
||||
* what the search returned, which is what the panel is for, and it is no longer
|
||||
* mixed in among the numbers the answer actually used.
|
||||
*
|
||||
* Returns new objects. Renumbering in place would corrupt a stored answer whose
|
||||
* text still holds the original markers.
|
||||
* Cited sources first. Anything retrieved and not cited follows, marked, and
|
||||
* still numbered so the list stays contiguous — it is evidence of what the
|
||||
* search returned, which is what the panel is for.
|
||||
*/
|
||||
// Fenced code, inline code and math, then a citation cluster. Alternation, so
|
||||
// a cluster inside one of those regions matches the protecting branch first and
|
||||
// is never rewritten — the same trick renderCitationLinks uses on HTML.
|
||||
//
|
||||
// This is not cosmetic. An answer containing arr[2][1] in a code block would
|
||||
// otherwise have its indices rewritten to whatever the source mapping said, and
|
||||
// nothing downstream could tell that the code had been altered.
|
||||
var CITATION_SCAN = /```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`|\$\$[\s\S]*?\$\$|\$[^$\n]*\$|\[((?:\d+\s*,\s*)*\d+)\]/g;
|
||||
|
||||
export function orderSourcesByCitation(text, sources) {
|
||||
export function orderSourcesByCitation(text, sources, options) {
|
||||
var list = Array.isArray(sources) ? sources : [];
|
||||
if (!list.length) return { text: String(text || ''), sources: list };
|
||||
|
||||
// First appearance wins, and only markers that resolve to a real source
|
||||
// count — an invented number must not reserve a position in the list.
|
||||
var order = [];
|
||||
String(text || '').replace(CITATION_SCAN, function (match, cluster) {
|
||||
if (!cluster) return match; // a protected region
|
||||
cluster.split(',').forEach(function (part) {
|
||||
var n = Number(part.trim());
|
||||
if (!Number.isInteger(n) || n < 1) return;
|
||||
if (!sourceByNumber(list, n)) return;
|
||||
if (order.indexOf(n) === -1) order.push(n);
|
||||
});
|
||||
return '';
|
||||
});
|
||||
if (!order.length) return { text: String(text || ''), sources: list };
|
||||
|
||||
var renumbered = [];
|
||||
var mapping = {};
|
||||
order.forEach(function (was, i) {
|
||||
var source = sourceByNumber(list, was);
|
||||
mapping[was] = i + 1;
|
||||
renumbered.push(Object.assign({}, source, { number: i + 1 }));
|
||||
var plain = String(text || '');
|
||||
var renderer = resolveRenderer(options || {});
|
||||
if (!list.length || !renderer) return { text: plain, sources: list };
|
||||
var env = { assistantSources: list, assistantOpts: {}, assistantCiteOrder: [] };
|
||||
renderer.parse(prepareMarkdown(plain, Object.assign({}, options || {}, { katex: null }), renderer).text, env);
|
||||
var order = env.assistantCiteOrder;
|
||||
if (!order.length) return { text: plain, sources: list };
|
||||
var out = order.map(function (n, i) {
|
||||
return Object.assign({}, sourceByNumber(list, n), { number: i + 1, sourceNumber: n });
|
||||
});
|
||||
list.forEach(function (source, i) {
|
||||
var was = Number(source && source.number) || i + 1;
|
||||
if (mapping[was]) return; // already placed
|
||||
renumbered.push(Object.assign({}, source, { number: renumbered.length + 1, uncited: true }));
|
||||
var n = Number(source && source.number);
|
||||
if (order.indexOf(n) !== -1) return;
|
||||
// Identity is whatever `number` was — kept raw, never coerced. A legacy
|
||||
// save can carry a non-numeric value here, and inventing a clean one in
|
||||
// its place would change which card a saved chip points at.
|
||||
var identity = source && source.number != null ? source.number : i + 1;
|
||||
out.push(Object.assign({}, source, { number: out.length + 1, sourceNumber: identity, uncited: true }));
|
||||
});
|
||||
|
||||
// Rewrite the markers in one pass. Doing it number by number would renumber
|
||||
// something twice — 2 becomes 1, then that 1 becomes whatever 1 maps to.
|
||||
var rewritten = String(text || '').replace(CITATION_SCAN, function (match, cluster) {
|
||||
if (!cluster) return match; // a protected region
|
||||
var nums = cluster.split(',').map(function (p) { return Number(p.trim()); });
|
||||
if (nums.some(function (n) { return !mapping[n]; })) return match; // leave anything unresolved alone
|
||||
return '[' + nums.map(function (n) { return mapping[n]; }).join(', ') + ']';
|
||||
});
|
||||
return { text: rewritten, sources: renumbered };
|
||||
return { text: plain, sources: out };
|
||||
}
|
||||
|
||||
export function renderCitationLinks(html, sources, options) {
|
||||
var opts = options || {};
|
||||
return String(html || '').replace(new RegExp(
|
||||
/<(pre|code|a)\b[^>]*>[\s\S]*?<\/\1>/.source + '|' + HTML_TAG_PATTERN.source + '|' + /\[((?:\d+\s*,\s*)*\d+)\]/.source,
|
||||
'gi'), function (match, tag, cluster) {
|
||||
if (!cluster) return match;
|
||||
var nums = cluster.split(',').map(function (n) { return Number(n.trim()); }).filter(function (n) { return Number.isInteger(n) && n > 0; });
|
||||
if (!nums.length || nums.some(function (n) { return !sourceByNumber(sources, n); })) return match;
|
||||
return nums.map(function (n) {
|
||||
var source = sourceByNumber(sources, n);
|
||||
var title = source ? source.title || source.resource || 'Source' : 'Source';
|
||||
var page = source && (source.page || source.page_number || source.pageNumber);
|
||||
var label = 'Source ' + n + ': ' + title + (page ? ', page ' + page : '');
|
||||
// The number, not "src". Every marker used to read the same, so the only
|
||||
// way to tell one citation from another was to hover it — and the list
|
||||
// below is numbered, which made the numbering useless to match against.
|
||||
// The export has always shown numbers; the screen now agrees with it.
|
||||
var text = opts.citationLabel === 'text' ? 'src' : String(n);
|
||||
return '<a class="assistant-cite" href="#' + escapeAttr(opts.citationTargetPrefix || 'assistant-source-') + n + '" data-source-number="' + n + '" title="' + escapeHtml(label) + '" aria-label="' + escapeAttr(label) + '">' + text + '</a>';
|
||||
}).join(' ');
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeAdjacentCitationClusters(text, sources) {
|
||||
var available = Array.isArray(sources) ? sources : [];
|
||||
var normalized = String(text || '').replace(/((?:\[(?:\d+\s*,\s*)*\d+\]\s*)+)\[((?:\d+\s*,\s*)*\d+)(?=$|[.,;:])/g, function(match, completeClusters, trailingCluster) {
|
||||
var nums = citationNumbers(completeClusters).concat(parseCitationCluster(trailingCluster));
|
||||
if (!allCitationsAvailable(nums, available)) return match;
|
||||
return formatCitationCluster(nums);
|
||||
});
|
||||
|
||||
return normalized.replace(/(?:\[(?:\d+\s*,\s*)*\d+\]\s*){2,}/g, function(match) {
|
||||
var nums = citationNumbers(match);
|
||||
if (!allCitationsAvailable(nums, available)) return match;
|
||||
return formatCitationCluster(nums);
|
||||
});
|
||||
}
|
||||
|
||||
function citationNumbers(text) {
|
||||
var clusters = String(text || '').match(/\[((?:\d+\s*,\s*)*\d+)\]/g) || [];
|
||||
var nums = [];
|
||||
clusters.forEach(function(cluster) {
|
||||
nums = nums.concat(parseCitationCluster(cluster.slice(1, -1)));
|
||||
});
|
||||
return nums;
|
||||
}
|
||||
|
||||
function parseCitationCluster(cluster) {
|
||||
return String(cluster || '').split(',').map(function(n) { return Number(n.trim()); }).filter(function(n) { return Number.isInteger(n) && n > 0; });
|
||||
}
|
||||
|
||||
function allCitationsAvailable(nums, sources) {
|
||||
return nums.length > 0 && nums.every(function(n) { return sourceByNumber(sources, n); });
|
||||
}
|
||||
|
||||
function formatCitationCluster(nums) {
|
||||
nums = Array.from(new Set(nums)).sort(function(a, b) { return a - b; });
|
||||
return '[' + nums.join(', ') + ']';
|
||||
}
|
||||
|
||||
// Keep Markdown structure and literal syntax out of prose cleanups. The installed
|
||||
// parser identifies complete blocks; the no-parser path only shields pipe lines.
|
||||
function textProtection(text, kind) {
|
||||
var prefix = '\uE000' + (kind || 'markdown') + ':';
|
||||
while (text.indexOf(prefix) !== -1) prefix += '\uE000';
|
||||
|
|
@ -410,91 +572,10 @@ export function stripOrphanMarkdownMarkers(text) {
|
|||
.trim();
|
||||
}
|
||||
|
||||
export function fallbackMarkdown(text) {
|
||||
var html = escapeHtml(text)
|
||||
.replace(/^### (.*)$/gm, '<h3>$1</h3>')
|
||||
.replace(/^## (.*)$/gm, '<h2>$1</h2>')
|
||||
.replace(/^# (.*)$/gm, '<h1>$1</h1>')
|
||||
.replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
|
||||
.replace(/\*(.*?)\*/g, '<em>$1</em>')
|
||||
.replace(/`([^`]+)`/g, '<code>$1</code>');
|
||||
html = html.split(/\n{2,}/).map(function (block) {
|
||||
if (/^\s*<h\d/.test(block) && block.indexOf('\n') !== -1) {
|
||||
var parts = block.split('\n');
|
||||
return parts.shift() + fallbackMarkdown(parts.join('\n'));
|
||||
}
|
||||
if (/^\s*<(h\d|ul|ol|pre|div)/.test(block)) return block;
|
||||
var lines = block.split('\n');
|
||||
if (isMarkdownTable(lines)) return renderFallbackTable(lines);
|
||||
if (lines.some(function (l) { return /^\s*[-*] /.test(l); })) return renderMixedList(lines);
|
||||
if (lines.every(function (l) { return /^\s*[-*] /.test(l) || !l.trim(); })) {
|
||||
return '<ul>' + lines.filter(Boolean).map(function (l) { return '<li>' + l.replace(/^\s*[-*] /, '') + '</li>'; }).join('') + '</ul>';
|
||||
}
|
||||
if (lines.every(function (l) { return /^\s*\d+\. /.test(l) || !l.trim(); })) {
|
||||
return '<ol>' + lines.filter(Boolean).map(function (l) { return '<li>' + l.replace(/^\s*\d+\. /, '') + '</li>'; }).join('') + '</ol>';
|
||||
}
|
||||
return '<p>' + block.replace(/\n/g, '<br>') + '</p>';
|
||||
}).join('');
|
||||
return html;
|
||||
}
|
||||
|
||||
function renderMixedList(lines) {
|
||||
var html = '';
|
||||
var list = [];
|
||||
var paragraph = [];
|
||||
lines.forEach(function(line) {
|
||||
if (/^\s*[-*] /.test(line)) {
|
||||
if (paragraph.length) {
|
||||
html += '<p>' + paragraph.join('<br>') + '</p>';
|
||||
paragraph = [];
|
||||
}
|
||||
list.push(line.replace(/^\s*[-*] /, ''));
|
||||
return;
|
||||
}
|
||||
if (list.length) {
|
||||
html += '<ul>' + list.map(function(item) { return '<li>' + item + '</li>'; }).join('') + '</ul>';
|
||||
list = [];
|
||||
}
|
||||
if (line.trim()) paragraph.push(line);
|
||||
});
|
||||
if (paragraph.length) html += '<p>' + paragraph.join('<br>') + '</p>';
|
||||
if (list.length) html += '<ul>' + list.map(function(item) { return '<li>' + item + '</li>'; }).join('') + '</ul>';
|
||||
return html;
|
||||
}
|
||||
|
||||
function isMarkdownTable(lines) {
|
||||
return lines.length >= 2 && /^\s*\|.*\|\s*$/.test(lines[0]) && /^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)+\|?\s*$/.test(lines[1]);
|
||||
}
|
||||
|
||||
function renderFallbackTable(lines) {
|
||||
var header = tableCells(lines[0]);
|
||||
var rows = lines.slice(2).filter(function(line) { return /^\s*\|.*\|\s*$/.test(line); }).map(tableCells);
|
||||
return '<table><thead><tr>' + header.map(function(cell) { return '<th>' + cell + '</th>'; }).join('') + '</tr></thead><tbody>' +
|
||||
rows.map(function(row) { return '<tr>' + row.map(function(cell) { return '<td>' + cell + '</td>'; }).join('') + '</tr>'; }).join('') +
|
||||
'</tbody></table>';
|
||||
}
|
||||
|
||||
function tableCells(line) {
|
||||
return String(line || '').trim().replace(/^\|/, '').replace(/\|$/, '').split(/(?<!\\)\|/).map(function(cell) { return cell.trim(); });
|
||||
}
|
||||
|
||||
export function renderLatexText(text, katex, hold) {
|
||||
if (!katex && !hold) return text;
|
||||
function render(raw, expr, display) {
|
||||
var html = katex ? safeKatex(katex, expr, display) : escapeHtml(raw);
|
||||
return hold ? hold(html, raw) : html;
|
||||
}
|
||||
return String(text || '')
|
||||
.replace(/\$\$([\s\S]+?)\$\$|\\\[([\s\S]+?)\\\]|\$([^$\n]+?)\$|\\\((.+?)\\\)|\\ce\{([^{}\n]*)\}|\\pu\{([^{}\n]*)\}/g, function(raw, dollars, brackets, inline, parens, ce, pu) {
|
||||
var display = !!(dollars || brackets);
|
||||
if (dollars || brackets) return render(raw, dollars || brackets, display);
|
||||
if (inline !== undefined) return render(raw, inline, false);
|
||||
if (parens !== undefined) return render(raw, parens, false);
|
||||
if (ce !== undefined) return render(raw, '\\ce{' + ce + '}', false);
|
||||
if (pu !== undefined) return render(raw, '\\pu{' + pu + '}', false);
|
||||
return raw;
|
||||
});
|
||||
}
|
||||
|
||||
function safeKatex(katex, expr, displayMode) {
|
||||
try { return katex.renderToString(expr, { displayMode: displayMode, throwOnError: false }); }
|
||||
|
|
|
|||
|
|
@ -317,11 +317,15 @@ function collectExportItems(messages, lastAnswer, lastSources, presentMessage) {
|
|||
|
||||
function renderExportRefs(sources, sectionNumber, cited) {
|
||||
return (sources || []).map(function (s, idx) {
|
||||
// Shown by display order; anchored by identity. A chip's href names the
|
||||
// source's original number, so the id must too, whatever the list order.
|
||||
var n = s.number || idx + 1;
|
||||
if (cited.length && !cited.includes(String(n))) return '';
|
||||
var anchor = s.sourceNumber != null ? s.sourceNumber : n;
|
||||
// `cited` holds the numbers written in the answer — identities.
|
||||
if (cited.length && !cited.includes(String(anchor))) return '';
|
||||
var title = s.title || s.resource || 'Untitled source';
|
||||
var page = s.page || s.page_number || s.pageNumber;
|
||||
return '<li id="ref-' + sectionNumber + '-' + escapeAttr(n) + '"><strong>[' + escapeHtml(n) + ']</strong> ' + escapeHtml(title) + (page ? ', page ' + escapeHtml(page) : '') + '.</li>';
|
||||
return '<li id="ref-' + sectionNumber + '-' + escapeAttr(anchor) + '"><strong>[' + escapeHtml(n) + ']</strong> ' + escapeHtml(title) + (page ? ', page ' + escapeHtml(page) : '') + '.</li>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ import { escapeAttr, escapeHtml } from './citations.js';
|
|||
export function renderSourcesList(sources) {
|
||||
if (!sources || sources.length === 0) return '<p class="assistant-muted">No citations returned.</p>';
|
||||
return sources.map(function (s, idx) {
|
||||
// `number` is what the reader sees — by first citation when the list has
|
||||
// been ordered. The card's id is the source's identity, which is what a
|
||||
// chip's href names, so a click lands on the right card whatever the order.
|
||||
var n = s.number || idx + 1;
|
||||
var identity = s.sourceNumber != null ? s.sourceNumber : n;
|
||||
var page = s.page || s.page_number || s.pageNumber;
|
||||
var meta = [];
|
||||
if (page) meta.push('page ' + page);
|
||||
|
|
@ -16,7 +20,7 @@ export function renderSourcesList(sources) {
|
|||
// is also a view of what the search found — but worth distinguishing from
|
||||
// the numbers the answer actually used.
|
||||
if (s.uncited) meta.unshift('not cited');
|
||||
return '<div class="assistant-source" id="assistant-source-' + escapeAttr(n) + '">' +
|
||||
return '<div class="assistant-source" id="assistant-source-' + escapeAttr(identity) + '">' +
|
||||
'<strong>[' + escapeHtml(n) + '] ' + escapeHtml(s.title || s.resource || 'Untitled source') + '</strong>' +
|
||||
renderSourceBadges(s) +
|
||||
'<div class="assistant-source-meta">' + escapeHtml(meta.join(' · ') || 'indexed source') + '</div>' +
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
// server can call native MCP directly without routing through mcpo.
|
||||
// ============================================================
|
||||
import { EMPTY_PROMPT_SETS } from './assistant/data.js';
|
||||
import { escapeAttr, escapeHtml, orderSourcesByCitation, renderAssistantMarkdown, renderCitationLinks, safeImageUrl, wrapTables } from './assistant/citations.js';
|
||||
import { escapeAttr, escapeHtml, linkCitationsInHtml, orderSourcesByCitation, renderAssistantMarkdown, safeImageUrl, wrapTables } from './assistant/citations.js';
|
||||
import { renderSourcesList } from './assistant/sources.js';
|
||||
import { createAssistantExporter } from './assistant/export.js';
|
||||
import { createAssistantImageStore } from './assistant/images.js';
|
||||
|
|
@ -602,14 +602,15 @@ import {
|
|||
// done while streaming: the order is the order of first citation, and a
|
||||
// citation that has not arrived yet cannot take its place — numbers would
|
||||
// shuffle under the reader mid-sentence.
|
||||
var ordered = orderSourcesByCitation(
|
||||
finalData.answer || finalData.markdown || '',
|
||||
finalData.sources || finalData.citations || streamSources);
|
||||
lastAnswer = ordered.text;
|
||||
lastSources = ordered.sources;
|
||||
replaceLoadingMessage(loading, lastAnswer, lastSources, finalData.suggestions || []);
|
||||
// Stored as the server sent them. Ordering by first citation happens at
|
||||
// the render points below and nowhere else, so the text and the saved
|
||||
// sources keep their identity: `[7]` stays `[7]`, and source 7 stays 7.
|
||||
lastAnswer = finalData.answer || finalData.markdown || '';
|
||||
lastSources = finalData.sources || finalData.citations || streamSources;
|
||||
var shown = displaySources(lastAnswer, lastSources);
|
||||
replaceLoadingMessage(loading, lastAnswer, shown, finalData.suggestions || []);
|
||||
attachImageJobs(loading, messages[messages.length - 1], finalData.imageJobs || []);
|
||||
renderSources(lastSources);
|
||||
renderSources(shown);
|
||||
// Last, not first. setBusy(false) announces assistant-answer-done, and
|
||||
// firing it before lastAnswer was assigned meant every listener — voice
|
||||
// mode among them — was handed the *previous* answer. It now fires once the
|
||||
|
|
@ -1003,7 +1004,6 @@ import {
|
|||
var opts = options || {};
|
||||
try {
|
||||
return renderAssistantMarkdown(md, sources || [], {
|
||||
marked: window.marked,
|
||||
markdownIt: getMarkdownRenderer(),
|
||||
katex: window.katex,
|
||||
sanitize: sanitize,
|
||||
|
|
@ -1022,8 +1022,10 @@ import {
|
|||
function getMarkdownRenderer() {
|
||||
if (markdownRenderer) return markdownRenderer;
|
||||
if (typeof window.markdownit === 'function') {
|
||||
// html:true, as marked rendered before it. An answer may carry a <span>
|
||||
// or a <br>; the single sanitisation boundary is what makes that safe.
|
||||
markdownRenderer = window.markdownit({
|
||||
html: false,
|
||||
html: true,
|
||||
linkify: true,
|
||||
typographer: true,
|
||||
breaks: true
|
||||
|
|
@ -1056,6 +1058,16 @@ import {
|
|||
if (label) label.textContent = count === 1 ? '1 source' : count + ' sources';
|
||||
}
|
||||
|
||||
// Sources as the reader should see them: ordered by first citation, with
|
||||
// `number` the display position and `sourceNumber` the identity. Used only
|
||||
// where something is rendered. What is stored — messages, lastSources, the
|
||||
// save payload — keeps the sources exactly as the server sent them, so a
|
||||
// saved chat is the same shape however many times it is opened and saved.
|
||||
function displaySources(text, sources) {
|
||||
if (!Array.isArray(sources) || !sources.length) return sources || [];
|
||||
return orderSourcesByCitation(text, sources, { markdownIt: getMarkdownRenderer() }).sources;
|
||||
}
|
||||
|
||||
function renderSources(sources) {
|
||||
var wrap = document.getElementById('assistant-sources');
|
||||
if (!wrap) return;
|
||||
|
|
@ -1175,9 +1187,14 @@ import {
|
|||
var bubble = citation.closest('.assistant-bubble');
|
||||
if (bubble && Array.isArray(bubble.assistantSources)) {
|
||||
renderSources(bubble.assistantSources);
|
||||
// data-source-number is the source's identity; the list is ordered by
|
||||
// first citation, so position no longer means anything.
|
||||
var number = Number(citation.getAttribute('data-source-number'));
|
||||
var source = bubble.assistantSources[number - 1];
|
||||
if (source) openSourceModal(source, number);
|
||||
var source = bubble.assistantSources.filter(function (s) {
|
||||
return Number(s && (s.sourceNumber || s.number)) === number;
|
||||
})[0] || bubble.assistantSources[number - 1];
|
||||
var shown = Number(citation.getAttribute('data-display-number')) || number;
|
||||
if (source) openSourceModal(source, shown);
|
||||
}
|
||||
return; // The native anchor navigates to the matching source in the refreshed panel.
|
||||
}
|
||||
|
|
@ -2301,13 +2318,16 @@ import {
|
|||
// Sanitize what the service returned, THEN turn the surviving [n] markers
|
||||
// into the usual chips bound to this message's sources, then restore the
|
||||
// scroll wrapper the simplification removed.
|
||||
var html = wrapTables(renderCitationLinks(sanitize(String(translated || '')), sources, {}));
|
||||
// The service returns HTML with the markers as plain text. Linked by
|
||||
// walking the document's text nodes — not the tokenizer, which never
|
||||
// sees this HTML, and not a pattern over the string.
|
||||
var html = wrapTables(linkCitationsInHtml(sanitize(String(translated || '')), sources, {}));
|
||||
var lost = expected.filter(function(n) { return citationNumbersIn(translated).indexOf(n) === -1; });
|
||||
if (lost.length) {
|
||||
// Keep dropped evidence reachable rather than letting it disappear.
|
||||
html += '<div class="assistant-translated-sources"><strong>' +
|
||||
escapeHtml('Sources not carried into the translation') + '</strong>' +
|
||||
renderCitationLinks(lost.map(function(n) { return '[' + n + ']'; }).join(' '), sources, {}) +
|
||||
linkCitationsInHtml(lost.map(function(n) { return '[' + n + ']'; }).join(' '), sources, {}) +
|
||||
'</div>';
|
||||
}
|
||||
bubble.innerHTML = html +
|
||||
|
|
@ -2680,13 +2700,17 @@ import {
|
|||
messages.forEach(function (m, index) {
|
||||
var display = savedMessagePresentation(m);
|
||||
display.attachments = m.attachments;
|
||||
var row = appendMessageNode(m.role, display.answer, m.sources && m.sources.length ? m.sources : lastSources, null, false, display);
|
||||
// Ordered for display here; m.sources itself stays as saved, so the
|
||||
// next save writes back exactly what was read.
|
||||
var turnSources = m.sources && m.sources.length ? m.sources : lastSources;
|
||||
var row = appendMessageNode(m.role, display.answer,
|
||||
m.role === 'assistant' ? displaySources(display.answer, turnSources) : turnSources, null, false, display);
|
||||
row.dataset.messageIndex = String(index);
|
||||
attachImageJobs(row, m, m.imageJobs);
|
||||
});
|
||||
wrap.scrollTop = wrap.scrollHeight;
|
||||
}
|
||||
renderSources(lastSources);
|
||||
renderSources(displaySources(lastAnswer, lastSources));
|
||||
exporter.invalidate();
|
||||
updateConversationBudget();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ test('client restores attachments as thumbnails and saves them plus the generate
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const saves = [];
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ function ui(t, options = {}) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const timers = fakeTimers();
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ function ui(t) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const toasts = [];
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ const MarkdownIt = require('markdown-it');
|
|||
|
||||
let modulePromise;
|
||||
|
||||
// The module is real ESM with no window; it finds its parser on globalThis,
|
||||
// which is what window is in the browser.
|
||||
globalThis.markdownit = require('markdown-it');
|
||||
|
||||
async function loadCitationModule() {
|
||||
if (!modulePromise) {
|
||||
modulePromise = fs.readFile(path.join(__dirname, '..', 'public', 'js', 'assistant', 'citations.js'), 'utf8')
|
||||
|
|
@ -69,21 +73,32 @@ test('does not merge separate citations across separate claims', async () => {
|
|||
assert.doesNotMatch(html, /data-source-number="1"[^>]*>\d+<\/a><a class="assistant-cite"[^>]*data-source-number="2"/);
|
||||
});
|
||||
|
||||
test('sorts adjacent citation tokens into one safe Vancouver cluster', async () => {
|
||||
test('adjacent citations render as one run of chips, numbered by first appearance', async () => {
|
||||
// They used to be sorted into ascending source order. Nothing is sorted now:
|
||||
// the chip shows the display number, which is first-appearance order, so a
|
||||
// reader sees 1 2 3 4 in either case — and the text underneath is untouched,
|
||||
// which is what keeps a saved answer's markers meaning what they meant.
|
||||
const { renderAssistantMarkdown } = await loadCitationModule();
|
||||
const html = renderAssistantMarkdown('Deteriorating course [1][4][2][3].', [
|
||||
{ title: 'A' }, { title: 'B' }, { title: 'C' }, { title: 'D' }
|
||||
]);
|
||||
assert.match(html, /data-source-number="1"[^>]*>1<\/a> <a class="assistant-cite"[^>]*data-source-number="4"[^>]*>2<\/a> <a class="assistant-cite"[^>]*data-source-number="2"[^>]*>3<\/a> <a class="assistant-cite"[^>]*data-source-number="3"[^>]*>4<\/a>/);
|
||||
assert.doesNotMatch(html, /\]<span|\]\[/);
|
||||
return;
|
||||
assert.match(html, /data-source-number="1"[^>]*>\d+<\/a> <a class="assistant-cite"[^>]*data-source-number="2"[^>]*>\d+<\/a> <a class="assistant-cite"[^>]*data-source-number="3"[^>]*>\d+<\/a> <a class="assistant-cite"[^>]*data-source-number="4"[^>]*>\d+<\/a>/);
|
||||
assert.doesNotMatch(html, /\]<span|\]\[/);
|
||||
});
|
||||
|
||||
test('repairs a clearly adjacent trailing citation missing its closing bracket', async () => {
|
||||
// "[1][4][2][3" at the very end is a bracket the model ran out of room for.
|
||||
// Repaired before parsing, so the rule sees a complete cluster; numbered by
|
||||
// first appearance like any other.
|
||||
const { renderAssistantMarkdown } = await loadCitationModule();
|
||||
const html = renderAssistantMarkdown('Deteriorating course [1][4][2][3', [
|
||||
{ title: 'A' }, { title: 'B' }, { title: 'C' }, { title: 'D' }
|
||||
]);
|
||||
assert.match(html, /data-source-number="1"[^>]*>\d+<\/a> <a class="assistant-cite"[^>]*data-source-number="2"[^>]*>\d+<\/a> <a class="assistant-cite"[^>]*data-source-number="3"[^>]*>\d+<\/a> <a class="assistant-cite"[^>]*data-source-number="4"[^>]*>\d+<\/a>/);
|
||||
assert.match(html, /data-source-number="1"[^>]*>1<\/a> <a class="assistant-cite"[^>]*data-source-number="4"[^>]*>2<\/a> <a class="assistant-cite"[^>]*data-source-number="2"[^>]*>3<\/a> <a class="assistant-cite"[^>]*data-source-number="3"[^>]*>4<\/a>/);
|
||||
assert.doesNotMatch(html, /\[3(?!\])/, 'the unterminated bracket is gone');
|
||||
});
|
||||
|
||||
test('does not normalize adjacent citations if any source number is unknown', async () => {
|
||||
|
|
@ -344,3 +359,13 @@ test('the sources toggle is a boolean the server enforces, with the legacy key h
|
|||
assert.match(fs.readFileSync(path.join(root, 'public/components/admin.html'), 'utf8'),
|
||||
/id="assistant-show-sources"/);
|
||||
});
|
||||
|
||||
test('a markdown link or footnote whose text is a bare number stays a link, not a citation', async () => {
|
||||
const { renderAssistantMarkdown } = await loadCitationModule();
|
||||
// The citation rule runs before markdown-it's link rule, so it has to step
|
||||
// aside for "[1](url)" and "[1]: url" or every numbered link would become a chip.
|
||||
const html = renderAssistantMarkdown('See [1](https://example.org/guide) and [2].', sources);
|
||||
assert.match(html, /<a[^>]+href="https:\/\/example\.org\/guide"[^>]*>1<\/a>/);
|
||||
assert.equal((html.match(/assistant-cite/g) || []).length, 1);
|
||||
assert.match(html, /data-source-number="2"/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const { test } = require('node:test');
|
|||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
globalThis.markdownit = require('markdown-it'); // the renderer finds its parser here, as the browser's window
|
||||
|
||||
let modulePromise;
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ function ui(t, options = {}) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const copies = [];
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ function ui(t, parser = marked) {
|
|||
style.textContent = read('public/css/assistant.css');
|
||||
window.document.head.appendChild(style);
|
||||
window.marked = parser;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const saves = [];
|
||||
|
|
@ -52,10 +53,14 @@ test('raw v2 actual save/load and SSE/fallback/append/export share all table cel
|
|||
c.fetchAssistantChat = async () => ({ success: true, answer: table, sources });
|
||||
const loading = c.appendLoadingMessage();
|
||||
await c.streamAssistantResponse({}, loading);
|
||||
// The chip shows its source number now rather than the word "src", so a
|
||||
// reader can match a marker to the numbered list under the answer.
|
||||
assert.deepEqual(rows(bubble(app)), [['Alpha', '1.25', 'A - B', '2'], ['Beta', '2-4', 'unchanged', '1']]);
|
||||
assert.match(bubble(app).querySelector('.assistant-cite').title, /Source 2: Synthetic B, page 19/);
|
||||
// The chip shows its display number — order of first appearance — not the
|
||||
// stored source number. Alpha's source 2 is cited first, so it reads 1.
|
||||
// The stored text still says 2; data-source-number still says 2.
|
||||
assert.deepEqual(rows(bubble(app)), [['Alpha', '1.25', 'A - B', '1'], ['Beta', '2-4', 'unchanged', '2']]);
|
||||
// The title names the number the reader sees on the chip — display order —
|
||||
// so hovering "1" says "Source 1". Identity is on data-source-number.
|
||||
assert.match(bubble(app).querySelector('.assistant-cite').title, /Source 1: Synthetic B, page 19/);
|
||||
assert.equal(bubble(app).querySelector('.assistant-cite').getAttribute('data-source-number'), '2');
|
||||
assert.equal(bubble(app).querySelectorAll('td')[1].getAttribute('align'), 'right');
|
||||
assert.equal(app.window.getComputedStyle(bubble(app).querySelectorAll('td')[1]).textAlign, 'right');
|
||||
assert.equal(bubble(app).querySelector('.assistant-table-scroll').getAttribute('tabindex'), '0');
|
||||
|
|
@ -67,7 +72,7 @@ test('raw v2 actual save/load and SSE/fallback/append/export share all table cel
|
|||
assert.equal(rows(bubble(app)).length, 2);
|
||||
c.exportAnswerPdf();
|
||||
const exported = app.document.querySelector('#assistant-export-modal');
|
||||
assert.deepEqual(rows(exported), [['Alpha', '1.25', 'A - B', '2'], ['Beta', '2-4', 'unchanged', '1']]);
|
||||
assert.deepEqual(rows(exported), [['Alpha', '1.25', 'A - B', '1'], ['Beta', '2-4', 'unchanged', '2']]);
|
||||
for (const cite of exported.querySelectorAll('.assistant-cite')) {
|
||||
assert.ok(exported.querySelector(cite.getAttribute('href')), 'export citation resolves to its own reference');
|
||||
}
|
||||
|
|
@ -112,7 +117,7 @@ test('code, math, escaped pipes and URLs cannot become lists or steal source-col
|
|||
assert.equal(rows(bubble(app)).length, 2);
|
||||
assert.equal(rows(bubble(app))[0][1], 'A - B [1] and $x - y$ and \\(a - b\\)');
|
||||
assert.equal(rows(bubble(app))[1][1], 'a|b and URL');
|
||||
assert.equal(rows(bubble(app))[1][2], '1');
|
||||
assert.equal(rows(bubble(app))[1][2], '2', 'row B cites source 1, cited second → reads 2');
|
||||
assert.equal(bubble(app).querySelector('code .assistant-cite'), null);
|
||||
assert.equal(bubble(app).querySelector('a[href^="https:"]').getAttribute('href'), 'https://example.test/a-b?q=1%7C2');
|
||||
for (const literal of ['`' + collapse(table) + '`', '~~~md\n' + collapse(table) + '\n~~~', ' ' + collapse(table), '$$' + collapse(table) + '$$', '\\[' + collapse(table) + '\\]', 'https://example.test/' + collapse(table).replace(/ /g, '%20')]) {
|
||||
|
|
@ -179,7 +184,7 @@ test('legacy recovery refuses missing boundaries; supports independent lines and
|
|||
reopen(app, collapse(escaped), 1);
|
||||
assert.equal(rows(bubble(app)).length, 2);
|
||||
assert.equal(rows(bubble(app))[0][1], 'a|b and x|y and $x - y$');
|
||||
assert.equal(rows(bubble(app))[0][2], '2');
|
||||
assert.equal(rows(bubble(app))[0][2], '1', 'row A cites source 2, cited first → reads 1');
|
||||
reopen(app, collapse(table) + '\n\nCaption two\n\n' + collapse(table), 1);
|
||||
assert.equal(bubble(app).querySelectorAll('table').length, 2);
|
||||
for (const raw of ['| A | B | --- | --- | x | y |', '| A | B | | --- | --- | | x | y', '| A | B | | --- | --- | | x | y | trailing prose']) {
|
||||
|
|
@ -200,7 +205,9 @@ test('math/code literals and sentinel-shaped input survive postprocessing withou
|
|||
assert.equal(bubble(app).querySelector('code').textContent, 'a|b [2][1] $notmath$');
|
||||
assert.equal(bubble(app).querySelector('code .katex, .katex .assistant-cite'), null);
|
||||
assert.deepEqual(expressions, ['x \\mid y [1]', 'a - b']);
|
||||
assert.equal(rows(bubble(app))[1][2], '1');
|
||||
// Alpha's source 2 is cited first (its [1] is inside math, held, not a
|
||||
// citation), so Beta's [1] is the second citation and reads 2.
|
||||
assert.equal(rows(bubble(app))[1][2], '2');
|
||||
for (const raw of ['$$\n' + table + '\n$$', '~~~md\n' + table + '\n[1][2] $code$\n~~~', '\uE000html:0\uE001 and `\uE000markdown:0\uE001 [2][1]`']) {
|
||||
reopen(app, raw);
|
||||
assert.equal(bubble(app).querySelectorAll('table, .assistant-cite').length, 0);
|
||||
|
|
@ -411,11 +418,9 @@ test('durable jobs preserve legacy provenance, provisional/clicked turn sources
|
|||
await c.performAutosave(); const saved = app.saves.at(-1);
|
||||
assert.equal(saved.messages[0].content, raw); assert.equal(saved.messages[0].retainedAnswer, retained); assert.equal(saved.messages[0].legacyClipped, true);
|
||||
assert.deepEqual(saved.messages[0].sources, sources);
|
||||
// Sources are reordered into citation order on the final render, and anything
|
||||
// retrieved but never cited is marked so the panel can say so. This answer
|
||||
// cites only the first, so the second is carried along as "not cited".
|
||||
assert.deepEqual(saved.sources,
|
||||
[secondSources[0], Object.assign({}, secondSources[1], { uncited: true })]);
|
||||
// Saved exactly as the server sent them. Ordering by citation is a display
|
||||
// matter and never reaches the stored sources or the save payload.
|
||||
assert.deepEqual(saved.sources, secondSources);
|
||||
assert.deepEqual(saved.messages[0].imageJobs, [{ jobId: id }]); assert.equal(saved.messages.at(-1).content, second);
|
||||
c.restoreSavedChat(saved); await tick();
|
||||
const before = JSON.stringify(c.messages);
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ function ui(t) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
window.AccountBoundary.enter({ id: 'synthetic-streaming-owner' }, true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const context = {
|
||||
|
|
|
|||
|
|
@ -126,6 +126,7 @@ function client(t, options = {}) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const calls = [];
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ function ui(t) {
|
|||
style.textContent = read('public/css/assistant.css');
|
||||
window.document.head.appendChild(style);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const context = { window, document: window.document, console, URL, Blob, TextDecoder, AbortController,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ function ui(t, options = {}) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-voice-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const transcriptions = [];
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ function workspace(t) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-rendering-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const fetched = [];
|
||||
|
|
|
|||
|
|
@ -1,142 +1,87 @@
|
|||
// Sources arrived in retrieval order — an order the reader never sees and
|
||||
// cannot follow. An answer whose first citation was [7] opened a list that
|
||||
// began at [1], so matching a marker to a source meant hunting for it.
|
||||
// cannot follow: an answer whose first citation was [7] opened a list that
|
||||
// began at [1]. Reference lists are numbered by first appearance for exactly
|
||||
// this reason.
|
||||
//
|
||||
// Borrowed from the quiz app, where validating citations and ordering them fall
|
||||
// out of the same pass: it collects the sources the answer actually used into an
|
||||
// insertion-ordered map, so the list comes back in first-citation order for
|
||||
// free. Reference lists in published writing are numbered by first appearance
|
||||
// for the same reason.
|
||||
// Ordering is derived from the same token walk that renders the chips, so the
|
||||
// list and the numbers on screen cannot disagree. The text is never rewritten:
|
||||
// `[7]` stays `[7]`, because that number is the source's identity in the saved
|
||||
// chat, the export and every chip. Two earlier versions rewrote the text and
|
||||
// both were wrong — one hit array indexing in code, the other an HTML
|
||||
// attribute — and a third would have hit the next literal context.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const MarkdownIt = require('markdown-it');
|
||||
globalThis.markdownit = MarkdownIt;
|
||||
|
||||
function load() {
|
||||
const src = fs.readFileSync(path.join(__dirname, '..', 'public/js/assistant/citations.js'), 'utf8')
|
||||
.replace(/^import[\s\S]*?from ['"][^'"]+['"];\s*/gm, '')
|
||||
.replace(/^export /gm, '');
|
||||
const ctx = { window: {}, document: undefined, console };
|
||||
vm.createContext(ctx);
|
||||
vm.runInContext(src + '\nthis.orderSourcesByCitation = orderSourcesByCitation;', ctx);
|
||||
return ctx;
|
||||
let mod;
|
||||
async function load() {
|
||||
if (!mod) {
|
||||
const fs = require('node:fs'); const path = require('node:path');
|
||||
const src = fs.readFileSync(path.join(__dirname, '..', 'public/js/assistant/citations.js'), 'utf8');
|
||||
mod = await import('data:text/javascript;charset=utf-8,' + encodeURIComponent(src));
|
||||
}
|
||||
return mod;
|
||||
}
|
||||
const four = [{ number: 1, title: 'Alpha' }, { number: 2, title: 'Beta' }, { number: 3, title: 'Gamma' }, { number: 4, title: 'Delta' }];
|
||||
const titles = out => Array.from(out.sources, s => s.title);
|
||||
|
||||
const four = [
|
||||
{ number: 1, title: 'Alpha' }, { number: 2, title: 'Beta' },
|
||||
{ number: 3, title: 'Gamma' }, { number: 4, title: 'Delta' }
|
||||
];
|
||||
|
||||
test('sources come back in the order the answer cites them', () => {
|
||||
const { orderSourcesByCitation } = load();
|
||||
test('sources come back in the order the answer cites them', async () => {
|
||||
const { orderSourcesByCitation } = await load();
|
||||
const out = orderSourcesByCitation('Third first [3]. Then the first [1].', four);
|
||||
assert.deepEqual(Array.from(out.sources.slice(0, 2), s => s.title), ['Gamma', 'Alpha']);
|
||||
assert.equal(out.text, 'Third first [1]. Then the first [2].');
|
||||
assert.deepEqual(titles(out).slice(0, 2), ['Gamma', 'Alpha']);
|
||||
assert.deepEqual(Array.from(out.sources.slice(0, 2), s => [s.number, s.sourceNumber]), [[1, 3], [2, 1]]);
|
||||
});
|
||||
|
||||
test('a cluster is renumbered as a whole, in its own order', () => {
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('Both of these [4, 2] agree.', four);
|
||||
assert.equal(out.text, 'Both of these [1, 2] agree.');
|
||||
assert.deepEqual(Array.from(out.sources.slice(0, 2), s => s.title), ['Delta', 'Beta']);
|
||||
test('the text is returned exactly as given', async () => {
|
||||
const { orderSourcesByCitation } = await load();
|
||||
const text = 'Third first [3]. Then the first [1].';
|
||||
assert.equal(orderSourcesByCitation(text, four).text, text);
|
||||
});
|
||||
|
||||
test('renumbering happens in one pass, so nothing is renumbered twice', () => {
|
||||
// Rewriting number by number turns 2 into 1, then that 1 into whatever 1
|
||||
// maps to. The whole text is rewritten once instead.
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('[2] then [1] then [2] again.', four);
|
||||
assert.equal(out.text, '[1] then [2] then [1] again.');
|
||||
test('a source cited twice keeps its first position', async () => {
|
||||
const { orderSourcesByCitation } = await load();
|
||||
assert.deepEqual(titles(orderSourcesByCitation('[3] ... [1] ... [3] again.', four)).slice(0, 2), ['Gamma', 'Alpha']);
|
||||
});
|
||||
|
||||
test('a source cited twice keeps its first position', () => {
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('[3] ... [1] ... [3] again.', four);
|
||||
assert.deepEqual(Array.from(out.sources.slice(0, 2), s => s.title), ['Gamma', 'Alpha']);
|
||||
});
|
||||
|
||||
test('retrieved but uncited sources follow, marked and still numbered', () => {
|
||||
// The panel is also a view of what the search returned, so they stay — just
|
||||
// no longer mixed in among the numbers the answer used.
|
||||
const { orderSourcesByCitation } = load();
|
||||
test('retrieved but uncited sources follow, marked and still numbered', async () => {
|
||||
const { orderSourcesByCitation } = await load();
|
||||
const out = orderSourcesByCitation('Only this one [2].', four);
|
||||
assert.equal(out.sources[0].title, 'Beta');
|
||||
assert.equal(out.sources[0].uncited, undefined);
|
||||
assert.equal(out.sources.length, 4, 'nothing is dropped');
|
||||
assert.deepEqual(Array.from(out.sources.slice(1), s => s.uncited), [true, true, true]);
|
||||
assert.deepEqual(Array.from(out.sources, s => s.number), [1, 2, 3, 4], 'numbering stays contiguous');
|
||||
});
|
||||
|
||||
test('an invented citation reserves no place and is left alone', () => {
|
||||
// It is not turned into a link either; the audit records it. What matters
|
||||
// here is that it cannot push a real source down the list.
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('Invented [9]. Real [2].', four);
|
||||
assert.equal(out.sources[0].title, 'Beta');
|
||||
assert.match(out.text, /Invented \[9\]/, 'the unresolved marker is untouched');
|
||||
assert.match(out.text, /Real \[1\]/);
|
||||
test('an invented citation reserves no place', async () => {
|
||||
const { orderSourcesByCitation } = await load();
|
||||
assert.equal(orderSourcesByCitation('Invented [9]. Real [2].', four).sources[0].title, 'Beta');
|
||||
});
|
||||
|
||||
test('a cluster containing an invented number is left whole', () => {
|
||||
// Renumbering half of it would silently change which source the good half
|
||||
// points at.
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('Mixed [2, 9].', four);
|
||||
assert.equal(out.text, 'Mixed [2, 9].');
|
||||
test('a bracket inside code, math or an HTML attribute is not a citation', async () => {
|
||||
// By construction, not by pattern: the rule only runs where the parser runs
|
||||
// inline rules, and those are places it never does.
|
||||
const { orderSourcesByCitation } = await load();
|
||||
const text = '```\nx = arr[3][1]\n```\n\n`m[4]` and $f[3]$ and <span title="see [3]">t</span>\n\nReal [2].';
|
||||
const out = orderSourcesByCitation(text, four);
|
||||
assert.equal(out.sources[0].title, 'Beta', 'the first real citation is first');
|
||||
assert.equal(out.sources[0].sourceNumber, 2);
|
||||
});
|
||||
|
||||
test('an answer that cites nothing is returned untouched', () => {
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('No citations here.', four);
|
||||
assert.equal(out.text, 'No citations here.');
|
||||
assert.deepEqual(Array.from(out.sources), four);
|
||||
test('the rendered chips agree with the ordered list', async () => {
|
||||
const { orderSourcesByCitation, renderAssistantMarkdown } = await load();
|
||||
const text = 'See [4], then [2], then [4] again.';
|
||||
const ordered = orderSourcesByCitation(text, four).sources;
|
||||
const html = renderAssistantMarkdown(text, ordered);
|
||||
// [4] is display 1, [2] is display 2 — in the chip text and in the list.
|
||||
assert.match(html, /data-source-number="4"[^>]*data-display-number="1"[^>]*>1<\/a>/);
|
||||
assert.match(html, /data-source-number="2"[^>]*data-display-number="2"[^>]*>2<\/a>/);
|
||||
assert.equal(ordered[0].sourceNumber, 4); assert.equal(ordered[0].number, 1);
|
||||
});
|
||||
|
||||
test('the originals are not mutated, so a stored answer stays readable', () => {
|
||||
// Its text still holds the original markers; renumbering in place would make
|
||||
// the two disagree.
|
||||
const { orderSourcesByCitation } = load();
|
||||
test('the originals are not mutated', async () => {
|
||||
const { orderSourcesByCitation } = await load();
|
||||
const before = JSON.parse(JSON.stringify(four));
|
||||
orderSourcesByCitation('[3] [1]', four);
|
||||
assert.deepEqual(four, before);
|
||||
});
|
||||
|
||||
test('sources with no number field fall back to position', () => {
|
||||
const { orderSourcesByCitation } = load();
|
||||
const bare = [{ title: 'One' }, { title: 'Two' }, { title: 'Three' }];
|
||||
const out = orderSourcesByCitation('Cite the third [3].', bare);
|
||||
assert.equal(out.sources[0].title, 'Three');
|
||||
assert.equal(out.text, 'Cite the third [1].');
|
||||
});
|
||||
|
||||
// ---- what must never be touched ---------------------------------------------
|
||||
|
||||
test('a citation-shaped index inside code is not a citation', () => {
|
||||
// arr[2][1] is array indexing. Renumbering it would alter the code and
|
||||
// nothing downstream could tell.
|
||||
const { orderSourcesByCitation } = load();
|
||||
const text = 'Use [3] here.\n\n```js\nconst x = arr[2][1];\n```\n\nAnd `m[1][2]` inline.';
|
||||
const out = orderSourcesByCitation(text, four);
|
||||
assert.match(out.text, /Use \[1\] here/);
|
||||
assert.match(out.text, /arr\[2\]\[1\]/, 'the fenced code was rewritten');
|
||||
assert.match(out.text, /`m\[1\]\[2\]`/, 'the inline code was rewritten');
|
||||
});
|
||||
|
||||
test('a bracket inside math is not a citation', () => {
|
||||
const { orderSourcesByCitation } = load();
|
||||
const text = 'See [2].\n\n$$ f[1] = x $$ and $g[3]$.';
|
||||
const out = orderSourcesByCitation(text, four);
|
||||
assert.match(out.text, /See \[1\]/);
|
||||
assert.match(out.text, /\$\$ f\[1\] = x \$\$/);
|
||||
assert.match(out.text, /\$g\[3\]\$/);
|
||||
});
|
||||
|
||||
test('a bracket inside code reserves no place in the order either', () => {
|
||||
// If [1] inside a code block counted as a citation, the first real citation
|
||||
// would come out numbered 2.
|
||||
const { orderSourcesByCitation } = load();
|
||||
const out = orderSourcesByCitation('```\nx[1]\n```\n\nReal [4].', four);
|
||||
assert.equal(out.sources[0].title, 'Delta');
|
||||
assert.match(out.text, /Real \[1\]/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -171,6 +171,8 @@ function browserUI(options = {}) {
|
|||
});
|
||||
const calls = { stream: [], save: [] };
|
||||
const escapeHtml = text => String(text).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
// The real renderer, when a test injects it, finds its parser on the window.
|
||||
dom.window.markdownit = require('markdown-it');
|
||||
const context = {
|
||||
window: dom.window, document: dom.window.document, navigator: dom.window.navigator,
|
||||
console: quiet, AbortController, TextDecoder, TextEncoder, URL, Blob, crypto: require("node:crypto").webcrypto,
|
||||
|
|
@ -287,7 +289,11 @@ test('real save/reopen and UI renderer contain malicious legacy numbers without
|
|||
const cards = ui.document.querySelectorAll('.assistant-source');
|
||||
assert.equal(cards.length, 2);
|
||||
assert.equal(cards[1].id, 'assistant-source-' + malicious);
|
||||
assert.equal(cards[1].querySelector('strong').textContent, '[' + malicious + '] Legacy markup');
|
||||
// The visible number is the card's position — this source is uncited, so it
|
||||
// follows the one cited source and reads [2]. Identity lives in the id and in
|
||||
// every chip's data-source-number, which is what the assertions around this
|
||||
// one check; the legacy value is never coerced or invented.
|
||||
assert.equal(cards[1].querySelector('strong').textContent, '[2] Legacy markup');
|
||||
assert.equal(ui.document.querySelector('img, script, [onerror], [data-injected]'), null);
|
||||
assert.equal(ui.document.querySelector('.assistant-cite').getAttribute('href'), '#assistant-source-1');
|
||||
assert.equal(ui.document.getElementById('assistant-source-1'), cards[0]);
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ test('native admin and assistant modules retain budget, table/source identity an
|
|||
window.showToast = values.showToast;
|
||||
window.confirm = () => true;
|
||||
window.marked = require('marked').marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.matchMedia = () => ({ matches: true }); // Exercise the real inline export, without printing/downloading.
|
||||
t.after(async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 2000)); // let autosave + late fetch chains settle against this mock
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ test('legacy HTTP/data and Share-only fallbacks keep the ORIGINAL owner across c
|
|||
test('selected sidebar B survives A-done -> B-queued -> save/reopen/export, without removing A from its conversation turn', async () => {
|
||||
const b='22345678-1234-1234-1234-123456789abc';let status='pending',saved;
|
||||
const ui=client();ui.dom.window.document.body.innerHTML=fs.readFileSync('public/components/assistant.html','utf8');
|
||||
Object.assign(ui.context,{escapeAttr:String,escapeHtml:String,EMPTY_PROMPT_SETS:[],renderAssistantMarkdown:text=>'<p>'+text+'</p>',renderSourcesList:()=>'',
|
||||
Object.assign(ui.context,{escapeAttr:String,escapeHtml:String,EMPTY_PROMPT_SETS:[],orderSourcesByCitation: (text, sources) => ({ text: text, sources: sources || [] }), renderAssistantMarkdown:text=>'<p>'+text+'</p>',renderSourcesList:()=>'',
|
||||
createAssistantExporter:()=>({invalidate(){}}),createAssistantImageStore:()=>({renderGeneratedImage:url=>'<img src="'+url+'">',clear(){}}),
|
||||
startAssistantImageJob:async()=>({success:true,jobId:b,status:'pending'}),
|
||||
saveAssistantChat:async payload=>{saved=JSON.parse(JSON.stringify(payload));return {success:true};},fetchSavedAssistantChats:async()=>({success:true,chats:[]})});
|
||||
|
|
|
|||
|
|
@ -185,6 +185,7 @@ function client(t, options = {}) {
|
|||
window.eval(read('public/js/accountBoundary.js'));
|
||||
assert.equal(window.AccountBoundary.enter({ id: 'synthetic-takehome-owner' }, true), true);
|
||||
window.marked = marked;
|
||||
window.markdownit = require('markdown-it');
|
||||
window.DOMPurify = require('dompurify')(window);
|
||||
window.matchMedia = () => ({ matches: true });
|
||||
const calls = [];
|
||||
|
|
|
|||
Loading…
Reference in a new issue