pediatric-ai-scribe-v3/src/utils/clinicalRetrieval.js

var sourceMarkdown = require('markdown-it')({ html: false });
var INTERNAL_TITLE_FALLBACKS = {
  '1586022': 'Respiratory Disease',
  'FABK010-fm[i-xiv].qxd': 'Respiratory Disease'
};

function normalizeMcpSearchResponse(result) {
  var data = result && (result.structuredContent || result.data || result);
  if ((!data || !Array.isArray(data.results)) && result && Array.isArray(result.content)) {
    for (var i = 0; i < result.content.length; i++) {
      var c = result.content[i];
      if (c && c.type === 'text' && c.text) {
        try {
          var parsed = JSON.parse(c.text);
          if (parsed && Array.isArray(parsed.results)) data = parsed;
        } catch (e) {}
      }
    }
  }
  if (!data || !Array.isArray(data.results)) return [];
  return data.results.map(function(r, idx) {
    var text = sourceExcerptWithContext(r, 1800);
    return {
      number: idx + 1,
      id: r.id,
      doc_type: r.doc_type,
      title: displayTitleForSource(r, 'Untitled resource'),
      page: r.page_number || r.pageNumber || null,
      page_count: r.page_count || r.pageCount || null,
      file_path: r.file_path || r.filePath || r.path || '',
      folder_path: r.folder_path || r.folderPath || '',
      category: r.category || '',
      source_priority: r.source_priority || r.sourcePriority || '',
      source_boost: r.source_boost || r.sourceBoost || null,
      tags: Array.isArray(r.tags) ? r.tags : [],
      excerpt: text,
      score: r.score,
      chunk_index: r.chunk_index,
      total_chunks: r.total_chunks
    };
  }).filter(function(r) { return r.excerpt; });
}

function dedupeSources(results) {
  var seen = new Map();
  var out = [];
  results.forEach(function(r) {
    var key = sourceDedupeKey(r);
    var existingIndex = seen.get(key);
    if (existingIndex != null) {
      out[existingIndex] = mergeDuplicateSource(out[existingIndex], r);
      return;
    }
    seen.set(key, out.length);
    out.push(Object.assign({}, r, { number: out.length + 1 }));
  });
  return out;
}

function sourceDedupeKey(source) {
  var title = cleanTitle(source && source.title || '').toLowerCase();
  var page = source && source.page ? String(source.page) : '';
  var kind = source && source.source_type ? source.source_type : 'text';
  var identity = source && (source.file_path || source.id) || title;
  return [kind, source.doc_type || '', identity, page, source.chunk_index == null ? '' : source.chunk_index].join('|');
}

function mergeDuplicateSource(existing, duplicate) {
  var merged = Object.assign({}, existing);
  var existingExcerpt = cleanSourceExcerpt(existing.excerpt || '');
  var duplicateExcerpt = cleanSourceExcerpt(duplicate.excerpt || '');
  if (duplicateExcerpt && existingExcerpt.indexOf(duplicateExcerpt.slice(0, 160)) === -1) {
    merged.excerpt = clipSourceExcerpt([existingExcerpt, duplicateExcerpt].filter(Boolean).join('\n\n'), 1800);
  }
  merged.score = Math.max(Number(existing.score) || 0, Number(duplicate.score) || 0) || existing.score || duplicate.score;
  return merged;
}

function clip(s, n) { return String(s || '').replace(/\s+/g, ' ').trim().substring(0, n); }

function displayTitleForSource(source, fallback) {
  source = source || {};
  var rawTitle = String(source.title || '').trim();
  var filePath = source.file_path || source.filePath || source.path || '';
  var fromPath = titleFromPath(filePath);
  if (fromPath) return cleanTitle(fromPath);
  if (looksLikePdfMetadataJunk(rawTitle)) {
    var mapped = INTERNAL_TITLE_FALLBACKS[String(source.id || '')] || INTERNAL_TITLE_FALLBACKS[rawTitle];
    if (mapped) return mapped;
  }
  return cleanTitle(rawTitle || filePath || fallback || 'Untitled resource');
}

function looksLikePdfMetadataJunk(title) {
  title = String(title || '').trim();
  return /\.qxd$/i.test(title) || /^https?:\/\//i.test(title) || /^www\./i.test(title);
}

function titleFromPath(filePath) {
  var text = String(filePath || '').trim();
  if (!text) return '';
  try { text = decodeURIComponent(text); } catch (e) {}
  var parts = text.split(/[\\/]+/).filter(Boolean);
  return parts.length ? parts[parts.length - 1] : text;
}

function cleanTitle(s) {
  var title = String(s || '').trim();
  try { title = decodeURIComponent(title); } catch (e) {}
  return title
    .replace(/\.(pdf|docx?|txt|md)$/i, '')
    .replace(/\s+/g, ' ')
    .replace(/\s*\(z-library\.sk,\s*1lib\.sk,\s*z-lib\.sk\)\s*/ig, '')
    .trim();
}

// Parse, don't guess from pipes: markdown-it excludes fenced code and handles escapes.
function sourceBlocks(text) {
  var lines = text.split('\n');
  var tables = sourceMarkdown.parse(text, {}).filter(function(t) { return t.type === 'table_open'; });
  var blocks = [];
  var end = 0;
  tables.forEach(function(t) {
    if (t.map[0] > end) blocks.push({ text: lines.slice(end, t.map[0]).join('\n'), table: false });
    blocks.push({ text: lines.slice(t.map[0], t.map[1]).join('\n'), table: true });
    end = t.map[1];
  });
  if (end < lines.length) blocks.push({ text: lines.slice(end).join('\n'), table: false });
  return blocks;
}

function cleanSourceExcerpt(text) {
  text = String(text || '')
    .replace(/^\[Page-image match\]\s*/i, '')
    .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
    .replace(/<br\s*\/?>/gi, ' ')
    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
    .replace(/<\/?[a-z][^>]*>/gi, '')
    .replace(/(?:file:\/\/)?\/(?:tmp|var|home)\/[^\s)<>|]+/g, '')
    .replace(/\*\*/g, '');
  return sourceBlocks(text).map(function(b) {
    return b.table ? b.text.trim() : b.text.split(/\n\s*\n/).map(function(paragraph) {
      return paragraph.replace(/\|\s*-{2,}\s*/g, ' ').replace(/\|/g, ' ').replace(/\s+/g, ' ').trim();
    }).filter(Boolean).join('\n\n');
  }).filter(Boolean).join('\n\n');
}

function clipSourceExcerpt(text, limit) {
  text = cleanSourceExcerpt(text);
  if (text.length <= limit) return text;
  var marker = '[Content omitted]';
  if (limit < marker.length) return '';
  var budget = limit - marker.length - 2;
  var blocks = sourceBlocks(text);
  var suffix = '';
  // Repeated clinical notes/units belong to the rows, even when some rows
  // are omitted. Never borrow notes across multiple unrelated tables.
  if (blocks.filter(function(b) { return b.table; }).length === 1) {
    var last = blocks[blocks.length - 1];
    if (last && !last.table && /^(?:notes?\b|footnotes?\b|units?\b|source\s*:|[*†‡]|\[\^?\w+\])/i.test(last.text.trim())) {
      suffix = blocks.pop().text.trim();
      if (suffix.length + 2 > budget) return marker;
      budget -= suffix.length + 2;
    }
  }
  var out = [];
  blocks.some(function(b) {
    var separator = out.length ? 2 : 0;
    var available = budget - separator;
    if (b.text.length <= available) {
      out.push(b.text);
      budget -= separator + b.text.length;
      return false;
    }
    if (b.table) {
      var rows = b.text.split('\n');
      var kept = rows.slice(0, 2);
      var size = kept.join('\n').length;
      for (var i = 2; i < rows.length && size + 1 + rows[i].length <= available; i++) {
        kept.push(rows[i]);
        size += 1 + rows[i].length;
      }
      // Never emit a partial cell or a header pretending to contain values.
      if (kept.length > 2) out.push(kept.join('\n'));
    } else if (available > 0) {
      out.push(b.text.slice(0, available).trim());
    }
    return true;
  });
  return out.concat(marker, suffix || []).join('\n\n');
}

function sourceExcerptWithContext(source, limit) {
  // Allocate the hit first, not the often-long before_context.
  var hit = clipSourceExcerpt(source.excerpt || source.marked_text || '', limit);
  var out = hit;
  [source.before_context, source.after_context].forEach(function(context) {
    if (!context) return;
    var extra = clipSourceExcerpt(context, limit - out.length - 2);
    if (extra) out += (out ? '\n\n' : '') + extra;
  });
  return out;
}

module.exports = {
  cleanSourceExcerpt: cleanSourceExcerpt,
  clipSourceExcerpt: clipSourceExcerpt,
  normalizeMcpSearchResponse: normalizeMcpSearchResponse,
  dedupeSources: dedupeSources
};