pdf-quiz-generator/backend/app/services/clinical_library.py

"""Read-only retrieval from the clinical library index.

The index is a Milvus collection of ~1.8M chunks from the reference library —
Nelson, Harrison's, Mandell's and the rest — embedded with bge-m3 at 1024
dimensions. PedsHub already embeds with the same model through the same LiteLLM
proxy, so a query vector generated here is directly comparable to what is
stored there; no second embedder and no re-indexing.

Two things this module will not do, deliberately:

* **It never writes.** The credentials are a query user and the collection
  belongs to another system. Articles are grounded in what comes back; nothing
  flows the other way.
* **It never hands back long verbatim passages for publication.** What it
  returns is source material for someone to write *from*, plus the citation
  needed to say where a fact came from. Copying a textbook into an article and
  publishing it is redistribution, whatever produced the copy.
"""
import json
import logging
import re
import urllib.parse

import requests

from app.config import settings

logger = logging.getLogger(__name__)

# Enough context to write a section from, without turning a prompt into a book.
SNIPPET_CHARS = 1200
DEFAULT_LIMIT = 12
# Below this a chunk is a heading, an index line or a running header, not prose.
# They rank well against a query that looks like a chapter title and then fill
# the shortlist with nothing — a topic named after a shelf retrieves ten of them.
MIN_PROSE_CHARS = 200
# Ask for more than we need, because the prose filter runs after ranking. A
# broad query — "Immunodeficiency", a specialty name — matches chapter titles
# and index lines first, and filtering inside a shortlist of fourteen left
# nothing at all.
POOL_MULTIPLIER = 5


TIMEOUT = 20


def _call(path: str, body: dict) -> dict | None:
    """Milvus over its HTTP API rather than gRPC.

    The gRPC client drags in a protobuf and grpcio version surface that fights
    with what this image already pins for other services. HTTP has neither
    problem and this module only ever reads.
    """
    if not settings.CLINICAL_MILVUS_URI:
        return None
    try:
        response = requests.post(
            f"{settings.CLINICAL_MILVUS_URI.rstrip('/')}{path}",
            headers={"Authorization": f"Bearer {settings.CLINICAL_MILVUS_TOKEN}",
                     "Content-Type": "application/json"},
            json=body, timeout=TIMEOUT,
        )
    except requests.RequestException:
        logger.warning("Clinical library unreachable", exc_info=True)
        return None
    if response.status_code != 200:
        logger.warning("Clinical library refused %s: %s %s", path,
                       response.status_code, response.text[:200])
        return None
    payload = response.json()
    if payload.get("code") not in (0, None):
        logger.warning("Clinical library error on %s: %s", path, str(payload)[:200])
        return None
    return payload


def available() -> bool:
    """Whether the index can be reached at all, for an honest error upstream."""
    payload = _call("/v2/vectordb/collections/list", {})
    return bool(payload) and settings.CLINICAL_MILVUS_COLLECTION in (payload.get("data") or [])


def _tidy(value: str | None) -> str:
    """File paths in this index are URL-encoded; a citation should not be."""
    if not value:
        return ""
    return re.sub(r"\s+", " ", urllib.parse.unquote(value)).strip()


def _source_of(payload: dict) -> dict:
    """The bibliographic details, separated from the prose."""
    title = _tidy(payload.get("title")) or _tidy(payload.get("file_path")).rsplit("/", 1)[-1]
    # Library filenames carry the shop they came from; a reference should not.
    title = re.sub(r"\s*\((?:z-library|1lib|z-lib)[^)]*\)", "", title, flags=re.I).strip()
    title = re.sub(r"\.pdf$", "", title, flags=re.I).strip()
    return {
        "title": title,
        "author": _tidy(payload.get("author")),
        "page": payload.get("page_number"),
        "specialty": _tidy(payload.get("subcategory")),
    }


def search(query: str, limit: int = DEFAULT_LIMIT,
           folder_contains: str | None = None) -> list[dict]:
    """Passages relevant to a topic, each with where it came from.

    `folder_contains` narrows to one shelf of the library — passing "Pediatrics"
    is how an article about a childhood condition is grounded in the paediatric
    texts rather than in the adult ones that outnumber them.
    """
    query = (query or "").strip()
    if not query:
        return []

    from app.services.embedding_service import generate_embedding

    vector = generate_embedding(query)
    if not vector:
        logger.warning("No query embedding; cannot search the clinical library")
        return []

    body = {
        "collectionName": settings.CLINICAL_MILVUS_COLLECTION,
        "data": [vector],
        "annsField": "dense",
        "limit": limit * POOL_MULTIPLIER,
        "outputFields": ["payload", "folder_path", "file_path", "chunk_index"],
    }
    # Filtering in the query is what keeps the shelf constraint honest: taking
    # the top hits and discarding the wrong folder afterwards would silently
    # return fewer results the more specific the request was.
    if folder_contains:
        safe = folder_contains.replace('"', "")
        body["filter"] = f'folder_path like "%{safe}%"'

    payload = _call("/v2/vectordb/entities/search", body)
    if not payload:
        return []

    results = []
    for hit in (payload.get("data") or []):
        raw = hit.get("payload") or {}
        if isinstance(raw, str):
            try:
                raw = json.loads(raw)
            except ValueError:
                raw = {}
        text = _tidy(raw.get("excerpt"))
        if len(text) < MIN_PROSE_CHARS:
            continue
        results.append({
            "text": text[:SNIPPET_CHARS],
            "score": float(hit.get("distance", 0.0)),
            "source": _source_of(raw),
            "folder": _tidy(hit.get("folder_path")),
        })
        if len(results) >= limit:
            break
    return results


def references_from(passages: list[dict], limit: int = 8) -> list[dict]:
    """One entry per book, with the pages that were actually used.

    Deduplicated by title because forty chunks of Nelson is one reference, not
    forty — a reference list that repeats a book is noise standing where
    provenance should be.
    """
    by_title: dict[str, dict] = {}
    for passage in passages:
        source = passage.get("source") or {}
        title = source.get("title")
        if not title:
            continue
        entry = by_title.setdefault(title, {
            "title": title, "author": source.get("author") or None, "pages": [],
        })
        page = source.get("page")
        if isinstance(page, int) and page not in entry["pages"]:
            entry["pages"].append(page)
    for entry in by_title.values():
        entry["pages"] = sorted(entry["pages"])[:6]
    return list(by_title.values())[:limit]