pdf-quiz-generator/backend/app/routers/search.py

"""One query across everything a learner can see.

Each corpus already has its own retrieval and its own rules about who may see
what. This endpoint runs them together and returns one answer, rather than
making somebody guess which of five pages holds the thing they are looking for.

Two decisions worth stating:

* Visibility is never re-implemented here. Questions go through the same bank
  predicate and exam scope as the question bank, articles through the same draft
  rule, cards through deck ownership, images through library grants. A search
  page that had its own idea of who may see what is how private content leaks.
* A section match is reported under its article, not beside it. Ten sections of
  one article are one result with ten places to start reading, not ten results
  that bury everything else.
* The results page is reranked by a cross-encoder; the typeahead below it is
  not. A page is a choice being made, and worth a third of a second to get
  right. A typeahead is a word being finished, runs on every keystroke, and has
  nothing to judge yet — it is prefix matching on titles, which is exactly what
  finishing a word wants.
"""
import logging
import re

from fastapi import APIRouter, Depends, Query
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Session

from app.database import get_db
from app.models.article import Article
from app.models.flashcard import Flashcard, FlashcardDeck
from app.models.media import MediaAsset
from app.models.question import Question
from app.models.user import User
from app.routers.media import readable_libraries
from app.services.quiz_builder import bank_query, exam_scope_predicate
from app.services.search_service import article_ids_with_sections, hybrid_ids, rerank_ids
from app.utils.auth import get_current_user

router = APIRouter()
log = logging.getLogger(__name__)

KINDS = ("article", "question", "flashcard", "media")
# Per-corpus retrieval pool. Wider than what is shown, because visibility
# filtering happens after ranking and can empty a page that had hits.
POOL = 60


def _strip_markup(value: str | None) -> str:
    text = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", value or "")
    text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"^\s{0,3}#{1,6}\s*", "", text, flags=re.M)
    text = re.sub(r"[*_`>|]", " ", text)
    return re.sub(r"\s+", " ", text).strip()


def _snippet(value: str | None, query: str, width: int = 180) -> str:
    """A window of the text around the first query word that appears in it.

    Showing the opening of every document makes results look identical; showing
    where the match is tells you whether it is the one you meant.
    """
    text = _strip_markup(value)
    if not text:
        return ""
    for word in sorted((w for w in re.findall(r"\w{4,}", query.lower())), key=len, reverse=True):
        found = text.lower().find(word)
        if found >= 0:
            start = max(0, found - width // 3)
            piece = text[start:start + width]
            return ("…" if start else "") + piece.strip() + ("…" if start + width < len(text) else "")
    return text[:width] + ("…" if len(text) > width else "")


def _ordered(rows, ranked: list[int]):
    """Rows back in the order retrieval put them, not the order the DB returned."""
    position = {row_id: index for index, row_id in enumerate(ranked)}
    return sorted(rows, key=lambda row: position.get(row.id, len(position)))


def _articles(db, user, q, limit):
    # A section match belongs to its article, so both rankers feed one result.
    wanted, by_article = article_ids_with_sections(db, q, limit=POOL)
    if not wanted:
        return []
    rows = db.query(Article).filter(Article.id.in_(wanted)).all()
    if not user.is_moderator:
        rows = [a for a in rows if a.status == "published" or a.user_id == user.id]
    results = []
    for article in _ordered(rows, wanted)[:limit]:
        results.append({
            "id": article.id,
            "slug": article.slug,
            "title": article.title,
            "snippet": _snippet(article.summary or article.content, q),
            "status": article.status,
            "section_count": len(article.sections or []),
            "sections": [
                {"section_id": s.section_id, "title": s.title, "snippet": _snippet(s.content, q)}
                for s in by_article.get(article.id, [])[:4]
            ],
        })
    return results


def _questions(db, user, q, limit):
    ranked, semantic = hybrid_ids(db, q, "question", limit=POOL)
    if not ranked:
        return []
    query = bank_query(db, user).filter(Question.id.in_(ranked))
    scope = exam_scope_predicate(db, user)
    if scope is not None:
        query = query.filter(scope)
    # Reranked after the visibility rules, not before: spending a cross-encoder's
    # candidate slots on rows this learner will never be shown is how a shortlist
    # of fifty turns into a page of three.
    visible = {row.id: row for row in query.all()}
    order = rerank_ids(db, q, "question", [row_id for row_id in ranked if row_id in visible])
    rows = [visible[row_id] for row_id in order][:limit]
    return [{
        "id": row.id,
        "snippet": _snippet(row.question_text, q),
        "difficulty": row.difficulty,
        # Worth saying: a hit nobody's words predicted came from the meaning.
        "match": "semantic" if row.id in semantic else "keyword",
    } for row in rows]


def _flashcards(db, user, q, limit):
    ranked, _ = hybrid_ids(db, q, "flashcard", limit=POOL)
    if not ranked:
        return []
    own = [d.id for d in db.query(FlashcardDeck.id).filter(
        FlashcardDeck.user_id == user.id, FlashcardDeck.deleted_at.is_(None)).all()]
    if not own:
        return []
    rows = db.query(Flashcard).filter(
        Flashcard.id.in_(ranked), Flashcard.deck_id.in_(own)).all()
    return [{"id": row.id, "deck_id": row.deck_id, "front": row.front,
             "snippet": _snippet(row.back, q)} for row in _ordered(rows, ranked)[:limit]]


def _media(db, user, q, limit):
    ranked, _ = hybrid_ids(db, q, "media", limit=POOL)
    if not ranked:
        return []
    query = db.query(MediaAsset).filter(MediaAsset.id.in_(ranked))
    scope = readable_libraries(db, user)
    if scope is not None:
        query = query.filter(MediaAsset.library_id.in_(scope or {0}))
    rows = _ordered(query.all(), ranked)[:limit]
    return [{"id": row.id, "path": row.path, "title": row.title,
             "snippet": _snippet(row.caption or row.alt_text, q)} for row in rows]


FINDERS = {"article": _articles, "question": _questions,
           "flashcard": _flashcards, "media": _media}


@router.get("")
def search_everything(
    q: str = Query("", max_length=500),
    kinds: str | None = Query(None, description="Comma-separated subset of article,question,flashcard,media"),
    limit: int = Query(10, ge=1, le=50),
    db: Session = Depends(get_db),
    current_user: User = Depends(get_current_user),
):
    """Search every corpus at once, grouped by what was found."""
    query_text = (q or "").strip()
    wanted = [k for k in (kinds.split(",") if kinds else KINDS) if k in FINDERS]
    empty = {"query": query_text, "results": {kind: [] for kind in wanted}, "total": 0}
    if len(query_text) < 2:
        return empty

    results = {}
    for kind in wanted:
        try:
            results[kind] = FINDERS[kind](db, current_user, query_text, limit)
        except Exception:
            # One corpus failing is a gap in the answer, not the end of it.
            log.warning("Search failed for %s", kind, exc_info=True)
            results[kind] = []
    return {"query": query_text, "results": results,
            "total": sum(len(rows) for rows in results.values())}


@router.get("/suggest")
def suggest(
    q: str = Query("", max_length=200),
    limit: int = Query(6, ge=1, le=12),
    db: Session = Depends(get_db),
    current_user: User = Depends(get_current_user),
):
    """Titles for a typeahead — cheap enough to run on every keystroke.

    Deliberately lexical and prefix-based: a typeahead is finishing the word you
    are typing, and a semantic neighbour of half a word is noise. No embedding
    and no reranker for the same reason — both would be asked to judge relevance
    to half a word, at a round trip each, several times a second.
    """
    query_text = (q or "").strip()
    if len(query_text) < 2:
        return {"suggestions": []}
    pattern = f"%{query_text.lower()}%"
    rows = db.execute(sa_text("""
        SELECT id, slug, title, status, user_id FROM articles
        WHERE lower(title) LIKE :pattern
        ORDER BY CASE WHEN lower(title) LIKE :prefix THEN 0 ELSE 1 END, length(title), title
        LIMIT :limit
    """), {"pattern": pattern, "prefix": f"{query_text.lower()}%", "limit": limit * 2}).fetchall()
    visible = [r for r in rows
               if r.status == "published" or current_user.is_moderator or r.user_id == current_user.id]
    return {"suggestions": [{"kind": "article", "id": r.id, "slug": r.slug, "title": r.title}
                            for r in visible[:limit]]}