"""One query across everything a learner can see. Each corpus already has its own retrieval and its own rules about who may see what. This endpoint runs them together and returns one answer, rather than making somebody guess which of five pages holds the thing they are looking for. Two decisions worth stating: * Visibility is never re-implemented here. Questions go through the same bank predicate and exam scope as the question bank, articles through the same draft rule, cards through deck ownership, images through library grants. A search page that had its own idea of who may see what is how private content leaks. * A section match is reported under its article, not beside it. Ten sections of one article are one result with ten places to start reading, not ten results that bury everything else. * The results page is reranked by a cross-encoder; the typeahead below it is not. A page is a choice being made, and worth a third of a second to get right. A typeahead is a word being finished, runs on every keystroke, and has nothing to judge yet — it is prefix matching on titles, which is exactly what finishing a word wants. """ import logging import re from fastapi import APIRouter, Depends, Query from sqlalchemy import text as sa_text from sqlalchemy.orm import Session from app.database import get_db from app.models.article import Article from app.models.flashcard import Flashcard, FlashcardDeck from app.models.media import MediaAsset from app.models.question import Question from app.models.user import User from app.routers.media import readable_libraries from app.services.quiz_builder import bank_query, exam_scope_predicate from app.services.search_service import article_ids_with_sections, hybrid_ids, rerank_ids from app.utils.auth import get_current_user router = APIRouter() log = logging.getLogger(__name__) KINDS = ("article", "question", "flashcard", "media") # Per-corpus retrieval pool. Wider than what is shown, because visibility # filtering happens after ranking and can empty a page that had hits. POOL = 60 def _strip_markup(value: str | None) -> str: text = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", value or "") text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"^\s{0,3}#{1,6}\s*", "", text, flags=re.M) text = re.sub(r"[*_`>|]", " ", text) return re.sub(r"\s+", " ", text).strip() def _snippet(value: str | None, query: str, width: int = 180) -> str: """A window of the text around the first query word that appears in it. Showing the opening of every document makes results look identical; showing where the match is tells you whether it is the one you meant. """ text = _strip_markup(value) if not text: return "" for word in sorted((w for w in re.findall(r"\w{4,}", query.lower())), key=len, reverse=True): found = text.lower().find(word) if found >= 0: start = max(0, found - width // 3) piece = text[start:start + width] return ("…" if start else "") + piece.strip() + ("…" if start + width < len(text) else "") return text[:width] + ("…" if len(text) > width else "") def _ordered(rows, ranked: list[int]): """Rows back in the order retrieval put them, not the order the DB returned.""" position = {row_id: index for index, row_id in enumerate(ranked)} return sorted(rows, key=lambda row: position.get(row.id, len(position))) def _articles(db, user, q, limit): # A section match belongs to its article, so both rankers feed one result. wanted, by_article = article_ids_with_sections(db, q, limit=POOL) if not wanted: return [] rows = db.query(Article).filter(Article.id.in_(wanted)).all() if not user.is_moderator: rows = [a for a in rows if a.status == "published" or a.user_id == user.id] results = [] for article in _ordered(rows, wanted)[:limit]: results.append({ "id": article.id, "slug": article.slug, "title": article.title, "snippet": _snippet(article.summary or article.content, q), "status": article.status, "section_count": len(article.sections or []), "sections": [ {"section_id": s.section_id, "title": s.title, "snippet": _snippet(s.content, q)} for s in by_article.get(article.id, [])[:4] ], }) return results def _questions(db, user, q, limit): ranked, semantic = hybrid_ids(db, q, "question", limit=POOL) if not ranked: return [] query = bank_query(db, user).filter(Question.id.in_(ranked)) scope = exam_scope_predicate(db, user) if scope is not None: query = query.filter(scope) # Reranked after the visibility rules, not before: spending a cross-encoder's # candidate slots on rows this learner will never be shown is how a shortlist # of fifty turns into a page of three. visible = {row.id: row for row in query.all()} order = rerank_ids(db, q, "question", [row_id for row_id in ranked if row_id in visible]) rows = [visible[row_id] for row_id in order][:limit] return [{ "id": row.id, "snippet": _snippet(row.question_text, q), "difficulty": row.difficulty, # Worth saying: a hit nobody's words predicted came from the meaning. "match": "semantic" if row.id in semantic else "keyword", } for row in rows] def _flashcards(db, user, q, limit): ranked, _ = hybrid_ids(db, q, "flashcard", limit=POOL) if not ranked: return [] own = [d.id for d in db.query(FlashcardDeck.id).filter( FlashcardDeck.user_id == user.id, FlashcardDeck.deleted_at.is_(None)).all()] if not own: return [] rows = db.query(Flashcard).filter( Flashcard.id.in_(ranked), Flashcard.deck_id.in_(own)).all() return [{"id": row.id, "deck_id": row.deck_id, "front": row.front, "snippet": _snippet(row.back, q)} for row in _ordered(rows, ranked)[:limit]] def _media(db, user, q, limit): ranked, _ = hybrid_ids(db, q, "media", limit=POOL) if not ranked: return [] query = db.query(MediaAsset).filter(MediaAsset.id.in_(ranked)) scope = readable_libraries(db, user) if scope is not None: query = query.filter(MediaAsset.library_id.in_(scope or {0})) rows = _ordered(query.all(), ranked)[:limit] return [{"id": row.id, "path": row.path, "title": row.title, "snippet": _snippet(row.caption or row.alt_text, q)} for row in rows] FINDERS = {"article": _articles, "question": _questions, "flashcard": _flashcards, "media": _media} @router.get("") def search_everything( q: str = Query("", max_length=500), kinds: str | None = Query(None, description="Comma-separated subset of article,question,flashcard,media"), limit: int = Query(10, ge=1, le=50), db: Session = Depends(get_db), current_user: User = Depends(get_current_user), ): """Search every corpus at once, grouped by what was found.""" query_text = (q or "").strip() wanted = [k for k in (kinds.split(",") if kinds else KINDS) if k in FINDERS] empty = {"query": query_text, "results": {kind: [] for kind in wanted}, "total": 0} if len(query_text) < 2: return empty results = {} for kind in wanted: try: results[kind] = FINDERS[kind](db, current_user, query_text, limit) except Exception: # One corpus failing is a gap in the answer, not the end of it. log.warning("Search failed for %s", kind, exc_info=True) results[kind] = [] return {"query": query_text, "results": results, "total": sum(len(rows) for rows in results.values())} @router.get("/suggest") def suggest( q: str = Query("", max_length=200), limit: int = Query(6, ge=1, le=12), db: Session = Depends(get_db), current_user: User = Depends(get_current_user), ): """Titles for a typeahead — cheap enough to run on every keystroke. Deliberately lexical and prefix-based: a typeahead is finishing the word you are typing, and a semantic neighbour of half a word is noise. No embedding and no reranker for the same reason — both would be asked to judge relevance to half a word, at a round trip each, several times a second. """ query_text = (q or "").strip() if len(query_text) < 2: return {"suggestions": []} pattern = f"%{query_text.lower()}%" rows = db.execute(sa_text(""" SELECT id, slug, title, status, user_id FROM articles WHERE lower(title) LIKE :pattern ORDER BY CASE WHEN lower(title) LIKE :prefix THEN 0 ELSE 1 END, length(title), title LIMIT :limit """), {"pattern": pattern, "prefix": f"{query_text.lower()}%", "limit": limit * 2}).fetchall() visible = [r for r in rows if r.status == "published" or current_user.is_moderator or r.user_id == current_user.id] return {"suggestions": [{"kind": "article", "id": r.id, "slug": r.slug, "title": r.title} for r in visible[:limit]]}