feat: section-level index so retrieval can cite a section, not a whole article

An article embedded as a single vector, which finds the article but not the
paragraph — so a citation could only ever point at the top of a page. Sections
live in a JSON column and cannot carry a vector or a full-text index, so they are
now projected into `article_section_index`: one row per section with its own
embedding and weighted tsvector (migration x6d7e8f9a0b1).

- Rows are keyed by section id, so editing a section updates it, removing one
  deletes it, and an unchanged section is not re-embedded on every save.
- `article_section` joins the embeddable kinds, so the retry task, the full
  regeneration and the health report cover it without further changes.
- `hybrid_ids(db, query, "article_section")` searches it like any other corpus.

This is the groundwork for grouped search results (article, then the sections
that matched) and for AI citations that deep-link to the right section.

Tests: 113 backend green.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PpfzbZ1QTLMeVYxM2kyq8m

backend/alembic/versions/x6d7e8f9a0b1_section_index.py

@@ -0,0 +1,42 @@
+"""Section-level index for articles, so a citation can point at the right part.
+
+An article embeds as one vector, which is enough to find the article but not the
+paragraph. Sections live in a JSON column, so they are projected into a derived
+table that can carry its own vector and full-text index.
+
+Revision ID: x6d7e8f9a0b1
+Revises: w5c6d7e8f9a0
+"""
+from alembic import op
+
+revision = "x6d7e8f9a0b1"
+down_revision = "w5c6d7e8f9a0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("""
+    CREATE TABLE IF NOT EXISTS article_section_index (
+        id SERIAL PRIMARY KEY,
+        article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
+        section_id VARCHAR(64) NOT NULL,
+        title TEXT,
+        content TEXT,
+        embedding vector(1024),
+        embedding_model VARCHAR(120),
+        embedded_at TIMESTAMP,
+        search_vector tsvector GENERATED ALWAYS AS (
+            setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
+            setweight(to_tsvector('english', coalesce(content, '')), 'B')
+        ) STORED,
+        CONSTRAINT uq_article_section_index UNIQUE (article_id, section_id)
+    )
+    """)
+    op.execute("CREATE INDEX IF NOT EXISTS ix_asi_search ON article_section_index USING GIN (search_vector)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_asi_article ON article_section_index(article_id)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_asi_model ON article_section_index(embedding_model)")
+
+
+def downgrade():
+    op.execute("DROP TABLE IF EXISTS article_section_index")

backend/app/models/article.py

@@ -56,3 +56,20 @@ class ArticleView(Base):
     user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
     article_id = Column(Integer, ForeignKey("articles.id", ondelete="CASCADE"), nullable=False)
     viewed_at = Column(DateTime, default=datetime.utcnow)
+
+
+class ArticleSectionIndex(Base, Embeddable):
+    """One row per article section, so retrieval can cite a section not a whole article.
+
+    Sections are stored in `Article.sections` as JSON; this is a projection of
+    them, rebuilt whenever the article is saved.
+    """
+
+    __tablename__ = "article_section_index"
+    __table_args__ = (UniqueConstraint("article_id", "section_id", name="uq_article_section_index"),)
+
+    id = Column(Integer, primary_key=True, index=True)
+    article_id = Column(Integer, ForeignKey("articles.id", ondelete="CASCADE"), nullable=False, index=True)
+    section_id = Column(String(64), nullable=False)
+    title = Column(Text, nullable=True)
+    content = Column(Text, nullable=True)

backend/app/routers/articles.py

@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
 from app.database import get_db
 from app.services.search_service import hybrid_ids
 from app.services import embedding_service
-from app.models.article import Article, ArticleView, QuestionArticleLink
+from app.models.article import Article, ArticleSectionIndex, ArticleView, QuestionArticleLink
 from app.models.flashcard import Flashcard, FlashcardDeck, FlashcardArticleLink
 from app.models.question import Question
 from app.models.section import Section
@@ -163,15 +163,51 @@ def _record_view(db, user, article) -> None:
 
 
 def _reembed(db, article) -> None:
-    """Embed on write. A failure is not fatal — the retry task sweeps it up."""
+    """Embed the article and reproject its sections. Failures wait for the retry task."""
     try:
-        if embedding_service.embed_record(article, "article"):
+        embedding_service.embed_record(article, "article")
-            db.commit()
+        _rebuild_section_index(db, article)
+        db.commit()
     except Exception:
         db.rollback()
         log.warning("Could not embed article %s; leaving it for the retry task", article.id, exc_info=True)
 
 
+def _rebuild_section_index(db, article) -> None:
+    """Mirror the article's JSON sections into their own searchable rows.
+
+    Sections live in a JSON column, so they cannot carry a vector or a full-text
+    index themselves. Projecting them lets a citation point at the right section
+    rather than the whole article. Rows are keyed by section id, so editing a
+    section updates it and removing one deletes it.
+    """
+    sections = article.sections or []
+    keep = set()
+    for section in sections:
+        section_id = section.get("id")
+        if not section_id:
+            continue
+        keep.add(section_id)
+        row = db.query(ArticleSectionIndex).filter_by(
+            article_id=article.id, section_id=section_id).first()
+        text_changed = True
+        if row is None:
+            row = ArticleSectionIndex(article_id=article.id, section_id=section_id)
+            db.add(row)
+        else:
+            text_changed = (row.title != section.get("title")) or (row.content != section.get("content"))
+        row.title = section.get("title")
+        row.content = section.get("content")
+        # Only pay for an embedding when the text actually changed.
+        if text_changed or row.embedding is None:
+            embedding_service.embed_record(row, "article_section")
+
+    stale = db.query(ArticleSectionIndex).filter(ArticleSectionIndex.article_id == article.id)
+    if keep:
+        stale = stale.filter(~ArticleSectionIndex.section_id.in_(keep))
+    stale.delete(synchronize_session=False)
+
+
 @router.get("/")
 def list_articles(
     category_id: int | None = Query(None),

backend/app/services/embedding_service.py

@@ -111,6 +111,7 @@ EMBEDDABLE = {
     "question": lambda row: _join(row.question_text, *(row.options or [])),
     "article": lambda row: _join(row.title, row.summary, row.content),
     "flashcard": lambda row: _join(row.front, row.back),
+    "article_section": lambda row: _join(row.title, row.content),
 }
 
 
@@ -149,11 +150,12 @@ def embed_question(question) -> bool:
 
 def embeddable_models() -> dict:
     """The ORM class behind each embeddable kind."""
-    from app.models.article import Article
+    from app.models.article import Article, ArticleSectionIndex
     from app.models.flashcard import Flashcard
     from app.models.question import Question
 
-    return {"question": Question, "article": Article, "flashcard": Flashcard}
+    return {"question": Question, "article": Article, "flashcard": Flashcard,
+            "article_section": ArticleSectionIndex}
 
 
 def stale_embedding_counts(db) -> dict:

backend/app/services/search_service.py

@@ -77,6 +77,7 @@ CORPORA = {
     "question": ("questions", ("question_text", "CAST(options AS TEXT)")),
     "article": ("articles", ("title", "summary", "content")),
     "flashcard": ("flashcards", ("front", "back")),
+    "article_section": ("article_section_index", ("title", "content")),
 }