diff --git a/backend/alembic/versions/x6d7e8f9a0b1_section_index.py b/backend/alembic/versions/x6d7e8f9a0b1_section_index.py new file mode 100644 index 0000000..3665478 --- /dev/null +++ b/backend/alembic/versions/x6d7e8f9a0b1_section_index.py @@ -0,0 +1,42 @@ +"""Section-level index for articles, so a citation can point at the right part. + +An article embeds as one vector, which is enough to find the article but not the +paragraph. Sections live in a JSON column, so they are projected into a derived +table that can carry its own vector and full-text index. + +Revision ID: x6d7e8f9a0b1 +Revises: w5c6d7e8f9a0 +""" +from alembic import op + +revision = "x6d7e8f9a0b1" +down_revision = "w5c6d7e8f9a0" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute(""" + CREATE TABLE IF NOT EXISTS article_section_index ( + id SERIAL PRIMARY KEY, + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + section_id VARCHAR(64) NOT NULL, + title TEXT, + content TEXT, + embedding vector(1024), + embedding_model VARCHAR(120), + embedded_at TIMESTAMP, + search_vector tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(title, '')), 'A') || + setweight(to_tsvector('english', coalesce(content, '')), 'B') + ) STORED, + CONSTRAINT uq_article_section_index UNIQUE (article_id, section_id) + ) + """) + op.execute("CREATE INDEX IF NOT EXISTS ix_asi_search ON article_section_index USING GIN (search_vector)") + op.execute("CREATE INDEX IF NOT EXISTS ix_asi_article ON article_section_index(article_id)") + op.execute("CREATE INDEX IF NOT EXISTS ix_asi_model ON article_section_index(embedding_model)") + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS article_section_index") diff --git a/backend/app/models/article.py b/backend/app/models/article.py index 6dcf30a..954a4b9 100644 --- a/backend/app/models/article.py +++ b/backend/app/models/article.py @@ -56,3 +56,20 @@ class ArticleView(Base): user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) article_id = Column(Integer, ForeignKey("articles.id", ondelete="CASCADE"), nullable=False) viewed_at = Column(DateTime, default=datetime.utcnow) + + +class ArticleSectionIndex(Base, Embeddable): + """One row per article section, so retrieval can cite a section not a whole article. + + Sections are stored in `Article.sections` as JSON; this is a projection of + them, rebuilt whenever the article is saved. + """ + + __tablename__ = "article_section_index" + __table_args__ = (UniqueConstraint("article_id", "section_id", name="uq_article_section_index"),) + + id = Column(Integer, primary_key=True, index=True) + article_id = Column(Integer, ForeignKey("articles.id", ondelete="CASCADE"), nullable=False, index=True) + section_id = Column(String(64), nullable=False) + title = Column(Text, nullable=True) + content = Column(Text, nullable=True) diff --git a/backend/app/routers/articles.py b/backend/app/routers/articles.py index 61a74b7..912a0fb 100644 --- a/backend/app/routers/articles.py +++ b/backend/app/routers/articles.py @@ -8,7 +8,7 @@ from sqlalchemy.orm import Session from app.database import get_db from app.services.search_service import hybrid_ids from app.services import embedding_service -from app.models.article import Article, ArticleView, QuestionArticleLink +from app.models.article import Article, ArticleSectionIndex, ArticleView, QuestionArticleLink from app.models.flashcard import Flashcard, FlashcardDeck, FlashcardArticleLink from app.models.question import Question from app.models.section import Section @@ -163,15 +163,51 @@ def _record_view(db, user, article) -> None: def _reembed(db, article) -> None: - """Embed on write. A failure is not fatal — the retry task sweeps it up.""" + """Embed the article and reproject its sections. Failures wait for the retry task.""" try: - if embedding_service.embed_record(article, "article"): - db.commit() + embedding_service.embed_record(article, "article") + _rebuild_section_index(db, article) + db.commit() except Exception: db.rollback() log.warning("Could not embed article %s; leaving it for the retry task", article.id, exc_info=True) +def _rebuild_section_index(db, article) -> None: + """Mirror the article's JSON sections into their own searchable rows. + + Sections live in a JSON column, so they cannot carry a vector or a full-text + index themselves. Projecting them lets a citation point at the right section + rather than the whole article. Rows are keyed by section id, so editing a + section updates it and removing one deletes it. + """ + sections = article.sections or [] + keep = set() + for section in sections: + section_id = section.get("id") + if not section_id: + continue + keep.add(section_id) + row = db.query(ArticleSectionIndex).filter_by( + article_id=article.id, section_id=section_id).first() + text_changed = True + if row is None: + row = ArticleSectionIndex(article_id=article.id, section_id=section_id) + db.add(row) + else: + text_changed = (row.title != section.get("title")) or (row.content != section.get("content")) + row.title = section.get("title") + row.content = section.get("content") + # Only pay for an embedding when the text actually changed. + if text_changed or row.embedding is None: + embedding_service.embed_record(row, "article_section") + + stale = db.query(ArticleSectionIndex).filter(ArticleSectionIndex.article_id == article.id) + if keep: + stale = stale.filter(~ArticleSectionIndex.section_id.in_(keep)) + stale.delete(synchronize_session=False) + + @router.get("/") def list_articles( category_id: int | None = Query(None), diff --git a/backend/app/services/embedding_service.py b/backend/app/services/embedding_service.py index 6f4fcc6..adbc613 100644 --- a/backend/app/services/embedding_service.py +++ b/backend/app/services/embedding_service.py @@ -111,6 +111,7 @@ EMBEDDABLE = { "question": lambda row: _join(row.question_text, *(row.options or [])), "article": lambda row: _join(row.title, row.summary, row.content), "flashcard": lambda row: _join(row.front, row.back), + "article_section": lambda row: _join(row.title, row.content), } @@ -149,11 +150,12 @@ def embed_question(question) -> bool: def embeddable_models() -> dict: """The ORM class behind each embeddable kind.""" - from app.models.article import Article + from app.models.article import Article, ArticleSectionIndex from app.models.flashcard import Flashcard from app.models.question import Question - return {"question": Question, "article": Article, "flashcard": Flashcard} + return {"question": Question, "article": Article, "flashcard": Flashcard, + "article_section": ArticleSectionIndex} def stale_embedding_counts(db) -> dict: diff --git a/backend/app/services/search_service.py b/backend/app/services/search_service.py index c3261aa..81708f0 100644 --- a/backend/app/services/search_service.py +++ b/backend/app/services/search_service.py @@ -77,6 +77,7 @@ CORPORA = { "question": ("questions", ("question_text", "CAST(options AS TEXT)")), "article": ("articles", ("title", "summary", "content")), "flashcard": ("flashcards", ("front", "back")), + "article_section": ("article_section_index", ("title", "content")), }