From 899ad5e879613615e9897e4d3d8c4ac72c72901b Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Sep 2026 02:22:52 +0200 Subject: [PATCH] feat: media as a searchable corpus, ready for a vision embedder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images were findable only by the filename someone typed. `media_assets` gives them a title, caption, alt text, a category on the shared tree and tags, with a weighted tsvector so they are searchable now (migration y7e8f9a0b1c2). The embedding column is filled from the caption today. A vision-capable model can fill it from the image itself later without another migration — and because `embedding_model` stamps every vector, a text-embedded caption and a vision-embedded image stay distinguishable instead of being silently mixed in one index. Adding "media" to the embeddable kinds is all the retry task, the full regeneration and the health report needed. `media_tag_links.tag_id` carries no ORM-level foreign key: `question_tags` is created by raw DDL rather than a model, so the constraint lives in the migration where the table actually exists. Tests: 113 backend green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PpfzbZ1QTLMeVYxM2kyq8m --- .../versions/y7e8f9a0b1c2_media_assets.py | 57 +++++++++++++++++++ backend/app/main.py | 2 +- backend/app/models/media.py | 38 +++++++++++++ backend/app/services/embedding_service.py | 8 ++- backend/app/services/search_service.py | 1 + backend/tests/test_hybrid_search.py | 1 + 6 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 backend/alembic/versions/y7e8f9a0b1c2_media_assets.py create mode 100644 backend/app/models/media.py diff --git a/backend/alembic/versions/y7e8f9a0b1c2_media_assets.py b/backend/alembic/versions/y7e8f9a0b1c2_media_assets.py new file mode 100644 index 0000000..54b13bf --- /dev/null +++ b/backend/alembic/versions/y7e8f9a0b1c2_media_assets.py @@ -0,0 +1,57 @@ +"""Media as a searchable corpus, ready for a vision embedder. + +Images are currently found only by the filename someone typed. This gives them a +caption, alt text and tags to search on now, and an embedding column that a +vision-capable model can fill later without another migration — `embedding_model` +records which model produced each vector, so a text-embedded caption and a +vision-embedded image are distinguishable rather than silently mixed. + +Revision ID: y7e8f9a0b1c2 +Revises: x6d7e8f9a0b1 +""" +from alembic import op + +revision = "y7e8f9a0b1c2" +down_revision = "x6d7e8f9a0b1" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute(""" + CREATE TABLE IF NOT EXISTS media_assets ( + id SERIAL PRIMARY KEY, + path VARCHAR(500) UNIQUE NOT NULL, + title VARCHAR(300), + caption TEXT, + alt_text TEXT, + kind VARCHAR(20) NOT NULL DEFAULT 'image', + category_id INTEGER REFERENCES question_categories(id) ON DELETE SET NULL, + user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + embedding vector(1024), + embedding_model VARCHAR(120), + embedded_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + search_vector tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english', coalesce(title, '')), 'A') || + setweight(to_tsvector('english', coalesce(caption, '')), 'B') || + setweight(to_tsvector('english', coalesce(alt_text, '')), 'C') + ) STORED + ) + """) + op.execute("CREATE INDEX IF NOT EXISTS ix_media_search ON media_assets USING GIN (search_vector)") + op.execute("CREATE INDEX IF NOT EXISTS ix_media_model ON media_assets(embedding_model)") + op.execute(""" + CREATE TABLE IF NOT EXISTS media_tag_links ( + id SERIAL PRIMARY KEY, + media_id INTEGER NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE, + tag_id INTEGER NOT NULL REFERENCES question_tags(id) ON DELETE CASCADE, + CONSTRAINT uq_media_tag UNIQUE (media_id, tag_id) + ) + """) + op.execute("CREATE INDEX IF NOT EXISTS ix_media_tag_media ON media_tag_links(media_id)") + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS media_tag_links") + op.execute("DROP TABLE IF EXISTS media_assets") diff --git a/backend/app/main.py b/backend/app/main.py index 76f0f72..97f490d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -168,7 +168,7 @@ def setup_pgvector(): # Import new models so create_all picks them up from app.models import quiz_category, quiz_question_link, question_category, favorite # noqa from app.models import flashcard, course # noqa - from app.models import category_grant # noqa + from app.models import category_grant, exam, media # noqa # Kill stale idle-in-transaction connections from previous killed startups. # They hold DDL locks and cause ALTER TABLE below to hang indefinitely. diff --git a/backend/app/models/media.py b/backend/app/models/media.py new file mode 100644 index 0000000..ff665ec --- /dev/null +++ b/backend/app/models/media.py @@ -0,0 +1,38 @@ +from datetime import datetime + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint + +from app.database import Base +from app.models.embeddable import Embeddable + + +class MediaAsset(Base, Embeddable): + """An image in the bank, searchable by what it shows rather than its filename. + + The embedding column is filled from the caption today. A vision-capable model + can fill it from the image itself later; `embedding_model` records which one + produced each vector, so the two are never mixed silently. + """ + + __tablename__ = "media_assets" + + id = Column(Integer, primary_key=True, index=True) + path = Column(String(500), unique=True, nullable=False) + title = Column(String(300), nullable=True) + caption = Column(Text, nullable=True) + alt_text = Column(Text, nullable=True) + kind = Column(String(20), default="image") + category_id = Column(Integer, ForeignKey("question_categories.id", ondelete="SET NULL"), nullable=True) + user_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + + +class MediaTagLink(Base): + __tablename__ = "media_tag_links" + __table_args__ = (UniqueConstraint("media_id", "tag_id", name="uq_media_tag"),) + + id = Column(Integer, primary_key=True, index=True) + media_id = Column(Integer, ForeignKey("media_assets.id", ondelete="CASCADE"), nullable=False, index=True) + # `question_tags` is created by raw DDL rather than an ORM model, so the + # constraint is declared in the migration; the mapper only stores the id. + tag_id = Column(Integer, nullable=False, index=True) diff --git a/backend/app/services/embedding_service.py b/backend/app/services/embedding_service.py index adbc613..fb9950d 100644 --- a/backend/app/services/embedding_service.py +++ b/backend/app/services/embedding_service.py @@ -112,6 +112,9 @@ EMBEDDABLE = { "article": lambda row: _join(row.title, row.summary, row.content), "flashcard": lambda row: _join(row.front, row.back), "article_section": lambda row: _join(row.title, row.content), + # Text today; a vision model can embed the image itself later without + # changing anything here but this line. + "media": lambda row: _join(row.title, row.caption, row.alt_text), } @@ -150,12 +153,15 @@ def embed_question(question) -> bool: def embeddable_models() -> dict: """The ORM class behind each embeddable kind.""" + from app.models import media # noqa — registers media_assets on the mapper. from app.models.article import Article, ArticleSectionIndex from app.models.flashcard import Flashcard from app.models.question import Question + from app.models.media import MediaAsset + return {"question": Question, "article": Article, "flashcard": Flashcard, - "article_section": ArticleSectionIndex} + "article_section": ArticleSectionIndex, "media": MediaAsset} def stale_embedding_counts(db) -> dict: diff --git a/backend/app/services/search_service.py b/backend/app/services/search_service.py index 81708f0..b4c5766 100644 --- a/backend/app/services/search_service.py +++ b/backend/app/services/search_service.py @@ -78,6 +78,7 @@ CORPORA = { "article": ("articles", ("title", "summary", "content")), "flashcard": ("flashcards", ("front", "back")), "article_section": ("article_section_index", ("title", "content")), + "media": ("media_assets", ("title", "caption", "alt_text")), } diff --git a/backend/tests/test_hybrid_search.py b/backend/tests/test_hybrid_search.py index 751acae..2116faf 100644 --- a/backend/tests/test_hybrid_search.py +++ b/backend/tests/test_hybrid_search.py @@ -14,6 +14,7 @@ from sqlalchemy.pool import StaticPool from app.database import Base from app.models.course import Course # noqa — Quiz.course_id FK needs the table in metadata. +from app.models.media import MediaAsset # noqa — the health report walks every embeddable table. from app.models.question import Question from app.models.user import User from app.services import search_service