feat: media as a searchable corpus, ready for a vision embedder
Images were findable only by the filename someone typed. `media_assets` gives them a title, caption, alt text, a category on the shared tree and tags, with a weighted tsvector so they are searchable now (migration y7e8f9a0b1c2). The embedding column is filled from the caption today. A vision-capable model can fill it from the image itself later without another migration — and because `embedding_model` stamps every vector, a text-embedded caption and a vision-embedded image stay distinguishable instead of being silently mixed in one index. Adding "media" to the embeddable kinds is all the retry task, the full regeneration and the health report needed. `media_tag_links.tag_id` carries no ORM-level foreign key: `question_tags` is created by raw DDL rather than a model, so the constraint lives in the migration where the table actually exists. Tests: 113 backend green. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PpfzbZ1QTLMeVYxM2kyq8m
This commit is contained in:
parent
4d0cdc8f2f
commit
899ad5e879
6 changed files with 105 additions and 2 deletions
57
backend/alembic/versions/y7e8f9a0b1c2_media_assets.py
Normal file
57
backend/alembic/versions/y7e8f9a0b1c2_media_assets.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
"""Media as a searchable corpus, ready for a vision embedder.
|
||||
|
||||
Images are currently found only by the filename someone typed. This gives them a
|
||||
caption, alt text and tags to search on now, and an embedding column that a
|
||||
vision-capable model can fill later without another migration — `embedding_model`
|
||||
records which model produced each vector, so a text-embedded caption and a
|
||||
vision-embedded image are distinguishable rather than silently mixed.
|
||||
|
||||
Revision ID: y7e8f9a0b1c2
|
||||
Revises: x6d7e8f9a0b1
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
revision = "y7e8f9a0b1c2"
|
||||
down_revision = "x6d7e8f9a0b1"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS media_assets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
path VARCHAR(500) UNIQUE NOT NULL,
|
||||
title VARCHAR(300),
|
||||
caption TEXT,
|
||||
alt_text TEXT,
|
||||
kind VARCHAR(20) NOT NULL DEFAULT 'image',
|
||||
category_id INTEGER REFERENCES question_categories(id) ON DELETE SET NULL,
|
||||
user_id INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
embedding vector(1024),
|
||||
embedding_model VARCHAR(120),
|
||||
embedded_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
search_vector tsvector GENERATED ALWAYS AS (
|
||||
setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(caption, '')), 'B') ||
|
||||
setweight(to_tsvector('english', coalesce(alt_text, '')), 'C')
|
||||
) STORED
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_media_search ON media_assets USING GIN (search_vector)")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_media_model ON media_assets(embedding_model)")
|
||||
op.execute("""
|
||||
CREATE TABLE IF NOT EXISTS media_tag_links (
|
||||
id SERIAL PRIMARY KEY,
|
||||
media_id INTEGER NOT NULL REFERENCES media_assets(id) ON DELETE CASCADE,
|
||||
tag_id INTEGER NOT NULL REFERENCES question_tags(id) ON DELETE CASCADE,
|
||||
CONSTRAINT uq_media_tag UNIQUE (media_id, tag_id)
|
||||
)
|
||||
""")
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_media_tag_media ON media_tag_links(media_id)")
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute("DROP TABLE IF EXISTS media_tag_links")
|
||||
op.execute("DROP TABLE IF EXISTS media_assets")
|
||||
|
|
@ -168,7 +168,7 @@ def setup_pgvector():
|
|||
# Import new models so create_all picks them up
|
||||
from app.models import quiz_category, quiz_question_link, question_category, favorite # noqa
|
||||
from app.models import flashcard, course # noqa
|
||||
from app.models import category_grant # noqa
|
||||
from app.models import category_grant, exam, media # noqa
|
||||
|
||||
# Kill stale idle-in-transaction connections from previous killed startups.
|
||||
# They hold DDL locks and cause ALTER TABLE below to hang indefinitely.
|
||||
|
|
|
|||
38
backend/app/models/media.py
Normal file
38
backend/app/models/media.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
from app.database import Base
|
||||
from app.models.embeddable import Embeddable
|
||||
|
||||
|
||||
class MediaAsset(Base, Embeddable):
|
||||
"""An image in the bank, searchable by what it shows rather than its filename.
|
||||
|
||||
The embedding column is filled from the caption today. A vision-capable model
|
||||
can fill it from the image itself later; `embedding_model` records which one
|
||||
produced each vector, so the two are never mixed silently.
|
||||
"""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
path = Column(String(500), unique=True, nullable=False)
|
||||
title = Column(String(300), nullable=True)
|
||||
caption = Column(Text, nullable=True)
|
||||
alt_text = Column(Text, nullable=True)
|
||||
kind = Column(String(20), default="image")
|
||||
category_id = Column(Integer, ForeignKey("question_categories.id", ondelete="SET NULL"), nullable=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class MediaTagLink(Base):
|
||||
__tablename__ = "media_tag_links"
|
||||
__table_args__ = (UniqueConstraint("media_id", "tag_id", name="uq_media_tag"),)
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
media_id = Column(Integer, ForeignKey("media_assets.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
# `question_tags` is created by raw DDL rather than an ORM model, so the
|
||||
# constraint is declared in the migration; the mapper only stores the id.
|
||||
tag_id = Column(Integer, nullable=False, index=True)
|
||||
|
|
@ -112,6 +112,9 @@ EMBEDDABLE = {
|
|||
"article": lambda row: _join(row.title, row.summary, row.content),
|
||||
"flashcard": lambda row: _join(row.front, row.back),
|
||||
"article_section": lambda row: _join(row.title, row.content),
|
||||
# Text today; a vision model can embed the image itself later without
|
||||
# changing anything here but this line.
|
||||
"media": lambda row: _join(row.title, row.caption, row.alt_text),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -150,12 +153,15 @@ def embed_question(question) -> bool:
|
|||
|
||||
def embeddable_models() -> dict:
|
||||
"""The ORM class behind each embeddable kind."""
|
||||
from app.models import media # noqa — registers media_assets on the mapper.
|
||||
from app.models.article import Article, ArticleSectionIndex
|
||||
from app.models.flashcard import Flashcard
|
||||
from app.models.question import Question
|
||||
|
||||
from app.models.media import MediaAsset
|
||||
|
||||
return {"question": Question, "article": Article, "flashcard": Flashcard,
|
||||
"article_section": ArticleSectionIndex}
|
||||
"article_section": ArticleSectionIndex, "media": MediaAsset}
|
||||
|
||||
|
||||
def stale_embedding_counts(db) -> dict:
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ CORPORA = {
|
|||
"article": ("articles", ("title", "summary", "content")),
|
||||
"flashcard": ("flashcards", ("front", "back")),
|
||||
"article_section": ("article_section_index", ("title", "content")),
|
||||
"media": ("media_assets", ("title", "caption", "alt_text")),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from sqlalchemy.pool import StaticPool
|
|||
|
||||
from app.database import Base
|
||||
from app.models.course import Course # noqa — Quiz.course_id FK needs the table in metadata.
|
||||
from app.models.media import MediaAsset # noqa — the health report walks every embeddable table.
|
||||
from app.models.question import Question
|
||||
from app.models.user import User
|
||||
from app.services import search_service
|
||||
|
|
|
|||
Loading…
Reference in a new issue