pdf-quiz-generator/backend/scripts/index_question_images.py

"""Register the extracted question figures in the image bank.

The figures have always been in object storage and attached to questions by
path, but they were never rows in `media_assets` — so nothing could browse them,
tag them, search them by what they show, or offer them when attaching an image
to another question. This puts every one of them in the bank.

Nothing is copied or moved: an asset row points at the key the question already
uses, so a question's picture and its library entry are the same file.

Each figure is described from the question it belongs to, which is the only
description that exists until someone writes a better one. Figures no question
currently uses are still listed — several were detached during the stem/answer
triage and are exactly what somebody reviewing that decision needs to look at.

    docker compose exec backend python -m scripts.index_question_images
    docker compose exec backend python -m scripts.index_question_images --apply
"""
import sys
from collections import defaultdict

from sqlalchemy import text as sa_text

from app.database import SessionLocal
from app.models.media import MediaAsset, MediaLibrary, MediaTagLink
from app.services import storage_service

LIBRARY = "Question figures"
PREFIX = "images/"


def _library(db):
    library = db.query(MediaLibrary).filter(MediaLibrary.name == LIBRARY).first()
    if library is None:
        library = MediaLibrary(name=LIBRARY,
                               description="Figures extracted from the source exam PDFs.")
        db.add(library)
        db.flush()
    return library


def _stored_keys() -> list[str]:
    keys = []
    client = storage_service._s3()
    from app.config import settings

    for page in client.get_paginator("list_objects_v2").paginate(
            Bucket=settings.S3_BUCKET, Prefix=PREFIX):
        keys.extend(obj["Key"] for obj in page.get("Contents", []))
    return keys


def _usage(db) -> tuple[dict, dict]:
    """Which question each figure belongs to now, and which it used to."""
    current, historical = {}, {}
    for qid, path, category in db.execute(sa_text("""
            SELECT q.id, q.image_path, c.name
            FROM questions q LEFT JOIN question_categories c ON c.id = q.question_category_id
            WHERE q.image_path IS NOT NULL AND q.image_path <> ''
    """)).fetchall():
        current[path] = (qid, category)
    for qid, path, category in db.execute(sa_text("""
            SELECT v.question_id, v.snapshot->>'image_path', c.name
            FROM question_versions v
            LEFT JOIN questions q ON q.id = v.question_id
            LEFT JOIN question_categories c ON c.id = q.question_category_id
            WHERE v.snapshot->>'image_path' IS NOT NULL AND v.snapshot->>'image_path' <> ''
    """)).fetchall():
        historical.setdefault(path, (qid, category))
    return current, historical


def main():
    apply_changes = "--apply" in sys.argv
    db = SessionLocal()
    try:
        keys = _stored_keys()
        known = {row[0] for row in db.query(MediaAsset.path).all()}
        current, historical = _usage(db)
        todo = [key for key in keys if key not in known]

        attached = sum(1 for k in todo if k in current)
        detached = sum(1 for k in todo if k not in current and k in historical)
        unused = len(todo) - attached - detached
        print(f"  figures in storage        : {len(keys)}")
        print(f"  already in the bank       : {len(keys) - len(todo)}")
        print(f"  to add                    : {len(todo)}")
        print(f"    attached to a question  : {attached}")
        print(f"    detached during triage  : {detached}")
        print(f"    never used by a question: {unused}")

        if not apply_changes:
            print("\n  Re-run with --apply to add them.")
            return 0

        library = _library(db)
        tag_cache: dict[str, int] = {}

        def tag_id(name: str) -> int:
            if name not in tag_cache:
                row = db.execute(sa_text(
                    "SELECT id FROM question_tags WHERE lower(name) = lower(:n) ORDER BY id LIMIT 1"
                ), {"n": name}).first()
                tag_cache[name] = row[0] if row else db.execute(sa_text(
                    "INSERT INTO question_tags (name, type) VALUES (:n, 'keyword') RETURNING id"
                ), {"n": name}).scalar()
            return tag_cache[name]

        added = 0
        for index, key in enumerate(todo, start=1):
            question = current.get(key)
            was = historical.get(key)
            source = question or was
            if question:
                # No caption unless an educator writes one. A generated one
                # ("Figure from question #3360") describes the database, not
                # the picture, and was shown to learners as though it were a
                # caption — file path and all.
                caption = None
                state = "in use"
            elif was:
                caption = (f"Detached from question #{was[0]} during the stem/answer review — "
                           f"kept so the decision can be checked")
                state = "detached"
            else:
                caption = "Extracted from a source PDF; no question uses it"
                state = "unused"

            asset = MediaAsset(
                path=key, title=key.rsplit("/", 1)[-1],
                caption=caption, alt_text=caption, kind="image",
                library_id=library.id, storage="s3",
            )
            db.add(asset)
            db.flush()
            # Tagged by where it came from and what it is doing, so the bank can
            # answer "show me what the triage detached" without a database query.
            for name in filter(None, [state, (source[1] if source else None)]):
                db.add(MediaTagLink(media_id=asset.id, tag_id=tag_id(name)))
            added += 1
            if index % 250 == 0:
                db.commit()
                print(f"    …{index}/{len(todo)}", flush=True)
        db.commit()
        print(f"\n  added to the bank: {added}")
        print("  Searchable by caption; tagged 'in use', 'detached' or 'unused'.")
    finally:
        db.close()
    return 0


if __name__ == "__main__":
    sys.exit(main())