"""Register the extracted question figures in the image bank. The figures have always been in object storage and attached to questions by path, but they were never rows in `media_assets` — so nothing could browse them, tag them, search them by what they show, or offer them when attaching an image to another question. This puts every one of them in the bank. Nothing is copied or moved: an asset row points at the key the question already uses, so a question's picture and its library entry are the same file. Each figure is described from the question it belongs to, which is the only description that exists until someone writes a better one. Figures no question currently uses are still listed — several were detached during the stem/answer triage and are exactly what somebody reviewing that decision needs to look at. docker compose exec backend python -m scripts.index_question_images docker compose exec backend python -m scripts.index_question_images --apply """ import sys from collections import defaultdict from sqlalchemy import text as sa_text from app.database import SessionLocal from app.models.media import MediaAsset, MediaLibrary, MediaTagLink from app.services import storage_service LIBRARY = "Question figures" PREFIX = "images/" def _library(db): library = db.query(MediaLibrary).filter(MediaLibrary.name == LIBRARY).first() if library is None: library = MediaLibrary(name=LIBRARY, description="Figures extracted from the source exam PDFs.") db.add(library) db.flush() return library def _stored_keys() -> list[str]: keys = [] client = storage_service._s3() from app.config import settings for page in client.get_paginator("list_objects_v2").paginate( Bucket=settings.S3_BUCKET, Prefix=PREFIX): keys.extend(obj["Key"] for obj in page.get("Contents", [])) return keys def _usage(db) -> tuple[dict, dict]: """Which question each figure belongs to now, and which it used to.""" current, historical = {}, {} for qid, path, category in db.execute(sa_text(""" SELECT q.id, q.image_path, c.name FROM questions q LEFT JOIN question_categories c ON c.id = q.question_category_id WHERE q.image_path IS NOT NULL AND q.image_path <> '' """)).fetchall(): current[path] = (qid, category) for qid, path, category in db.execute(sa_text(""" SELECT v.question_id, v.snapshot->>'image_path', c.name FROM question_versions v LEFT JOIN questions q ON q.id = v.question_id LEFT JOIN question_categories c ON c.id = q.question_category_id WHERE v.snapshot->>'image_path' IS NOT NULL AND v.snapshot->>'image_path' <> '' """)).fetchall(): historical.setdefault(path, (qid, category)) return current, historical def main(): apply_changes = "--apply" in sys.argv db = SessionLocal() try: keys = _stored_keys() known = {row[0] for row in db.query(MediaAsset.path).all()} current, historical = _usage(db) todo = [key for key in keys if key not in known] attached = sum(1 for k in todo if k in current) detached = sum(1 for k in todo if k not in current and k in historical) unused = len(todo) - attached - detached print(f" figures in storage : {len(keys)}") print(f" already in the bank : {len(keys) - len(todo)}") print(f" to add : {len(todo)}") print(f" attached to a question : {attached}") print(f" detached during triage : {detached}") print(f" never used by a question: {unused}") if not apply_changes: print("\n Re-run with --apply to add them.") return 0 library = _library(db) tag_cache: dict[str, int] = {} def tag_id(name: str) -> int: if name not in tag_cache: row = db.execute(sa_text( "SELECT id FROM question_tags WHERE lower(name) = lower(:n) ORDER BY id LIMIT 1" ), {"n": name}).first() tag_cache[name] = row[0] if row else db.execute(sa_text( "INSERT INTO question_tags (name, type) VALUES (:n, 'keyword') RETURNING id" ), {"n": name}).scalar() return tag_cache[name] added = 0 for index, key in enumerate(todo, start=1): question = current.get(key) was = historical.get(key) source = question or was if question: # No caption unless an educator writes one. A generated one # ("Figure from question #3360") describes the database, not # the picture, and was shown to learners as though it were a # caption — file path and all. caption = None state = "in use" elif was: caption = (f"Detached from question #{was[0]} during the stem/answer review — " f"kept so the decision can be checked") state = "detached" else: caption = "Extracted from a source PDF; no question uses it" state = "unused" asset = MediaAsset( path=key, title=key.rsplit("/", 1)[-1], caption=caption, alt_text=caption, kind="image", library_id=library.id, storage="s3", ) db.add(asset) db.flush() # Tagged by where it came from and what it is doing, so the bank can # answer "show me what the triage detached" without a database query. for name in filter(None, [state, (source[1] if source else None)]): db.add(MediaTagLink(media_id=asset.id, tag_id=tag_id(name))) added += 1 if index % 250 == 0: db.commit() print(f" …{index}/{len(todo)}", flush=True) db.commit() print(f"\n added to the bank: {added}") print(" Searchable by caption; tagged 'in use', 'detached' or 'unused'.") finally: db.close() return 0 if __name__ == "__main__": sys.exit(main())