f2b5e146eb
The objective did almost nothing It scoped question counts and nothing else, which is why changing it appeared to have no effect. An exam now carries a family (USMLE, COMLEX, boards), a description, and the article views it offers, and `/exams/` reports what the current objective actually changes rather than leaving the learner to guess. Reading follows from it: an article returns only the views its objective allows, so someone revising a basic-science step is never shown bedside dosing they must not act on — a view you can open but must never use is worse than one you were never offered. An editor still gets the whole article, because they cannot edit what they cannot see. An objective configured to show nothing falls back to all three; that is a configuration mistake, not a preference worth honouring. Unused figures deleted, at the user's request 3,262 figures — 334 MB — that nothing had ever used. "Unused" was defined by exclusion and every exclusion was checked rather than assumed: kept if any question uses it as a stem or explanation image, if any question version mentions it, or if it appears in article prose or a flashcard. 440 kept, and five question figures spot-checked as still readable afterwards. MinIO is now 596 objects, 520 MB, down from 3,858 and 854 MB. This is not reversible from the application; the nightly borg backup of the volume is the only way back, and that is stated in the script rather than assumed. For the record, since it was asked: the extraction is PyMuPDF, with an MD5 skip list for repeated branding images. It pulled every embedded image from all 18 source PDFs, which is why one 767-page document alone produced 908 of them. 208 backend tests green. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
106 lines
4.3 KiB
Python
106 lines
4.3 KiB
Python
"""Delete extracted figures that nothing has ever used.
|
|
|
|
The extractor pulled every embedded image out of 18 source PDFs — figures,
|
|
photographs, logos, page rules — and only the ones it could tie to a question
|
|
were ever attached. The rest have sat in storage since import.
|
|
|
|
"Unused" is defined by exclusion, and every exclusion is checked here rather
|
|
than assumed: a figure is kept if any question uses it as a stem or explanation
|
|
image, if any question version mentions it, if it appears in article prose or a
|
|
flashcard, or if it is not an extracted figure at all. What is left is deleted
|
|
from object storage and from the image bank.
|
|
|
|
This is not reversible from the application. The volume is in the nightly borg
|
|
backup, which is the only way back.
|
|
|
|
docker compose exec backend python -m scripts.purge_unused_figures
|
|
docker compose exec backend python -m scripts.purge_unused_figures --apply
|
|
"""
|
|
import sys
|
|
|
|
from sqlalchemy import text as sa_text
|
|
|
|
from app.config import settings
|
|
from app.database import SessionLocal
|
|
from app.models.media import MediaAsset, MediaTagLink
|
|
from app.services import storage_service
|
|
|
|
PREFIX = "images/"
|
|
|
|
|
|
def _referenced(db) -> set[str]:
|
|
"""Every figure any part of the platform points at, however indirectly."""
|
|
keys: set[str] = set()
|
|
for statement in (
|
|
"SELECT image_path FROM questions WHERE image_path LIKE 'images/%'",
|
|
"SELECT explanation_image_path FROM questions WHERE explanation_image_path LIKE 'images/%'",
|
|
"SELECT snapshot->>'image_path' FROM question_versions WHERE snapshot->>'image_path' LIKE 'images/%'",
|
|
"SELECT snapshot->>'explanation_image_path' FROM question_versions"
|
|
" WHERE snapshot->>'explanation_image_path' LIKE 'images/%'",
|
|
):
|
|
keys.update(row[0] for row in db.execute(sa_text(statement)).fetchall() if row[0])
|
|
|
|
# Prose can embed a figure by path; a substring search is the honest check
|
|
# because the path is inside Markdown rather than in a column of its own.
|
|
prose = db.execute(sa_text("""
|
|
SELECT COALESCE(content, '') || ' ' || COALESCE(sections::text, '') FROM articles
|
|
UNION ALL SELECT COALESCE(front, '') || ' ' || COALESCE(back, '') FROM flashcards
|
|
""")).fetchall()
|
|
body = " ".join(row[0] or "" for row in prose)
|
|
if PREFIX in body:
|
|
for key in list(_stored_keys()):
|
|
if key in body:
|
|
keys.add(key)
|
|
return keys
|
|
|
|
|
|
def _stored_keys() -> list[str]:
|
|
client = storage_service._s3()
|
|
keys = []
|
|
for page in client.get_paginator("list_objects_v2").paginate(
|
|
Bucket=settings.S3_BUCKET, Prefix=PREFIX):
|
|
keys.extend((obj["Key"], obj["Size"]) for obj in page.get("Contents", []))
|
|
return [k for k, _ in keys], dict(keys)
|
|
|
|
|
|
def main():
|
|
apply_changes = "--apply" in sys.argv
|
|
db = SessionLocal()
|
|
try:
|
|
keys, sizes = _stored_keys()
|
|
keep = _referenced(db)
|
|
doomed = [k for k in keys if k not in keep]
|
|
freed = sum(sizes.get(k, 0) for k in doomed)
|
|
|
|
print(f" figures in storage : {len(keys)}")
|
|
print(f" referenced, kept : {len(keys) - len(doomed)}")
|
|
print(f" unused, to delete : {len(doomed)} ({freed / 1024 / 1024:.0f} MB)")
|
|
if not apply_changes:
|
|
print("\n Re-run with --apply to delete them. Only the nightly backup"
|
|
"\n holds them afterwards.")
|
|
return 0
|
|
|
|
client = storage_service._s3()
|
|
removed = 0
|
|
for start in range(0, len(doomed), 900):
|
|
batch = doomed[start:start + 900]
|
|
client.delete_objects(Bucket=settings.S3_BUCKET,
|
|
Delete={"Objects": [{"Key": k} for k in batch]})
|
|
ids = [row[0] for row in db.query(MediaAsset.id).filter(
|
|
MediaAsset.path.in_(batch)).all()]
|
|
if ids:
|
|
db.query(MediaTagLink).filter(MediaTagLink.media_id.in_(ids)).delete(
|
|
synchronize_session=False)
|
|
db.query(MediaAsset).filter(MediaAsset.id.in_(ids)).delete(
|
|
synchronize_session=False)
|
|
db.commit()
|
|
removed += len(batch)
|
|
print(f" …{removed}/{len(doomed)}", flush=True)
|
|
print(f"\n deleted {removed} figures, {freed / 1024 / 1024:.0f} MB freed.")
|
|
finally:
|
|
db.close()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|