refactor: name the study plans ourselves, and stop reserving 64k tokens a call
"PREP" is the American Academy of Pediatrics' trademark for their own product. The plans here are our own sets of questions grouped by year, so they are now named for what they are: Board Review 2021, and Mixed Review for the plan that draws from every year at once. Renamed in the database as well as the code — 13 plans, 14 quizzes a learner had already generated from a block, and the 12 year tags, which appear in the question bank's filters and are as visible as the plans. The seeder matches both the old and new names so a fresh import still finds its material, and the tagger mints the new one so the next run cannot undo this. Prompts and comments that described the source PDFs by that name now describe them by what they are. The generation run's 377 failures were not a bug Every call was reserving the model's full 64k output ceiling, and OpenRouter refuses the whole request when the balance is below the reservation — "you requested up to 64000 tokens, but can only afford 52017" — however short the answer would actually be. `_call_model` now takes a max_tokens, and the article writer asks for 4000, which is comfortable for three views of one topic and keeps each request small enough to be affordable. 98 articles were written before the balance ran down; 158 exist in total. Generation is paused at the user's request while credits are topped up. 208 backend, 243 frontend green. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
This commit is contained in:
parent
1d5962b40e
commit
a1459b2965
17 changed files with 182 additions and 53 deletions
|
|
@ -15,7 +15,7 @@ class QuizCreate(BaseModel):
|
||||||
extraction_mode: str = "standard"
|
extraction_mode: str = "standard"
|
||||||
# standard — current working mode (inline Correct Answer / Preferred Response)
|
# standard — current working mode (inline Correct Answer / Preferred Response)
|
||||||
# questions_only — extract Q+options only, no answers (admin fills later)
|
# questions_only — extract Q+options only, no answers (admin fills later)
|
||||||
# two_step — separate answer key section (PREP 2013 style)
|
# two_step — separate answer key section (2013-style layout)
|
||||||
# regex — AI analyses format then extracts answer key with regex
|
# regex — AI analyses format then extracts answer key with regex
|
||||||
# ai_decide — AI reads a sample and decides which approach to use
|
# ai_decide — AI reads a sample and decides which approach to use
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ def _proxy_model(model_id: str) -> str:
|
||||||
return f"openai/{model_id}"
|
return f"openai/{model_id}"
|
||||||
return model_id
|
return model_id
|
||||||
|
|
||||||
EXTRACTION_PROMPT = """You are extracting questions from a PREP (Pediatric Review and Education Program) exam PDF.
|
EXTRACTION_PROMPT = """You are extracting questions from a pediatric board review exam PDF.
|
||||||
|
|
||||||
These PDFs follow a strict format:
|
These PDFs follow a strict format:
|
||||||
1. A numbered question with a clinical vignette (patient scenario)
|
1. A numbered question with a clinical vignette (patient scenario)
|
||||||
|
|
@ -70,7 +70,7 @@ CRITICAL RULES — follow exactly:
|
||||||
Content from page(s) {page_info}:
|
Content from page(s) {page_info}:
|
||||||
{content}"""
|
{content}"""
|
||||||
|
|
||||||
ANSWER_KEY_PROMPT = """Extract the answer key from this PREP exam content.
|
ANSWER_KEY_PROMPT = """Extract the answer key from this board review exam content.
|
||||||
|
|
||||||
The answer key lists items with their correct answer letters, like:
|
The answer key lists items with their correct answer letters, like:
|
||||||
"Item 193 Preferred Response: D"
|
"Item 193 Preferred Response: D"
|
||||||
|
|
@ -217,13 +217,18 @@ def extract_questions(
|
||||||
|
|
||||||
|
|
||||||
def _call_model(prompt: str, model_id: str | None, api_key: str | None,
|
def _call_model(prompt: str, model_id: str | None, api_key: str | None,
|
||||||
timeout: int = 180) -> str:
|
timeout: int = 180, max_tokens: int | None = None) -> str:
|
||||||
"""Call the configured LLM and return raw text response.
|
"""Call the configured LLM and return raw text response.
|
||||||
|
|
||||||
The timeout is not optional in practice: every other call in this module has
|
The timeout is not optional in practice: every other call in this module has
|
||||||
one, and this one did not. A stalled connection to the proxy hung the caller
|
one, and this one did not. A stalled connection to the proxy hung the caller
|
||||||
for good — which an interactive request survives by the user giving up, and
|
for good — which an interactive request survives by the user giving up, and
|
||||||
an unattended run of several hundred topics does not.
|
an unattended run of several hundred topics does not.
|
||||||
|
|
||||||
|
`max_tokens` is worth setting for the same reason. Left unset, the request
|
||||||
|
reserves the model's full output ceiling — 64k on the current default — and
|
||||||
|
a provider that bills against reserved capacity refuses the whole call when
|
||||||
|
the balance is below that, however short the answer would actually be.
|
||||||
"""
|
"""
|
||||||
use_model = _proxy_model(model_id or settings.LITELLM_MODEL)
|
use_model = _proxy_model(model_id or settings.LITELLM_MODEL)
|
||||||
use_key = api_key or settings.LITELLM_API_KEY
|
use_key = api_key or settings.LITELLM_API_KEY
|
||||||
|
|
@ -233,6 +238,8 @@ def _call_model(prompt: str, model_id: str | None, api_key: str | None,
|
||||||
"temperature": 0.1,
|
"temperature": 0.1,
|
||||||
"timeout": timeout,
|
"timeout": timeout,
|
||||||
}
|
}
|
||||||
|
if max_tokens:
|
||||||
|
kwargs["max_tokens"] = max_tokens
|
||||||
if use_key:
|
if use_key:
|
||||||
kwargs["api_key"] = use_key
|
kwargs["api_key"] = use_key
|
||||||
if settings.LITELLM_API_BASE:
|
if settings.LITELLM_API_BASE:
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,9 @@ MIN_SOURCE_CHARS = 3000
|
||||||
# Generous for a long article, short enough that a stalled call is noticed in
|
# Generous for a long article, short enough that a stalled call is noticed in
|
||||||
# minutes rather than discovered hours later with nothing written since.
|
# minutes rather than discovered hours later with nothing written since.
|
||||||
WRITE_TIMEOUT = 150
|
WRITE_TIMEOUT = 150
|
||||||
|
# Three views of one topic, the long one about 550 words. Four thousand tokens is
|
||||||
|
# comfortable for that and keeps each request small enough to be affordable.
|
||||||
|
WRITE_MAX_TOKENS = 4000
|
||||||
# Long enough to be worth reading, short enough that nobody skims past the point.
|
# Long enough to be worth reading, short enough that nobody skims past the point.
|
||||||
LONG_WORDS = 550
|
LONG_WORDS = 550
|
||||||
SHELF = "Pediatrics"
|
SHELF = "Pediatrics"
|
||||||
|
|
@ -152,7 +155,8 @@ def write_article(db: Session, topic: str, category_id: int | None = None,
|
||||||
|
|
||||||
model_id, api_key = get_model_for_task(db, "extraction")
|
model_id, api_key = get_model_for_task(db, "extraction")
|
||||||
# One topic must not be able to stall a run of five hundred.
|
# One topic must not be able to stall a run of five hundred.
|
||||||
raw = _call_model(_prompt(topic, passages), model_id, api_key, timeout=WRITE_TIMEOUT)
|
raw = _call_model(_prompt(topic, passages), model_id, api_key,
|
||||||
|
timeout=WRITE_TIMEOUT, max_tokens=WRITE_MAX_TOKENS)
|
||||||
text = raw.strip()
|
text = raw.strip()
|
||||||
if text.startswith("```"):
|
if text.startswith("```"):
|
||||||
text = re.sub(r"^```[a-z]*\n?|```$", "", text).strip()
|
text = re.sub(r"^```[a-z]*\n?|```$", "", text).strip()
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
Modes
|
Modes
|
||||||
-----
|
-----
|
||||||
questions_only Extract Q+options with no answers. User fills answers later via QuizEditPage.
|
questions_only Extract Q+options with no answers. User fills answers later via QuizEditPage.
|
||||||
two_step Separate answer key section (PREP 2013): Phase 1 = questions, Phase 2 = key, Phase 3 = match.
|
two_step Separate answer key section (2013-style): Phase 1 = questions, Phase 2 = key, Phase 3 = match.
|
||||||
regex AI generates a regex pattern for the document's answer format, then we apply it.
|
regex AI generates a regex pattern for the document's answer format, then we apply it.
|
||||||
ai_decide AI samples the document and picks standard / two_step / questions_only.
|
ai_decide AI samples the document and picks standard / two_step / questions_only.
|
||||||
generate AI reads plain text/study material and creates MCQ questions from scratch.
|
generate AI reads plain text/study material and creates MCQ questions from scratch.
|
||||||
|
|
@ -25,7 +25,7 @@ def _normalize(text: str) -> str:
|
||||||
|
|
||||||
# ─── QUESTIONS ONLY ──────────────────────────────────────────────────────────
|
# ─── QUESTIONS ONLY ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
QUESTIONS_ONLY_PROMPT = """Extract every question from this PREP exam content.
|
QUESTIONS_ONLY_PROMPT = """Extract every question from this board review exam content.
|
||||||
Do NOT look for correct answers — we only need the question text and answer options.
|
Do NOT look for correct answers — we only need the question text and answer options.
|
||||||
|
|
||||||
Return ONLY JSON:
|
Return ONLY JSON:
|
||||||
|
|
@ -195,7 +195,7 @@ def extract_two_step(
|
||||||
) -> tuple[list[dict], list[str]]:
|
) -> tuple[list[dict], list[str]]:
|
||||||
"""
|
"""
|
||||||
Two-phase extraction for PDFs with questions in the first half
|
Two-phase extraction for PDFs with questions in the first half
|
||||||
and a separate answer key section (e.g. PREP 2013 "Preferred Response:").
|
and a separate answer key section (e.g. a 2013-style "Preferred Response:").
|
||||||
|
|
||||||
Returns (valid_questions, skipped_list).
|
Returns (valid_questions, skipped_list).
|
||||||
Raises ValueError if answer section not found or no questions matched.
|
Raises ValueError if answer section not found or no questions matched.
|
||||||
|
|
@ -291,7 +291,7 @@ def extract_two_step(
|
||||||
|
|
||||||
# ─── REGEX MODE ──────────────────────────────────────────────────────────────
|
# ─── REGEX MODE ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
REGEX_ANALYSIS_PROMPT = """Look at this PREP exam PDF content and identify the pattern used to mark correct answers.
|
REGEX_ANALYSIS_PROMPT = """Look at this board review exam PDF content and identify the pattern used to mark correct answers.
|
||||||
|
|
||||||
Describe:
|
Describe:
|
||||||
1. The exact text pattern before the correct answer letter (e.g. "Correct Answer:" or "Preferred Response:")
|
1. The exact text pattern before the correct answer letter (e.g. "Correct Answer:" or "Preferred Response:")
|
||||||
|
|
|
||||||
|
|
@ -40,9 +40,9 @@ def extract_text_for_range(file_path: str, start: int, end: int) -> str:
|
||||||
|
|
||||||
|
|
||||||
# MD5 hashes of known repeated branding images (logos, headers) to skip during extraction.
|
# MD5 hashes of known repeated branding images (logos, headers) to skip during extraction.
|
||||||
# These appear on every page of PREP PDFs and are not clinical images.
|
# These appear on every page of the source PDFs and are not clinical images.
|
||||||
_SKIP_IMAGE_HASHES = {
|
_SKIP_IMAGE_HASHES = {
|
||||||
"f48b094ec260f0aa8d7c52bc3cf562e4", # AAP logo (34300 bytes, appears 869 times across PREP PDFs)
|
"f48b094ec260f0aa8d7c52bc3cf562e4", # AAP logo (34300 bytes, appears 869 times across the source PDFs)
|
||||||
"82c449d72791fe181fc9964bb8efad0f", # Sepsis document header/logo (20397 bytes, repeated per page)
|
"82c449d72791fe181fc9964bb8efad0f", # Sepsis document header/logo (20397 bytes, repeated per page)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ def _push_step(r, job_id: str, step: str, message: str):
|
||||||
|
|
||||||
|
|
||||||
def _normalize_ocr(text: str) -> str:
|
def _normalize_ocr(text: str) -> str:
|
||||||
"""Fix common OCR artifacts in PREP PDFs."""
|
"""Fix common OCR artifacts in the source PDFs."""
|
||||||
return (text
|
return (text
|
||||||
.replace("Pref erred", "Preferred")
|
.replace("Pref erred", "Preferred")
|
||||||
.replace("Pre ferred", "Preferred")
|
.replace("Pre ferred", "Preferred")
|
||||||
|
|
|
||||||
|
|
@ -187,7 +187,7 @@ def main():
|
||||||
systems = specific
|
systems = specific
|
||||||
else:
|
else:
|
||||||
# Untagged and Pediatrics-only questions fall back to General Pediatrics;
|
# Untagged and Pediatrics-only questions fall back to General Pediatrics;
|
||||||
# PREP categories are being retired so nothing is left dangling.
|
# The source-set categories are being retired so nothing is left dangling.
|
||||||
systems = ["General Pediatrics"]
|
systems = ["General Pediatrics"]
|
||||||
if not (subjects.get(qid) or diseases.get(qid)):
|
if not (subjects.get(qid) or diseases.get(qid)):
|
||||||
skipped += 1
|
skipped += 1
|
||||||
|
|
@ -208,10 +208,10 @@ def main():
|
||||||
for cid in sorted(extras - existing):
|
for cid in sorted(extras - existing):
|
||||||
db.add(QuestionCategoryLink(question_id=qid, category_id=cid))
|
db.add(QuestionCategoryLink(question_id=qid, category_id=cid))
|
||||||
links_added += 1
|
links_added += 1
|
||||||
# PREP provenance becomes a keyword tag; the question categories are retired.
|
# Provenance becomes a keyword tag; the question categories are retired.
|
||||||
prep_categories = db.query(QuestionCategory).filter(QuestionCategory.name.ilike("prep %")).all()
|
source_categories = db.query(QuestionCategory).filter(QuestionCategory.name.ilike("prep %")).all()
|
||||||
prep_tagged = 0
|
source_tagged = 0
|
||||||
for category in prep_categories:
|
for category in source_categories:
|
||||||
linked = {row[0] for row in db.query(QuestionCategoryLink.question_id).filter_by(category_id=category.id).all()}
|
linked = {row[0] for row in db.query(QuestionCategoryLink.question_id).filter_by(category_id=category.id).all()}
|
||||||
linked |= {row[0] for row in db.query(Question.id).filter(Question.question_category_id == category.id).all()}
|
linked |= {row[0] for row in db.query(Question.id).filter(Question.question_category_id == category.id).all()}
|
||||||
linked.discard(None)
|
linked.discard(None)
|
||||||
|
|
@ -225,11 +225,11 @@ def main():
|
||||||
for qid in linked:
|
for qid in linked:
|
||||||
db.execute(text("INSERT INTO question_tag_links (question_id, tag_id) VALUES (:q, :t) "
|
db.execute(text("INSERT INTO question_tag_links (question_id, tag_id) VALUES (:q, :t) "
|
||||||
"ON CONFLICT DO NOTHING"), {"q": qid, "t": tag_id})
|
"ON CONFLICT DO NOTHING"), {"q": qid, "t": tag_id})
|
||||||
prep_tagged += len(linked)
|
source_tagged += len(linked)
|
||||||
db.delete(category)
|
db.delete(category)
|
||||||
db.commit()
|
db.commit()
|
||||||
print(f"Reassigned primary for {changed} questions; added {links_added} extra links; "
|
print(f"Reassigned primary for {changed} questions; added {links_added} extra links; "
|
||||||
f"tagged {prep_tagged} question-links across {len(prep_categories)} retired PREP categories; "
|
f"tagged {source_tagged} question-links across {len(source_categories)} retired source categories; "
|
||||||
f"skipped {skipped}; {db.query(QuestionCategory).count()} categories total.")
|
f"skipped {skipped}; {db.query(QuestionCategory).count()} categories total.")
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
"""Repair OCR-damaged units and turn inline lab panels into markdown tables.
|
"""Repair OCR-damaged units and turn inline lab panels into markdown tables.
|
||||||
|
|
||||||
The PREP PDFs were scanned, so the extracted stems carry two separate injuries.
|
The source PDFs were scanned, so the extracted stems carry two separate injuries.
|
||||||
|
|
||||||
1. Unit corruption. The scanner confuses letter pairs that share a shape —
|
1. Unit corruption. The scanner confuses letter pairs that share a shape —
|
||||||
"m" reads as "rn" or "in", "µ" as "p" or "4" — and superscripts are lost
|
"m" reads as "rn" or "in", "µ" as "p" or "4" — and superscripts are lost
|
||||||
|
|
|
||||||
108
backend/scripts/rename_study_plans.py
Normal file
108
backend/scripts/rename_study_plans.py
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
"""Rename the study plans away from the vendor's programme name.
|
||||||
|
|
||||||
|
"PREP" is the American Academy of Pediatrics' trademark for its own product.
|
||||||
|
The plans here are our own sets of questions grouped by year, so they get names
|
||||||
|
that describe what they are: "Board Review 2021", and "Mixed Review" for the
|
||||||
|
plan that draws from every year at once.
|
||||||
|
|
||||||
|
Slugs change with them, which is safe because a study plan is reached by id and
|
||||||
|
the slug is not a public address. Quizzes a learner already generated from a
|
||||||
|
block are renamed too, so a title in their history matches the plan it came from
|
||||||
|
rather than referring to something that no longer exists. So are the year tags,
|
||||||
|
which appear in the question bank's filters and are as visible as the plans.
|
||||||
|
|
||||||
|
docker compose exec backend python -m scripts.rename_study_plans
|
||||||
|
docker compose exec backend python -m scripts.rename_study_plans --apply
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from sqlalchemy import text as sa_text
|
||||||
|
|
||||||
|
from app.database import SessionLocal
|
||||||
|
|
||||||
|
YEAR_NAME = "Board Review {year}"
|
||||||
|
YEAR_SLUG = "board-review-{year}"
|
||||||
|
MIXED_NAME = "Mixed Review"
|
||||||
|
MIXED_SLUG = "mixed-review"
|
||||||
|
|
||||||
|
|
||||||
|
def planned(db):
|
||||||
|
"""(id, old name, new name, old slug, new slug) for everything to rename."""
|
||||||
|
rows = db.execute(sa_text(
|
||||||
|
"SELECT id, slug, name, kind FROM study_plans ORDER BY sort_order, name")).fetchall()
|
||||||
|
changes = []
|
||||||
|
for plan in rows:
|
||||||
|
year = re.search(r"(\d{4})", plan.name)
|
||||||
|
if plan.kind == "mixed" or "mixed" in plan.name.lower():
|
||||||
|
new_name, new_slug = MIXED_NAME, MIXED_SLUG
|
||||||
|
elif year:
|
||||||
|
new_name = YEAR_NAME.format(year=year.group(1))
|
||||||
|
new_slug = YEAR_SLUG.format(year=year.group(1))
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
if (new_name, new_slug) != (plan.name, plan.slug):
|
||||||
|
changes.append((plan.id, plan.name, new_name, plan.slug, new_slug))
|
||||||
|
return changes
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
apply_changes = "--apply" in sys.argv
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
# No early return when the plans are already done: the quizzes and the
|
||||||
|
# year tags are renamed by the same pass, and a second run has to be able
|
||||||
|
# to finish what a first one left.
|
||||||
|
changes = planned(db)
|
||||||
|
print(f" plans to rename: {len(changes)}\n")
|
||||||
|
for _pid, old_name, new_name, old_slug, new_slug in changes:
|
||||||
|
print(f" {old_name:<16} -> {new_name:<22} ({old_slug} -> {new_slug})")
|
||||||
|
|
||||||
|
quizzes = db.execute(sa_text(
|
||||||
|
"SELECT id, title FROM quizzes WHERE title LIKE '%PREP%'")).fetchall()
|
||||||
|
print(f"\n quizzes already generated from a block: {len(quizzes)}")
|
||||||
|
|
||||||
|
tags = db.execute(sa_text(
|
||||||
|
"SELECT id, name FROM question_tags WHERE name ~ '^PREP [0-9]{4}$'")).fetchall()
|
||||||
|
print(f" year tags shown in the bank's filters : {len(tags)}")
|
||||||
|
|
||||||
|
if not apply_changes:
|
||||||
|
print("\n Re-run with --apply to rename them.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
for plan_id, _old_name, new_name, _old_slug, new_slug in changes:
|
||||||
|
db.execute(sa_text("UPDATE study_plans SET name = :n, slug = :s WHERE id = :i"),
|
||||||
|
{"n": new_name, "s": new_slug, "i": plan_id})
|
||||||
|
|
||||||
|
renamed = 0
|
||||||
|
for quiz in quizzes:
|
||||||
|
title = quiz.title
|
||||||
|
year = re.search(r"PREP (\d{4})", title)
|
||||||
|
if year:
|
||||||
|
title = title.replace(f"PREP {year.group(1)}", YEAR_NAME.format(year=year.group(1)))
|
||||||
|
else:
|
||||||
|
title = title.replace("PREP Mixed", MIXED_NAME).replace("PREP", "Board Review")
|
||||||
|
if title != quiz.title:
|
||||||
|
db.execute(sa_text("UPDATE quizzes SET title = :t WHERE id = :i"),
|
||||||
|
{"t": title, "i": quiz.id})
|
||||||
|
renamed += 1
|
||||||
|
retagged = 0
|
||||||
|
for tag in tags:
|
||||||
|
year = re.search(r"(\d{4})", tag.name)
|
||||||
|
if not year:
|
||||||
|
continue
|
||||||
|
db.execute(sa_text("UPDATE question_tags SET name = :n WHERE id = :i"),
|
||||||
|
{"n": YEAR_NAME.format(year=year.group(1)), "i": tag.id})
|
||||||
|
retagged += 1
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
print(f"\n plans renamed : {len(changes)}")
|
||||||
|
print(f" quizzes renamed : {renamed}")
|
||||||
|
print(f" tags renamed : {retagged}")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
|
|
@ -1,14 +1,14 @@
|
||||||
"""Turn the PREP question sets into study plans of numbered blocks.
|
"""Turn the year question sets into study plans of numbered blocks.
|
||||||
|
|
||||||
One plan per PREP year, split into blocks of BLOCK_SIZE, plus a mixed plan that
|
One plan per year, split into blocks of BLOCK_SIZE, plus a mixed plan that
|
||||||
draws MIXED_SIZE questions at random across every year.
|
draws MIXED_SIZE questions at random across every year.
|
||||||
|
|
||||||
Block membership is snapshotted, not a live filter: a plan you are part-way
|
Block membership is snapshotted, not a live filter: a plan you are part-way
|
||||||
through must not reshuffle between visits. Re-running updates block contents for
|
through must not reshuffle between visits. Re-running updates block contents for
|
||||||
years that changed, and leaves the mixed plan's draw alone unless --reshuffle.
|
years that changed, and leaves the mixed plan's draw alone unless --reshuffle.
|
||||||
|
|
||||||
docker compose exec backend python -m scripts.seed_prep_study_plans
|
docker compose exec backend python -m scripts.seed_study_plans
|
||||||
docker compose exec backend python -m scripts.seed_prep_study_plans --apply
|
docker compose exec backend python -m scripts.seed_study_plans --apply
|
||||||
"""
|
"""
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
|
|
@ -22,14 +22,22 @@ from app.models.study_plan import StudyPlan, StudyPlanBlock
|
||||||
|
|
||||||
BLOCK_SIZE = 50
|
BLOCK_SIZE = 50
|
||||||
MIXED_SIZE = 300
|
MIXED_SIZE = 300
|
||||||
MIXED_SLUG = "prep-mixed"
|
MIXED_SLUG = "mixed-review"
|
||||||
|
# The imported material is tagged with the source programme's name; the plans
|
||||||
|
# built from it are ours and are named for what they are.
|
||||||
|
YEAR_NAME = "Board Review {year}"
|
||||||
|
YEAR_SLUG = "board-review-{year}"
|
||||||
|
|
||||||
|
|
||||||
def prep_tags(db):
|
def prep_tags(db):
|
||||||
"""PREP year tags, newest first — 'PREP 2021', not 'Preparticipation Exam'."""
|
"""Year tags, newest first.
|
||||||
|
|
||||||
|
The tags still carry the source programme's name because that is what the
|
||||||
|
imported material was labelled with; the plans built from them do not.
|
||||||
|
"""
|
||||||
rows = db.execute(sa_text("""
|
rows = db.execute(sa_text("""
|
||||||
SELECT t.id, t.name FROM question_tags t
|
SELECT t.id, t.name FROM question_tags t
|
||||||
WHERE t.name ~ '^PREP [0-9]{4}$'
|
WHERE t.name ~ '^(Board Review|PREP) [0-9]{4}$'
|
||||||
ORDER BY t.name DESC
|
ORDER BY t.name DESC
|
||||||
""")).fetchall()
|
""")).fetchall()
|
||||||
return [(row[0], row[1]) for row in rows]
|
return [(row[0], row[1]) for row in rows]
|
||||||
|
|
@ -72,7 +80,7 @@ def main():
|
||||||
exam_id = exam.id if exam else None
|
exam_id = exam.id if exam else None
|
||||||
tags = prep_tags(db)
|
tags = prep_tags(db)
|
||||||
if not tags:
|
if not tags:
|
||||||
print("No 'PREP <year>' tags found; nothing to do.")
|
print("No year tags found; nothing to do.")
|
||||||
return
|
return
|
||||||
|
|
||||||
everything, summary = [], []
|
everything, summary = [], []
|
||||||
|
|
@ -83,7 +91,7 @@ def main():
|
||||||
summary.append((name, len(ids), len(chunks)))
|
summary.append((name, len(ids), len(chunks)))
|
||||||
if apply_changes:
|
if apply_changes:
|
||||||
year = re.search(r"(\d{4})", name).group(1)
|
year = re.search(r"(\d{4})", name).group(1)
|
||||||
plan = upsert_plan(db, f"prep-{year}", name,
|
plan = upsert_plan(db, YEAR_SLUG.format(year=year), YEAR_NAME.format(year=year),
|
||||||
f"{len(ids)} questions in {len(chunks)} blocks of up to {BLOCK_SIZE}.",
|
f"{len(ids)} questions in {len(chunks)} blocks of up to {BLOCK_SIZE}.",
|
||||||
"set", order, exam_id)
|
"set", order, exam_id)
|
||||||
set_blocks(db, plan, chunks)
|
set_blocks(db, plan, chunks)
|
||||||
|
|
@ -96,8 +104,8 @@ def main():
|
||||||
random.shuffle(pool)
|
random.shuffle(pool)
|
||||||
mixed_ids = pool[:MIXED_SIZE]
|
mixed_ids = pool[:MIXED_SIZE]
|
||||||
if apply_changes:
|
if apply_changes:
|
||||||
plan = upsert_plan(db, MIXED_SLUG, "PREP Mixed",
|
plan = upsert_plan(db, MIXED_SLUG, "Mixed Review",
|
||||||
f"{MIXED_SIZE} questions drawn at random from every PREP year.",
|
f"{MIXED_SIZE} questions drawn at random from every year.",
|
||||||
"mixed", 0, exam_id)
|
"mixed", 0, exam_id)
|
||||||
if draw_needed:
|
if draw_needed:
|
||||||
set_blocks(db, plan, [mixed_ids])
|
set_blocks(db, plan, [mixed_ids])
|
||||||
|
|
@ -105,8 +113,10 @@ def main():
|
||||||
|
|
||||||
print("APPLIED" if apply_changes else "DRY RUN")
|
print("APPLIED" if apply_changes else "DRY RUN")
|
||||||
for name, count, blocks in summary:
|
for name, count, blocks in summary:
|
||||||
print(f" {name:12s} {count:4d} questions -> {blocks} blocks")
|
year = re.search(r"(\d{4})", name)
|
||||||
print(f" {'PREP Mixed':12s} {len(mixed_ids) or MIXED_SIZE:4d} questions -> 1 block"
|
shown = YEAR_NAME.format(year=year.group(1)) if year else name
|
||||||
|
print(f" {shown:18s} {count:4d} questions -> {blocks} blocks")
|
||||||
|
print(f" {'Mixed Review':18s} {len(mixed_ids) or MIXED_SIZE:4d} questions -> 1 block"
|
||||||
f"{'' if draw_needed else ' (existing draw kept)'}")
|
f"{'' if draw_needed else ' (existing draw kept)'}")
|
||||||
if not apply_changes:
|
if not apply_changes:
|
||||||
print("\n Re-run with --apply to write these plans.")
|
print("\n Re-run with --apply to write these plans.")
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
"""Tag questions by their PREP source quiz and retire any leftover PREP categories.
|
"""Tag questions by their source quiz and retire any leftover source categories.
|
||||||
|
|
||||||
Run after the tag→category conversion: PREP provenance moves from categories to
|
Run after the tag→category conversion: provenance moves from categories to
|
||||||
keyword tags named after the PREP quiz (e.g. 'PREP 2020').
|
keyword tags named after the source quiz (e.g. 'Board Review 2020').
|
||||||
"""
|
"""
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
@ -20,7 +20,7 @@ def main():
|
||||||
tagged = 0
|
tagged = 0
|
||||||
for quiz in prep_quizzes:
|
for quiz in prep_quizzes:
|
||||||
year = re.search(r"\b(19|20)\d{2}\b", quiz.title or "")
|
year = re.search(r"\b(19|20)\d{2}\b", quiz.title or "")
|
||||||
tag_name = f"PREP {year.group(0)}" if year else (quiz.title or f"PREP {quiz.id}").strip()
|
tag_name = f"Board Review {year.group(0)}" if year else (quiz.title or f"Board Review {quiz.id}").strip()
|
||||||
db.execute(text("INSERT INTO question_tags (name, type) VALUES (:name, 'keyword') "
|
db.execute(text("INSERT INTO question_tags (name, type) VALUES (:name, 'keyword') "
|
||||||
"ON CONFLICT (LOWER(name), type) DO NOTHING"), {"name": tag_name})
|
"ON CONFLICT (LOWER(name), type) DO NOTHING"), {"name": tag_name})
|
||||||
tag_id = db.execute(text("SELECT id FROM question_tags WHERE LOWER(name) = LOWER(:name) AND type = 'keyword'"),
|
tag_id = db.execute(text("SELECT id FROM question_tags WHERE LOWER(name) = LOWER(:name) AND type = 'keyword'"),
|
||||||
|
|
@ -11,7 +11,7 @@ Three groups, decided in this order:
|
||||||
1. the stem itself refers to a figure → the image belongs there, leave it;
|
1. the stem itself refers to a figure → the image belongs there, leave it;
|
||||||
2. only the explanation refers to one → the image belongs to the explanation,
|
2. only the explanation refers to one → the image belongs to the explanation,
|
||||||
so it comes off the stem;
|
so it comes off the stem;
|
||||||
(PREP labels its figures by where they are printed — "Item Q37A" beside the
|
(the source material labels its figures by where they are printed — "Item Q37A" beside the
|
||||||
question, "Item C37B" beside the critique — which decides most of these;)
|
question, "Item C37B" beside the critique — which decides most of these;)
|
||||||
3. neither says anything → ask a vision model what the picture
|
3. neither says anything → ask a vision model what the picture
|
||||||
shows and which half of the question it illustrates.
|
shows and which half of the question it illustrates.
|
||||||
|
|
@ -65,7 +65,7 @@ STEM_CUE = re.compile(
|
||||||
# printed with the critique, not with the vignette.
|
# printed with the critique, not with the vignette.
|
||||||
EXPLANATION_CUE = STEM_CUE
|
EXPLANATION_CUE = STEM_CUE
|
||||||
|
|
||||||
# The strongest signal in this bank, and one specific to PREP: figures are
|
# The strongest signal in this bank, and one specific to the source: figures are
|
||||||
# labelled by where they are printed. "Item Q37A" is a figure beside the
|
# labelled by where they are printed. "Item Q37A" is a figure beside the
|
||||||
# question; "Item C37B" is a figure beside the critique. The stem citing an
|
# question; "Item C37B" is a figure beside the critique. The stem citing an
|
||||||
# Item Q settles it on its own, and only a critique cites an Item C.
|
# Item Q settles it on its own, and only a critique cites an Item C.
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ const categories = [
|
||||||
{ id: 1, name: 'Pediatrics', question_count: 30, breadcrumbs: [{ id: 1, name: 'Pediatrics' }] },
|
{ id: 1, name: 'Pediatrics', question_count: 30, breadcrumbs: [{ id: 1, name: 'Pediatrics' }] },
|
||||||
{ id: 2, name: 'Neonatal', question_count: 10, breadcrumbs: [{ id: 1, name: 'Pediatrics' }, { id: 2, name: 'Neonatal' }] },
|
{ id: 2, name: 'Neonatal', question_count: 10, breadcrumbs: [{ id: 1, name: 'Pediatrics' }, { id: 2, name: 'Neonatal' }] },
|
||||||
]
|
]
|
||||||
const TAGS = { subjects: [{ id: 7, name: 'Cardiology' }], keywords: [{ id: 8, name: 'PREP 2019' }] }
|
const TAGS = { subjects: [{ id: 7, name: 'Cardiology' }], keywords: [{ id: 8, name: 'Board Review 2019' }] }
|
||||||
function setupCount(count = 30) {
|
function setupCount(count = 30) {
|
||||||
api.get.mockImplementation(url => {
|
api.get.mockImplementation(url => {
|
||||||
if (url === '/question-categories/') return Promise.resolve({ data: categories })
|
if (url === '/question-categories/') return Promise.resolve({ data: categories })
|
||||||
|
|
|
||||||
|
|
@ -406,16 +406,16 @@ export default function DocumentDetailPage() {
|
||||||
<option value="standard">Standard — inline answers (Correct Answer / Preferred Response)</option>
|
<option value="standard">Standard — inline answers (Correct Answer / Preferred Response)</option>
|
||||||
<option value="questions_only">Questions Only — no answers (fill in manually later)</option>
|
<option value="questions_only">Questions Only — no answers (fill in manually later)</option>
|
||||||
<option value="ai_answer">AI Answer — extract questions, AI determines correct answers</option>
|
<option value="ai_answer">AI Answer — extract questions, AI determines correct answers</option>
|
||||||
<option value="two_step">Two-Step — separate answer key section (PREP 2013 style)</option>
|
<option value="two_step">Two-Step — separate answer key section (2013-style layout)</option>
|
||||||
<option value="regex">AI + Regex — AI analyses format then applies regex for answers</option>
|
<option value="regex">AI + Regex — AI analyses format then applies regex for answers</option>
|
||||||
<option value="ai_decide">AI Decides — AI reads the document and picks best strategy</option>
|
<option value="ai_decide">AI Decides — AI reads the document and picks best strategy</option>
|
||||||
<option value="generate">Generate — AI creates questions from plain text / textbook chapters</option>
|
<option value="generate">Generate — AI creates questions from plain text / textbook chapters</option>
|
||||||
</select>
|
</select>
|
||||||
<p style={{ fontSize: '0.75rem', color: 'var(--text-subtle)', marginTop: 4 }}>
|
<p style={{ fontSize: '0.75rem', color: 'var(--text-subtle)', marginTop: 4 }}>
|
||||||
{extractionMode === 'standard' && 'Best for PREP 2012, 2014 and most PDFs with answers inline.'}
|
{extractionMode === 'standard' && 'Best for 2012 and 2014 sets, and most PDFs with answers inline.'}
|
||||||
{extractionMode === 'questions_only' && 'Extracts questions + options only. Answer each question manually in Edit mode.'}
|
{extractionMode === 'questions_only' && 'Extracts questions + options only. Answer each question manually in Edit mode.'}
|
||||||
{extractionMode === 'ai_answer' && 'For Q&A PDFs with no answer key. AI extracts questions then determines the correct answer and explanation from document context + medical knowledge.'}
|
{extractionMode === 'ai_answer' && 'For Q&A PDFs with no answer key. AI extracts questions then determines the correct answer and explanation from document context + medical knowledge.'}
|
||||||
{extractionMode === 'two_step' && 'For PDFs where all questions come first, then all answers at the back (PREP 2013 style).'}
|
{extractionMode === 'two_step' && 'For PDFs where all questions come first, then all answers at the back (2013-style layout).'}
|
||||||
{extractionMode === 'regex' && 'AI detects the answer pattern, then uses regex for fast reliable extraction.'}
|
{extractionMode === 'regex' && 'AI detects the answer pattern, then uses regex for fast reliable extraction.'}
|
||||||
{extractionMode === 'ai_decide' && 'AI samples the document and automatically picks the right strategy (standard, two_step, or ai_answer).'}
|
{extractionMode === 'ai_decide' && 'AI samples the document and automatically picks the right strategy (standard, two_step, or ai_answer).'}
|
||||||
{extractionMode === 'generate' && 'For textbook chapters, lecture notes, or any material without a Q&A format. AI creates MCQ questions with correct answers from the text.'}
|
{extractionMode === 'generate' && 'For textbook chapters, lecture notes, or any material without a Q&A format. AI creates MCQ questions with correct answers from the text.'}
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ const FEATURES = [
|
||||||
{
|
{
|
||||||
icon: '📄',
|
icon: '📄',
|
||||||
title: 'Quiz from Any PDF',
|
title: 'Quiz from Any PDF',
|
||||||
desc: 'Upload PREP materials, textbook chapters, or lecture slides. AI extracts questions with answers and explanations — no formatting required.',
|
desc: 'Upload board review material, textbook chapters, or lecture slides. AI extracts questions with answers and explanations — no formatting required.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
icon: '🎓',
|
icon: '🎓',
|
||||||
|
|
@ -394,7 +394,7 @@ export default function LandingPage() {
|
||||||
</span>
|
</span>
|
||||||
</h1>
|
</h1>
|
||||||
<p style={{ fontSize: '1.15rem', color: 'rgba(226,232,240,0.75)', lineHeight: 1.7, marginBottom: 40, maxWidth: 560, margin: '0 auto 40px' }}>
|
<p style={{ fontSize: '1.15rem', color: 'rgba(226,232,240,0.75)', lineHeight: 1.7, marginBottom: 40, maxWidth: 560, margin: '0 auto 40px' }}>
|
||||||
Upload any PDF — PREP materials, lecture notes, textbook chapters.
|
Upload any PDF — board review material, lecture notes, textbook chapters.
|
||||||
AI extracts questions, reads them aloud, and explains every answer.
|
AI extracts questions, reads them aloud, and explains every answer.
|
||||||
</p>
|
</p>
|
||||||
<div style={{ display: 'flex', gap: 14, justifyContent: 'center', flexWrap: 'wrap' }}>
|
<div style={{ display: 'flex', gap: 14, justifyContent: 'center', flexWrap: 'wrap' }}>
|
||||||
|
|
|
||||||
|
|
@ -11,14 +11,14 @@ let currentUser = { id: 1, name: 'Learner', is_moderator: false }
|
||||||
vi.mock('../context/AuthContext', () => ({ useAuth: () => ({ user: currentUser }) }))
|
vi.mock('../context/AuthContext', () => ({ useAuth: () => ({ user: currentUser }) }))
|
||||||
|
|
||||||
const plans = [
|
const plans = [
|
||||||
{ id: 1, slug: 'prep-2025', name: 'PREP 2025', kind: 'set', exam_name: 'Pediatrics Boards',
|
{ id: 1, slug: 'board-review-2025', name: 'Board Review 2025', kind: 'set', exam_name: 'Pediatrics Boards',
|
||||||
is_published: true, block_count: 4, question_count: 200, blocks_completed: 1 },
|
is_published: true, block_count: 4, question_count: 200, blocks_completed: 1 },
|
||||||
{ id: 13, slug: 'prep-mixed', name: 'PREP Mixed', kind: 'mixed', exam_name: null,
|
{ id: 13, slug: 'mixed-review', name: 'Mixed Review', kind: 'mixed', exam_name: null,
|
||||||
is_published: true, block_count: 1, question_count: 300, blocks_completed: 0 },
|
is_published: true, block_count: 1, question_count: 300, blocks_completed: 0 },
|
||||||
]
|
]
|
||||||
|
|
||||||
const plan = {
|
const plan = {
|
||||||
id: 1, slug: 'prep-2025', name: 'PREP 2025', description: null, kind: 'set', is_published: true,
|
id: 1, slug: 'board-review-2025', name: 'Board Review 2025', description: null, kind: 'set', is_published: true,
|
||||||
blocks: [
|
blocks: [
|
||||||
{ id: 10, position: 0, title: 'Block 1', question_count: 50, quiz_id: 77, completed: true,
|
{ id: 10, position: 0, title: 'Block 1', question_count: 50, quiz_id: 77, completed: true,
|
||||||
articles: [{ link_id: 100, article_id: 5, slug: 'asthma', title: 'Asthma', status: 'published', read: true }] },
|
articles: [{ link_id: 100, article_id: 5, slug: 'asthma', title: 'Asthma', status: 'published', read: true }] },
|
||||||
|
|
@ -53,7 +53,7 @@ describe('study plans', () => {
|
||||||
|
|
||||||
it('states progress in blocks, which is something you can act on', async () => {
|
it('states progress in blocks, which is something you can act on', async () => {
|
||||||
mountList()
|
mountList()
|
||||||
const card = (await screen.findByText('PREP 2025')).closest('.plan-card')
|
const card = (await screen.findByText('Board Review 2025')).closest('.plan-card')
|
||||||
expect(within(card).getByText('1 of 4 blocks done')).toBeInTheDocument()
|
expect(within(card).getByText('1 of 4 blocks done')).toBeInTheDocument()
|
||||||
expect(within(card).getByText(/4 blocks · 200 questions/)).toBeInTheDocument()
|
expect(within(card).getByText(/4 blocks · 200 questions/)).toBeInTheDocument()
|
||||||
})
|
})
|
||||||
|
|
@ -120,13 +120,13 @@ describe('study plans, as an educator', () => {
|
||||||
|
|
||||||
it('creates a plan as a draft, because an empty plan is not for a learner', async () => {
|
it('creates a plan as a draft, because an empty plan is not for a learner', async () => {
|
||||||
mountList()
|
mountList()
|
||||||
await screen.findByText('PREP 2025')
|
await screen.findByText('Board Review 2025')
|
||||||
api.post.mockResolvedValue({ data: { id: 20 } })
|
api.post.mockResolvedValue({ data: { id: 20 } })
|
||||||
await userEvent.click(screen.getByRole('button', { name: 'New plan' }))
|
await userEvent.click(screen.getByRole('button', { name: 'New plan' }))
|
||||||
await userEvent.type(screen.getByLabelText('New plan name'), 'PREP 2026')
|
await userEvent.type(screen.getByLabelText('New plan name'), 'Board Review 2026')
|
||||||
await userEvent.click(screen.getByRole('button', { name: 'Create' }))
|
await userEvent.click(screen.getByRole('button', { name: 'Create' }))
|
||||||
await waitFor(() => expect(api.post).toHaveBeenCalledWith('/study-plans/', {
|
await waitFor(() => expect(api.post).toHaveBeenCalledWith('/study-plans/', {
|
||||||
name: 'PREP 2026', slug: 'prep-2026', kind: 'set', is_published: false,
|
name: 'Board Review 2026', slug: 'board-review-2026', kind: 'set', is_published: false,
|
||||||
}))
|
}))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@ export default function StudyPlansPage() {
|
||||||
|
|
||||||
{creating && (
|
{creating && (
|
||||||
<div className="plans-create">
|
<div className="plans-create">
|
||||||
<input value={name} autoFocus placeholder="Plan name, e.g. PREP 2026" aria-label="New plan name"
|
<input value={name} autoFocus placeholder="Plan name, e.g. Board Review 2026" aria-label="New plan name"
|
||||||
onChange={e => setName(e.target.value)}
|
onChange={e => setName(e.target.value)}
|
||||||
onKeyDown={e => { if (e.key === 'Enter') create() }} />
|
onKeyDown={e => { if (e.key === 'Enter') create() }} />
|
||||||
{/* Created unpublished: a plan with no blocks is not something to
|
{/* Created unpublished: a plan with no blocks is not something to
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue