From 8a220bb12e7eed080e27efbb8d8d5bf2aeebcb63 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 1 Apr 2026 04:09:55 +0200 Subject: [PATCH] Fix PREP 2013 extraction: OCR normalization, correct chunk boundary; category UI fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extraction fixes: - OCR normalization: 'Pref erred' → 'Preferred', 'ltem' → 'Item' applied to boundary scan, Phase 1 questions, and Phase 2 answer key content before AI processing - Chunk boundary: Phase 1 chunks now capped at (answer_section_start - 1) so no chunk bleeds into the answer section — (51, 100) becomes (51, 55) for PREP 2013 - Result: Phase 1 gets 2 clean chunks (1-50 and 51-55), Phase 2 gets pages 56-227 Category creation in DocumentDetailPage: - Replaced window.prompt() with inline input form (more reliable, no browser quirks) - Fixed option value type: String(c.id) ensures consistent string comparison with selectedQuestionCategoryId state (prevents type mismatch in controlled select) - "+ New" button toggles inline form; Enter key or Add button submits Deletion safety (confirmed): - Deleting a quiz: questions detached to bank if exclusive, kept if shared — NEVER deleted - Deleting a question category: questions uncategorized or moved — NEVER deleted Co-Authored-By: Claude Sonnet 4.6 (1M context) --- backend/app/tasks/quiz_tasks.py | 29 +++++++++++++----- frontend/src/pages/DocumentDetailPage.jsx | 36 +++++++++++++++++------ 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/backend/app/tasks/quiz_tasks.py b/backend/app/tasks/quiz_tasks.py index 00218cf..63ae57d 100644 --- a/backend/app/tasks/quiz_tasks.py +++ b/backend/app/tasks/quiz_tasks.py @@ -95,17 +95,28 @@ def extract_quiz( if n_chunks > 1: _push_step(r, job_id, "text", f"Large section: splitting into {n_chunks} chunks of up to {CHUNK_PAGES} pages each.") + def _normalize_ocr(text: str) -> str: + """Normalize common OCR artifacts in PREP PDFs.""" + return (text + .replace("Pref erred", "Preferred") + .replace("Pre ferred", "Preferred") + .replace("Prefer red", "Preferred") + .replace("ltem", "Item") + .replace("ltcm", "Item")) + # --- Detect separate answer key section (e.g. PREP 2013) --- # Scan document in 10-page steps to find where "Preferred Response:" first appears answer_section_start = None scan_step = 10 for scan_p in range(section.start_page, section.end_page, scan_step): scan_end = min(scan_p + scan_step - 1, section.end_page) - scan_chunk = vector_service.get_pages_text( + raw_chunk = vector_service.get_pages_text( document_id=section.document_id, start_page=scan_p, end_page=scan_end, ) - if scan_chunk and ("Preferred Response:" in scan_chunk or "preferred response:" in scan_chunk.lower()): - # Found it — but go back a few pages to be safe + if not raw_chunk: + continue + scan_chunk = _normalize_ocr(raw_chunk) + if "Preferred Response:" in scan_chunk or "preferred response:" in scan_chunk.lower(): answer_section_start = max(section.start_page, scan_p - 5) break @@ -113,10 +124,12 @@ def extract_quiz( if has_end_answer_key: _push_step(r, job_id, "ai", f"Detected separate answer key section starting around page {answer_section_start}. Using two-phase extraction.") - # Restrict question chunks to BEFORE the answer section - chunks = [(s, e) for s, e in chunks if s < answer_section_start] + # Restrict question chunks to pages strictly BEFORE the answer section + # Cap the end page so no chunk bleeds into the answer section + q_end = answer_section_start - 1 + chunks = [(s, min(e, q_end)) for s, e in chunks if s <= q_end] if not chunks: - chunks = [(section.start_page, answer_section_start - 1)] + chunks = [(section.start_page, q_end)] all_valid_questions = [] all_skipped = [] @@ -133,7 +146,7 @@ def extract_quiz( continue try: chunk_qs = ai_service.extract_questions_no_answers( - chunk_content, page_info=f"{start_p}-{end_p}", + _normalize_ocr(chunk_content), page_info=f"{start_p}-{end_p}", page_ref=start_p, model_id=model_id, api_key=api_key, ) raw_questions.extend(chunk_qs) @@ -155,7 +168,7 @@ def extract_quiz( if not answer_content: continue chunk_key = ai_service.extract_answer_key( - answer_content, page_info=f"{ans_start}-{ans_end}", + _normalize_ocr(answer_content), page_info=f"{ans_start}-{ans_end}", model_id=model_id, api_key=api_key, ) answer_key.update(chunk_key) diff --git a/frontend/src/pages/DocumentDetailPage.jsx b/frontend/src/pages/DocumentDetailPage.jsx index 2f0494a..24ef5b3 100644 --- a/frontend/src/pages/DocumentDetailPage.jsx +++ b/frontend/src/pages/DocumentDetailPage.jsx @@ -119,6 +119,8 @@ export default function DocumentDetailPage() { const [availableModels, setAvailableModels] = useState([]) const [questionCategories, setQuestionCategories] = useState([]) const [selectedQuestionCategoryId, setSelectedQuestionCategoryId] = useState('') + const [newCatInline, setNewCatInline] = useState('') + const [showNewCat, setShowNewCat] = useState(false) const [error, setError] = useState('') const isModerator = user?.role === 'admin' || user?.role === 'moderator' @@ -217,12 +219,15 @@ export default function DocumentDetailPage() { } finally { setDeletingSection(null) } } - const createCategoryAndSelect = async (name) => { - if (!name.trim()) return + const createCategoryAndSelect = async () => { + const name = newCatInline.trim() + if (!name) return try { - const res = await api.post('/question-categories/', { name: name.trim() }) + const res = await api.post('/question-categories/', { name }) setQuestionCategories(prev => [...prev, res.data]) setSelectedQuestionCategoryId(String(res.data.id)) + setNewCatInline('') + setShowNewCat(false) } catch (err) { setError(err.response?.data?.detail || 'Failed to create category') } @@ -369,20 +374,33 @@ export default function DocumentDetailPage() {
- setSelectedQuestionCategoryId(e.target.value)} + style={{ flex: 1 }}> {questionCategories.map(c => ( - + ))}
+ {showNewCat && ( +
+ setNewCatInline(e.target.value)} + onKeyDown={e => e.key === 'Enter' && createCategoryAndSelect()} + placeholder="Category name…" + style={{ flex: 1, padding: '6px 10px', border: '1px solid var(--border)', borderRadius: 6, fontSize: '0.85rem', background: 'var(--input-bg)', color: 'var(--text)' }} + autoFocus /> + + +
+ )}

Extracted questions will be tagged with this category in the Question Bank.