Fix PREP 2013 extraction: OCR normalization, correct chunk boundary; category UI fix
Extraction fixes: - OCR normalization: 'Pref erred' → 'Preferred', 'ltem' → 'Item' applied to boundary scan, Phase 1 questions, and Phase 2 answer key content before AI processing - Chunk boundary: Phase 1 chunks now capped at (answer_section_start - 1) so no chunk bleeds into the answer section — (51, 100) becomes (51, 55) for PREP 2013 - Result: Phase 1 gets 2 clean chunks (1-50 and 51-55), Phase 2 gets pages 56-227 Category creation in DocumentDetailPage: - Replaced window.prompt() with inline input form (more reliable, no browser quirks) - Fixed option value type: String(c.id) ensures consistent string comparison with selectedQuestionCategoryId state (prevents type mismatch in controlled select) - "+ New" button toggles inline form; Enter key or Add button submits Deletion safety (confirmed): - Deleting a quiz: questions detached to bank if exclusive, kept if shared — NEVER deleted - Deleting a question category: questions uncategorized or moved — NEVER deleted Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel
parent
d07be64f59
commit
8a220bb12e
2 changed files with 48 additions and 17 deletions
|
|
@ -95,17 +95,28 @@ def extract_quiz(
|
|||
if n_chunks > 1:
|
||||
_push_step(r, job_id, "text", f"Large section: splitting into {n_chunks} chunks of up to {CHUNK_PAGES} pages each.")
|
||||
|
||||
def _normalize_ocr(text: str) -> str:
|
||||
"""Normalize common OCR artifacts in PREP PDFs."""
|
||||
return (text
|
||||
.replace("Pref erred", "Preferred")
|
||||
.replace("Pre ferred", "Preferred")
|
||||
.replace("Prefer red", "Preferred")
|
||||
.replace("ltem", "Item")
|
||||
.replace("ltcm", "Item"))
|
||||
|
||||
# --- Detect separate answer key section (e.g. PREP 2013) ---
|
||||
# Scan document in 10-page steps to find where "Preferred Response:" first appears
|
||||
answer_section_start = None
|
||||
scan_step = 10
|
||||
for scan_p in range(section.start_page, section.end_page, scan_step):
|
||||
scan_end = min(scan_p + scan_step - 1, section.end_page)
|
||||
scan_chunk = vector_service.get_pages_text(
|
||||
raw_chunk = vector_service.get_pages_text(
|
||||
document_id=section.document_id, start_page=scan_p, end_page=scan_end,
|
||||
)
|
||||
if scan_chunk and ("Preferred Response:" in scan_chunk or "preferred response:" in scan_chunk.lower()):
|
||||
# Found it — but go back a few pages to be safe
|
||||
if not raw_chunk:
|
||||
continue
|
||||
scan_chunk = _normalize_ocr(raw_chunk)
|
||||
if "Preferred Response:" in scan_chunk or "preferred response:" in scan_chunk.lower():
|
||||
answer_section_start = max(section.start_page, scan_p - 5)
|
||||
break
|
||||
|
||||
|
|
@ -113,10 +124,12 @@ def extract_quiz(
|
|||
|
||||
if has_end_answer_key:
|
||||
_push_step(r, job_id, "ai", f"Detected separate answer key section starting around page {answer_section_start}. Using two-phase extraction.")
|
||||
# Restrict question chunks to BEFORE the answer section
|
||||
chunks = [(s, e) for s, e in chunks if s < answer_section_start]
|
||||
# Restrict question chunks to pages strictly BEFORE the answer section
|
||||
# Cap the end page so no chunk bleeds into the answer section
|
||||
q_end = answer_section_start - 1
|
||||
chunks = [(s, min(e, q_end)) for s, e in chunks if s <= q_end]
|
||||
if not chunks:
|
||||
chunks = [(section.start_page, answer_section_start - 1)]
|
||||
chunks = [(section.start_page, q_end)]
|
||||
|
||||
all_valid_questions = []
|
||||
all_skipped = []
|
||||
|
|
@ -133,7 +146,7 @@ def extract_quiz(
|
|||
continue
|
||||
try:
|
||||
chunk_qs = ai_service.extract_questions_no_answers(
|
||||
chunk_content, page_info=f"{start_p}-{end_p}",
|
||||
_normalize_ocr(chunk_content), page_info=f"{start_p}-{end_p}",
|
||||
page_ref=start_p, model_id=model_id, api_key=api_key,
|
||||
)
|
||||
raw_questions.extend(chunk_qs)
|
||||
|
|
@ -155,7 +168,7 @@ def extract_quiz(
|
|||
if not answer_content:
|
||||
continue
|
||||
chunk_key = ai_service.extract_answer_key(
|
||||
answer_content, page_info=f"{ans_start}-{ans_end}",
|
||||
_normalize_ocr(answer_content), page_info=f"{ans_start}-{ans_end}",
|
||||
model_id=model_id, api_key=api_key,
|
||||
)
|
||||
answer_key.update(chunk_key)
|
||||
|
|
|
|||
|
|
@ -119,6 +119,8 @@ export default function DocumentDetailPage() {
|
|||
const [availableModels, setAvailableModels] = useState([])
|
||||
const [questionCategories, setQuestionCategories] = useState([])
|
||||
const [selectedQuestionCategoryId, setSelectedQuestionCategoryId] = useState('')
|
||||
const [newCatInline, setNewCatInline] = useState('')
|
||||
const [showNewCat, setShowNewCat] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
|
||||
const isModerator = user?.role === 'admin' || user?.role === 'moderator'
|
||||
|
|
@ -217,12 +219,15 @@ export default function DocumentDetailPage() {
|
|||
} finally { setDeletingSection(null) }
|
||||
}
|
||||
|
||||
const createCategoryAndSelect = async (name) => {
|
||||
if (!name.trim()) return
|
||||
const createCategoryAndSelect = async () => {
|
||||
const name = newCatInline.trim()
|
||||
if (!name) return
|
||||
try {
|
||||
const res = await api.post('/question-categories/', { name: name.trim() })
|
||||
const res = await api.post('/question-categories/', { name })
|
||||
setQuestionCategories(prev => [...prev, res.data])
|
||||
setSelectedQuestionCategoryId(String(res.data.id))
|
||||
setNewCatInline('')
|
||||
setShowNewCat(false)
|
||||
} catch (err) {
|
||||
setError(err.response?.data?.detail || 'Failed to create category')
|
||||
}
|
||||
|
|
@ -369,20 +374,33 @@ export default function DocumentDetailPage() {
|
|||
<div className="form-group">
|
||||
<label>Add Questions to Bank Category <span style={{ fontWeight: 400, color: 'var(--text-muted)' }}>(optional)</span></label>
|
||||
<div style={{ display: 'flex', gap: 8 }}>
|
||||
<select value={selectedQuestionCategoryId} onChange={e => setSelectedQuestionCategoryId(e.target.value)} style={{ flex: 1 }}>
|
||||
<select value={selectedQuestionCategoryId}
|
||||
onChange={e => setSelectedQuestionCategoryId(e.target.value)}
|
||||
style={{ flex: 1 }}>
|
||||
<option value="">— No category —</option>
|
||||
{questionCategories.map(c => (
|
||||
<option key={c.id} value={c.id}>{c.name}{c.question_count > 0 ? ` (${c.question_count})` : ''}</option>
|
||||
<option key={c.id} value={String(c.id)}>
|
||||
{c.name}{c.question_count > 0 ? ` (${c.question_count})` : ''}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<button type="button" className="btn btn-secondary btn-sm" style={{ whiteSpace: 'nowrap' }}
|
||||
onClick={() => {
|
||||
const name = prompt('New category name:')
|
||||
if (name) createCategoryAndSelect(name)
|
||||
}}>
|
||||
onClick={() => setShowNewCat(v => !v)}>
|
||||
+ New
|
||||
</button>
|
||||
</div>
|
||||
{showNewCat && (
|
||||
<div style={{ display: 'flex', gap: 6, marginTop: 6 }}>
|
||||
<input type="text" value={newCatInline}
|
||||
onChange={e => setNewCatInline(e.target.value)}
|
||||
onKeyDown={e => e.key === 'Enter' && createCategoryAndSelect()}
|
||||
placeholder="Category name…"
|
||||
style={{ flex: 1, padding: '6px 10px', border: '1px solid var(--border)', borderRadius: 6, fontSize: '0.85rem', background: 'var(--input-bg)', color: 'var(--text)' }}
|
||||
autoFocus />
|
||||
<button type="button" className="btn btn-primary btn-sm" onClick={createCategoryAndSelect}>Add</button>
|
||||
<button type="button" className="btn btn-secondary btn-sm" onClick={() => { setShowNewCat(false); setNewCatInline('') }}>✕</button>
|
||||
</div>
|
||||
)}
|
||||
<p style={{ fontSize: '0.75rem', color: 'var(--text-subtle)', marginTop: 4 }}>
|
||||
Extracted questions will be tagged with this category in the Question Bank.
|
||||
</p>
|
||||
|
|
|
|||
Loading…
Reference in a new issue