2cbbfe00c3
- Fix tag filtering (sa_text import shadowing caused UnboundLocalError) - Add TagBrowser component with per-section search - Multi-category selection (OR within categories, AND with tags) - AI image validation: has_figure field in extraction prompt - Skip known branding images by MD5 hash + dimension filters - Fix quiz timer auto-submit (wrong useEffect dependency) - Fix QuizResponse schema: section_id nullable - Fix Question.quiz_id → source_quiz_id attribute name - Fix SQL injection in quizzes.py vector search - Add PDF processing progress steps via Redis - Add delete user from admin panel - Admin page: no spinner flash on data refresh - Upload progress: axios 1.x e.progress, remove manual Content-Type - Duplicate model error: 409 with clear message - Backend startup: retry DDL migration on lock timeout - Replace all silent except:pass with warning logs - Comprehensive multi-page documentation (docs/) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
82 lines
2.9 KiB
Python
82 lines
2.9 KiB
Python
import logging
|
|
import json
|
|
import time
|
|
|
|
from app.tasks import celery_app
|
|
from app.database import SessionLocal
|
|
from app.models.pdf_document import PDFDocument
|
|
from app.services import pdf_service, vector_service
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _redis():
|
|
import redis
|
|
return redis.from_url(settings.REDIS_URL, decode_responses=True)
|
|
|
|
|
|
def _push_step(r, doc_id: int, step: str, message: str):
|
|
r.rpush(f"pdf:steps:{doc_id}", json.dumps({"step": step, "message": message, "ts": time.time()}))
|
|
r.expire(f"pdf:steps:{doc_id}", 3600)
|
|
|
|
|
|
@celery_app.task(name="process_pdf")
|
|
def process_pdf(document_id: int, file_path: str):
|
|
"""Background task: extract PDF text and store in ChromaDB."""
|
|
db = SessionLocal()
|
|
r = _redis()
|
|
try:
|
|
doc = db.query(PDFDocument).filter(PDFDocument.id == document_id).first()
|
|
if not doc:
|
|
logger.error(f"Document {document_id} not found")
|
|
return
|
|
|
|
_push_step(r, document_id, "start", "Starting PDF processing…")
|
|
|
|
# Get page count
|
|
total_pages = pdf_service.get_page_count(file_path)
|
|
doc.total_pages = total_pages
|
|
db.commit()
|
|
_push_step(r, document_id, "pages", f"Found {total_pages} pages")
|
|
|
|
# Extract text from all pages
|
|
_push_step(r, document_id, "text", "Extracting text from pages…")
|
|
pages = pdf_service.extract_text_by_page(file_path)
|
|
if not pages:
|
|
doc.status = "error"
|
|
doc.error_message = "No text could be extracted from the PDF"
|
|
db.commit()
|
|
_push_step(r, document_id, "error", "No text could be extracted")
|
|
return
|
|
|
|
_push_step(r, document_id, "text", f"Extracted text from {len(pages)} pages")
|
|
|
|
# Store in ChromaDB
|
|
total_chunks = 0
|
|
for page_num, text in pages.items():
|
|
chunks = vector_service.chunk_text(text)
|
|
total_chunks += len(chunks)
|
|
_push_step(r, document_id, "vector", f"Vectorizing {total_chunks} chunks across {len(pages)} pages…")
|
|
vector_service.store_pages(document_id, pages)
|
|
_push_step(r, document_id, "vector", "Vectorization complete")
|
|
|
|
# Mark as ready
|
|
doc.status = "ready"
|
|
db.commit()
|
|
_push_step(r, document_id, "done", f"Ready — {total_pages} pages, {len(pages)} with text")
|
|
logger.info(f"Document {document_id} processed: {total_pages} pages, {len(pages)} with text")
|
|
|
|
except Exception as e:
|
|
logger.exception(f"Error processing document {document_id}")
|
|
_push_step(r, document_id, "error", f"Failed: {str(e)[:200]}")
|
|
try:
|
|
doc = db.query(PDFDocument).filter(PDFDocument.id == document_id).first()
|
|
if doc:
|
|
doc.status = "error"
|
|
doc.error_message = str(e)[:500]
|
|
db.commit()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
db.close()
|