from fastapi import FastAPI, UploadFile, File, HTTPException, Depends, Response, Request from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, RedirectResponse from pydantic import BaseModel from sqlalchemy.orm import Session from typing import Optional import httpx, re, uuid, requests as req_lib, secrets, smtplib from datetime import datetime from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from models import get_db, init_db, User, Document, Progress, Setting, VerificationToken, UserSetting from auth import ( hash_password, verify_password, create_token, get_current_user, require_admin ) # ── EMAIL CONFIG ───────────────────────────────────────────────────── SMTP_HOST = "mail.smtp2go.com" SMTP_PORT = 587 SMTP_USER = "danvics.com" SMTP_PASS = "YFZ2L8F06YUYBfBR" SMTP_FROM = "noreply@pedshub.com" APP_NAME = "Speechify" APP_URL = "https://read.pedshub.com" # ── CONFIG ────────────────────────────────────────────────────────── LITELLM_BASE = "http://localhost:4000" LITELLM_KEY = "sk-c5c58adcc8dda288067f034ee927be23509142d274fe48418c25009dfb507a52" TURNSTILE_SECRET = "0x4AAAAAAC0VtE0VOo0vpGoTvmHSk8yRN4Y" TTS_MODELS = { "openai-tts-1": { "name": "OpenAI TTS", "quality": "Standard", "voices": [ {"id": "alloy", "name": "Alloy"}, {"id": "ash", "name": "Ash"}, {"id": "coral", "name": "Coral"}, {"id": "echo", "name": "Echo"}, {"id": "fable", "name": "Fable"}, {"id": "nova", "name": "Nova"}, {"id": "onyx", "name": "Onyx"}, {"id": "sage", "name": "Sage"}, {"id": "shimmer", "name": "Shimmer"}, ], }, "openai-tts-1-hd": { "name": "OpenAI TTS HD", "quality": "High Definition", "voices": [ {"id": "alloy", "name": "Alloy"}, {"id": "ash", "name": "Ash"}, {"id": "coral", "name": "Coral"}, {"id": "echo", "name": "Echo"}, {"id": "fable", "name": "Fable"}, {"id": "nova", "name": "Nova"}, {"id": "onyx", "name": "Onyx"}, {"id": "sage", "name": "Sage"}, {"id": "shimmer", "name": "Shimmer"}, ], }, "elevenlabs-v3": { "name": "ElevenLabs v3", "quality": "Premium", "voices": [ {"id": "Rachel", "name": "Rachel"}, {"id": "Drew", "name": "Drew"}, {"id": "Clyde", "name": "Clyde"}, {"id": "Paul", "name": "Paul"}, {"id": "Domi", "name": "Domi"}, {"id": "Dave", "name": "Dave"}, {"id": "Fin", "name": "Fin"}, {"id": "Bella", "name": "Bella"}, {"id": "Antoni", "name": "Antoni"}, {"id": "Thomas", "name": "Thomas"}, {"id": "Charlie", "name": "Charlie"}, {"id": "Emily", "name": "Emily"}, {"id": "Elli", "name": "Elli"}, {"id": "Callum", "name": "Callum"}, {"id": "Patrick", "name": "Patrick"}, {"id": "Harry", "name": "Harry"}, {"id": "Liam", "name": "Liam"}, {"id": "Dorothy", "name": "Dorothy"}, {"id": "Josh", "name": "Josh"}, {"id": "Arnold", "name": "Arnold"}, {"id": "Adam", "name": "Adam"}, {"id": "Sam", "name": "Sam"}, ], }, "elevenlabs-multilingual": { "name": "ElevenLabs Multilingual", "quality": "Premium Multilingual", "voices": [ {"id": "Rachel", "name": "Rachel"}, {"id": "Drew", "name": "Drew"}, {"id": "Clyde", "name": "Clyde"}, {"id": "Paul", "name": "Paul"}, {"id": "Domi", "name": "Domi"}, {"id": "Dave", "name": "Dave"}, {"id": "Fin", "name": "Fin"}, {"id": "Bella", "name": "Bella"}, {"id": "Antoni", "name": "Antoni"}, {"id": "Thomas", "name": "Thomas"}, {"id": "Charlie", "name": "Charlie"}, {"id": "Emily", "name": "Emily"}, {"id": "Elli", "name": "Elli"}, {"id": "Callum", "name": "Callum"}, {"id": "Patrick", "name": "Patrick"}, {"id": "Harry", "name": "Harry"}, {"id": "Liam", "name": "Liam"}, {"id": "Dorothy", "name": "Dorothy"}, {"id": "Josh", "name": "Josh"}, {"id": "Arnold", "name": "Arnold"}, {"id": "Adam", "name": "Adam"}, {"id": "Sam", "name": "Sam"}, ], }, "vertex-gemini-2.5-flash-tts": { "name": "Gemini 2.5 Flash TTS", "quality": "Fast", "voices": [ {"id": "Aoede", "name": "Aoede"}, {"id": "Charon", "name": "Charon"}, {"id": "Fenrir", "name": "Fenrir"}, {"id": "Kore", "name": "Kore"}, {"id": "Puck", "name": "Puck"}, {"id": "Orbit", "name": "Orbit"}, {"id": "Zephyr", "name": "Zephyr"}, {"id": "Autonoe", "name": "Autonoe"}, {"id": "Enceladus", "name": "Enceladus"}, {"id": "Laomedeia", "name": "Laomedeia"}, {"id": "Achernar", "name": "Achernar"}, {"id": "Rasalgethi", "name": "Rasalgethi"}, {"id": "Schedar", "name": "Schedar"}, {"id": "Sulafat", "name": "Sulafat"}, ], }, "vertex-gemini-2.5-pro-tts": { "name": "Gemini 2.5 Pro TTS", "quality": "High Quality", "voices": [ {"id": "Aoede", "name": "Aoede"}, {"id": "Charon", "name": "Charon"}, {"id": "Fenrir", "name": "Fenrir"}, {"id": "Kore", "name": "Kore"}, {"id": "Puck", "name": "Puck"}, {"id": "Orbit", "name": "Orbit"}, {"id": "Zephyr", "name": "Zephyr"}, {"id": "Autonoe", "name": "Autonoe"}, {"id": "Enceladus", "name": "Enceladus"}, {"id": "Laomedeia", "name": "Laomedeia"}, {"id": "Achernar", "name": "Achernar"}, {"id": "Rasalgethi", "name": "Rasalgethi"}, {"id": "Schedar", "name": "Schedar"}, {"id": "Sulafat", "name": "Sulafat"}, ], }, "groq-playai-tts": { "name": "Groq PlayAI TTS", "quality": "Ultra-fast", "voices": [ {"id": "Fritz-PlayAI", "name": "Fritz"}, {"id": "Celeste-PlayAI", "name": "Celeste"}, {"id": "Ava-PlayAI", "name": "Ava"}, {"id": "Atlas-PlayAI", "name": "Atlas"}, {"id": "Jade-PlayAI", "name": "Jade"}, {"id": "Perseus-PlayAI", "name": "Perseus"}, {"id": "Luna-PlayAI", "name": "Luna"}, {"id": "Caelus-PlayAI", "name": "Caelus"}, {"id": "Orion-PlayAI", "name": "Orion"}, {"id": "Aaliyah-PlayAI", "name": "Aaliyah"}, {"id": "Adelaide-PlayAI", "name": "Adelaide"}, {"id": "Briggs-PlayAI", "name": "Briggs"}, {"id": "Deedee-PlayAI", "name": "Deedee"}, {"id": "Gail-PlayAI", "name": "Gail"}, {"id": "Jennifer-PlayAI", "name": "Jennifer"}, {"id": "Mamaw-PlayAI", "name": "Mamaw"}, {"id": "Mitch-PlayAI", "name": "Mitch"}, {"id": "Nia-PlayAI", "name": "Nia"}, {"id": "Quinn-PlayAI", "name": "Quinn"}, {"id": "Thunder-PlayAI", "name": "Thunder"}, ], }, "groq-playai-tts-arabic": { "name": "Groq PlayAI Arabic", "quality": "Ultra-fast · Arabic", "voices": [ {"id": "Ahmad-PlayAI", "name": "Ahmad"}, {"id": "Amira-PlayAI", "name": "Amira"}, ], }, } # ── APP ───────────────────────────────────────────────────────────── app = FastAPI(title="Speechify Clone") @app.on_event("startup") def startup(): init_db() db = next(get_db()) # Create default admin if no users exist if db.query(User).count() == 0: admin = User( username="admin", email="admin@speechify.local", password_hash=hash_password("admin123"), role="admin", ) db.add(admin) db.commit() print("\n" + "="*50) print("Default admin created: admin / admin123") print("="*50 + "\n") db.close() # ── EMAIL ──────────────────────────────────────────────────────────── def send_email(to: str, subject: str, html: str): try: msg = MIMEMultipart("alternative") msg["Subject"] = subject msg["From"] = f"{APP_NAME} <{SMTP_FROM}>" msg["To"] = to msg.attach(MIMEText(html, "html")) with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as s: s.starttls() s.login(SMTP_USER, SMTP_PASS) s.sendmail(SMTP_FROM, to, msg.as_string()) except Exception as e: print(f"Email error: {e}") def verification_email_html(username: str, link: str) -> str: return f"""
{APP_NAME}

Verify your email

Hi {username}, thanks for signing up!
Click the button below to verify your email address and activate your account.

Verify Email Address

If you didn't create an account, you can safely ignore this email.
This link will remain valid as long as your account exists.

{APP_NAME} · {APP_URL}

""" def create_verification_token(db: Session, user_id: int) -> str: # Remove old tokens for this user db.query(VerificationToken).filter(VerificationToken.user_id == user_id).delete() token = secrets.token_urlsafe(32) db.add(VerificationToken(user_id=user_id, token=token)) db.commit() return token # ── HELPERS ───────────────────────────────────────────────────────── def verify_turnstile(token: str, ip: str = "") -> bool: try: r = req_lib.post( "https://challenges.cloudflare.com/turnstile/v0/siteverify", data={"secret": TURNSTILE_SECRET, "response": token, "remoteip": ip}, timeout=10, ) return r.json().get("success", False) except Exception: return False def get_setting(db: Session, key: str, default: str = "") -> str: s = db.query(Setting).filter(Setting.key == key).first() return s.value if s else default def set_setting(db: Session, key: str, value: str): s = db.query(Setting).filter(Setting.key == key).first() if s: s.value = value else: db.add(Setting(key=key, value=value)) db.commit() def get_user_setting(db: Session, user_id: int, key: str, default: str = "") -> str: s = db.query(UserSetting).filter(UserSetting.user_id == user_id, UserSetting.key == key).first() return s.value if s else default def set_user_setting(db: Session, user_id: int, key: str, value: str): s = db.query(UserSetting).filter(UserSetting.user_id == user_id, UserSetting.key == key).first() if s: s.value = value else: db.add(UserSetting(user_id=user_id, key=key, value=value)) db.commit() def resolve_nextcloud(db: Session, user: User): """Return (url, username, password) using user settings if set, else admin settings.""" url = get_user_setting(db, user.id, "nc_url") or get_setting(db, "nextcloud_url") uname = get_user_setting(db, user.id, "nc_user") or get_setting(db, "nextcloud_username") pw = get_user_setting(db, user.id, "nc_pass") or get_setting(db, "nextcloud_password") return url, uname, pw # ── PYDANTIC MODELS ────────────────────────────────────────────────── class LoginRequest(BaseModel): username: str password: str turnstile_token: str class RegisterRequest(BaseModel): username: str email: str password: str turnstile_token: str class TTSRequest(BaseModel): text: str model: str voice: str speed: float = 1.0 class URLRequest(BaseModel): url: str class SaveDocRequest(BaseModel): title: str content: str source_type: str = "text" class ProgressRequest(BaseModel): sentence_index: int class NextcloudConfigRequest(BaseModel): url: str username: str password: str class UserSettingsRequest(BaseModel): nc_url: Optional[str] = None nc_user: Optional[str] = None nc_pass: Optional[str] = None class NextcloudDownloadRequest(BaseModel): path: str class UserUpdateRequest(BaseModel): role: Optional[str] = None is_active: Optional[bool] = None class SettingsRequest(BaseModel): nextcloud_url: Optional[str] = None nextcloud_username: Optional[str] = None nextcloud_password: Optional[str] = None # ── AUTH ROUTES ───────────────────────────────────────────────────── @app.post("/api/auth/login") async def login(req: LoginRequest, response: Response, request: Request, db: Session = Depends(get_db)): ip = request.client.host if request.client else "" if not verify_turnstile(req.turnstile_token, ip): raise HTTPException(status_code=400, detail="Turnstile verification failed") user = db.query(User).filter(User.username == req.username).first() if not user or not verify_password(req.password, user.password_hash): raise HTTPException(status_code=401, detail="Invalid credentials") if not user.is_active: raise HTTPException(status_code=403, detail="Account disabled") token = create_token(user.id, user.username, user.role) response.set_cookie("auth_token", token, httponly=True, samesite="lax", max_age=172800) return {"username": user.username, "role": user.role, "email_verified": user.email_verified} @app.post("/api/auth/register") async def register(req: RegisterRequest, response: Response, request: Request, db: Session = Depends(get_db)): ip = request.client.host if request.client else "" if not verify_turnstile(req.turnstile_token, ip): raise HTTPException(status_code=400, detail="Turnstile verification failed") if db.query(User).filter(User.username == req.username).first(): raise HTTPException(status_code=400, detail="Username already taken") if db.query(User).filter(User.email == req.email).first(): raise HTTPException(status_code=400, detail="Email already registered") user = User( username=req.username, email=req.email, password_hash=hash_password(req.password), role="user", email_verified=False, ) db.add(user) db.commit() db.refresh(user) # Send verification email vtoken = create_verification_token(db, user.id) link = f"{APP_URL}/api/auth/verify?token={vtoken}" html = verification_email_html(user.username, link) import threading threading.Thread(target=send_email, args=(user.email, f"Verify your {APP_NAME} account", html), daemon=True).start() token = create_token(user.id, user.username, user.role) response.set_cookie("auth_token", token, httponly=True, samesite="lax", max_age=172800) return {"username": user.username, "role": user.role, "email_verified": False} @app.get("/api/auth/verify") async def verify_email(token: str, db: Session = Depends(get_db)): vt = db.query(VerificationToken).filter(VerificationToken.token == token).first() if not vt: return FileResponse("static/verify-fail.html") user = db.query(User).filter(User.id == vt.user_id).first() if user: user.email_verified = True db.commit() db.delete(vt) db.commit() return FileResponse("static/verify-ok.html") @app.post("/api/auth/resend-verification") async def resend_verification(user: User = Depends(get_current_user), db: Session = Depends(get_db)): if user.email_verified: return {"ok": True, "message": "Already verified"} vtoken = create_verification_token(db, user.id) link = f"{APP_URL}/api/auth/verify?token={vtoken}" html = verification_email_html(user.username, link) import threading threading.Thread(target=send_email, args=(user.email, f"Verify your {APP_NAME} account", html), daemon=True).start() return {"ok": True} @app.post("/api/auth/logout") async def logout(response: Response): response.delete_cookie("auth_token") return {"ok": True} @app.get("/api/auth/me") async def me(user: User = Depends(get_current_user)): return { "id": user.id, "username": user.username, "email": user.email, "role": user.role, "email_verified": user.email_verified, } # ── ADMIN ROUTES ───────────────────────────────────────────────────── @app.get("/api/admin/users") async def list_users(admin: User = Depends(require_admin), db: Session = Depends(get_db)): users = db.query(User).all() return [{"id": u.id, "username": u.username, "email": u.email, "role": u.role, "is_active": u.is_active, "created_at": u.created_at.isoformat()} for u in users] @app.patch("/api/admin/users/{user_id}") async def update_user(user_id: int, req: UserUpdateRequest, admin: User = Depends(require_admin), db: Session = Depends(get_db)): user = db.query(User).filter(User.id == user_id).first() if not user: raise HTTPException(status_code=404, detail="User not found") if req.role is not None: user.role = req.role if req.is_active is not None: user.is_active = req.is_active db.commit() return {"ok": True} @app.delete("/api/admin/users/{user_id}") async def delete_user(user_id: int, admin: User = Depends(require_admin), db: Session = Depends(get_db)): user = db.query(User).filter(User.id == user_id).first() if not user: raise HTTPException(status_code=404, detail="User not found") if user.id == admin.id: raise HTTPException(status_code=400, detail="Cannot delete yourself") db.delete(user) db.commit() return {"ok": True} @app.get("/api/admin/settings") async def get_settings(admin: User = Depends(require_admin), db: Session = Depends(get_db)): return { "nextcloud_url": get_setting(db, "nextcloud_url"), "nextcloud_username": get_setting(db, "nextcloud_username"), "nextcloud_password": get_setting(db, "nextcloud_password"), } @app.post("/api/admin/settings") async def save_settings(req: SettingsRequest, admin: User = Depends(require_admin), db: Session = Depends(get_db)): if req.nextcloud_url is not None: set_setting(db, "nextcloud_url", req.nextcloud_url) if req.nextcloud_username is not None: set_setting(db, "nextcloud_username", req.nextcloud_username) if req.nextcloud_password is not None: set_setting(db, "nextcloud_password", req.nextcloud_password) return {"ok": True} # ── DOCUMENT ROUTES ────────────────────────────────────────────────── @app.get("/api/documents") async def list_documents(user: User = Depends(get_current_user), db: Session = Depends(get_db)): docs = db.query(Document).filter(Document.user_id == user.id).order_by(Document.created_at.desc()).all() return [{"id": d.id, "title": d.title, "source_type": d.source_type, "created_at": d.created_at.isoformat()} for d in docs] @app.post("/api/documents") async def save_document(req: SaveDocRequest, user: User = Depends(get_current_user), db: Session = Depends(get_db)): doc = Document( id=str(uuid.uuid4()), user_id=user.id, title=req.title, content=req.content, source_type=req.source_type, ) db.add(doc) db.commit() return {"id": doc.id} @app.put("/api/documents/{doc_id}") async def update_document(doc_id: str, req: SaveDocRequest, user: User = Depends(get_current_user), db: Session = Depends(get_db)): doc = db.query(Document).filter(Document.id == doc_id, Document.user_id == user.id).first() if not doc: raise HTTPException(status_code=404, detail="Document not found") doc.title = req.title doc.content = req.content db.commit() return {"id": doc.id} @app.get("/api/documents/{doc_id}") async def get_document(doc_id: str, user: User = Depends(get_current_user), db: Session = Depends(get_db)): doc = db.query(Document).filter(Document.id == doc_id, Document.user_id == user.id).first() if not doc: raise HTTPException(status_code=404, detail="Document not found") return {"id": doc.id, "title": doc.title, "content": doc.content, "source_type": doc.source_type} @app.delete("/api/documents/{doc_id}") async def delete_document(doc_id: str, user: User = Depends(get_current_user), db: Session = Depends(get_db)): doc = db.query(Document).filter(Document.id == doc_id, Document.user_id == user.id).first() if not doc: raise HTTPException(status_code=404, detail="Document not found") db.query(Progress).filter(Progress.document_id == doc_id).delete() db.delete(doc) db.commit() return {"ok": True} # ── PROGRESS ROUTES ────────────────────────────────────────────────── @app.get("/api/progress/{doc_id}") async def get_progress(doc_id: str, user: User = Depends(get_current_user), db: Session = Depends(get_db)): p = db.query(Progress).filter(Progress.user_id == user.id, Progress.document_id == doc_id).first() return {"sentence_index": p.sentence_index if p else 0} @app.post("/api/progress/{doc_id}") async def save_progress(doc_id: str, req: ProgressRequest, user: User = Depends(get_current_user), db: Session = Depends(get_db)): p = db.query(Progress).filter(Progress.user_id == user.id, Progress.document_id == doc_id).first() if p: p.sentence_index = req.sentence_index p.updated_at = datetime.utcnow() else: db.add(Progress(user_id=user.id, document_id=doc_id, sentence_index=req.sentence_index)) db.commit() return {"ok": True} # ── USER SETTINGS ROUTES ───────────────────────────────────────────── @app.get("/api/user/settings") async def get_user_settings(user: User = Depends(get_current_user), db: Session = Depends(get_db)): return { "nc_url": get_user_setting(db, user.id, "nc_url"), "nc_user": get_user_setting(db, user.id, "nc_user"), # Don't return password } @app.post("/api/user/settings") async def save_user_settings(req: UserSettingsRequest, user: User = Depends(get_current_user), db: Session = Depends(get_db)): if req.nc_url is not None: set_user_setting(db, user.id, "nc_url", req.nc_url) if req.nc_user is not None: set_user_setting(db, user.id, "nc_user", req.nc_user) if req.nc_pass is not None: set_user_setting(db, user.id, "nc_pass", req.nc_pass) return {"ok": True} # ── NEXTCLOUD ROUTES ───────────────────────────────────────────────── @app.get("/api/nextcloud/files") async def list_nextcloud_files(path: str = "/", user: User = Depends(get_current_user), db: Session = Depends(get_db)): nc_url, nc_user, nc_pass = resolve_nextcloud(db, user) if not nc_url: raise HTTPException(status_code=400, detail="Nextcloud not configured") dav_url = f"{nc_url.rstrip('/')}/remote.php/dav/files/{nc_user}/{path.lstrip('/')}" try: r = req_lib.request( "PROPFIND", dav_url, auth=(nc_user, nc_pass), headers={"Depth": "1"}, timeout=15, ) if r.status_code not in (207, 200): raise HTTPException(status_code=r.status_code, detail="Nextcloud error") # Parse WebDAV XML response from xml.etree import ElementTree as ET ns = {"d": "DAV:"} tree = ET.fromstring(r.text) items = [] for resp in tree.findall("d:response", ns): href = resp.find("d:href", ns).text prop = resp.find(".//d:prop", ns) display_name = prop.find("d:displayname", ns) content_type = prop.find("d:getcontenttype", ns) is_collection = prop.find("d:resourcetype/d:collection", ns) is not None name = display_name.text if display_name is not None else href.split("/")[-1] mime = content_type.text if content_type is not None else "" if not name: continue # Extract path relative to user's root dav_root = f"/remote.php/dav/files/{nc_user}/" rel_path = href.split(dav_root)[-1] if dav_root in href else href if is_collection or mime == "application/pdf" or rel_path.lower().endswith(".pdf"): items.append({ "name": name, "path": rel_path, "is_dir": is_collection, "mime": mime, }) return items except HTTPException: raise except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @app.post("/api/nextcloud/download") async def download_nextcloud_file(req: NextcloudDownloadRequest, user: User = Depends(get_current_user), db: Session = Depends(get_db)): nc_url, nc_user, nc_pass = resolve_nextcloud(db, user) if not nc_url: raise HTTPException(status_code=400, detail="Nextcloud not configured") try: import fitz dav_url = f"{nc_url.rstrip('/')}/remote.php/dav/files/{nc_user}/{req.path.lstrip('/')}" r = req_lib.get(dav_url, auth=(nc_user, nc_pass), timeout=30) if r.status_code != 200: raise HTTPException(status_code=r.status_code, detail="File download failed") import base64 full_text, page_texts = _extract_pdf(r.content) pdf_b64 = base64.b64encode(r.content).decode() title = req.path.split("/")[-1].removesuffix(".pdf").removesuffix(".PDF") return {"text": full_text, "title": title, "page_texts": page_texts, "pdf_b64": pdf_b64} except HTTPException: raise except Exception as e: raise HTTPException(status_code=500, detail=str(e)) # ── VOICES ─────────────────────────────────────────────────────────── @app.get("/api/voices") async def get_voices(_: User = Depends(get_current_user)): return TTS_MODELS # ── TTS ────────────────────────────────────────────────────────────── @app.post("/api/tts") async def text_to_speech(req: TTSRequest, _: User = Depends(get_current_user)): payload = {"model": req.model, "input": req.text, "voice": req.voice} if req.speed != 1.0: payload["speed"] = req.speed async with httpx.AsyncClient(timeout=60.0) as client: resp = await client.post( f"{LITELLM_BASE}/v1/audio/speech", json=payload, headers={"Authorization": f"Bearer {LITELLM_KEY}"}, ) if resp.status_code != 200: raise HTTPException(status_code=resp.status_code, detail=resp.text) from fastapi.responses import Response return Response(content=resp.content, media_type="audio/mpeg") # ── URL EXTRACTION ──────────────────────────────────────────────────── @app.post("/api/extract-url") async def extract_from_url(req: URLRequest, _: User = Depends(get_current_user)): try: from bs4 import BeautifulSoup import html2text async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: resp = await client.get(req.url, headers={"User-Agent": "Mozilla/5.0"}) soup = BeautifulSoup(resp.text, "html.parser") for tag in soup(["script", "style", "nav", "footer", "header", "aside", "iframe"]): tag.decompose() main = soup.find("article") or soup.find("main") or soup.find(id="content") or soup.body h = html2text.HTML2Text() h.ignore_links = True h.ignore_images = True h.body_width = 0 text = re.sub(r'\n{3,}', '\n\n', h.handle(str(main))).strip() title = soup.title.string.strip() if soup.title else req.url return {"text": text, "title": title} except Exception as e: raise HTTPException(status_code=400, detail=str(e)) # ── PDF EXTRACTION ──────────────────────────────────────────────────── def _extract_pdf(content: bytes) -> tuple[str, list[str]]: """ Extract text from PDF bytes using block-aware reading order. Returns (full_text, page_texts[]). For multi-column layouts, detects column boundaries per page and reads left column top-to-bottom, then right column top-to-bottom, which matches human reading order. """ import fitz import base64 doc = fitz.open(stream=content, filetype="pdf") page_texts = [] for page in doc: pw = page.rect.width # Get text blocks: (x0,y0,x1,y1, text, block_no, block_type) blocks = [b for b in page.get_text("blocks", sort=True) if b[6] == 0 and b[4].strip()] if not blocks: page_texts.append("") continue # Detect two-column: if significant blocks appear on both sides of page midpoint mid = pw * 0.52 left_blocks = [b for b in blocks if b[2] <= mid] # right edge ≤ midpoint right_blocks = [b for b in blocks if b[0] >= mid * 0.48] # left edge ≥ ~midpoint is_two_col = ( len(left_blocks) >= 3 and len(right_blocks) >= 3 and len(left_blocks) + len(right_blocks) >= len(blocks) * 0.75 ) if is_two_col: # Segregate by horizontal midpoint col_left = [b for b in blocks if (b[0] + b[2]) / 2 < mid] col_right = [b for b in blocks if (b[0] + b[2]) / 2 >= mid] ordered = sorted(col_left, key=lambda b: b[1]) + sorted(col_right, key=lambda b: b[1]) else: ordered = sorted(blocks, key=lambda b: (round(b[1] / 20), b[0])) # row-then-col txt = "\n\n".join(b[4].strip() for b in ordered) txt = re.sub(r'[ \t]+', ' ', txt) # collapse whitespace within lines txt = re.sub(r'\n{3,}', '\n\n', txt) # max double newlines # de-hyphenate: "compli-\ncated" → "complicated" txt = re.sub(r'-\n(\w)', r'\1', txt) page_texts.append(txt.strip()) full_text = re.sub(r'\n{3,}', '\n\n', "\n\n".join(page_texts)).strip() return full_text, page_texts @app.post("/api/extract-pdf") async def extract_from_pdf(file: UploadFile = File(...), _: User = Depends(get_current_user)): try: import base64 content = await file.read() full_text, page_texts = _extract_pdf(content) pdf_b64 = base64.b64encode(content).decode() return { "text": full_text, "title": file.filename.removesuffix(".pdf").removesuffix(".PDF"), "page_texts": page_texts, "pdf_b64": pdf_b64, } except Exception as e: raise HTTPException(status_code=400, detail=str(e)) # ── STATIC / PAGES ──────────────────────────────────────────────────── app.mount("/static", StaticFiles(directory="static"), name="static") @app.get("/login") async def login_page(): return FileResponse("static/login.html") @app.get("/settings") async def settings_page(): return FileResponse("static/settings.html") @app.get("/") async def root(request: Request): return FileResponse("static/index.html") @app.get("/admin") async def admin_page(): return FileResponse("static/admin.html")