diff --git a/.claude/plan.md b/.claude/plan.md new file mode 100644 index 0000000..ef6a2b2 --- /dev/null +++ b/.claude/plan.md @@ -0,0 +1,155 @@ +# Plan : Architecture Simplification — 2 services (Vue + FastAPI/SQLite) + +## Objectif +Supprimer le backend Spring Boot (passe-plat) et PostgreSQL. Le document-parser Python absorbe toute la logique backend. On passe de **4 services** à **2 services** : Vue frontend + FastAPI Python. + +## Architecture Python — Clean Architecture légère + +``` +document-parser/ +├── main.py # FastAPI app, CORS, lifespan, routers mount +├── bbox.py # (existant) coord conversion +├── test_bbox.py # (existant) bbox tests +├── requirements.txt # + aiosqlite, aiofiles +├── Dockerfile # (adapté) +├── .dockerignore # (existant) +│ +├── domain/ # 🧠 Modèles métier purs (pas de dépendance framework) +│ ├── __init__.py +│ ├── models.py # Document, AnalysisJob, Status (dataclasses/Pydantic) +│ └── parsing.py # Logique d'extraction Docling (déplacée depuis main.py) +│ +├── api/ # 🌐 Couche HTTP (FastAPI routers) +│ ├── __init__.py +│ ├── schemas.py # Pydantic request/response schemas (DTOs) +│ ├── documents.py # Router /api/documents (CRUD + upload + preview) +│ └── analyses.py # Router /api/analyses (CRUD + async processing) +│ +├── persistence/ # 💾 Couche données (SQLite via aiosqlite) +│ ├── __init__.py +│ ├── database.py # SQLite connection, init schema, get_db() +│ ├── document_repo.py # CRUD documents +│ └── analysis_repo.py # CRUD analysis jobs +│ +└── services/ # ⚙️ Orchestration (use cases) + ├── __init__.py + ├── document_service.py # Upload, delete, preview (file I/O + persistence) + └── analysis_service.py # Create job, background parse, update status +``` + +### Pourquoi cette structure plutôt qu'hexagonale ? +- **domain/** : modèles purs, testables, zéro import framework → l'esprit de l'hexagonale +- **api/** : adaptateur HTTP (port entrant) +- **persistence/** : adaptateur stockage (port sortant) +- **services/** : orchestration des use cases +- Pas de ports/adapters formels (interfaces abstraites) → overkill pour le scope +- Un mec de Docling voit ça, il comprend en 5 secondes. Clean, pas over-engineered. + +## Endpoints conservés (contrat API identique) + +Le frontend ne change quasiment pas — mêmes URLs, mêmes payloads : + +| Method | Path | Description | +|--------|------|-------------| +| POST | `/api/documents/upload` | Upload PDF (multipart) | +| GET | `/api/documents` | List documents | +| GET | `/api/documents/{id}` | Get document | +| DELETE | `/api/documents/{id}` | Delete document + file | +| GET | `/api/documents/{id}/preview?page=&dpi=` | Page preview PNG | +| POST | `/api/analyses` | Create analysis (body: {documentId}) | +| GET | `/api/analyses` | List analyses | +| GET | `/api/analyses/{id}` | Get analysis (polling) | +| DELETE | `/api/analyses/{id}` | Delete analysis | +| GET | `/health` | Health check | + +## Détails d'implémentation + +### 1. SQLite (persistence/database.py) +- `aiosqlite` pour async natif avec FastAPI +- Schema identique au Liquibase actuel (2 tables: documents, analysis_jobs) +- DB file dans volume Docker : `/app/data/docling_studio.db` +- Init schema au startup (CREATE TABLE IF NOT EXISTS) +- Pas besoin d'Alembic pour un projet de cette taille + +### 2. File storage +- Même logique : `./uploads/{uuid}_{filename}` +- Volume Docker monté sur `/app/uploads` + +### 3. Async analysis (services/analysis_service.py) +- `asyncio.create_task()` pour le background processing (pas besoin de Celery) +- Status polling identique : PENDING → RUNNING → COMPLETED | FAILED +- Le parse Docling tourne dans un thread via `asyncio.to_thread()` (car bloquant + lock) + +### 4. CORS +- `fastapi.middleware.cors.CORSMiddleware` dans main.py +- Origins configurables via env var + +### 5. domain/parsing.py +- Déplace depuis main.py : `_build_converter()`, `_get_element_type()`, `extract_pages_detail()`, `_process_content_item()` +- Le converter et le lock restent globaux (singleton pattern) + +## Changements Frontend + +Minimes — seulement la configuration : + +1. **vite.config.js** : proxy target change `8081` → `8000` +2. **frontend/Dockerfile** (nginx) : proxy_pass change `backend:8081` → `document-parser:8000` +3. **api.js** : AUCUN changement (mêmes paths `/api/...`) +4. **stores** : AUCUN changement + +## docker-compose.yml simplifié + +```yaml +services: + document-parser: + build: ./document-parser + ports: + - "8000:8000" + volumes: + - uploads_data:/app/uploads + - db_data:/app/data + deploy: + resources: + limits: + memory: 4g + + frontend: + build: ./frontend + ports: + - "3000:80" + depends_on: + - document-parser + +volumes: + uploads_data: + db_data: +``` + +→ **Plus de postgres, plus de backend Java.** 2 services, clean. + +## Étapes d'exécution + +1. **Créer la structure Python** (domain/, api/, persistence/, services/) +2. **persistence/database.py** — SQLite async init + schema +3. **persistence/document_repo.py** — CRUD documents +4. **persistence/analysis_repo.py** — CRUD analyses +5. **domain/models.py** — Dataclasses Document, AnalysisJob, Status +6. **domain/parsing.py** — Extraire la logique Docling depuis main.py +7. **api/schemas.py** — Pydantic DTOs (DocumentResponse, AnalysisResponse, etc.) +8. **services/document_service.py** — Upload, delete, preview, list, get +9. **services/analysis_service.py** — Create, run background, status tracking +10. **api/documents.py** — Router documents (5 endpoints) +11. **api/analyses.py** — Router analyses (4 endpoints) +12. **main.py** — Réécrire : CORS, lifespan (DB init), mount routers +13. **requirements.txt** — Ajouter aiosqlite, aiofiles +14. **docker-compose.yml** — Simplifier (2 services) +15. **frontend/vite.config.js** — Changer proxy target +16. **frontend/Dockerfile** — Changer nginx proxy_pass +17. **Supprimer le dossier backend/** entier +18. **Mettre à jour README.md** — Nouvelle architecture + +## Ce qu'on ne touche PAS +- bbox.py, test_bbox.py (inchangés) +- Toute la logique Vue (stores, components, pages) +- Logique Docling (parsing, extraction) — juste déplacée +- Dockerfile du parser (juste adapter le CMD si nécessaire) diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8ff39a2 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# CORS (comma-separated origins, only needed for custom deployments) +# CORS_ORIGINS=http://localhost:3000,https://your-domain.com + +# File storage path (inside container) +# UPLOAD_DIR=./uploads + +# Database path (inside container) +# DB_PATH=./data/docling_studio.db diff --git a/.gitignore b/.gitignore index c256853..ccc1bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,11 +18,23 @@ Thumbs.db # Env .env .env.local +.env.production # Uploads uploads/ +backend/uploads/ # Python __pycache__/ *.pyc .venv/ +*.egg-info/ + +# Java +*.class +*.jar +*.log +hs_err_pid* + +# Docker +docker-compose.override.yml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5b3bab8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Pier-Jean Malandrino + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index e0d4a42..a4364ca 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,64 @@ # Docling Studio -A professional document analysis studio powered by [Docling](https://github.com/DS4SD/docling), inspired by MistralAI Studio. +A visual document analysis studio powered by [Docling](https://github.com/DS4SD/docling). +Upload a PDF, configure the extraction pipeline, and visualize the results — text, tables, images, formulas, bounding boxes — all from your browser. + +![Docling Studio — Visual Mode](docs/screenshots/visual-mode.png) + +## Features + +- **PDF viewer** with page navigation and visual overlay toggle +- **Configurable Docling pipeline** — OCR on/off, table extraction mode (fast/accurate) +- **Bounding box visualization** — overlay extracted elements directly on the PDF with color-coded types +- **Per-page results** — right panel syncs with the current PDF page +- **Document hierarchy** — heading levels and structure preserved from Docling's `iterate_items()` API +- **Markdown & HTML export** of extracted content +- **Analysis history** — re-visit past analyses + +
+More screenshots + +| Import | Configure | Results | +|--------|-----------|---------| +| ![Import](docs/screenshots/import.png) | ![Configure](docs/screenshots/configure.png) | ![Results](docs/screenshots/results.png) | + +
## Architecture ``` -frontend/ → Vue 3 + Vite + Pinia (port 3000) -backend/ → Spring Boot 3.3.5 / Java 21 (port 8081) -document-parser/ → FastAPI + Docling (port 8000) +┌────────────┐ ┌───────────────────────┐ +│ Frontend │────────▶│ Document Parser │ +│ Vue 3 │ /api/* │ FastAPI + Docling │ +│ port 3000 │ │ SQLite + file storage │ +└────────────┘ │ port 8000 │ + └───────────────────────┘ +``` + +| Service | Stack | Role | +|---------|-------|------| +| **frontend** | Vue 3, Vite, Pinia | UI, PDF viewer, results display | +| **document-parser** | FastAPI, Docling, SQLite, pdf2image | REST API, document parsing, storage, persistence | + +### Python project structure (clean architecture) + +``` +document-parser/ +├── main.py # FastAPI app, CORS, lifespan +├── domain/ # Pure domain models & Docling logic +│ ├── models.py # Document, AnalysisJob dataclasses +│ └── parsing.py # Docling conversion & page extraction +├── api/ # HTTP layer (FastAPI routers) +│ ├── schemas.py # Pydantic DTOs (camelCase serialization) +│ ├── documents.py # /api/documents endpoints +│ └── analyses.py # /api/analyses endpoints +├── persistence/ # Data layer (SQLite) +│ ├── database.py # Connection management, schema init +│ ├── document_repo.py # Document CRUD +│ └── analysis_repo.py # AnalysisJob CRUD +└── services/ # Use case orchestration + ├── document_service.py # Upload, delete, preview + └── analysis_service.py # Async Docling processing ``` ## Quick Start @@ -15,37 +66,106 @@ document-parser/ → FastAPI + Docling (port 8000) ### Docker Compose (recommended) ```bash -docker-compose up --build +# Clone the repo +git clone https://github.com/scub-france/docling-studio.git +cd docling-studio + +# (Optional) customize settings +cp .env.example .env + +# Start all services +docker compose up --build ``` Open [http://localhost:3000](http://localhost:3000) +> **Note:** First analysis may take a few minutes as Docling downloads its ML models (~40 MB) on first run. + ### Local Development -**Document Parser:** +**Document Parser** (Python 3.12+): ```bash cd document-parser +python -m venv .venv && source .venv/bin/activate pip install -r requirements.txt uvicorn main:app --reload --port 8000 ``` -**Backend:** -```bash -cd backend -./mvnw spring-boot:run -``` - -**Frontend:** +**Frontend** (Node 20+): ```bash cd frontend npm install npm run dev ``` -## Features +## Docling Integration -- PDF upload and document analysis via Docling -- Extracted content viewing (Markdown, HTML) -- Document structure visualization with bounding boxes -- Image detection -- Analysis history +The document parser wraps [Docling](https://github.com/DS4SD/docling) with configurable pipeline options exposed as query parameters on the `/parse` endpoint: + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `do_ocr` | `true` | Enable OCR for scanned documents | +| `do_table_structure` | `true` | Enable table structure extraction | +| `table_mode` | `accurate` | Table extraction mode: `accurate` or `fast` | + +Element types are detected using `isinstance()` checks against Docling's type hierarchy (`TextItem`, `TableItem`, `PictureItem`, `SectionHeaderItem`, etc.) and the document tree depth from `iterate_items()` is preserved for heading-level reconstruction. + +## Configuration + +All configuration is done via environment variables. See [`.env.example`](.env.example) for available options. + +| Variable | Default | Description | +|----------|---------|-------------| +| `CORS_ORIGINS` | `http://localhost:3000,...` | CORS allowed origins (comma-separated) | +| `UPLOAD_DIR` | `./uploads` | File storage directory | +| `DB_PATH` | `./data/docling_studio.db` | SQLite database path | + +## Performance & System Requirements + +Docling runs ML models (layout analysis, OCR, table structure) on **CPU by default**. Processing time depends on document size and complexity. + +| Document type | Pages | Approx. time (CPU) | +|---------------|-------|---------------------| +| Simple report | 5-10 | 1-3 min | +| Research paper | 15-30 | 5-10 min | +| Dense PDF with tables | 30+ | 10-20 min | + +### Docker Desktop settings + +The document parser needs **at least 4 GB of RAM**. Recommended Docker Desktop allocation: + +| Resource | Minimum | Recommended | +|----------|---------|-------------| +| Memory | 6 GB | 8 GB+ | +| CPUs | 4 | 8+ | + +> On **macOS**: Docker Desktop > Settings > Resources +> On **Windows**: Docker Desktop > Settings > Resources > WSL 2 + +### Platform support + +All Docker images are **multi-arch** (linux/amd64 + linux/arm64). Works natively on: + +| Platform | Architecture | GPU acceleration | +|----------|-------------|-----------------| +| **macOS Apple Silicon** (M1/M2/M3) | arm64 | Not in Docker (MPS unavailable). Run parser locally for GPU. | +| **macOS Intel** | amd64 | N/A | +| **Linux x86_64** | amd64 | NVIDIA GPU via `docker compose --profile gpu` (coming soon) | +| **Linux ARM** (Raspberry Pi 5, Ampere) | arm64 | CPU only | +| **Windows + WSL2** | amd64 | NVIDIA GPU passthrough supported | + +> **Tip for Mac users:** For faster processing, run the document parser **locally** (outside Docker) to leverage Apple Silicon's MPS acceleration when supported by PyTorch/Docling. + +## Tech Stack + +- **Frontend**: Vue 3 + Vite + Pinia +- **Backend**: FastAPI + Docling 2.x + SQLite + pdf2image +- **Infra**: Docker Compose + Nginx + +## Contributing + +Contributions are welcome! Please open an issue first to discuss what you'd like to change. + +## License + +[MIT](LICENSE) — Pier-Jean Malandrino diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index 11f3ea3..0000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM eclipse-temurin:21-jdk AS build - -WORKDIR /app -COPY pom.xml . -COPY src ./src - -RUN apt-get update && apt-get install -y maven && \ - mvn package -DskipTests - -FROM eclipse-temurin:21-jre - -WORKDIR /app -COPY --from=build /app/target/*.jar app.jar - -EXPOSE 8081 - -ENTRYPOINT ["java", "-jar", "app.jar"] diff --git a/backend/pom.xml b/backend/pom.xml deleted file mode 100644 index 6303db3..0000000 --- a/backend/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - 4.0.0 - - - org.springframework.boot - spring-boot-starter-parent - 3.3.5 - - - - com.docling - docling-studio - 0.1.0 - Docling Studio Backend - - - 21 - - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-data-jpa - - - org.springframework.boot - spring-boot-starter-webflux - - - org.springframework.boot - spring-boot-starter-actuator - - - - org.postgresql - postgresql - runtime - - - org.liquibase - liquibase-core - - - - org.springframework.boot - spring-boot-starter-test - test - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - - diff --git a/backend/src/main/java/com/docling/studio/DoclingStudioApplication.java b/backend/src/main/java/com/docling/studio/DoclingStudioApplication.java deleted file mode 100644 index b8e6711..0000000 --- a/backend/src/main/java/com/docling/studio/DoclingStudioApplication.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.docling.studio; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class DoclingStudioApplication { - public static void main(String[] args) { - SpringApplication.run(DoclingStudioApplication.class, args); - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisController.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisController.java deleted file mode 100644 index 0dc25ff..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisController.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.analysis.dto.AnalysisResponse; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.util.List; -import java.util.Map; -import java.util.UUID; - -@RestController -@RequestMapping("/api/analyses") -public class AnalysisController { - - private final AnalysisService service; - - public AnalysisController(AnalysisService service) { - this.service = service; - } - - @PostMapping - public AnalysisResponse create(@RequestBody Map body) { - UUID documentId = UUID.fromString(body.get("documentId")); - AnalysisJob job = service.create(documentId); - return AnalysisResponse.from(job); - } - - @GetMapping - public List list() { - return service.findAll().stream().map(AnalysisResponse::from).toList(); - } - - @GetMapping("/{id}") - public AnalysisResponse get(@PathVariable UUID id) { - return AnalysisResponse.from(service.findById(id)); - } - - @DeleteMapping("/{id}") - public ResponseEntity delete(@PathVariable UUID id) { - service.delete(id); - return ResponseEntity.noContent().build(); - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisJob.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisJob.java deleted file mode 100644 index 070e893..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisJob.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.document.Document; -import jakarta.persistence.*; -import java.time.Instant; -import java.util.UUID; - -@Entity -@Table(name = "analysis_jobs") -public class AnalysisJob { - - public enum Status { PENDING, RUNNING, COMPLETED, FAILED } - - @Id - private UUID id; - - @ManyToOne(fetch = FetchType.EAGER) - @JoinColumn(name = "document_id", nullable = false) - private Document document; - - @Enumerated(EnumType.STRING) - @Column(nullable = false) - private Status status; - - @Column(columnDefinition = "text") - private String contentMarkdown; - - @Column(columnDefinition = "text") - private String contentHtml; - - @Column(columnDefinition = "text") - private String pagesJson; - - private String errorMessage; - - private Instant startedAt; - private Instant completedAt; - private Instant createdAt; - - protected AnalysisJob() {} - - public AnalysisJob(Document document) { - this.id = UUID.randomUUID(); - this.document = document; - this.status = Status.PENDING; - this.createdAt = Instant.now(); - } - - public void markRunning() { - this.status = Status.RUNNING; - this.startedAt = Instant.now(); - } - - public void markCompleted(String markdown, String html, String pagesJson) { - this.status = Status.COMPLETED; - this.contentMarkdown = markdown; - this.contentHtml = html; - this.pagesJson = pagesJson; - this.completedAt = Instant.now(); - } - - public void markFailed(String error) { - this.status = Status.FAILED; - this.errorMessage = error; - this.completedAt = Instant.now(); - } - - public UUID getId() { return id; } - public Document getDocument() { return document; } - public Status getStatus() { return status; } - public String getContentMarkdown() { return contentMarkdown; } - public String getContentHtml() { return contentHtml; } - public String getPagesJson() { return pagesJson; } - public String getErrorMessage() { return errorMessage; } - public Instant getStartedAt() { return startedAt; } - public Instant getCompletedAt() { return completedAt; } - public Instant getCreatedAt() { return createdAt; } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisJobRepository.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisJobRepository.java deleted file mode 100644 index 044bc3b..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisJobRepository.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.docling.studio.analysis; - -import org.springframework.data.jpa.repository.JpaRepository; -import java.util.List; -import java.util.UUID; - -public interface AnalysisJobRepository extends JpaRepository { - List findAllByOrderByCreatedAtDesc(); - List findByDocumentIdOrderByCreatedAtDesc(UUID documentId); -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisService.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisService.java deleted file mode 100644 index 133b3ba..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisService.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.document.Document; -import com.docling.studio.document.DocumentParserClient; -import com.docling.studio.document.DocumentService; -import com.docling.studio.shared.exception.ResourceNotFoundException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; - -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -@Service -public class AnalysisService { - - private final AnalysisJobRepository repository; - private final DocumentService documentService; - private final DocumentParserClient parserClient; - private final ObjectMapper objectMapper; - - public AnalysisService( - AnalysisJobRepository repository, - DocumentService documentService, - DocumentParserClient parserClient, - ObjectMapper objectMapper - ) { - this.repository = repository; - this.documentService = documentService; - this.parserClient = parserClient; - this.objectMapper = objectMapper; - } - - public AnalysisJob create(UUID documentId) { - Document doc = documentService.findById(documentId); - AnalysisJob job = new AnalysisJob(doc); - repository.save(job); - runAnalysis(job.getId()); - return job; - } - - @Async("analysisExecutor") - public void runAnalysis(UUID jobId) { - AnalysisJob job = repository.findById(jobId).orElseThrow(); - job.markRunning(); - repository.save(job); - - try { - Path filePath = documentService.getFilePath(job.getDocument().getId()); - Map result = parserClient.parse(filePath, job.getDocument().getFilename()); - - String markdown = (String) result.getOrDefault("content_markdown", ""); - String html = (String) result.getOrDefault("content_html", ""); - Object pages = result.get("pages"); - String pagesJson = pages != null ? objectMapper.writeValueAsString(pages) : "[]"; - - // Update page count on document if available - Object pageCount = result.get("page_count"); - if (pageCount instanceof Number n && n.intValue() > 0) { - Document doc = job.getDocument(); - doc.setPageCount(n.intValue()); - documentService.save(doc); - } - - job.markCompleted(markdown, html, pagesJson); - repository.save(job); - - } catch (Exception e) { - job.markFailed(e.getMessage()); - repository.save(job); - } - } - - public AnalysisJob findById(UUID id) { - return repository.findById(id) - .orElseThrow(() -> new ResourceNotFoundException("Analysis not found: " + id)); - } - - public List findAll() { - return repository.findAllByOrderByCreatedAtDesc(); - } - - public void delete(UUID id) { - AnalysisJob job = findById(id); - repository.delete(job); - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/dto/AnalysisResponse.java b/backend/src/main/java/com/docling/studio/analysis/dto/AnalysisResponse.java deleted file mode 100644 index 0e4f6a0..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/dto/AnalysisResponse.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.docling.studio.analysis.dto; - -import com.docling.studio.analysis.AnalysisJob; -import java.time.Instant; -import java.util.UUID; - -public record AnalysisResponse( - UUID id, - UUID documentId, - String documentFilename, - String status, - String contentMarkdown, - String contentHtml, - String pagesJson, - String errorMessage, - Instant startedAt, - Instant completedAt, - Instant createdAt -) { - public static AnalysisResponse from(AnalysisJob job) { - return new AnalysisResponse( - job.getId(), - job.getDocument().getId(), - job.getDocument().getFilename(), - job.getStatus().name(), - job.getContentMarkdown(), - job.getContentHtml(), - job.getPagesJson(), - job.getErrorMessage(), - job.getStartedAt(), - job.getCompletedAt(), - job.getCreatedAt() - ); - } -} diff --git a/backend/src/main/java/com/docling/studio/config/AsyncConfig.java b/backend/src/main/java/com/docling/studio/config/AsyncConfig.java deleted file mode 100644 index 828475b..0000000 --- a/backend/src/main/java/com/docling/studio/config/AsyncConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.docling.studio.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; - -@Configuration -@EnableAsync -public class AsyncConfig { - - @Bean(name = "analysisExecutor") - public Executor analysisExecutor() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(2); - executor.setMaxPoolSize(4); - executor.setQueueCapacity(20); - executor.setThreadNamePrefix("analysis-"); - executor.initialize(); - return executor; - } -} diff --git a/backend/src/main/java/com/docling/studio/config/DocumentParserProperties.java b/backend/src/main/java/com/docling/studio/config/DocumentParserProperties.java deleted file mode 100644 index 6b6723f..0000000 --- a/backend/src/main/java/com/docling/studio/config/DocumentParserProperties.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.docling.studio.config; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Component -@ConfigurationProperties(prefix = "app.document-parser") -public class DocumentParserProperties { - - private String baseUrl = "http://localhost:8000"; - - public String getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } -} diff --git a/backend/src/main/java/com/docling/studio/config/WebConfig.java b/backend/src/main/java/com/docling/studio/config/WebConfig.java deleted file mode 100644 index a6a3113..0000000 --- a/backend/src/main/java/com/docling/studio/config/WebConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.docling.studio.config; - -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.CorsRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class WebConfig implements WebMvcConfigurer { - - @Override - public void addCorsMappings(CorsRegistry registry) { - registry.addMapping("/api/**") - .allowedOrigins("http://localhost:3000", "http://localhost:5173") - .allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS") - .allowedHeaders("*") - .allowCredentials(true); - } -} diff --git a/backend/src/main/java/com/docling/studio/document/Document.java b/backend/src/main/java/com/docling/studio/document/Document.java deleted file mode 100644 index acfb293..0000000 --- a/backend/src/main/java/com/docling/studio/document/Document.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.docling.studio.document; - -import jakarta.persistence.*; -import java.time.Instant; -import java.util.UUID; - -@Entity -@Table(name = "documents") -public class Document { - - @Id - private UUID id; - - @Column(nullable = false) - private String filename; - - private String contentType; - - private Long fileSize; - - private Integer pageCount; - - @Column(nullable = false) - private String storagePath; - - private Instant createdAt; - - protected Document() {} - - public Document(String filename, String contentType, Long fileSize, String storagePath) { - this.id = UUID.randomUUID(); - this.filename = filename; - this.contentType = contentType; - this.fileSize = fileSize; - this.storagePath = storagePath; - this.createdAt = Instant.now(); - } - - public UUID getId() { return id; } - public String getFilename() { return filename; } - public String getContentType() { return contentType; } - public Long getFileSize() { return fileSize; } - public Integer getPageCount() { return pageCount; } - public void setPageCount(Integer pageCount) { this.pageCount = pageCount; } - public String getStoragePath() { return storagePath; } - public Instant getCreatedAt() { return createdAt; } -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentController.java b/backend/src/main/java/com/docling/studio/document/DocumentController.java deleted file mode 100644 index 32fc4a4..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentController.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.docling.studio.document; - -import com.docling.studio.document.dto.DocumentResponse; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; - -import java.util.List; -import java.util.UUID; - -@RestController -@RequestMapping("/api/documents") -public class DocumentController { - - private final DocumentService service; - - public DocumentController(DocumentService service) { - this.service = service; - } - - @PostMapping("/upload") - public DocumentResponse upload(@RequestParam("file") MultipartFile file) { - Document doc = service.upload(file); - return DocumentResponse.from(doc); - } - - @GetMapping - public List list() { - return service.findAll().stream().map(DocumentResponse::from).toList(); - } - - @GetMapping("/{id}") - public DocumentResponse get(@PathVariable UUID id) { - return DocumentResponse.from(service.findById(id)); - } - - @DeleteMapping("/{id}") - public ResponseEntity delete(@PathVariable UUID id) { - service.delete(id); - return ResponseEntity.noContent().build(); - } - - @GetMapping(value = "/{id}/preview") - public ResponseEntity preview( - @PathVariable UUID id, - @RequestParam(defaultValue = "1") int page, - @RequestParam(defaultValue = "150") int dpi - ) { - byte[] bytes = service.getPreview(id, page, dpi); - if (bytes == null) { - return ResponseEntity.notFound().build(); - } - return ResponseEntity.ok().contentType(MediaType.IMAGE_PNG).body(bytes); - } -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentParserClient.java b/backend/src/main/java/com/docling/studio/document/DocumentParserClient.java deleted file mode 100644 index 8dbed2e..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentParserClient.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.docling.studio.document; - -import com.docling.studio.config.DocumentParserProperties; -import org.springframework.core.io.FileSystemResource; -import org.springframework.http.MediaType; -import org.springframework.http.client.MultipartBodyBuilder; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.BodyInserters; -import org.springframework.web.reactive.function.client.WebClient; - -import java.nio.file.Path; -import java.util.Map; - -@Component -public class DocumentParserClient { - - private final WebClient webClient; - - public DocumentParserClient(DocumentParserProperties props) { - this.webClient = WebClient.builder() - .baseUrl(props.getBaseUrl()) - .codecs(config -> config.defaultCodecs().maxInMemorySize(50 * 1024 * 1024)) - .build(); - } - - @SuppressWarnings("unchecked") - public Map parse(Path filePath, String filename) { - MultipartBodyBuilder builder = new MultipartBodyBuilder(); - builder.part("file", new FileSystemResource(filePath)) - .filename(filename) - .contentType(MediaType.APPLICATION_PDF); - - return webClient.post() - .uri("/parse") - .body(BodyInserters.fromMultipartData(builder.build())) - .retrieve() - .bodyToMono(Map.class) - .block(); - } - - public byte[] preview(Path filePath, String filename, int page, int dpi) { - MultipartBodyBuilder builder = new MultipartBodyBuilder(); - builder.part("file", new FileSystemResource(filePath)) - .filename(filename) - .contentType(MediaType.APPLICATION_PDF); - - try { - return webClient.post() - .uri(uri -> uri.path("/preview") - .queryParam("page", page) - .queryParam("dpi", dpi) - .build()) - .body(BodyInserters.fromMultipartData(builder.build())) - .retrieve() - .bodyToMono(byte[].class) - .block(); - } catch (Exception e) { - return null; - } - } -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentRepository.java b/backend/src/main/java/com/docling/studio/document/DocumentRepository.java deleted file mode 100644 index 0926527..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentRepository.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.docling.studio.document; - -import org.springframework.data.jpa.repository.JpaRepository; -import java.util.UUID; - -public interface DocumentRepository extends JpaRepository { -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentService.java b/backend/src/main/java/com/docling/studio/document/DocumentService.java deleted file mode 100644 index ef6073d..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentService.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.docling.studio.document; - -import com.docling.studio.shared.exception.ResourceNotFoundException; -import com.docling.studio.shared.exception.ServiceException; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.UUID; - -@Service -public class DocumentService { - - private final DocumentRepository repository; - private final DocumentParserClient parserClient; - private final Path storagePath; - - public DocumentService( - DocumentRepository repository, - DocumentParserClient parserClient, - @Value("${app.storage.path:./uploads}") String storagePath - ) { - this.repository = repository; - this.parserClient = parserClient; - this.storagePath = Path.of(storagePath); - } - - public Document upload(MultipartFile file) { - if (file.isEmpty() || file.getOriginalFilename() == null) { - throw new IllegalArgumentException("File is empty or has no name"); - } - - try { - Files.createDirectories(storagePath); - String safeName = UUID.randomUUID() + "_" + file.getOriginalFilename(); - Path target = storagePath.resolve(safeName); - file.transferTo(target); - - Document doc = new Document( - file.getOriginalFilename(), - file.getContentType(), - file.getSize(), - target.toString() - ); - return repository.save(doc); - - } catch (IOException e) { - throw new ServiceException("Failed to store file", e); - } - } - - public Document save(Document document) { - return repository.save(document); - } - - public List findAll() { - return repository.findAll(); - } - - public Document findById(UUID id) { - return repository.findById(id) - .orElseThrow(() -> new ResourceNotFoundException("Document not found: " + id)); - } - - public void delete(UUID id) { - Document doc = findById(id); - try { - Files.deleteIfExists(Path.of(doc.getStoragePath())); - } catch (IOException e) { - // Log but continue - } - repository.delete(doc); - } - - public byte[] getPreview(UUID id, int page, int dpi) { - Document doc = findById(id); - return parserClient.preview(Path.of(doc.getStoragePath()), doc.getFilename(), page, dpi); - } - - public Path getFilePath(UUID id) { - Document doc = findById(id); - return Path.of(doc.getStoragePath()); - } -} diff --git a/backend/src/main/java/com/docling/studio/document/dto/DocumentResponse.java b/backend/src/main/java/com/docling/studio/document/dto/DocumentResponse.java deleted file mode 100644 index 6b0ec63..0000000 --- a/backend/src/main/java/com/docling/studio/document/dto/DocumentResponse.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.docling.studio.document.dto; - -import com.docling.studio.document.Document; -import java.time.Instant; -import java.util.UUID; - -public record DocumentResponse( - UUID id, - String filename, - String contentType, - Long fileSize, - Integer pageCount, - Instant createdAt -) { - public static DocumentResponse from(Document doc) { - return new DocumentResponse( - doc.getId(), doc.getFilename(), doc.getContentType(), - doc.getFileSize(), doc.getPageCount(), doc.getCreatedAt() - ); - } -} diff --git a/backend/src/main/java/com/docling/studio/shared/exception/GlobalExceptionHandler.java b/backend/src/main/java/com/docling/studio/shared/exception/GlobalExceptionHandler.java deleted file mode 100644 index d598726..0000000 --- a/backend/src/main/java/com/docling/studio/shared/exception/GlobalExceptionHandler.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.docling.studio.shared.exception; - -import org.springframework.http.HttpStatus; -import org.springframework.http.ProblemDetail; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -@RestControllerAdvice -public class GlobalExceptionHandler { - - @ExceptionHandler(ResourceNotFoundException.class) - public ProblemDetail handleNotFound(ResourceNotFoundException ex) { - return ProblemDetail.forStatusAndDetail(HttpStatus.NOT_FOUND, ex.getMessage()); - } - - @ExceptionHandler(ServiceException.class) - public ProblemDetail handleService(ServiceException ex) { - return ProblemDetail.forStatusAndDetail(HttpStatus.INTERNAL_SERVER_ERROR, ex.getMessage()); - } - - @ExceptionHandler(IllegalArgumentException.class) - public ProblemDetail handleBadRequest(IllegalArgumentException ex) { - return ProblemDetail.forStatusAndDetail(HttpStatus.BAD_REQUEST, ex.getMessage()); - } -} diff --git a/backend/src/main/java/com/docling/studio/shared/exception/ResourceNotFoundException.java b/backend/src/main/java/com/docling/studio/shared/exception/ResourceNotFoundException.java deleted file mode 100644 index 013d442..0000000 --- a/backend/src/main/java/com/docling/studio/shared/exception/ResourceNotFoundException.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.docling.studio.shared.exception; - -public class ResourceNotFoundException extends RuntimeException { - public ResourceNotFoundException(String message) { - super(message); - } -} diff --git a/backend/src/main/java/com/docling/studio/shared/exception/ServiceException.java b/backend/src/main/java/com/docling/studio/shared/exception/ServiceException.java deleted file mode 100644 index 44f326a..0000000 --- a/backend/src/main/java/com/docling/studio/shared/exception/ServiceException.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.docling.studio.shared.exception; - -public class ServiceException extends RuntimeException { - public ServiceException(String message) { - super(message); - } - - public ServiceException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml deleted file mode 100644 index a7e3374..0000000 --- a/backend/src/main/resources/application.yml +++ /dev/null @@ -1,24 +0,0 @@ -server: - port: 8081 - -spring: - datasource: - url: jdbc:postgresql://localhost:5432/docling_studio - username: app - password: app - jpa: - hibernate: - ddl-auto: validate - open-in-view: false - liquibase: - change-log: classpath:db/changelog/db.changelog-master.yml - servlet: - multipart: - max-file-size: 50MB - max-request-size: 50MB - -app: - document-parser: - base-url: http://localhost:8000 - storage: - path: ./uploads diff --git a/backend/src/main/resources/db/changelog/001-create-documents.yml b/backend/src/main/resources/db/changelog/001-create-documents.yml deleted file mode 100644 index 3f0f317..0000000 --- a/backend/src/main/resources/db/changelog/001-create-documents.yml +++ /dev/null @@ -1,36 +0,0 @@ -databaseChangeLog: - - changeSet: - id: 001-create-documents - author: docling-studio - changes: - - createTable: - tableName: documents - columns: - - column: - name: id - type: uuid - constraints: - primaryKey: true - - column: - name: filename - type: varchar(255) - constraints: - nullable: false - - column: - name: content_type - type: varchar(100) - - column: - name: file_size - type: bigint - - column: - name: page_count - type: int - - column: - name: storage_path - type: varchar(500) - constraints: - nullable: false - - column: - name: created_at - type: timestamp - defaultValueComputed: NOW() diff --git a/backend/src/main/resources/db/changelog/002-create-analysis-jobs.yml b/backend/src/main/resources/db/changelog/002-create-analysis-jobs.yml deleted file mode 100644 index d695702..0000000 --- a/backend/src/main/resources/db/changelog/002-create-analysis-jobs.yml +++ /dev/null @@ -1,47 +0,0 @@ -databaseChangeLog: - - changeSet: - id: 002-create-analysis-jobs - author: docling-studio - changes: - - createTable: - tableName: analysis_jobs - columns: - - column: - name: id - type: uuid - constraints: - primaryKey: true - - column: - name: document_id - type: uuid - constraints: - nullable: false - foreignKeyName: fk_analysis_document - references: documents(id) - - column: - name: status - type: varchar(20) - constraints: - nullable: false - - column: - name: content_markdown - type: text - - column: - name: content_html - type: text - - column: - name: pages_json - type: text - - column: - name: error_message - type: text - - column: - name: started_at - type: timestamp - - column: - name: completed_at - type: timestamp - - column: - name: created_at - type: timestamp - defaultValueComputed: NOW() diff --git a/backend/src/main/resources/db/changelog/db.changelog-master.yml b/backend/src/main/resources/db/changelog/db.changelog-master.yml deleted file mode 100644 index 70cba0f..0000000 --- a/backend/src/main/resources/db/changelog/db.changelog-master.yml +++ /dev/null @@ -1,5 +0,0 @@ -databaseChangeLog: - - include: - file: db/changelog/001-create-documents.yml - - include: - file: db/changelog/002-create-analysis-jobs.yml diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 622529a..4294634 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,15 +2,15 @@ services: postgres: image: postgres:16-alpine environment: - POSTGRES_USER: app - POSTGRES_PASSWORD: app - POSTGRES_DB: docling_studio + POSTGRES_USER: ${POSTGRES_USER:-app} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-app} + POSTGRES_DB: ${POSTGRES_DB:-docling_studio} ports: - "5432:5432" volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U app -d docling_studio"] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-app} -d ${POSTGRES_DB:-docling_studio}"] interval: 5s timeout: 5s retries: 10 diff --git a/docker-compose.yml b/docker-compose.yml index 990e20c..86819f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,39 +1,18 @@ services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: app - POSTGRES_PASSWORD: app - POSTGRES_DB: docling_studio - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U app -d docling_studio"] - interval: 5s - timeout: 5s - retries: 10 - document-parser: build: context: ./document-parser ports: - "8000:8000" - - backend: - build: - context: ./backend - ports: - - "8081:8081" + volumes: + - uploads_data:/app/uploads + - db_data:/app/data environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/docling_studio - SPRING_DATASOURCE_USERNAME: app - SPRING_DATASOURCE_PASSWORD: app - APP_DOCUMENT-PARSER_BASE-URL: http://document-parser:8000 - depends_on: - postgres: - condition: service_healthy + CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173} + deploy: + resources: + limits: + memory: 4g frontend: build: @@ -41,7 +20,8 @@ services: ports: - "3000:80" depends_on: - - backend + - document-parser volumes: - postgres_data: + uploads_data: + db_data: diff --git a/document-parser/.dockerignore b/document-parser/.dockerignore new file mode 100644 index 0000000..643c3a7 --- /dev/null +++ b/document-parser/.dockerignore @@ -0,0 +1,12 @@ +.venv/ +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +.git/ +.gitignore +.env +*.log +.mypy_cache/ +.pytest_cache/ +.ruff_cache/ diff --git a/document-parser/Dockerfile b/document-parser/Dockerfile index b047eee..6ab8e94 100644 --- a/document-parser/Dockerfile +++ b/document-parser/Dockerfile @@ -2,6 +2,8 @@ FROM python:3.12-slim RUN apt-get update && apt-get install -y --no-install-recommends \ poppler-utils \ + libgl1 \ + libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -11,6 +13,11 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . +RUN mkdir -p /app/uploads /app/data + EXPOSE 8000 +ENV UPLOAD_DIR=/app/uploads +ENV DB_PATH=/app/data/docling_studio.db + CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/document-parser/api/__init__.py b/document-parser/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/api/analyses.py b/document-parser/api/analyses.py new file mode 100644 index 0000000..72c4712 --- /dev/null +++ b/document-parser/api/analyses.py @@ -0,0 +1,67 @@ +"""Analysis API router — create, list, get, delete analysis jobs.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, HTTPException + +from api.schemas import AnalysisResponse, CreateAnalysisRequest +from services import analysis_service + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/analyses", tags=["analyses"]) + + +def _to_response(job) -> AnalysisResponse: + return AnalysisResponse( + id=job.id, + document_id=job.document_id, + document_filename=job.document_filename, + status=job.status.value, + content_markdown=job.content_markdown, + content_html=job.content_html, + pages_json=job.pages_json, + error_message=job.error_message, + started_at=str(job.started_at) if job.started_at else None, + completed_at=str(job.completed_at) if job.completed_at else None, + created_at=str(job.created_at), + ) + + +@router.post("", response_model=AnalysisResponse) +async def create_analysis(body: CreateAnalysisRequest): + """Create a new analysis job for a document.""" + if not body.documentId or not body.documentId.strip(): + raise HTTPException(status_code=400, detail="documentId is required") + + try: + job = await analysis_service.create(body.documentId) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + + return _to_response(job) + + +@router.get("", response_model=list[AnalysisResponse]) +async def list_analyses(): + """List all analysis jobs.""" + jobs = await analysis_service.find_all() + return [_to_response(j) for j in jobs] + + +@router.get("/{job_id}", response_model=AnalysisResponse) +async def get_analysis(job_id: str): + """Get a single analysis job.""" + job = await analysis_service.find_by_id(job_id) + if not job: + raise HTTPException(status_code=404, detail="Analysis not found") + return _to_response(job) + + +@router.delete("/{job_id}", status_code=204) +async def delete_analysis(job_id: str): + """Delete an analysis job.""" + deleted = await analysis_service.delete(job_id) + if not deleted: + raise HTTPException(status_code=404, detail="Analysis not found") diff --git a/document-parser/api/documents.py b/document-parser/api/documents.py new file mode 100644 index 0000000..2e94c0b --- /dev/null +++ b/document-parser/api/documents.py @@ -0,0 +1,92 @@ +"""Document API router — upload, list, get, delete, preview.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, HTTPException, Query, UploadFile +from fastapi.responses import Response + +from api.schemas import DocumentResponse +from services import document_service + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/documents", tags=["documents"]) + + +def _to_response(doc) -> DocumentResponse: + return DocumentResponse( + id=doc.id, + filename=doc.filename, + content_type=doc.content_type, + file_size=doc.file_size, + page_count=doc.page_count, + created_at=str(doc.created_at), + ) + + +@router.post("/upload", response_model=DocumentResponse) +async def upload(file: UploadFile): + """Upload a PDF document.""" + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + + content = await file.read() + + try: + doc = await document_service.upload( + filename=file.filename, + content_type=file.content_type or "application/pdf", + file_content=content, + ) + except ValueError as e: + raise HTTPException(status_code=413, detail=str(e)) + + return _to_response(doc) + + +@router.get("", response_model=list[DocumentResponse]) +async def list_documents(): + """List all documents.""" + docs = await document_service.find_all() + return [_to_response(d) for d in docs] + + +@router.get("/{doc_id}", response_model=DocumentResponse) +async def get_document(doc_id: str): + """Get a single document.""" + doc = await document_service.find_by_id(doc_id) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + return _to_response(doc) + + +@router.delete("/{doc_id}", status_code=204) +async def delete_document(doc_id: str): + """Delete a document and its file.""" + deleted = await document_service.delete(doc_id) + if not deleted: + raise HTTPException(status_code=404, detail="Document not found") + + +@router.get("/{doc_id}/preview") +async def preview( + doc_id: str, + page: int = Query(1, ge=1), + dpi: int = Query(150, ge=72, le=300), +): + """Generate a PNG preview of a specific PDF page.""" + doc = await document_service.find_by_id(doc_id) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + + try: + with open(doc.storage_path, "rb") as f: + file_content = f.read() + png_bytes = document_service.generate_preview(file_content, page=page, dpi=dpi) + return Response(content=png_bytes, media_type="image/png") + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + except Exception as e: + logger.exception("Failed to generate preview") + raise HTTPException(status_code=422, detail=f"Failed to generate preview: {e}") diff --git a/document-parser/api/schemas.py b/document-parser/api/schemas.py new file mode 100644 index 0000000..e9a981b --- /dev/null +++ b/document-parser/api/schemas.py @@ -0,0 +1,52 @@ +"""Pydantic schemas — API request/response DTOs. + +All responses use camelCase serialization to match the existing frontend contract +(originally served by the Spring Boot backend). +""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict + + +def _to_camel(name: str) -> str: + parts = name.split("_") + return parts[0] + "".join(w.capitalize() for w in parts[1:]) + + +class _CamelModel(BaseModel): + """Base model that serializes field names to camelCase.""" + model_config = ConfigDict( + alias_generator=_to_camel, + populate_by_name=True, + serialize_by_alias=True, + ) + + +class DocumentResponse(_CamelModel): + id: str + filename: str + content_type: str | None = None + file_size: int | None = None + page_count: int | None = None + created_at: str | datetime + + +class AnalysisResponse(_CamelModel): + id: str + document_id: str = "" + document_filename: str | None = None + status: str + content_markdown: str | None = None + content_html: str | None = None + pages_json: str | None = None + error_message: str | None = None + started_at: str | datetime | None = None + completed_at: str | datetime | None = None + created_at: str | datetime + + +class CreateAnalysisRequest(BaseModel): + documentId: str # camelCase to match existing frontend contract diff --git a/document-parser/data/docling_studio.db b/document-parser/data/docling_studio.db new file mode 100644 index 0000000..b3fd80f Binary files /dev/null and b/document-parser/data/docling_studio.db differ diff --git a/document-parser/domain/__init__.py b/document-parser/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/bbox.py b/document-parser/domain/bbox.py similarity index 93% rename from document-parser/bbox.py rename to document-parser/domain/bbox.py index b1498f2..c36c590 100644 --- a/document-parser/bbox.py +++ b/document-parser/domain/bbox.py @@ -9,7 +9,7 @@ bboxes are normalized to TOPLEFT [left, top, right, bottom] before being sent to the frontend. """ -from docling_core.types.doc.base import BoundingBox, CoordOrigin +from docling_core.types.doc.base import BoundingBox def to_topleft_list(bbox: BoundingBox, page_height: float) -> list[float]: diff --git a/document-parser/domain/models.py b/document-parser/domain/models.py new file mode 100644 index 0000000..a26dac0 --- /dev/null +++ b/document-parser/domain/models.py @@ -0,0 +1,69 @@ +"""Domain models — pure data structures with no framework dependencies.""" + +from __future__ import annotations + +import enum +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone + + +class AnalysisStatus(str, enum.Enum): + PENDING = "PENDING" + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _new_id() -> str: + return str(uuid.uuid4()) + + +@dataclass +class Document: + id: str = field(default_factory=_new_id) + filename: str = "" + content_type: str | None = None + file_size: int | None = None + page_count: int | None = None + storage_path: str = "" + created_at: datetime = field(default_factory=_utcnow) + + +@dataclass +class AnalysisJob: + id: str = field(default_factory=_new_id) + document_id: str = "" + status: AnalysisStatus = AnalysisStatus.PENDING + content_markdown: str | None = None + content_html: str | None = None + pages_json: str | None = None + error_message: str | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + created_at: datetime = field(default_factory=_utcnow) + + # Joined from document (not persisted separately) + document_filename: str | None = None + + def mark_running(self) -> None: + self.status = AnalysisStatus.RUNNING + self.started_at = _utcnow() + + def mark_completed( + self, markdown: str, html: str, pages_json: str, + ) -> None: + self.status = AnalysisStatus.COMPLETED + self.content_markdown = markdown + self.content_html = html + self.pages_json = pages_json + self.completed_at = _utcnow() + + def mark_failed(self, error: str) -> None: + self.status = AnalysisStatus.FAILED + self.error_message = error + self.completed_at = _utcnow() diff --git a/document-parser/domain/parsing.py b/document-parser/domain/parsing.py new file mode 100644 index 0000000..e7c5faf --- /dev/null +++ b/document-parser/domain/parsing.py @@ -0,0 +1,275 @@ +"""Docling document extraction logic — pure domain, no HTTP concerns. + +Wraps the Docling library to convert documents and extract structured +per-page elements with bounding boxes and hierarchy levels. +""" + +from __future__ import annotations + +import logging +import threading +from dataclasses import dataclass, field + +from docling.document_converter import DocumentConverter, PdfFormatOption +from docling.datamodel.base_models import InputFormat +from docling.datamodel.pipeline_options import ( + PdfPipelineOptions, + TableFormerMode, + TableStructureOptions, +) +from docling_core.types.doc import ( + CodeItem, + DocItem, + FloatingItem, + FormulaItem, + GroupItem, + ListItem, + PictureItem, + SectionHeaderItem, + TableItem, + TextItem, + TitleItem, +) + +from domain.bbox import to_topleft_list + +logger = logging.getLogger(__name__) + +# Thread lock — DocumentConverter is not thread-safe +_converter_lock = threading.Lock() + +# Default converter (lazy-init on first request) +_default_converter: DocumentConverter | None = None + + +# --------------------------------------------------------------------------- +# Domain value objects +# --------------------------------------------------------------------------- + +@dataclass +class PageElement: + type: str + bbox: list[float] + content: str + level: int = 0 + + +@dataclass +class PageDetail: + page_number: int + width: float + height: float + elements: list[PageElement] = field(default_factory=list) + + +@dataclass +class ConversionResult: + page_count: int + content_markdown: str + content_html: str + pages: list[PageDetail] + skipped_items: int = 0 + + +# --------------------------------------------------------------------------- +# Element type detection +# --------------------------------------------------------------------------- + +# Mapping from Docling type to element type string. +# Order matters: most specific types before their parents. +_ELEMENT_TYPE_MAP: list[tuple[type, str]] = [ + (TableItem, "table"), + (PictureItem, "picture"), + (TitleItem, "title"), + (SectionHeaderItem, "section_header"), + (ListItem, "list"), + (FormulaItem, "formula"), + (CodeItem, "code"), + (FloatingItem, "floating"), + (TextItem, "text"), +] + + +def _get_element_type(item: DocItem) -> str: + """Determine element type via isinstance on Docling's type hierarchy.""" + for cls, type_name in _ELEMENT_TYPE_MAP: + if isinstance(item, cls): + return type_name + return "text" + + +# --------------------------------------------------------------------------- +# Pipeline factory +# --------------------------------------------------------------------------- + +def build_converter( + do_ocr: bool = True, + do_table_structure: bool = True, + table_mode: str = "accurate", +) -> DocumentConverter: + """Build a DocumentConverter with the given pipeline options. + + Only exposes options that work out of the box (no extra model downloads). + """ + table_options = TableStructureOptions( + do_cell_matching=True, + mode=TableFormerMode.ACCURATE if table_mode == "accurate" else TableFormerMode.FAST, + ) + + pipeline_options = PdfPipelineOptions( + do_ocr=do_ocr, + do_table_structure=do_table_structure, + table_structure_options=table_options, + do_code_enrichment=False, + do_formula_enrichment=False, + do_picture_classification=False, + do_picture_description=False, + generate_page_images=False, + generate_picture_images=False, + ) + + return DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options), + } + ) + + +def get_default_converter() -> DocumentConverter: + global _default_converter + if _default_converter is None: + _default_converter = build_converter() + return _default_converter + + +# --------------------------------------------------------------------------- +# Page extraction +# --------------------------------------------------------------------------- + +def extract_pages_detail(doc_result) -> tuple[list[PageDetail], int]: + """Extract per-page element details with bounding boxes from Docling result. + + Returns (pages, skipped_count) for transparent error reporting. + """ + pages: dict[int, PageDetail] = {} + document = doc_result.document + skipped = 0 + + for page_key, page_obj in document.pages.items(): + page_no = int(page_key) if isinstance(page_key, str) else page_key + pages[page_no] = PageDetail( + page_number=page_no, + width=page_obj.size.width, + height=page_obj.size.height, + ) + + for item, level in document.iterate_items(): + ok = _process_content_item(item, level, pages) + if not ok: + skipped += 1 + + sorted_pages = sorted(pages.values(), key=lambda p: p.page_number) + return sorted_pages, skipped + + +def _process_content_item( + item: DocItem | GroupItem, level: int, pages: dict[int, PageDetail], +) -> bool: + """Process a single content item and add it to the appropriate page.""" + if isinstance(item, GroupItem): + return True + + if not isinstance(item, DocItem) or not item.prov: + return False + + for prov in item.prov: + try: + page_no = prov.page_no + if page_no not in pages: + pages[page_no] = PageDetail(page_number=page_no, width=612.0, height=792.0) + + page_height = pages[page_no].height + + bbox = [0.0, 0.0, 0.0, 0.0] + if prov.bbox: + bbox = to_topleft_list(prov.bbox, page_height) + + element_type = _get_element_type(item) + + content = getattr(item, "text", "") or "" + if isinstance(item, TableItem): + try: + content = item.export_to_markdown() + except Exception: + pass + + pages[page_no].elements.append( + PageElement(type=element_type, bbox=bbox, content=content, level=level) + ) + except Exception: + logger.warning( + "Skipping item %s on page %s", + type(item).__name__, + getattr(prov, "page_no", "?"), + exc_info=True, + ) + return False + + return True + + +# --------------------------------------------------------------------------- +# Main conversion entry point +# --------------------------------------------------------------------------- + +def convert_document( + file_path: str, + *, + do_ocr: bool = True, + do_table_structure: bool = True, + table_mode: str = "accurate", +) -> ConversionResult: + """Convert a document and return structured results. + + This is the main entry point for document parsing. Runs synchronously + (caller should use asyncio.to_thread for non-blocking execution). + """ + if do_ocr and do_table_structure and table_mode == "accurate": + conv = get_default_converter() + else: + conv = build_converter( + do_ocr=do_ocr, + do_table_structure=do_table_structure, + table_mode=table_mode, + ) + + with _converter_lock: + result = conv.convert(file_path) + + doc = result.document + content_markdown = doc.export_to_markdown() + content_html = doc.export_to_html() + page_count = len(doc.pages) + + pages_detail, skipped = extract_pages_detail(result) + + if not pages_detail and page_count > 0: + pages_detail = [ + PageDetail( + page_number=i + 1, + width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else 612.0, + height=doc.pages[i + 1].size.height if (i + 1) in doc.pages else 792.0, + ) + for i in range(page_count) + ] + + if skipped > 0: + logger.info("Parsed: %d pages, %d items skipped", page_count, skipped) + + return ConversionResult( + page_count=page_count or len(pages_detail) or 1, + content_markdown=content_markdown, + content_html=content_html, + pages=pages_detail, + skipped_items=skipped, + ) diff --git a/document-parser/main.py b/document-parser/main.py index efa66bf..f1b4038 100644 --- a/document-parser/main.py +++ b/document-parser/main.py @@ -1,238 +1,63 @@ -import io +"""Docling Studio — unified FastAPI backend. + +Single service replacing both the Spring Boot backend and the document parser. +Provides document management (upload, CRUD), analysis orchestration (async Docling +processing), and PDF preview — all backed by SQLite. +""" + +from __future__ import annotations + import logging import os -import tempfile -from pathlib import Path +from contextlib import asynccontextmanager -from fastapi import FastAPI, UploadFile, HTTPException, Query -from fastapi.responses import StreamingResponse -from pydantic import BaseModel -from docling.document_converter import DocumentConverter -from pdf2image import convert_from_bytes -from PIL import Image +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware -from bbox import to_topleft_list +from persistence.database import init_db +from api.documents import router as documents_router +from api.analyses import router as analyses_router +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s — %(message)s", +) logger = logging.getLogger(__name__) -app = FastAPI(title="Docling Studio - Document Parser") -converter = DocumentConverter() - -MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB +@asynccontextmanager +async def lifespan(app: FastAPI): + """Startup: initialize database. Shutdown: nothing special needed.""" + await init_db() + logger.info("Docling Studio backend ready") + yield -# --- Response models --- +app = FastAPI( + title="Docling Studio", + description="Document analysis studio powered by Docling", + lifespan=lifespan, +) -class PageElement(BaseModel): - type: str - bbox: list[float] - content: str +# CORS — configurable via env, defaults for local dev +allowed_origins = os.environ.get( + "CORS_ORIGINS", "http://localhost:3000,http://localhost:5173" +).split(",") +app.add_middleware( + CORSMiddleware, + allow_origins=[o.strip() for o in allowed_origins], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) -class PageDetail(BaseModel): - page_number: int - width: float - height: float - elements: list[PageElement] - - -class ParseResponse(BaseModel): - filename: str - page_count: int - content_markdown: str - content_html: str - pages: list[PageDetail] - - -# --- Helpers --- - -def extract_pages_detail(doc_result) -> list[PageDetail]: - """Extract per-page element details with bounding boxes from Docling result. - - Uses Docling's iterate_items() API (preferred) or falls back to document.texts. - Both provide a flat iteration over all content items, avoiding duplicates. - """ - pages: dict[int, PageDetail] = {} - document = doc_result.document - - # Get page dimensions from document pages - if hasattr(document, 'pages') and document.pages: - for page_key, page_obj in document.pages.items(): - page_no = int(page_key) if isinstance(page_key, str) else page_key - width = page_obj.size.width if hasattr(page_obj, 'size') and page_obj.size else 612.0 - height = page_obj.size.height if hasattr(page_obj, 'size') and page_obj.size else 792.0 - pages[page_no] = PageDetail( - page_number=page_no, - width=width, - height=height, - elements=[] - ) - - # Use iterate_items() (Docling v2 API) — avoids duplicates - if hasattr(document, 'iterate_items'): - for item, _level in document.iterate_items(): - _process_content_item(item, pages) - elif hasattr(document, 'texts'): - for text_item in document.texts: - _process_content_item(text_item, pages) - - # Sort by page number - return sorted(pages.values(), key=lambda p: p.page_number) - - -def _process_content_item(item, pages: dict[int, PageDetail]): - """Process a single content item and add it to the appropriate page. - - Silently skips items that lack provenance or fail to process, - so one bad item doesn't break the whole extraction. - """ - if not hasattr(item, 'prov') or not item.prov: - return - - for prov in item.prov: - try: - page_no = prov.page_no if hasattr(prov, 'page_no') else 1 - - if page_no not in pages: - pages[page_no] = PageDetail( - page_number=page_no, width=612.0, height=792.0, elements=[] - ) - - page_height = pages[page_no].height - - bbox = [0, 0, 0, 0] - if hasattr(prov, 'bbox') and prov.bbox: - b = prov.bbox - if hasattr(b, 'l'): - bbox = to_topleft_list(b, page_height) - elif isinstance(b, (list, tuple)) and len(b) >= 4: - bbox = list(b[:4]) - - element_type = _get_element_type(item) - content = "" - if hasattr(item, 'text'): - content = item.text or "" - - pages[page_no].elements.append(PageElement( - type=element_type, - bbox=bbox, - content=content[:500] - )) - except Exception: - logger.warning("Skipping item %s: failed to process", type(item).__name__, exc_info=True) - - - -def _get_element_type(item) -> str: - """Determine the element type from a Docling document item.""" - type_name = type(item).__name__.lower() - if 'table' in type_name: - return 'table' - if 'picture' in type_name or 'image' in type_name or 'figure' in type_name: - return 'picture' - if 'section' in type_name or 'heading' in type_name: - return 'section_header' - if 'list' in type_name: - return 'list' - if 'formula' in type_name or 'equation' in type_name: - return 'formula' - if 'caption' in type_name: - return 'caption' - if hasattr(item, 'label'): - label = str(item.label).lower() - if 'table' in label: - return 'table' - if 'picture' in label or 'figure' in label: - return 'picture' - if 'section' in label or 'head' in label: - return 'section_header' - if 'list' in label: - return 'list' - if 'formula' in label: - return 'formula' - return 'text' - - -# --- Endpoints --- - -@app.post("/parse", response_model=ParseResponse) -async def parse(file: UploadFile): - if not file.filename: - raise HTTPException(status_code=400, detail="No filename provided") - - content = await file.read() - if len(content) > MAX_FILE_SIZE: - raise HTTPException(status_code=413, detail="File too large (max 50MB)") - - suffix = Path(file.filename).suffix - tmp_path = None - try: - with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: - tmp.write(content) - tmp_path = tmp.name - - result = converter.convert(tmp_path) - doc = result.document - - content_markdown = doc.export_to_markdown() - content_html = doc.export_to_html() if hasattr(doc, 'export_to_html') else "" - - page_count = 0 - if hasattr(doc, 'pages') and doc.pages: - page_count = len(doc.pages) - - pages_detail = extract_pages_detail(result) - if not pages_detail and page_count > 0: - pages_detail = [ - PageDetail(page_number=i + 1, width=612.0, height=792.0, elements=[]) - for i in range(page_count) - ] - - return ParseResponse( - filename=file.filename, - page_count=page_count or len(pages_detail) or 1, - content_markdown=content_markdown, - content_html=content_html, - pages=pages_detail, - ) - - except Exception as e: - logger.exception("Failed to parse document: %s", file.filename) - raise HTTPException(status_code=422, detail=f"Failed to parse document: {str(e)}") - finally: - if tmp_path and os.path.exists(tmp_path): - os.unlink(tmp_path) - - -@app.post("/preview") -async def preview( - file: UploadFile, - page: int = Query(1, ge=1), - dpi: int = Query(150, ge=72, le=300), -): - """Generate a PNG preview of a specific page.""" - if not file.filename: - raise HTTPException(status_code=400, detail="No filename provided") - - content = await file.read() - if len(content) > MAX_FILE_SIZE: - raise HTTPException(status_code=413, detail="File too large (max 50MB)") - - try: - images = convert_from_bytes(content, first_page=page, last_page=page, dpi=dpi) - if not images: - raise HTTPException(status_code=404, detail=f"Page {page} not found") - - buf = io.BytesIO() - images[0].save(buf, format="PNG") - buf.seek(0) - return StreamingResponse(buf, media_type="image/png") - - except Exception as e: - raise HTTPException(status_code=422, detail=f"Failed to generate preview: {str(e)}") +# Mount routers +app.include_router(documents_router) +app.include_router(analyses_router) @app.get("/health") def health(): + """Health check endpoint.""" return {"status": "ok"} diff --git a/document-parser/persistence/__init__.py b/document-parser/persistence/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/persistence/analysis_repo.py b/document-parser/persistence/analysis_repo.py new file mode 100644 index 0000000..fa3202e --- /dev/null +++ b/document-parser/persistence/analysis_repo.py @@ -0,0 +1,108 @@ +"""Analysis job repository — SQLite CRUD for analysis_jobs table.""" + +from __future__ import annotations + +from domain.models import AnalysisJob, AnalysisStatus +from persistence.database import get_db + + +def _row_to_job(row) -> AnalysisJob: + return AnalysisJob( + id=row["id"], + document_id=row["document_id"], + status=AnalysisStatus(row["status"]), + content_markdown=row["content_markdown"], + content_html=row["content_html"], + pages_json=row["pages_json"], + error_message=row["error_message"], + started_at=row["started_at"], + completed_at=row["completed_at"], + created_at=row["created_at"], + document_filename=row["filename"] if "filename" in row.keys() else None, + ) + + +_SELECT_WITH_DOC = """ + SELECT aj.*, d.filename + FROM analysis_jobs aj + JOIN documents d ON d.id = aj.document_id +""" + + +async def insert(job: AnalysisJob) -> None: + db = await get_db() + try: + await db.execute( + """INSERT INTO analysis_jobs (id, document_id, status, created_at) + VALUES (?, ?, ?, ?)""", + (job.id, job.document_id, job.status.value, str(job.created_at)), + ) + await db.commit() + finally: + await db.close() + + +async def find_all() -> list[AnalysisJob]: + db = await get_db() + try: + cursor = await db.execute( + f"{_SELECT_WITH_DOC} ORDER BY aj.created_at DESC" + ) + rows = await cursor.fetchall() + return [_row_to_job(r) for r in rows] + finally: + await db.close() + + +async def find_by_id(job_id: str) -> AnalysisJob | None: + db = await get_db() + try: + cursor = await db.execute( + f"{_SELECT_WITH_DOC} WHERE aj.id = ?", (job_id,) + ) + row = await cursor.fetchone() + return _row_to_job(row) if row else None + finally: + await db.close() + + +async def update_status(job: AnalysisJob) -> None: + db = await get_db() + try: + await db.execute( + """UPDATE analysis_jobs + SET status = ?, content_markdown = ?, content_html = ?, + pages_json = ?, error_message = ?, started_at = ?, completed_at = ? + WHERE id = ?""", + (job.status.value, job.content_markdown, job.content_html, + job.pages_json, job.error_message, + str(job.started_at) if job.started_at else None, + str(job.completed_at) if job.completed_at else None, + job.id), + ) + await db.commit() + finally: + await db.close() + + +async def delete(job_id: str) -> bool: + db = await get_db() + try: + cursor = await db.execute("DELETE FROM analysis_jobs WHERE id = ?", (job_id,)) + await db.commit() + return cursor.rowcount > 0 + finally: + await db.close() + + +async def delete_by_document(document_id: str) -> int: + """Delete all analysis jobs for a given document. Returns count deleted.""" + db = await get_db() + try: + cursor = await db.execute( + "DELETE FROM analysis_jobs WHERE document_id = ?", (document_id,) + ) + await db.commit() + return cursor.rowcount + finally: + await db.close() diff --git a/document-parser/persistence/database.py b/document-parser/persistence/database.py new file mode 100644 index 0000000..fa7f154 --- /dev/null +++ b/document-parser/persistence/database.py @@ -0,0 +1,54 @@ +"""SQLite database management — async via aiosqlite.""" + +from __future__ import annotations + +import logging +import os + +import aiosqlite + +logger = logging.getLogger(__name__) + +DB_PATH = os.environ.get("DB_PATH", "./data/docling_studio.db") + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, + filename TEXT NOT NULL, + content_type TEXT, + file_size INTEGER, + page_count INTEGER, + storage_path TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS analysis_jobs ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL REFERENCES documents(id), + status TEXT NOT NULL DEFAULT 'PENDING', + content_markdown TEXT, + content_html TEXT, + pages_json TEXT, + error_message TEXT, + started_at TEXT, + completed_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +""" + + +async def init_db() -> None: + """Create database file and tables if they don't exist.""" + os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True) + async with aiosqlite.connect(DB_PATH) as db: + await db.executescript(_SCHEMA) + await db.commit() + logger.info("Database initialized at %s", DB_PATH) + + +async def get_db() -> aiosqlite.Connection: + """Open a new database connection with row factory and FK enforcement.""" + db = await aiosqlite.connect(DB_PATH) + db.row_factory = aiosqlite.Row + await db.execute("PRAGMA foreign_keys = ON") + return db diff --git a/document-parser/persistence/document_repo.py b/document-parser/persistence/document_repo.py new file mode 100644 index 0000000..94a91f3 --- /dev/null +++ b/document-parser/persistence/document_repo.py @@ -0,0 +1,76 @@ +"""Document repository — SQLite CRUD for documents table.""" + +from __future__ import annotations + +from domain.models import Document +from persistence.database import get_db + + +def _row_to_document(row) -> Document: + return Document( + id=row["id"], + filename=row["filename"], + content_type=row["content_type"], + file_size=row["file_size"], + page_count=row["page_count"], + storage_path=row["storage_path"], + created_at=row["created_at"], + ) + + +async def insert(doc: Document) -> None: + db = await get_db() + try: + await db.execute( + """INSERT INTO documents (id, filename, content_type, file_size, page_count, storage_path, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (doc.id, doc.filename, doc.content_type, doc.file_size, + doc.page_count, doc.storage_path, str(doc.created_at)), + ) + await db.commit() + finally: + await db.close() + + +async def find_all() -> list[Document]: + db = await get_db() + try: + cursor = await db.execute( + "SELECT * FROM documents ORDER BY created_at DESC" + ) + rows = await cursor.fetchall() + return [_row_to_document(r) for r in rows] + finally: + await db.close() + + +async def find_by_id(doc_id: str) -> Document | None: + db = await get_db() + try: + cursor = await db.execute("SELECT * FROM documents WHERE id = ?", (doc_id,)) + row = await cursor.fetchone() + return _row_to_document(row) if row else None + finally: + await db.close() + + +async def update_page_count(doc_id: str, page_count: int) -> None: + db = await get_db() + try: + await db.execute( + "UPDATE documents SET page_count = ? WHERE id = ?", + (page_count, doc_id), + ) + await db.commit() + finally: + await db.close() + + +async def delete(doc_id: str) -> bool: + db = await get_db() + try: + cursor = await db.execute("DELETE FROM documents WHERE id = ?", (doc_id,)) + await db.commit() + return cursor.rowcount > 0 + finally: + await db.close() diff --git a/document-parser/requirements.txt b/document-parser/requirements.txt index e1a213d..46d50d0 100644 --- a/document-parser/requirements.txt +++ b/document-parser/requirements.txt @@ -1,6 +1,8 @@ -docling -fastapi -uvicorn[standard] -python-multipart -pdf2image -pillow +docling>=2.80.0,<3.0.0 +docling-core>=2.0.0,<3.0.0 +fastapi>=0.115.0,<1.0.0 +uvicorn[standard]>=0.32.0,<1.0.0 +python-multipart>=0.0.12 +pdf2image>=1.17.0,<2.0.0 +pillow>=10.0.0,<11.0.0 +aiosqlite>=0.20.0,<1.0.0 diff --git a/document-parser/services/__init__.py b/document-parser/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/services/analysis_service.py b/document-parser/services/analysis_service.py new file mode 100644 index 0000000..12708d3 --- /dev/null +++ b/document-parser/services/analysis_service.py @@ -0,0 +1,78 @@ +"""Analysis service — async document parsing orchestration.""" + +from __future__ import annotations + +import asyncio +import json +import logging +from dataclasses import asdict + +from domain.models import AnalysisJob +from domain.parsing import ConversionResult, convert_document +from persistence import analysis_repo, document_repo + +logger = logging.getLogger(__name__) + + +async def create(document_id: str) -> AnalysisJob: + """Create a new analysis job and launch background processing.""" + doc = await document_repo.find_by_id(document_id) + if not doc: + raise ValueError(f"Document not found: {document_id}") + + job = AnalysisJob(document_id=document_id) + job.document_filename = doc.filename + await analysis_repo.insert(job) + + # Fire-and-forget background task + asyncio.create_task(_run_analysis(job.id, doc.storage_path, doc.filename)) + + return job + + +async def find_all() -> list[AnalysisJob]: + return await analysis_repo.find_all() + + +async def find_by_id(job_id: str) -> AnalysisJob | None: + return await analysis_repo.find_by_id(job_id) + + +async def delete(job_id: str) -> bool: + return await analysis_repo.delete(job_id) + + +async def _run_analysis(job_id: str, file_path: str, filename: str) -> None: + """Background task: run Docling conversion and update job status.""" + job = await analysis_repo.find_by_id(job_id) + if not job: + logger.error("Analysis job %s not found", job_id) + return + + job.mark_running() + await analysis_repo.update_status(job) + logger.info("Analysis started: %s (file: %s)", job_id, filename) + + try: + # Run blocking Docling conversion in a thread + result: ConversionResult = await asyncio.to_thread(convert_document, file_path) + + pages_json = json.dumps([asdict(p) for p in result.pages]) + + job.mark_completed( + markdown=result.content_markdown, + html=result.content_html, + pages_json=pages_json, + ) + await analysis_repo.update_status(job) + + # Update document page count if available + if result.page_count: + await document_repo.update_page_count(job.document_id, result.page_count) + + logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count) + + except Exception as e: + logger.exception("Analysis failed: %s", job_id) + job.mark_failed(str(e)) + await analysis_repo.update_status(job) diff --git a/document-parser/services/document_service.py b/document-parser/services/document_service.py new file mode 100644 index 0000000..d1bdeba --- /dev/null +++ b/document-parser/services/document_service.py @@ -0,0 +1,93 @@ +"""Document service — file upload, storage, and preview orchestration.""" + +from __future__ import annotations + +import io +import logging +import os +import uuid + +from pdf2image import convert_from_bytes, pdfinfo_from_bytes + +from domain.models import Document +from persistence import analysis_repo, document_repo + +logger = logging.getLogger(__name__) + +UPLOAD_DIR = os.environ.get("UPLOAD_DIR", "./uploads") +MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB + + +async def upload(filename: str, content_type: str, file_content: bytes) -> Document: + """Save uploaded file to disk and persist metadata.""" + if len(file_content) > MAX_FILE_SIZE: + raise ValueError("File too large (max 50 MB)") + + os.makedirs(UPLOAD_DIR, exist_ok=True) + + safe_name = f"{uuid.uuid4()}_{filename}" + file_path = os.path.join(UPLOAD_DIR, safe_name) + + with open(file_path, "wb") as f: + f.write(file_content) + + # Count PDF pages + page_count = _count_pages(file_content) + + doc = Document( + filename=filename, + content_type=content_type, + file_size=len(file_content), + page_count=page_count, + storage_path=os.path.abspath(file_path), + ) + await document_repo.insert(doc) + return doc + + +async def find_all() -> list[Document]: + return await document_repo.find_all() + + +async def find_by_id(doc_id: str) -> Document | None: + return await document_repo.find_by_id(doc_id) + + +async def delete(doc_id: str) -> bool: + """Delete document file, associated analyses, and database record.""" + doc = await document_repo.find_by_id(doc_id) + if not doc: + return False + + # Delete associated analyses first (cascade) + await analysis_repo.delete_by_document(doc_id) + + # Delete file from disk + try: + if os.path.exists(doc.storage_path): + os.unlink(doc.storage_path) + except OSError: + logger.warning("Could not delete file: %s", doc.storage_path) + + return await document_repo.delete(doc_id) + + +def generate_preview(file_content: bytes, page: int = 1, dpi: int = 150) -> bytes: + """Generate a PNG preview of a specific PDF page.""" + images = convert_from_bytes(file_content, first_page=page, last_page=page, dpi=dpi) + if not images: + raise ValueError(f"Page {page} not found") + + buf = io.BytesIO() + images[0].save(buf, format="PNG") + return buf.getvalue() + + +def _count_pages(file_content: bytes) -> int | None: + """Count PDF pages using poppler via pdf2image.""" + try: + info = pdfinfo_from_bytes(file_content) + return info.get("Pages") + except Exception: + logger.warning("Could not count pages", exc_info=True) + return None diff --git a/document-parser/test_bbox.py b/document-parser/tests/test_bbox.py similarity index 98% rename from document-parser/test_bbox.py rename to document-parser/tests/test_bbox.py index 2bfd5a7..d088f2d 100644 --- a/document-parser/test_bbox.py +++ b/document-parser/tests/test_bbox.py @@ -3,7 +3,7 @@ import pytest from docling_core.types.doc.base import BoundingBox, CoordOrigin -from bbox import to_topleft_list +from domain.bbox import to_topleft_list class TestToTopleftList: diff --git a/frontend/nginx.conf b/frontend/nginx.conf index ca50c32..515b230 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -9,9 +9,11 @@ server { } location /api/ { - proxy_pass http://backend:8081; + proxy_pass http://document-parser:8000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 600s; + proxy_send_timeout 600s; client_max_body_size 50M; } } diff --git a/frontend/src/features/analysis/ui/BboxOverlay.vue b/frontend/src/features/analysis/ui/BboxOverlay.vue index 131dfca..a3badf7 100644 --- a/frontend/src/features/analysis/ui/BboxOverlay.vue +++ b/frontend/src/features/analysis/ui/BboxOverlay.vue @@ -40,12 +40,14 @@ import { ref, computed, watch, nextTick, reactive, onMounted, onBeforeUnmount } import { computeScale, bboxToRect, pointInRect } from '../bboxScaling.js' const ELEMENT_COLORS = { + title: '#EF4444', section_header: '#F97316', text: '#3B82F6', table: '#8B5CF6', picture: '#22C55E', list: '#06B6D4', formula: '#EC4899', + code: '#14B8A6', caption: '#EAB308' } diff --git a/frontend/src/features/analysis/ui/ResultTabs.vue b/frontend/src/features/analysis/ui/ResultTabs.vue index 98c4c8a..177262e 100644 --- a/frontend/src/features/analysis/ui/ResultTabs.vue +++ b/frontend/src/features/analysis/ui/ResultTabs.vue @@ -83,17 +83,27 @@ const pageMarkdown = computed(() => { function formatElement(el) { if (!el.content) return '' + const indent = ' '.repeat(Math.max(0, (el.level || 0) - 1)) switch (el.type) { - case 'section_header': - return `## ${el.content}` + case 'title': + return `# ${el.content}` + case 'section_header': { + // Use hierarchy level for heading depth (h2-h4) + const depth = Math.min(Math.max(el.level || 2, 2), 4) + return `${'#'.repeat(depth)} ${el.content}` + } case 'caption': - return `*${el.content}*` + return `${indent}*${el.content}*` case 'table': return el.content case 'formula': - return `$$${el.content}$$` + return `${indent}$$${el.content}$$` + case 'code': + return `${indent}\`\`\`\n${el.content}\n\`\`\`` + case 'list': + return `${indent}- ${el.content}` default: - return el.content + return `${indent}${el.content}` } } diff --git a/frontend/src/features/settings/store.js b/frontend/src/features/settings/store.js index 2cb9a10..011cc4f 100644 --- a/frontend/src/features/settings/store.js +++ b/frontend/src/features/settings/store.js @@ -2,8 +2,7 @@ import { defineStore } from 'pinia' import { ref } from 'vue' export const useSettingsStore = defineStore('settings', () => { - const parserUrl = ref('http://localhost:8000') - const backendUrl = ref('http://localhost:8081') + const apiUrl = ref('http://localhost:8000') - return { parserUrl, backendUrl } + return { apiUrl } }) diff --git a/frontend/src/features/settings/ui/SettingsPanel.vue b/frontend/src/features/settings/ui/SettingsPanel.vue index 5f0c428..02d561b 100644 --- a/frontend/src/features/settings/ui/SettingsPanel.vue +++ b/frontend/src/features/settings/ui/SettingsPanel.vue @@ -1,12 +1,8 @@