diff --git a/.claude/plan.md b/.claude/plan.md new file mode 100644 index 0000000..ef6a2b2 --- /dev/null +++ b/.claude/plan.md @@ -0,0 +1,155 @@ +# Plan : Architecture Simplification — 2 services (Vue + FastAPI/SQLite) + +## Objectif +Supprimer le backend Spring Boot (passe-plat) et PostgreSQL. Le document-parser Python absorbe toute la logique backend. On passe de **4 services** à **2 services** : Vue frontend + FastAPI Python. + +## Architecture Python — Clean Architecture légère + +``` +document-parser/ +├── main.py # FastAPI app, CORS, lifespan, routers mount +├── bbox.py # (existant) coord conversion +├── test_bbox.py # (existant) bbox tests +├── requirements.txt # + aiosqlite, aiofiles +├── Dockerfile # (adapté) +├── .dockerignore # (existant) +│ +├── domain/ # 🧠 Modèles métier purs (pas de dépendance framework) +│ ├── __init__.py +│ ├── models.py # Document, AnalysisJob, Status (dataclasses/Pydantic) +│ └── parsing.py # Logique d'extraction Docling (déplacée depuis main.py) +│ +├── api/ # 🌐 Couche HTTP (FastAPI routers) +│ ├── __init__.py +│ ├── schemas.py # Pydantic request/response schemas (DTOs) +│ ├── documents.py # Router /api/documents (CRUD + upload + preview) +│ └── analyses.py # Router /api/analyses (CRUD + async processing) +│ +├── persistence/ # 💾 Couche données (SQLite via aiosqlite) +│ ├── __init__.py +│ ├── database.py # SQLite connection, init schema, get_db() +│ ├── document_repo.py # CRUD documents +│ └── analysis_repo.py # CRUD analysis jobs +│ +└── services/ # ⚙️ Orchestration (use cases) + ├── __init__.py + ├── document_service.py # Upload, delete, preview (file I/O + persistence) + └── analysis_service.py # Create job, background parse, update status +``` + +### Pourquoi cette structure plutôt qu'hexagonale ? +- **domain/** : modèles purs, testables, zéro import framework → l'esprit de l'hexagonale +- **api/** : adaptateur HTTP (port entrant) +- **persistence/** : adaptateur stockage (port sortant) +- **services/** : orchestration des use cases +- Pas de ports/adapters formels (interfaces abstraites) → overkill pour le scope +- Un mec de Docling voit ça, il comprend en 5 secondes. Clean, pas over-engineered. + +## Endpoints conservés (contrat API identique) + +Le frontend ne change quasiment pas — mêmes URLs, mêmes payloads : + +| Method | Path | Description | +|--------|------|-------------| +| POST | `/api/documents/upload` | Upload PDF (multipart) | +| GET | `/api/documents` | List documents | +| GET | `/api/documents/{id}` | Get document | +| DELETE | `/api/documents/{id}` | Delete document + file | +| GET | `/api/documents/{id}/preview?page=&dpi=` | Page preview PNG | +| POST | `/api/analyses` | Create analysis (body: {documentId}) | +| GET | `/api/analyses` | List analyses | +| GET | `/api/analyses/{id}` | Get analysis (polling) | +| DELETE | `/api/analyses/{id}` | Delete analysis | +| GET | `/health` | Health check | + +## Détails d'implémentation + +### 1. SQLite (persistence/database.py) +- `aiosqlite` pour async natif avec FastAPI +- Schema identique au Liquibase actuel (2 tables: documents, analysis_jobs) +- DB file dans volume Docker : `/app/data/docling_studio.db` +- Init schema au startup (CREATE TABLE IF NOT EXISTS) +- Pas besoin d'Alembic pour un projet de cette taille + +### 2. File storage +- Même logique : `./uploads/{uuid}_{filename}` +- Volume Docker monté sur `/app/uploads` + +### 3. Async analysis (services/analysis_service.py) +- `asyncio.create_task()` pour le background processing (pas besoin de Celery) +- Status polling identique : PENDING → RUNNING → COMPLETED | FAILED +- Le parse Docling tourne dans un thread via `asyncio.to_thread()` (car bloquant + lock) + +### 4. CORS +- `fastapi.middleware.cors.CORSMiddleware` dans main.py +- Origins configurables via env var + +### 5. domain/parsing.py +- Déplace depuis main.py : `_build_converter()`, `_get_element_type()`, `extract_pages_detail()`, `_process_content_item()` +- Le converter et le lock restent globaux (singleton pattern) + +## Changements Frontend + +Minimes — seulement la configuration : + +1. **vite.config.js** : proxy target change `8081` → `8000` +2. **frontend/Dockerfile** (nginx) : proxy_pass change `backend:8081` → `document-parser:8000` +3. **api.js** : AUCUN changement (mêmes paths `/api/...`) +4. **stores** : AUCUN changement + +## docker-compose.yml simplifié + +```yaml +services: + document-parser: + build: ./document-parser + ports: + - "8000:8000" + volumes: + - uploads_data:/app/uploads + - db_data:/app/data + deploy: + resources: + limits: + memory: 4g + + frontend: + build: ./frontend + ports: + - "3000:80" + depends_on: + - document-parser + +volumes: + uploads_data: + db_data: +``` + +→ **Plus de postgres, plus de backend Java.** 2 services, clean. + +## Étapes d'exécution + +1. **Créer la structure Python** (domain/, api/, persistence/, services/) +2. **persistence/database.py** — SQLite async init + schema +3. **persistence/document_repo.py** — CRUD documents +4. **persistence/analysis_repo.py** — CRUD analyses +5. **domain/models.py** — Dataclasses Document, AnalysisJob, Status +6. **domain/parsing.py** — Extraire la logique Docling depuis main.py +7. **api/schemas.py** — Pydantic DTOs (DocumentResponse, AnalysisResponse, etc.) +8. **services/document_service.py** — Upload, delete, preview, list, get +9. **services/analysis_service.py** — Create, run background, status tracking +10. **api/documents.py** — Router documents (5 endpoints) +11. **api/analyses.py** — Router analyses (4 endpoints) +12. **main.py** — Réécrire : CORS, lifespan (DB init), mount routers +13. **requirements.txt** — Ajouter aiosqlite, aiofiles +14. **docker-compose.yml** — Simplifier (2 services) +15. **frontend/vite.config.js** — Changer proxy target +16. **frontend/Dockerfile** — Changer nginx proxy_pass +17. **Supprimer le dossier backend/** entier +18. **Mettre à jour README.md** — Nouvelle architecture + +## Ce qu'on ne touche PAS +- bbox.py, test_bbox.py (inchangés) +- Toute la logique Vue (stores, components, pages) +- Logique Docling (parsing, extraction) — juste déplacée +- Dockerfile du parser (juste adapter le CMD si nécessaire) diff --git a/.env.example b/.env.example index df0b3fe..8ff39a2 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,8 @@ -# PostgreSQL -POSTGRES_USER=app -POSTGRES_PASSWORD=changeme -POSTGRES_DB=docling_studio - -# Backend (Spring Boot) -SPRING_DATASOURCE_USERNAME=app -SPRING_DATASOURCE_PASSWORD=changeme - -# Document parser URL (only change if running parser on a different host) -# APP_DOCUMENT-PARSER_BASE-URL=http://document-parser:8000 - # CORS (comma-separated origins, only needed for custom deployments) -# APP_CORS_ALLOWED_ORIGINS=http://localhost:3000,https://your-domain.com +# CORS_ORIGINS=http://localhost:3000,https://your-domain.com -# File storage path (inside backend container) -# APP_STORAGE_PATH=./uploads +# File storage path (inside container) +# UPLOAD_DIR=./uploads + +# Database path (inside container) +# DB_PATH=./data/docling_studio.db diff --git a/README.md b/README.md index ec14cce..a4364ca 100644 --- a/README.md +++ b/README.md @@ -27,24 +27,39 @@ Upload a PDF, configure the extraction pipeline, and visualize the results — t ## Architecture ``` -┌────────────┐ ┌──────────────┐ ┌──────────────────┐ -│ Frontend │────▶│ Backend │────▶│ Document Parser │ -│ Vue 3 │ │ Spring Boot │ │ FastAPI + Docling │ -│ port 3000 │ │ port 8081 │ │ port 8000 │ -└────────────┘ └──────┬───────┘ └──────────────────┘ - │ - ┌──────▼───────┐ - │ PostgreSQL │ - │ port 5432 │ - └──────────────┘ +┌────────────┐ ┌───────────────────────┐ +│ Frontend │────────▶│ Document Parser │ +│ Vue 3 │ /api/* │ FastAPI + Docling │ +│ port 3000 │ │ SQLite + file storage │ +└────────────┘ │ port 8000 │ + └───────────────────────┘ ``` | Service | Stack | Role | |---------|-------|------| | **frontend** | Vue 3, Vite, Pinia | UI, PDF viewer, results display | -| **backend** | Spring Boot 3.3, Java 21, Liquibase | REST API, storage, orchestration | -| **document-parser** | FastAPI, Docling, pdf2image | PDF parsing with configurable pipeline | -| **postgres** | PostgreSQL 16 | Documents & analysis persistence | +| **document-parser** | FastAPI, Docling, SQLite, pdf2image | REST API, document parsing, storage, persistence | + +### Python project structure (clean architecture) + +``` +document-parser/ +├── main.py # FastAPI app, CORS, lifespan +├── domain/ # Pure domain models & Docling logic +│ ├── models.py # Document, AnalysisJob dataclasses +│ └── parsing.py # Docling conversion & page extraction +├── api/ # HTTP layer (FastAPI routers) +│ ├── schemas.py # Pydantic DTOs (camelCase serialization) +│ ├── documents.py # /api/documents endpoints +│ └── analyses.py # /api/analyses endpoints +├── persistence/ # Data layer (SQLite) +│ ├── database.py # Connection management, schema init +│ ├── document_repo.py # Document CRUD +│ └── analysis_repo.py # AnalysisJob CRUD +└── services/ # Use case orchestration + ├── document_service.py # Upload, delete, preview + └── analysis_service.py # Async Docling processing +``` ## Quick Start @@ -52,10 +67,10 @@ Upload a PDF, configure the extraction pipeline, and visualize the results — t ```bash # Clone the repo -git clone https://github.com/pjmalandrino/docling-studio.git +git clone https://github.com/scub-france/docling-studio.git cd docling-studio -# (Optional) customize credentials +# (Optional) customize settings cp .env.example .env # Start all services @@ -68,14 +83,6 @@ Open [http://localhost:3000](http://localhost:3000) ### Local Development -Start only PostgreSQL via Docker: - -```bash -docker compose -f docker-compose.dev.yml up -d -``` - -Then run each service locally: - **Document Parser** (Python 3.12+): ```bash cd document-parser @@ -84,12 +91,6 @@ pip install -r requirements.txt uvicorn main:app --reload --port 8000 ``` -**Backend** (Java 21+): -```bash -cd backend -./mvnw spring-boot:run -``` - **Frontend** (Node 20+): ```bash cd frontend @@ -115,12 +116,9 @@ All configuration is done via environment variables. See [`.env.example`](.env.e | Variable | Default | Description | |----------|---------|-------------| -| `POSTGRES_USER` | `app` | Database user | -| `POSTGRES_PASSWORD` | `app` | Database password | -| `POSTGRES_DB` | `docling_studio` | Database name | -| `APP_CORS_ALLOWED_ORIGINS` | `http://localhost:3000,...` | CORS allowed origins (comma-separated) | -| `APP_DOCUMENT-PARSER_BASE-URL` | `http://localhost:8000` | Document parser URL | -| `APP_STORAGE_PATH` | `./uploads` | File storage directory | +| `CORS_ORIGINS` | `http://localhost:3000,...` | CORS allowed origins (comma-separated) | +| `UPLOAD_DIR` | `./uploads` | File storage directory | +| `DB_PATH` | `./data/docling_studio.db` | SQLite database path | ## Performance & System Requirements @@ -161,9 +159,7 @@ All Docker images are **multi-arch** (linux/amd64 + linux/arm64). Works natively ## Tech Stack - **Frontend**: Vue 3 + Vite + Pinia -- **Backend**: Spring Boot 3.3 + Java 21 + Liquibase + PDFBox -- **Parser**: FastAPI + Docling 2.x + PyTorch + pdf2image -- **Database**: PostgreSQL 16 +- **Backend**: FastAPI + Docling 2.x + SQLite + pdf2image - **Infra**: Docker Compose + Nginx ## Contributing diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index 11f3ea3..0000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM eclipse-temurin:21-jdk AS build - -WORKDIR /app -COPY pom.xml . -COPY src ./src - -RUN apt-get update && apt-get install -y maven && \ - mvn package -DskipTests - -FROM eclipse-temurin:21-jre - -WORKDIR /app -COPY --from=build /app/target/*.jar app.jar - -EXPOSE 8081 - -ENTRYPOINT ["java", "-jar", "app.jar"] diff --git a/backend/pom.xml b/backend/pom.xml deleted file mode 100644 index 5fa447f..0000000 --- a/backend/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - 4.0.0 - - - org.springframework.boot - spring-boot-starter-parent - 3.3.5 - - - - com.docling - docling-studio - 0.1.0 - Docling Studio Backend - - - 21 - - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-data-jpa - - - org.springframework.boot - spring-boot-starter-webflux - - - org.springframework.boot - spring-boot-starter-actuator - - - - org.postgresql - postgresql - runtime - - - org.liquibase - liquibase-core - - - org.apache.pdfbox - pdfbox - 3.0.3 - - - - org.springframework.boot - spring-boot-starter-test - test - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - - diff --git a/backend/src/main/java/com/docling/studio/DoclingStudioApplication.java b/backend/src/main/java/com/docling/studio/DoclingStudioApplication.java deleted file mode 100644 index b8e6711..0000000 --- a/backend/src/main/java/com/docling/studio/DoclingStudioApplication.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.docling.studio; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@SpringBootApplication -public class DoclingStudioApplication { - public static void main(String[] args) { - SpringApplication.run(DoclingStudioApplication.class, args); - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisController.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisController.java deleted file mode 100644 index 7769273..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisController.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.analysis.dto.AnalysisResponse; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.util.List; -import java.util.Map; -import java.util.UUID; - -@RestController -@RequestMapping("/api/analyses") -public class AnalysisController { - - private final AnalysisService service; - - public AnalysisController(AnalysisService service) { - this.service = service; - } - - @PostMapping - public AnalysisResponse create(@RequestBody Map body) { - String raw = body.get("documentId"); - if (raw == null || raw.isBlank()) { - throw new IllegalArgumentException("documentId is required"); - } - UUID documentId = UUID.fromString(raw); - AnalysisJob job = service.create(documentId); - return AnalysisResponse.from(job); - } - - @GetMapping - public List list() { - return service.findAll().stream().map(AnalysisResponse::from).toList(); - } - - @GetMapping("/{id}") - public AnalysisResponse get(@PathVariable UUID id) { - return AnalysisResponse.from(service.findById(id)); - } - - @DeleteMapping("/{id}") - public ResponseEntity delete(@PathVariable UUID id) { - service.delete(id); - return ResponseEntity.noContent().build(); - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisJob.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisJob.java deleted file mode 100644 index 070e893..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisJob.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.document.Document; -import jakarta.persistence.*; -import java.time.Instant; -import java.util.UUID; - -@Entity -@Table(name = "analysis_jobs") -public class AnalysisJob { - - public enum Status { PENDING, RUNNING, COMPLETED, FAILED } - - @Id - private UUID id; - - @ManyToOne(fetch = FetchType.EAGER) - @JoinColumn(name = "document_id", nullable = false) - private Document document; - - @Enumerated(EnumType.STRING) - @Column(nullable = false) - private Status status; - - @Column(columnDefinition = "text") - private String contentMarkdown; - - @Column(columnDefinition = "text") - private String contentHtml; - - @Column(columnDefinition = "text") - private String pagesJson; - - private String errorMessage; - - private Instant startedAt; - private Instant completedAt; - private Instant createdAt; - - protected AnalysisJob() {} - - public AnalysisJob(Document document) { - this.id = UUID.randomUUID(); - this.document = document; - this.status = Status.PENDING; - this.createdAt = Instant.now(); - } - - public void markRunning() { - this.status = Status.RUNNING; - this.startedAt = Instant.now(); - } - - public void markCompleted(String markdown, String html, String pagesJson) { - this.status = Status.COMPLETED; - this.contentMarkdown = markdown; - this.contentHtml = html; - this.pagesJson = pagesJson; - this.completedAt = Instant.now(); - } - - public void markFailed(String error) { - this.status = Status.FAILED; - this.errorMessage = error; - this.completedAt = Instant.now(); - } - - public UUID getId() { return id; } - public Document getDocument() { return document; } - public Status getStatus() { return status; } - public String getContentMarkdown() { return contentMarkdown; } - public String getContentHtml() { return contentHtml; } - public String getPagesJson() { return pagesJson; } - public String getErrorMessage() { return errorMessage; } - public Instant getStartedAt() { return startedAt; } - public Instant getCompletedAt() { return completedAt; } - public Instant getCreatedAt() { return createdAt; } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisJobRepository.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisJobRepository.java deleted file mode 100644 index 044bc3b..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisJobRepository.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.docling.studio.analysis; - -import org.springframework.data.jpa.repository.JpaRepository; -import java.util.List; -import java.util.UUID; - -public interface AnalysisJobRepository extends JpaRepository { - List findAllByOrderByCreatedAtDesc(); - List findByDocumentIdOrderByCreatedAtDesc(UUID documentId); -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisRunner.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisRunner.java deleted file mode 100644 index 4861eef..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisRunner.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.document.Document; -import com.docling.studio.document.DocumentParserClient; -import com.docling.studio.document.DocumentService; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -import java.nio.file.Path; -import java.util.Map; -import java.util.UUID; - -/** - * Separated from AnalysisService so that Spring's @Async proxy works correctly. - * (Self-invocation within the same class bypasses the AOP proxy.) - */ -@Component -public class AnalysisRunner { - - private static final Logger log = LoggerFactory.getLogger(AnalysisRunner.class); - - private final AnalysisJobRepository repository; - private final DocumentService documentService; - private final DocumentParserClient parserClient; - private final ObjectMapper objectMapper; - - public AnalysisRunner( - AnalysisJobRepository repository, - DocumentService documentService, - DocumentParserClient parserClient, - ObjectMapper objectMapper - ) { - this.repository = repository; - this.documentService = documentService; - this.parserClient = parserClient; - this.objectMapper = objectMapper; - } - - @Async("analysisExecutor") - public void runAnalysis(UUID jobId) { - AnalysisJob job = repository.findById(jobId).orElseThrow(); - job.markRunning(); - repository.save(job); - - log.info("Starting analysis for document: {}", job.getDocument().getFilename()); - - try { - Path filePath = documentService.getFilePath(job.getDocument().getId()); - Map result = parserClient.parse(filePath, job.getDocument().getFilename()); - - String markdown = (String) result.getOrDefault("content_markdown", ""); - String html = (String) result.getOrDefault("content_html", ""); - Object pages = result.get("pages"); - String pagesJson = pages != null ? objectMapper.writeValueAsString(pages) : "[]"; - - // Update page count on document if available - Object pageCount = result.get("page_count"); - if (pageCount instanceof Number n && n.intValue() > 0) { - Document doc = job.getDocument(); - doc.setPageCount(n.intValue()); - documentService.save(doc); - } - - job.markCompleted(markdown, html, pagesJson); - repository.save(job); - - log.info("Analysis completed for document: {}", job.getDocument().getFilename()); - - } catch (Exception e) { - log.error("Analysis failed for document: {}", job.getDocument().getFilename(), e); - job.markFailed(e.getMessage()); - repository.save(job); - } - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/AnalysisService.java b/backend/src/main/java/com/docling/studio/analysis/AnalysisService.java deleted file mode 100644 index 7001def..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/AnalysisService.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.docling.studio.analysis; - -import com.docling.studio.document.Document; -import com.docling.studio.document.DocumentService; -import com.docling.studio.shared.exception.ResourceNotFoundException; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.List; -import java.util.UUID; - -@Service -public class AnalysisService { - - private final AnalysisJobRepository repository; - private final DocumentService documentService; - private final AnalysisRunner runner; - - public AnalysisService( - AnalysisJobRepository repository, - DocumentService documentService, - AnalysisRunner runner - ) { - this.repository = repository; - this.documentService = documentService; - this.runner = runner; - } - - @Transactional - public AnalysisJob create(UUID documentId) { - Document doc = documentService.findById(documentId); - AnalysisJob job = new AnalysisJob(doc); - repository.save(job); - // Delegate to a separate bean so @Async proxy works (no self-invocation) - runner.runAnalysis(job.getId()); - return job; - } - - @Transactional(readOnly = true) - public AnalysisJob findById(UUID id) { - return repository.findById(id) - .orElseThrow(() -> new ResourceNotFoundException("Analysis not found: " + id)); - } - - @Transactional(readOnly = true) - public List findAll() { - return repository.findAllByOrderByCreatedAtDesc(); - } - - @Transactional - public void delete(UUID id) { - AnalysisJob job = findById(id); - repository.delete(job); - } -} diff --git a/backend/src/main/java/com/docling/studio/analysis/dto/AnalysisResponse.java b/backend/src/main/java/com/docling/studio/analysis/dto/AnalysisResponse.java deleted file mode 100644 index 0e4f6a0..0000000 --- a/backend/src/main/java/com/docling/studio/analysis/dto/AnalysisResponse.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.docling.studio.analysis.dto; - -import com.docling.studio.analysis.AnalysisJob; -import java.time.Instant; -import java.util.UUID; - -public record AnalysisResponse( - UUID id, - UUID documentId, - String documentFilename, - String status, - String contentMarkdown, - String contentHtml, - String pagesJson, - String errorMessage, - Instant startedAt, - Instant completedAt, - Instant createdAt -) { - public static AnalysisResponse from(AnalysisJob job) { - return new AnalysisResponse( - job.getId(), - job.getDocument().getId(), - job.getDocument().getFilename(), - job.getStatus().name(), - job.getContentMarkdown(), - job.getContentHtml(), - job.getPagesJson(), - job.getErrorMessage(), - job.getStartedAt(), - job.getCompletedAt(), - job.getCreatedAt() - ); - } -} diff --git a/backend/src/main/java/com/docling/studio/config/AsyncConfig.java b/backend/src/main/java/com/docling/studio/config/AsyncConfig.java deleted file mode 100644 index 828475b..0000000 --- a/backend/src/main/java/com/docling/studio/config/AsyncConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.docling.studio.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; - -@Configuration -@EnableAsync -public class AsyncConfig { - - @Bean(name = "analysisExecutor") - public Executor analysisExecutor() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(2); - executor.setMaxPoolSize(4); - executor.setQueueCapacity(20); - executor.setThreadNamePrefix("analysis-"); - executor.initialize(); - return executor; - } -} diff --git a/backend/src/main/java/com/docling/studio/config/DocumentParserProperties.java b/backend/src/main/java/com/docling/studio/config/DocumentParserProperties.java deleted file mode 100644 index 6b6723f..0000000 --- a/backend/src/main/java/com/docling/studio/config/DocumentParserProperties.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.docling.studio.config; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Component -@ConfigurationProperties(prefix = "app.document-parser") -public class DocumentParserProperties { - - private String baseUrl = "http://localhost:8000"; - - public String getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(String baseUrl) { - this.baseUrl = baseUrl; - } -} diff --git a/backend/src/main/java/com/docling/studio/config/WebConfig.java b/backend/src/main/java/com/docling/studio/config/WebConfig.java deleted file mode 100644 index 0d75d39..0000000 --- a/backend/src/main/java/com/docling/studio/config/WebConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.docling.studio.config; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.CorsRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class WebConfig implements WebMvcConfigurer { - - @Value("${app.cors.allowed-origins:http://localhost:3000,http://localhost:5173}") - private String allowedOrigins; - - @Override - public void addCorsMappings(CorsRegistry registry) { - registry.addMapping("/api/**") - .allowedOrigins(allowedOrigins.split(",")) - .allowedMethods("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS") - .allowedHeaders("*") - .allowCredentials(true); - } -} diff --git a/backend/src/main/java/com/docling/studio/document/Document.java b/backend/src/main/java/com/docling/studio/document/Document.java deleted file mode 100644 index acfb293..0000000 --- a/backend/src/main/java/com/docling/studio/document/Document.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.docling.studio.document; - -import jakarta.persistence.*; -import java.time.Instant; -import java.util.UUID; - -@Entity -@Table(name = "documents") -public class Document { - - @Id - private UUID id; - - @Column(nullable = false) - private String filename; - - private String contentType; - - private Long fileSize; - - private Integer pageCount; - - @Column(nullable = false) - private String storagePath; - - private Instant createdAt; - - protected Document() {} - - public Document(String filename, String contentType, Long fileSize, String storagePath) { - this.id = UUID.randomUUID(); - this.filename = filename; - this.contentType = contentType; - this.fileSize = fileSize; - this.storagePath = storagePath; - this.createdAt = Instant.now(); - } - - public UUID getId() { return id; } - public String getFilename() { return filename; } - public String getContentType() { return contentType; } - public Long getFileSize() { return fileSize; } - public Integer getPageCount() { return pageCount; } - public void setPageCount(Integer pageCount) { this.pageCount = pageCount; } - public String getStoragePath() { return storagePath; } - public Instant getCreatedAt() { return createdAt; } -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentController.java b/backend/src/main/java/com/docling/studio/document/DocumentController.java deleted file mode 100644 index 32fc4a4..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentController.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.docling.studio.document; - -import com.docling.studio.document.dto.DocumentResponse; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; - -import java.util.List; -import java.util.UUID; - -@RestController -@RequestMapping("/api/documents") -public class DocumentController { - - private final DocumentService service; - - public DocumentController(DocumentService service) { - this.service = service; - } - - @PostMapping("/upload") - public DocumentResponse upload(@RequestParam("file") MultipartFile file) { - Document doc = service.upload(file); - return DocumentResponse.from(doc); - } - - @GetMapping - public List list() { - return service.findAll().stream().map(DocumentResponse::from).toList(); - } - - @GetMapping("/{id}") - public DocumentResponse get(@PathVariable UUID id) { - return DocumentResponse.from(service.findById(id)); - } - - @DeleteMapping("/{id}") - public ResponseEntity delete(@PathVariable UUID id) { - service.delete(id); - return ResponseEntity.noContent().build(); - } - - @GetMapping(value = "/{id}/preview") - public ResponseEntity preview( - @PathVariable UUID id, - @RequestParam(defaultValue = "1") int page, - @RequestParam(defaultValue = "150") int dpi - ) { - byte[] bytes = service.getPreview(id, page, dpi); - if (bytes == null) { - return ResponseEntity.notFound().build(); - } - return ResponseEntity.ok().contentType(MediaType.IMAGE_PNG).body(bytes); - } -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentParserClient.java b/backend/src/main/java/com/docling/studio/document/DocumentParserClient.java deleted file mode 100644 index 1db79c4..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentParserClient.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.docling.studio.document; - -import com.docling.studio.config.DocumentParserProperties; -import com.docling.studio.shared.exception.ServiceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.FileSystemResource; -import org.springframework.http.MediaType; -import org.springframework.http.client.MultipartBodyBuilder; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.BodyInserters; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientResponseException; - -import java.nio.file.Path; -import java.time.Duration; -import java.util.Map; - -@Component -public class DocumentParserClient { - - private static final Logger log = LoggerFactory.getLogger(DocumentParserClient.class); - - private final WebClient webClient; - - public DocumentParserClient(DocumentParserProperties props) { - this.webClient = WebClient.builder() - .baseUrl(props.getBaseUrl()) - .codecs(config -> config.defaultCodecs().maxInMemorySize(50 * 1024 * 1024)) - .build(); - } - - @SuppressWarnings("unchecked") - public Map parse(Path filePath, String filename) { - MultipartBodyBuilder builder = new MultipartBodyBuilder(); - builder.part("file", new FileSystemResource(filePath)) - .filename(filename) - .contentType(MediaType.APPLICATION_PDF); - - try { - return webClient.post() - .uri("/parse") - .body(BodyInserters.fromMultipartData(builder.build())) - .retrieve() - .bodyToMono(Map.class) - .block(Duration.ofMinutes(10)); - } catch (WebClientResponseException e) { - log.error("Parser returned HTTP {}: {}", e.getStatusCode().value(), e.getResponseBodyAsString()); - throw new ServiceException("Document parser error: " + extractDetail(e)); - } catch (java.lang.IllegalStateException e) { - if (e.getMessage() != null && e.getMessage().contains("Timeout")) { - log.error("Parser timed out after 10 minutes for file: {}", filename); - throw new ServiceException("Document parsing timed out. The document may be too large or complex."); - } - throw new ServiceException("Document parser error: " + e.getMessage()); - } catch (Exception e) { - log.error("Failed to reach document parser", e); - throw new ServiceException("Document parser unavailable: " + e.getMessage()); - } - } - - public byte[] preview(Path filePath, String filename, int page, int dpi) { - MultipartBodyBuilder builder = new MultipartBodyBuilder(); - builder.part("file", new FileSystemResource(filePath)) - .filename(filename) - .contentType(MediaType.APPLICATION_PDF); - - try { - return webClient.post() - .uri(uri -> uri.path("/preview") - .queryParam("page", page) - .queryParam("dpi", dpi) - .build()) - .body(BodyInserters.fromMultipartData(builder.build())) - .retrieve() - .bodyToMono(byte[].class) - .block(Duration.ofSeconds(30)); - } catch (Exception e) { - log.warn("Preview generation failed for page {}: {}", page, e.getMessage()); - return null; - } - } - - /** Extract the "detail" field from a FastAPI error response, or fall back to the raw body. */ - private String extractDetail(WebClientResponseException e) { - String body = e.getResponseBodyAsString(); - try { - var tree = new com.fasterxml.jackson.databind.ObjectMapper().readTree(body); - if (tree.has("detail")) { - return tree.get("detail").asText(); - } - } catch (Exception ignored) { - // Not valid JSON — fall through to raw body - } - return body.length() > 200 ? body.substring(0, 200) : body; - } -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentRepository.java b/backend/src/main/java/com/docling/studio/document/DocumentRepository.java deleted file mode 100644 index 0926527..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentRepository.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.docling.studio.document; - -import org.springframework.data.jpa.repository.JpaRepository; -import java.util.UUID; - -public interface DocumentRepository extends JpaRepository { -} diff --git a/backend/src/main/java/com/docling/studio/document/DocumentService.java b/backend/src/main/java/com/docling/studio/document/DocumentService.java deleted file mode 100644 index 400dfa1..0000000 --- a/backend/src/main/java/com/docling/studio/document/DocumentService.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.docling.studio.document; - -import com.docling.studio.shared.exception.ResourceNotFoundException; -import com.docling.studio.shared.exception.ServiceException; -import org.apache.pdfbox.Loader; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.multipart.MultipartFile; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.UUID; - -@Service -public class DocumentService { - - private static final Logger log = LoggerFactory.getLogger(DocumentService.class); - - private final DocumentRepository repository; - private final DocumentParserClient parserClient; - private final Path storagePath; - - public DocumentService( - DocumentRepository repository, - DocumentParserClient parserClient, - @Value("${app.storage.path:./uploads}") String storagePath - ) { - this.repository = repository; - this.parserClient = parserClient; - this.storagePath = Path.of(storagePath); - } - - @Transactional - public Document upload(MultipartFile file) { - if (file.isEmpty() || file.getOriginalFilename() == null) { - throw new IllegalArgumentException("File is empty or has no name"); - } - - try { - Files.createDirectories(storagePath); - String safeName = UUID.randomUUID() + "_" + file.getOriginalFilename(); - Path target = storagePath.resolve(safeName); - file.transferTo(target); - - Document doc = new Document( - file.getOriginalFilename(), - file.getContentType(), - file.getSize(), - target.toString() - ); - doc.setPageCount(countPages(target)); - return repository.save(doc); - - } catch (IOException e) { - throw new ServiceException("Failed to store file", e); - } - } - - public Document save(Document document) { - return repository.save(document); - } - - public List findAll() { - return repository.findAll(); - } - - public Document findById(UUID id) { - return repository.findById(id) - .orElseThrow(() -> new ResourceNotFoundException("Document not found: " + id)); - } - - @Transactional - public void delete(UUID id) { - Document doc = findById(id); - try { - Files.deleteIfExists(Path.of(doc.getStoragePath())); - } catch (IOException e) { - // Log but continue - } - repository.delete(doc); - } - - public byte[] getPreview(UUID id, int page, int dpi) { - Document doc = findById(id); - return parserClient.preview(Path.of(doc.getStoragePath()), doc.getFilename(), page, dpi); - } - - public Path getFilePath(UUID id) { - Document doc = findById(id); - return Path.of(doc.getStoragePath()); - } - - private int countPages(Path pdfPath) { - try (PDDocument pdf = Loader.loadPDF(pdfPath.toFile())) { - return pdf.getNumberOfPages(); - } catch (Exception e) { - log.warn("Could not count pages for {}: {}", pdfPath.getFileName(), e.getMessage()); - return 0; - } - } -} diff --git a/backend/src/main/java/com/docling/studio/document/dto/DocumentResponse.java b/backend/src/main/java/com/docling/studio/document/dto/DocumentResponse.java deleted file mode 100644 index 6b0ec63..0000000 --- a/backend/src/main/java/com/docling/studio/document/dto/DocumentResponse.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.docling.studio.document.dto; - -import com.docling.studio.document.Document; -import java.time.Instant; -import java.util.UUID; - -public record DocumentResponse( - UUID id, - String filename, - String contentType, - Long fileSize, - Integer pageCount, - Instant createdAt -) { - public static DocumentResponse from(Document doc) { - return new DocumentResponse( - doc.getId(), doc.getFilename(), doc.getContentType(), - doc.getFileSize(), doc.getPageCount(), doc.getCreatedAt() - ); - } -} diff --git a/backend/src/main/java/com/docling/studio/shared/exception/GlobalExceptionHandler.java b/backend/src/main/java/com/docling/studio/shared/exception/GlobalExceptionHandler.java deleted file mode 100644 index d598726..0000000 --- a/backend/src/main/java/com/docling/studio/shared/exception/GlobalExceptionHandler.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.docling.studio.shared.exception; - -import org.springframework.http.HttpStatus; -import org.springframework.http.ProblemDetail; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -@RestControllerAdvice -public class GlobalExceptionHandler { - - @ExceptionHandler(ResourceNotFoundException.class) - public ProblemDetail handleNotFound(ResourceNotFoundException ex) { - return ProblemDetail.forStatusAndDetail(HttpStatus.NOT_FOUND, ex.getMessage()); - } - - @ExceptionHandler(ServiceException.class) - public ProblemDetail handleService(ServiceException ex) { - return ProblemDetail.forStatusAndDetail(HttpStatus.INTERNAL_SERVER_ERROR, ex.getMessage()); - } - - @ExceptionHandler(IllegalArgumentException.class) - public ProblemDetail handleBadRequest(IllegalArgumentException ex) { - return ProblemDetail.forStatusAndDetail(HttpStatus.BAD_REQUEST, ex.getMessage()); - } -} diff --git a/backend/src/main/java/com/docling/studio/shared/exception/ResourceNotFoundException.java b/backend/src/main/java/com/docling/studio/shared/exception/ResourceNotFoundException.java deleted file mode 100644 index 013d442..0000000 --- a/backend/src/main/java/com/docling/studio/shared/exception/ResourceNotFoundException.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.docling.studio.shared.exception; - -public class ResourceNotFoundException extends RuntimeException { - public ResourceNotFoundException(String message) { - super(message); - } -} diff --git a/backend/src/main/java/com/docling/studio/shared/exception/ServiceException.java b/backend/src/main/java/com/docling/studio/shared/exception/ServiceException.java deleted file mode 100644 index 44f326a..0000000 --- a/backend/src/main/java/com/docling/studio/shared/exception/ServiceException.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.docling.studio.shared.exception; - -public class ServiceException extends RuntimeException { - public ServiceException(String message) { - super(message); - } - - public ServiceException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml deleted file mode 100644 index e1e49dc..0000000 --- a/backend/src/main/resources/application.yml +++ /dev/null @@ -1,26 +0,0 @@ -server: - port: 8081 - -spring: - datasource: - url: jdbc:postgresql://${DB_HOST:localhost}:5432/${DB_NAME:docling_studio} - username: ${SPRING_DATASOURCE_USERNAME:app} - password: ${SPRING_DATASOURCE_PASSWORD:app} - jpa: - hibernate: - ddl-auto: validate - open-in-view: false - liquibase: - change-log: classpath:db/changelog/db.changelog-master.yml - servlet: - multipart: - max-file-size: 50MB - max-request-size: 50MB - -app: - document-parser: - base-url: ${APP_DOCUMENT-PARSER_BASE-URL:http://localhost:8000} - storage: - path: ${APP_STORAGE_PATH:./uploads} - cors: - allowed-origins: ${APP_CORS_ALLOWED_ORIGINS:http://localhost:3000,http://localhost:5173} diff --git a/backend/src/main/resources/db/changelog/001-create-documents.yml b/backend/src/main/resources/db/changelog/001-create-documents.yml deleted file mode 100644 index 3f0f317..0000000 --- a/backend/src/main/resources/db/changelog/001-create-documents.yml +++ /dev/null @@ -1,36 +0,0 @@ -databaseChangeLog: - - changeSet: - id: 001-create-documents - author: docling-studio - changes: - - createTable: - tableName: documents - columns: - - column: - name: id - type: uuid - constraints: - primaryKey: true - - column: - name: filename - type: varchar(255) - constraints: - nullable: false - - column: - name: content_type - type: varchar(100) - - column: - name: file_size - type: bigint - - column: - name: page_count - type: int - - column: - name: storage_path - type: varchar(500) - constraints: - nullable: false - - column: - name: created_at - type: timestamp - defaultValueComputed: NOW() diff --git a/backend/src/main/resources/db/changelog/002-create-analysis-jobs.yml b/backend/src/main/resources/db/changelog/002-create-analysis-jobs.yml deleted file mode 100644 index d695702..0000000 --- a/backend/src/main/resources/db/changelog/002-create-analysis-jobs.yml +++ /dev/null @@ -1,47 +0,0 @@ -databaseChangeLog: - - changeSet: - id: 002-create-analysis-jobs - author: docling-studio - changes: - - createTable: - tableName: analysis_jobs - columns: - - column: - name: id - type: uuid - constraints: - primaryKey: true - - column: - name: document_id - type: uuid - constraints: - nullable: false - foreignKeyName: fk_analysis_document - references: documents(id) - - column: - name: status - type: varchar(20) - constraints: - nullable: false - - column: - name: content_markdown - type: text - - column: - name: content_html - type: text - - column: - name: pages_json - type: text - - column: - name: error_message - type: text - - column: - name: started_at - type: timestamp - - column: - name: completed_at - type: timestamp - - column: - name: created_at - type: timestamp - defaultValueComputed: NOW() diff --git a/backend/src/main/resources/db/changelog/db.changelog-master.yml b/backend/src/main/resources/db/changelog/db.changelog-master.yml deleted file mode 100644 index 70cba0f..0000000 --- a/backend/src/main/resources/db/changelog/db.changelog-master.yml +++ /dev/null @@ -1,5 +0,0 @@ -databaseChangeLog: - - include: - file: db/changelog/001-create-documents.yml - - include: - file: db/changelog/002-create-analysis-jobs.yml diff --git a/docker-compose.yml b/docker-compose.yml index 613a90c..86819f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,54 +1,27 @@ services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: ${POSTGRES_USER:-app} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-app} - POSTGRES_DB: ${POSTGRES_DB:-docling_studio} - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-app} -d ${POSTGRES_DB:-docling_studio}"] - interval: 5s - timeout: 5s - retries: 10 - document-parser: build: context: ./document-parser ports: - "8000:8000" + volumes: + - uploads_data:/app/uploads + - db_data:/app/data + environment: + CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173} deploy: resources: limits: memory: 4g - backend: - build: - context: ./backend - ports: - - "8081:8081" - environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-docling_studio} - SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER:-app} - SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:-app} - APP_DOCUMENT-PARSER_BASE-URL: http://document-parser:8000 - volumes: - - uploads_data:/app/uploads - depends_on: - postgres: - condition: service_healthy - frontend: build: context: ./frontend ports: - "3000:80" depends_on: - - backend + - document-parser volumes: - postgres_data: uploads_data: + db_data: diff --git a/document-parser/Dockerfile b/document-parser/Dockerfile index a58d6be..6ab8e94 100644 --- a/document-parser/Dockerfile +++ b/document-parser/Dockerfile @@ -13,6 +13,11 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . +RUN mkdir -p /app/uploads /app/data + EXPOSE 8000 +ENV UPLOAD_DIR=/app/uploads +ENV DB_PATH=/app/data/docling_studio.db + CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/document-parser/api/__init__.py b/document-parser/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/api/analyses.py b/document-parser/api/analyses.py new file mode 100644 index 0000000..72c4712 --- /dev/null +++ b/document-parser/api/analyses.py @@ -0,0 +1,67 @@ +"""Analysis API router — create, list, get, delete analysis jobs.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, HTTPException + +from api.schemas import AnalysisResponse, CreateAnalysisRequest +from services import analysis_service + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/analyses", tags=["analyses"]) + + +def _to_response(job) -> AnalysisResponse: + return AnalysisResponse( + id=job.id, + document_id=job.document_id, + document_filename=job.document_filename, + status=job.status.value, + content_markdown=job.content_markdown, + content_html=job.content_html, + pages_json=job.pages_json, + error_message=job.error_message, + started_at=str(job.started_at) if job.started_at else None, + completed_at=str(job.completed_at) if job.completed_at else None, + created_at=str(job.created_at), + ) + + +@router.post("", response_model=AnalysisResponse) +async def create_analysis(body: CreateAnalysisRequest): + """Create a new analysis job for a document.""" + if not body.documentId or not body.documentId.strip(): + raise HTTPException(status_code=400, detail="documentId is required") + + try: + job = await analysis_service.create(body.documentId) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + + return _to_response(job) + + +@router.get("", response_model=list[AnalysisResponse]) +async def list_analyses(): + """List all analysis jobs.""" + jobs = await analysis_service.find_all() + return [_to_response(j) for j in jobs] + + +@router.get("/{job_id}", response_model=AnalysisResponse) +async def get_analysis(job_id: str): + """Get a single analysis job.""" + job = await analysis_service.find_by_id(job_id) + if not job: + raise HTTPException(status_code=404, detail="Analysis not found") + return _to_response(job) + + +@router.delete("/{job_id}", status_code=204) +async def delete_analysis(job_id: str): + """Delete an analysis job.""" + deleted = await analysis_service.delete(job_id) + if not deleted: + raise HTTPException(status_code=404, detail="Analysis not found") diff --git a/document-parser/api/documents.py b/document-parser/api/documents.py new file mode 100644 index 0000000..2e94c0b --- /dev/null +++ b/document-parser/api/documents.py @@ -0,0 +1,92 @@ +"""Document API router — upload, list, get, delete, preview.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, HTTPException, Query, UploadFile +from fastapi.responses import Response + +from api.schemas import DocumentResponse +from services import document_service + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/documents", tags=["documents"]) + + +def _to_response(doc) -> DocumentResponse: + return DocumentResponse( + id=doc.id, + filename=doc.filename, + content_type=doc.content_type, + file_size=doc.file_size, + page_count=doc.page_count, + created_at=str(doc.created_at), + ) + + +@router.post("/upload", response_model=DocumentResponse) +async def upload(file: UploadFile): + """Upload a PDF document.""" + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + + content = await file.read() + + try: + doc = await document_service.upload( + filename=file.filename, + content_type=file.content_type or "application/pdf", + file_content=content, + ) + except ValueError as e: + raise HTTPException(status_code=413, detail=str(e)) + + return _to_response(doc) + + +@router.get("", response_model=list[DocumentResponse]) +async def list_documents(): + """List all documents.""" + docs = await document_service.find_all() + return [_to_response(d) for d in docs] + + +@router.get("/{doc_id}", response_model=DocumentResponse) +async def get_document(doc_id: str): + """Get a single document.""" + doc = await document_service.find_by_id(doc_id) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + return _to_response(doc) + + +@router.delete("/{doc_id}", status_code=204) +async def delete_document(doc_id: str): + """Delete a document and its file.""" + deleted = await document_service.delete(doc_id) + if not deleted: + raise HTTPException(status_code=404, detail="Document not found") + + +@router.get("/{doc_id}/preview") +async def preview( + doc_id: str, + page: int = Query(1, ge=1), + dpi: int = Query(150, ge=72, le=300), +): + """Generate a PNG preview of a specific PDF page.""" + doc = await document_service.find_by_id(doc_id) + if not doc: + raise HTTPException(status_code=404, detail="Document not found") + + try: + with open(doc.storage_path, "rb") as f: + file_content = f.read() + png_bytes = document_service.generate_preview(file_content, page=page, dpi=dpi) + return Response(content=png_bytes, media_type="image/png") + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + except Exception as e: + logger.exception("Failed to generate preview") + raise HTTPException(status_code=422, detail=f"Failed to generate preview: {e}") diff --git a/document-parser/api/schemas.py b/document-parser/api/schemas.py new file mode 100644 index 0000000..e9a981b --- /dev/null +++ b/document-parser/api/schemas.py @@ -0,0 +1,52 @@ +"""Pydantic schemas — API request/response DTOs. + +All responses use camelCase serialization to match the existing frontend contract +(originally served by the Spring Boot backend). +""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict + + +def _to_camel(name: str) -> str: + parts = name.split("_") + return parts[0] + "".join(w.capitalize() for w in parts[1:]) + + +class _CamelModel(BaseModel): + """Base model that serializes field names to camelCase.""" + model_config = ConfigDict( + alias_generator=_to_camel, + populate_by_name=True, + serialize_by_alias=True, + ) + + +class DocumentResponse(_CamelModel): + id: str + filename: str + content_type: str | None = None + file_size: int | None = None + page_count: int | None = None + created_at: str | datetime + + +class AnalysisResponse(_CamelModel): + id: str + document_id: str = "" + document_filename: str | None = None + status: str + content_markdown: str | None = None + content_html: str | None = None + pages_json: str | None = None + error_message: str | None = None + started_at: str | datetime | None = None + completed_at: str | datetime | None = None + created_at: str | datetime + + +class CreateAnalysisRequest(BaseModel): + documentId: str # camelCase to match existing frontend contract diff --git a/document-parser/data/docling_studio.db b/document-parser/data/docling_studio.db new file mode 100644 index 0000000..b3fd80f Binary files /dev/null and b/document-parser/data/docling_studio.db differ diff --git a/document-parser/domain/__init__.py b/document-parser/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/bbox.py b/document-parser/domain/bbox.py similarity index 93% rename from document-parser/bbox.py rename to document-parser/domain/bbox.py index b1498f2..c36c590 100644 --- a/document-parser/bbox.py +++ b/document-parser/domain/bbox.py @@ -9,7 +9,7 @@ bboxes are normalized to TOPLEFT [left, top, right, bottom] before being sent to the frontend. """ -from docling_core.types.doc.base import BoundingBox, CoordOrigin +from docling_core.types.doc.base import BoundingBox def to_topleft_list(bbox: BoundingBox, page_height: float) -> list[float]: diff --git a/document-parser/domain/models.py b/document-parser/domain/models.py new file mode 100644 index 0000000..a26dac0 --- /dev/null +++ b/document-parser/domain/models.py @@ -0,0 +1,69 @@ +"""Domain models — pure data structures with no framework dependencies.""" + +from __future__ import annotations + +import enum +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone + + +class AnalysisStatus(str, enum.Enum): + PENDING = "PENDING" + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _new_id() -> str: + return str(uuid.uuid4()) + + +@dataclass +class Document: + id: str = field(default_factory=_new_id) + filename: str = "" + content_type: str | None = None + file_size: int | None = None + page_count: int | None = None + storage_path: str = "" + created_at: datetime = field(default_factory=_utcnow) + + +@dataclass +class AnalysisJob: + id: str = field(default_factory=_new_id) + document_id: str = "" + status: AnalysisStatus = AnalysisStatus.PENDING + content_markdown: str | None = None + content_html: str | None = None + pages_json: str | None = None + error_message: str | None = None + started_at: datetime | None = None + completed_at: datetime | None = None + created_at: datetime = field(default_factory=_utcnow) + + # Joined from document (not persisted separately) + document_filename: str | None = None + + def mark_running(self) -> None: + self.status = AnalysisStatus.RUNNING + self.started_at = _utcnow() + + def mark_completed( + self, markdown: str, html: str, pages_json: str, + ) -> None: + self.status = AnalysisStatus.COMPLETED + self.content_markdown = markdown + self.content_html = html + self.pages_json = pages_json + self.completed_at = _utcnow() + + def mark_failed(self, error: str) -> None: + self.status = AnalysisStatus.FAILED + self.error_message = error + self.completed_at = _utcnow() diff --git a/document-parser/domain/parsing.py b/document-parser/domain/parsing.py new file mode 100644 index 0000000..e7c5faf --- /dev/null +++ b/document-parser/domain/parsing.py @@ -0,0 +1,275 @@ +"""Docling document extraction logic — pure domain, no HTTP concerns. + +Wraps the Docling library to convert documents and extract structured +per-page elements with bounding boxes and hierarchy levels. +""" + +from __future__ import annotations + +import logging +import threading +from dataclasses import dataclass, field + +from docling.document_converter import DocumentConverter, PdfFormatOption +from docling.datamodel.base_models import InputFormat +from docling.datamodel.pipeline_options import ( + PdfPipelineOptions, + TableFormerMode, + TableStructureOptions, +) +from docling_core.types.doc import ( + CodeItem, + DocItem, + FloatingItem, + FormulaItem, + GroupItem, + ListItem, + PictureItem, + SectionHeaderItem, + TableItem, + TextItem, + TitleItem, +) + +from domain.bbox import to_topleft_list + +logger = logging.getLogger(__name__) + +# Thread lock — DocumentConverter is not thread-safe +_converter_lock = threading.Lock() + +# Default converter (lazy-init on first request) +_default_converter: DocumentConverter | None = None + + +# --------------------------------------------------------------------------- +# Domain value objects +# --------------------------------------------------------------------------- + +@dataclass +class PageElement: + type: str + bbox: list[float] + content: str + level: int = 0 + + +@dataclass +class PageDetail: + page_number: int + width: float + height: float + elements: list[PageElement] = field(default_factory=list) + + +@dataclass +class ConversionResult: + page_count: int + content_markdown: str + content_html: str + pages: list[PageDetail] + skipped_items: int = 0 + + +# --------------------------------------------------------------------------- +# Element type detection +# --------------------------------------------------------------------------- + +# Mapping from Docling type to element type string. +# Order matters: most specific types before their parents. +_ELEMENT_TYPE_MAP: list[tuple[type, str]] = [ + (TableItem, "table"), + (PictureItem, "picture"), + (TitleItem, "title"), + (SectionHeaderItem, "section_header"), + (ListItem, "list"), + (FormulaItem, "formula"), + (CodeItem, "code"), + (FloatingItem, "floating"), + (TextItem, "text"), +] + + +def _get_element_type(item: DocItem) -> str: + """Determine element type via isinstance on Docling's type hierarchy.""" + for cls, type_name in _ELEMENT_TYPE_MAP: + if isinstance(item, cls): + return type_name + return "text" + + +# --------------------------------------------------------------------------- +# Pipeline factory +# --------------------------------------------------------------------------- + +def build_converter( + do_ocr: bool = True, + do_table_structure: bool = True, + table_mode: str = "accurate", +) -> DocumentConverter: + """Build a DocumentConverter with the given pipeline options. + + Only exposes options that work out of the box (no extra model downloads). + """ + table_options = TableStructureOptions( + do_cell_matching=True, + mode=TableFormerMode.ACCURATE if table_mode == "accurate" else TableFormerMode.FAST, + ) + + pipeline_options = PdfPipelineOptions( + do_ocr=do_ocr, + do_table_structure=do_table_structure, + table_structure_options=table_options, + do_code_enrichment=False, + do_formula_enrichment=False, + do_picture_classification=False, + do_picture_description=False, + generate_page_images=False, + generate_picture_images=False, + ) + + return DocumentConverter( + format_options={ + InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options), + } + ) + + +def get_default_converter() -> DocumentConverter: + global _default_converter + if _default_converter is None: + _default_converter = build_converter() + return _default_converter + + +# --------------------------------------------------------------------------- +# Page extraction +# --------------------------------------------------------------------------- + +def extract_pages_detail(doc_result) -> tuple[list[PageDetail], int]: + """Extract per-page element details with bounding boxes from Docling result. + + Returns (pages, skipped_count) for transparent error reporting. + """ + pages: dict[int, PageDetail] = {} + document = doc_result.document + skipped = 0 + + for page_key, page_obj in document.pages.items(): + page_no = int(page_key) if isinstance(page_key, str) else page_key + pages[page_no] = PageDetail( + page_number=page_no, + width=page_obj.size.width, + height=page_obj.size.height, + ) + + for item, level in document.iterate_items(): + ok = _process_content_item(item, level, pages) + if not ok: + skipped += 1 + + sorted_pages = sorted(pages.values(), key=lambda p: p.page_number) + return sorted_pages, skipped + + +def _process_content_item( + item: DocItem | GroupItem, level: int, pages: dict[int, PageDetail], +) -> bool: + """Process a single content item and add it to the appropriate page.""" + if isinstance(item, GroupItem): + return True + + if not isinstance(item, DocItem) or not item.prov: + return False + + for prov in item.prov: + try: + page_no = prov.page_no + if page_no not in pages: + pages[page_no] = PageDetail(page_number=page_no, width=612.0, height=792.0) + + page_height = pages[page_no].height + + bbox = [0.0, 0.0, 0.0, 0.0] + if prov.bbox: + bbox = to_topleft_list(prov.bbox, page_height) + + element_type = _get_element_type(item) + + content = getattr(item, "text", "") or "" + if isinstance(item, TableItem): + try: + content = item.export_to_markdown() + except Exception: + pass + + pages[page_no].elements.append( + PageElement(type=element_type, bbox=bbox, content=content, level=level) + ) + except Exception: + logger.warning( + "Skipping item %s on page %s", + type(item).__name__, + getattr(prov, "page_no", "?"), + exc_info=True, + ) + return False + + return True + + +# --------------------------------------------------------------------------- +# Main conversion entry point +# --------------------------------------------------------------------------- + +def convert_document( + file_path: str, + *, + do_ocr: bool = True, + do_table_structure: bool = True, + table_mode: str = "accurate", +) -> ConversionResult: + """Convert a document and return structured results. + + This is the main entry point for document parsing. Runs synchronously + (caller should use asyncio.to_thread for non-blocking execution). + """ + if do_ocr and do_table_structure and table_mode == "accurate": + conv = get_default_converter() + else: + conv = build_converter( + do_ocr=do_ocr, + do_table_structure=do_table_structure, + table_mode=table_mode, + ) + + with _converter_lock: + result = conv.convert(file_path) + + doc = result.document + content_markdown = doc.export_to_markdown() + content_html = doc.export_to_html() + page_count = len(doc.pages) + + pages_detail, skipped = extract_pages_detail(result) + + if not pages_detail and page_count > 0: + pages_detail = [ + PageDetail( + page_number=i + 1, + width=doc.pages[i + 1].size.width if (i + 1) in doc.pages else 612.0, + height=doc.pages[i + 1].size.height if (i + 1) in doc.pages else 792.0, + ) + for i in range(page_count) + ] + + if skipped > 0: + logger.info("Parsed: %d pages, %d items skipped", page_count, skipped) + + return ConversionResult( + page_count=page_count or len(pages_detail) or 1, + content_markdown=content_markdown, + content_html=content_html, + pages=pages_detail, + skipped_items=skipped, + ) diff --git a/document-parser/main.py b/document-parser/main.py index 35f91e8..f1b4038 100644 --- a/document-parser/main.py +++ b/document-parser/main.py @@ -1,379 +1,60 @@ -"""Docling Studio — Document Parser service. +"""Docling Studio — unified FastAPI backend. -A FastAPI microservice wrapping the Docling library for structured document -extraction. Provides parse (full extraction) and preview (page image) endpoints -with configurable pipeline options. +Single service replacing both the Spring Boot backend and the document parser. +Provides document management (upload, CRUD), analysis orchestration (async Docling +processing), and PDF preview — all backed by SQLite. """ -import io +from __future__ import annotations + import logging import os -import tempfile -import threading -from pathlib import Path -from typing import Optional +from contextlib import asynccontextmanager -from fastapi import FastAPI, UploadFile, HTTPException, Query -from fastapi.responses import StreamingResponse -from pydantic import BaseModel -from pdf2image import convert_from_bytes +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware -from docling.document_converter import DocumentConverter, PdfFormatOption -from docling.datamodel.base_models import InputFormat -from docling.datamodel.pipeline_options import ( - PdfPipelineOptions, - TableFormerMode, - TableStructureOptions, +from persistence.database import init_db +from api.documents import router as documents_router +from api.analyses import router as analyses_router + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s — %(message)s", ) -from docling_core.types.doc import ( - CodeItem, - FloatingItem, - FormulaItem, - GroupItem, - ListItem, - PictureItem, - SectionHeaderItem, - TableItem, - TextItem, - TitleItem, -) - -from bbox import to_topleft_list - logger = logging.getLogger(__name__) -app = FastAPI(title="Docling Studio — Document Parser") -MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB - -# Thread lock for converter — DocumentConverter is not thread-safe -_converter_lock = threading.Lock() +@asynccontextmanager +async def lifespan(app: FastAPI): + """Startup: initialize database. Shutdown: nothing special needed.""" + await init_db() + logger.info("Docling Studio backend ready") + yield -# --------------------------------------------------------------------------- -# Pipeline factory -# --------------------------------------------------------------------------- +app = FastAPI( + title="Docling Studio", + description="Document analysis studio powered by Docling", + lifespan=lifespan, +) -def _build_converter( - do_ocr: bool = True, - do_table_structure: bool = True, - table_mode: str = "accurate", -) -> DocumentConverter: - """Build a DocumentConverter with the given pipeline options. +# CORS — configurable via env, defaults for local dev +allowed_origins = os.environ.get( + "CORS_ORIGINS", "http://localhost:3000,http://localhost:5173" +).split(",") - Only exposes options that work out of the box (no extra model downloads). - """ - table_options = TableStructureOptions( - do_cell_matching=True, - mode=TableFormerMode.ACCURATE if table_mode == "accurate" else TableFormerMode.FAST, - ) +app.add_middleware( + CORSMiddleware, + allow_origins=[o.strip() for o in allowed_origins], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) - pipeline_options = PdfPipelineOptions( - do_ocr=do_ocr, - do_table_structure=do_table_structure, - table_structure_options=table_options, - # These require VLM model downloads — disabled by default - do_code_enrichment=False, - do_formula_enrichment=False, - do_picture_classification=False, - do_picture_description=False, - # Page images are handled by pdf2image in /preview, not needed here - generate_page_images=False, - generate_picture_images=False, - ) - - return DocumentConverter( - format_options={ - InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options), - } - ) - - -# Default converter (lazy-init on first request) -_default_converter: Optional[DocumentConverter] = None - - -def _get_default_converter() -> DocumentConverter: - global _default_converter - if _default_converter is None: - _default_converter = _build_converter() - return _default_converter - - -# --------------------------------------------------------------------------- -# Response models -# --------------------------------------------------------------------------- - -class PageElement(BaseModel): - type: str - bbox: list[float] - content: str - level: int = 0 # Hierarchy depth from iterate_items() - - -class PageDetail(BaseModel): - page_number: int - width: float - height: float - elements: list[PageElement] - - -class ParseResponse(BaseModel): - filename: str - page_count: int - content_markdown: str - content_html: str - pages: list[PageDetail] - skipped_items: int = 0 # Transparency on extraction failures - - -# --------------------------------------------------------------------------- -# Element type detection — isinstance-based (no string matching) -# --------------------------------------------------------------------------- - -def _get_element_type(item) -> str: - """Determine the element type using isinstance checks on Docling classes. - - Order matters: more specific types are checked before their parent classes. - """ - if isinstance(item, TableItem): - return "table" - if isinstance(item, PictureItem): - return "picture" - if isinstance(item, TitleItem): - return "title" - if isinstance(item, SectionHeaderItem): - return "section_header" - if isinstance(item, ListItem): - return "list" - if isinstance(item, FormulaItem): - return "formula" - if isinstance(item, CodeItem): - return "code" - if isinstance(item, FloatingItem): - return "floating" - if isinstance(item, GroupItem): - return "group" - if isinstance(item, TextItem): - return "text" - return "text" - - -# --------------------------------------------------------------------------- -# Page extraction -# --------------------------------------------------------------------------- - -def extract_pages_detail(doc_result) -> tuple[list[PageDetail], int]: - """Extract per-page element details with bounding boxes from Docling result. - - Returns (pages, skipped_count) for transparent error reporting. - """ - pages: dict[int, PageDetail] = {} - document = doc_result.document - skipped = 0 - - # Populate page dimensions from document metadata - if hasattr(document, "pages") and document.pages: - for page_key, page_obj in document.pages.items(): - page_no = int(page_key) if isinstance(page_key, str) else page_key - width = page_obj.size.width if hasattr(page_obj, "size") and page_obj.size else 612.0 - height = page_obj.size.height if hasattr(page_obj, "size") and page_obj.size else 792.0 - pages[page_no] = PageDetail( - page_number=page_no, width=width, height=height, elements=[] - ) - - # Use iterate_items() — the Docling v2 API that yields (item, level) - if hasattr(document, "iterate_items"): - for item, level in document.iterate_items(): - ok = _process_content_item(item, level, pages) - if not ok: - skipped += 1 - elif hasattr(document, "texts"): - for text_item in document.texts: - ok = _process_content_item(text_item, 0, pages) - if not ok: - skipped += 1 - - sorted_pages = sorted(pages.values(), key=lambda p: p.page_number) - return sorted_pages, skipped - - -def _process_content_item(item, level: int, pages: dict[int, PageDetail]) -> bool: - """Process a single content item and add it to the appropriate page. - - Returns True on success, False if the item was skipped. - """ - # Skip groups and items without provenance - if isinstance(item, GroupItem): - return True # Groups are structural, not content — not an error - if not hasattr(item, "prov") or not item.prov: - return False - - for prov in item.prov: - try: - page_no = prov.page_no if hasattr(prov, "page_no") else 1 - - if page_no not in pages: - pages[page_no] = PageDetail( - page_number=page_no, width=612.0, height=792.0, elements=[] - ) - - page_height = pages[page_no].height - - bbox = [0.0, 0.0, 0.0, 0.0] - if hasattr(prov, "bbox") and prov.bbox: - b = prov.bbox - if hasattr(b, "l"): - bbox = to_topleft_list(b, page_height) - elif isinstance(b, (list, tuple)) and len(b) >= 4: - bbox = list(b[:4]) - - element_type = _get_element_type(item) - - content = "" - if hasattr(item, "text"): - content = item.text or "" - # For tables, try to export structured content - if isinstance(item, TableItem) and hasattr(item, "export_to_markdown"): - try: - content = item.export_to_markdown() - except Exception: - pass # Fall back to .text - - pages[page_no].elements.append( - PageElement( - type=element_type, - bbox=bbox, - content=content, - level=level, - ) - ) - except Exception: - logger.warning( - "Skipping item %s on page %s", - type(item).__name__, - getattr(prov, "page_no", "?"), - exc_info=True, - ) - return False - - return True - - -# --------------------------------------------------------------------------- -# Endpoints -# --------------------------------------------------------------------------- - -@app.post("/parse", response_model=ParseResponse) -async def parse( - file: UploadFile, - do_ocr: bool = Query(True, description="Enable OCR for scanned documents"), - do_table_structure: bool = Query(True, description="Enable table structure extraction"), - table_mode: str = Query("accurate", regex="^(accurate|fast)$", description="Table extraction mode"), -): - """Parse a document and return structured content with per-page elements.""" - if not file.filename: - raise HTTPException(status_code=400, detail="No filename provided") - - file_content = await file.read() - if len(file_content) > MAX_FILE_SIZE: - raise HTTPException(status_code=413, detail="File too large (max 50 MB)") - - suffix = Path(file.filename).suffix - tmp_path = None - try: - with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: - tmp.write(file_content) - tmp_path = tmp.name - - # Build converter with requested options (or reuse default) - if do_ocr and do_table_structure and table_mode == "accurate": - conv = _get_default_converter() - else: - conv = _build_converter( - do_ocr=do_ocr, - do_table_structure=do_table_structure, - table_mode=table_mode, - ) - - with _converter_lock: - result = conv.convert(tmp_path) - - doc = result.document - - content_markdown = doc.export_to_markdown() - content_html = doc.export_to_html() if hasattr(doc, "export_to_html") else "" - - page_count = len(doc.pages) if hasattr(doc, "pages") and doc.pages else 0 - - pages_detail, skipped = extract_pages_detail(result) - - # Ensure we have page entries even if no elements were extracted - if not pages_detail and page_count > 0: - pages_detail = [ - PageDetail(page_number=i + 1, width=612.0, height=792.0, elements=[]) - for i in range(page_count) - ] - - if skipped > 0: - logger.info( - "Parsed %s: %d pages, %d items skipped", - file.filename, - page_count, - skipped, - ) - - return ParseResponse( - filename=file.filename, - page_count=page_count or len(pages_detail) or 1, - content_markdown=content_markdown, - content_html=content_html, - pages=pages_detail, - skipped_items=skipped, - ) - - except HTTPException: - raise - except Exception as e: - logger.exception("Failed to parse document: %s", file.filename) - raise HTTPException(status_code=422, detail=f"Failed to parse document: {e}") - finally: - if tmp_path and os.path.exists(tmp_path): - try: - os.unlink(tmp_path) - except OSError: - logger.warning("Could not delete temp file: %s", tmp_path) - - -@app.post("/preview") -async def preview( - file: UploadFile, - page: int = Query(1, ge=1), - dpi: int = Query(150, ge=72, le=300), -): - """Generate a PNG preview of a specific PDF page.""" - if not file.filename: - raise HTTPException(status_code=400, detail="No filename provided") - - file_content = await file.read() - if len(file_content) > MAX_FILE_SIZE: - raise HTTPException(status_code=413, detail="File too large (max 50 MB)") - - try: - images = convert_from_bytes(file_content, first_page=page, last_page=page, dpi=dpi) - if not images: - raise HTTPException(status_code=404, detail=f"Page {page} not found") - - buf = io.BytesIO() - images[0].save(buf, format="PNG") - buf.seek(0) - return StreamingResponse(buf, media_type="image/png") - - except HTTPException: - raise - except Exception as e: - logger.exception("Failed to generate preview for page %d", page) - raise HTTPException(status_code=422, detail=f"Failed to generate preview: {e}") +# Mount routers +app.include_router(documents_router) +app.include_router(analyses_router) @app.get("/health") diff --git a/document-parser/persistence/__init__.py b/document-parser/persistence/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/persistence/analysis_repo.py b/document-parser/persistence/analysis_repo.py new file mode 100644 index 0000000..fa3202e --- /dev/null +++ b/document-parser/persistence/analysis_repo.py @@ -0,0 +1,108 @@ +"""Analysis job repository — SQLite CRUD for analysis_jobs table.""" + +from __future__ import annotations + +from domain.models import AnalysisJob, AnalysisStatus +from persistence.database import get_db + + +def _row_to_job(row) -> AnalysisJob: + return AnalysisJob( + id=row["id"], + document_id=row["document_id"], + status=AnalysisStatus(row["status"]), + content_markdown=row["content_markdown"], + content_html=row["content_html"], + pages_json=row["pages_json"], + error_message=row["error_message"], + started_at=row["started_at"], + completed_at=row["completed_at"], + created_at=row["created_at"], + document_filename=row["filename"] if "filename" in row.keys() else None, + ) + + +_SELECT_WITH_DOC = """ + SELECT aj.*, d.filename + FROM analysis_jobs aj + JOIN documents d ON d.id = aj.document_id +""" + + +async def insert(job: AnalysisJob) -> None: + db = await get_db() + try: + await db.execute( + """INSERT INTO analysis_jobs (id, document_id, status, created_at) + VALUES (?, ?, ?, ?)""", + (job.id, job.document_id, job.status.value, str(job.created_at)), + ) + await db.commit() + finally: + await db.close() + + +async def find_all() -> list[AnalysisJob]: + db = await get_db() + try: + cursor = await db.execute( + f"{_SELECT_WITH_DOC} ORDER BY aj.created_at DESC" + ) + rows = await cursor.fetchall() + return [_row_to_job(r) for r in rows] + finally: + await db.close() + + +async def find_by_id(job_id: str) -> AnalysisJob | None: + db = await get_db() + try: + cursor = await db.execute( + f"{_SELECT_WITH_DOC} WHERE aj.id = ?", (job_id,) + ) + row = await cursor.fetchone() + return _row_to_job(row) if row else None + finally: + await db.close() + + +async def update_status(job: AnalysisJob) -> None: + db = await get_db() + try: + await db.execute( + """UPDATE analysis_jobs + SET status = ?, content_markdown = ?, content_html = ?, + pages_json = ?, error_message = ?, started_at = ?, completed_at = ? + WHERE id = ?""", + (job.status.value, job.content_markdown, job.content_html, + job.pages_json, job.error_message, + str(job.started_at) if job.started_at else None, + str(job.completed_at) if job.completed_at else None, + job.id), + ) + await db.commit() + finally: + await db.close() + + +async def delete(job_id: str) -> bool: + db = await get_db() + try: + cursor = await db.execute("DELETE FROM analysis_jobs WHERE id = ?", (job_id,)) + await db.commit() + return cursor.rowcount > 0 + finally: + await db.close() + + +async def delete_by_document(document_id: str) -> int: + """Delete all analysis jobs for a given document. Returns count deleted.""" + db = await get_db() + try: + cursor = await db.execute( + "DELETE FROM analysis_jobs WHERE document_id = ?", (document_id,) + ) + await db.commit() + return cursor.rowcount + finally: + await db.close() diff --git a/document-parser/persistence/database.py b/document-parser/persistence/database.py new file mode 100644 index 0000000..fa7f154 --- /dev/null +++ b/document-parser/persistence/database.py @@ -0,0 +1,54 @@ +"""SQLite database management — async via aiosqlite.""" + +from __future__ import annotations + +import logging +import os + +import aiosqlite + +logger = logging.getLogger(__name__) + +DB_PATH = os.environ.get("DB_PATH", "./data/docling_studio.db") + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, + filename TEXT NOT NULL, + content_type TEXT, + file_size INTEGER, + page_count INTEGER, + storage_path TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS analysis_jobs ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL REFERENCES documents(id), + status TEXT NOT NULL DEFAULT 'PENDING', + content_markdown TEXT, + content_html TEXT, + pages_json TEXT, + error_message TEXT, + started_at TEXT, + completed_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +""" + + +async def init_db() -> None: + """Create database file and tables if they don't exist.""" + os.makedirs(os.path.dirname(DB_PATH) or ".", exist_ok=True) + async with aiosqlite.connect(DB_PATH) as db: + await db.executescript(_SCHEMA) + await db.commit() + logger.info("Database initialized at %s", DB_PATH) + + +async def get_db() -> aiosqlite.Connection: + """Open a new database connection with row factory and FK enforcement.""" + db = await aiosqlite.connect(DB_PATH) + db.row_factory = aiosqlite.Row + await db.execute("PRAGMA foreign_keys = ON") + return db diff --git a/document-parser/persistence/document_repo.py b/document-parser/persistence/document_repo.py new file mode 100644 index 0000000..94a91f3 --- /dev/null +++ b/document-parser/persistence/document_repo.py @@ -0,0 +1,76 @@ +"""Document repository — SQLite CRUD for documents table.""" + +from __future__ import annotations + +from domain.models import Document +from persistence.database import get_db + + +def _row_to_document(row) -> Document: + return Document( + id=row["id"], + filename=row["filename"], + content_type=row["content_type"], + file_size=row["file_size"], + page_count=row["page_count"], + storage_path=row["storage_path"], + created_at=row["created_at"], + ) + + +async def insert(doc: Document) -> None: + db = await get_db() + try: + await db.execute( + """INSERT INTO documents (id, filename, content_type, file_size, page_count, storage_path, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (doc.id, doc.filename, doc.content_type, doc.file_size, + doc.page_count, doc.storage_path, str(doc.created_at)), + ) + await db.commit() + finally: + await db.close() + + +async def find_all() -> list[Document]: + db = await get_db() + try: + cursor = await db.execute( + "SELECT * FROM documents ORDER BY created_at DESC" + ) + rows = await cursor.fetchall() + return [_row_to_document(r) for r in rows] + finally: + await db.close() + + +async def find_by_id(doc_id: str) -> Document | None: + db = await get_db() + try: + cursor = await db.execute("SELECT * FROM documents WHERE id = ?", (doc_id,)) + row = await cursor.fetchone() + return _row_to_document(row) if row else None + finally: + await db.close() + + +async def update_page_count(doc_id: str, page_count: int) -> None: + db = await get_db() + try: + await db.execute( + "UPDATE documents SET page_count = ? WHERE id = ?", + (page_count, doc_id), + ) + await db.commit() + finally: + await db.close() + + +async def delete(doc_id: str) -> bool: + db = await get_db() + try: + cursor = await db.execute("DELETE FROM documents WHERE id = ?", (doc_id,)) + await db.commit() + return cursor.rowcount > 0 + finally: + await db.close() diff --git a/document-parser/requirements.txt b/document-parser/requirements.txt index a9eb762..46d50d0 100644 --- a/document-parser/requirements.txt +++ b/document-parser/requirements.txt @@ -5,3 +5,4 @@ uvicorn[standard]>=0.32.0,<1.0.0 python-multipart>=0.0.12 pdf2image>=1.17.0,<2.0.0 pillow>=10.0.0,<11.0.0 +aiosqlite>=0.20.0,<1.0.0 diff --git a/document-parser/services/__init__.py b/document-parser/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/document-parser/services/analysis_service.py b/document-parser/services/analysis_service.py new file mode 100644 index 0000000..12708d3 --- /dev/null +++ b/document-parser/services/analysis_service.py @@ -0,0 +1,78 @@ +"""Analysis service — async document parsing orchestration.""" + +from __future__ import annotations + +import asyncio +import json +import logging +from dataclasses import asdict + +from domain.models import AnalysisJob +from domain.parsing import ConversionResult, convert_document +from persistence import analysis_repo, document_repo + +logger = logging.getLogger(__name__) + + +async def create(document_id: str) -> AnalysisJob: + """Create a new analysis job and launch background processing.""" + doc = await document_repo.find_by_id(document_id) + if not doc: + raise ValueError(f"Document not found: {document_id}") + + job = AnalysisJob(document_id=document_id) + job.document_filename = doc.filename + await analysis_repo.insert(job) + + # Fire-and-forget background task + asyncio.create_task(_run_analysis(job.id, doc.storage_path, doc.filename)) + + return job + + +async def find_all() -> list[AnalysisJob]: + return await analysis_repo.find_all() + + +async def find_by_id(job_id: str) -> AnalysisJob | None: + return await analysis_repo.find_by_id(job_id) + + +async def delete(job_id: str) -> bool: + return await analysis_repo.delete(job_id) + + +async def _run_analysis(job_id: str, file_path: str, filename: str) -> None: + """Background task: run Docling conversion and update job status.""" + job = await analysis_repo.find_by_id(job_id) + if not job: + logger.error("Analysis job %s not found", job_id) + return + + job.mark_running() + await analysis_repo.update_status(job) + logger.info("Analysis started: %s (file: %s)", job_id, filename) + + try: + # Run blocking Docling conversion in a thread + result: ConversionResult = await asyncio.to_thread(convert_document, file_path) + + pages_json = json.dumps([asdict(p) for p in result.pages]) + + job.mark_completed( + markdown=result.content_markdown, + html=result.content_html, + pages_json=pages_json, + ) + await analysis_repo.update_status(job) + + # Update document page count if available + if result.page_count: + await document_repo.update_page_count(job.document_id, result.page_count) + + logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count) + + except Exception as e: + logger.exception("Analysis failed: %s", job_id) + job.mark_failed(str(e)) + await analysis_repo.update_status(job) diff --git a/document-parser/services/document_service.py b/document-parser/services/document_service.py new file mode 100644 index 0000000..d1bdeba --- /dev/null +++ b/document-parser/services/document_service.py @@ -0,0 +1,93 @@ +"""Document service — file upload, storage, and preview orchestration.""" + +from __future__ import annotations + +import io +import logging +import os +import uuid + +from pdf2image import convert_from_bytes, pdfinfo_from_bytes + +from domain.models import Document +from persistence import analysis_repo, document_repo + +logger = logging.getLogger(__name__) + +UPLOAD_DIR = os.environ.get("UPLOAD_DIR", "./uploads") +MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB + + +async def upload(filename: str, content_type: str, file_content: bytes) -> Document: + """Save uploaded file to disk and persist metadata.""" + if len(file_content) > MAX_FILE_SIZE: + raise ValueError("File too large (max 50 MB)") + + os.makedirs(UPLOAD_DIR, exist_ok=True) + + safe_name = f"{uuid.uuid4()}_{filename}" + file_path = os.path.join(UPLOAD_DIR, safe_name) + + with open(file_path, "wb") as f: + f.write(file_content) + + # Count PDF pages + page_count = _count_pages(file_content) + + doc = Document( + filename=filename, + content_type=content_type, + file_size=len(file_content), + page_count=page_count, + storage_path=os.path.abspath(file_path), + ) + await document_repo.insert(doc) + return doc + + +async def find_all() -> list[Document]: + return await document_repo.find_all() + + +async def find_by_id(doc_id: str) -> Document | None: + return await document_repo.find_by_id(doc_id) + + +async def delete(doc_id: str) -> bool: + """Delete document file, associated analyses, and database record.""" + doc = await document_repo.find_by_id(doc_id) + if not doc: + return False + + # Delete associated analyses first (cascade) + await analysis_repo.delete_by_document(doc_id) + + # Delete file from disk + try: + if os.path.exists(doc.storage_path): + os.unlink(doc.storage_path) + except OSError: + logger.warning("Could not delete file: %s", doc.storage_path) + + return await document_repo.delete(doc_id) + + +def generate_preview(file_content: bytes, page: int = 1, dpi: int = 150) -> bytes: + """Generate a PNG preview of a specific PDF page.""" + images = convert_from_bytes(file_content, first_page=page, last_page=page, dpi=dpi) + if not images: + raise ValueError(f"Page {page} not found") + + buf = io.BytesIO() + images[0].save(buf, format="PNG") + return buf.getvalue() + + +def _count_pages(file_content: bytes) -> int | None: + """Count PDF pages using poppler via pdf2image.""" + try: + info = pdfinfo_from_bytes(file_content) + return info.get("Pages") + except Exception: + logger.warning("Could not count pages", exc_info=True) + return None diff --git a/document-parser/test_bbox.py b/document-parser/tests/test_bbox.py similarity index 98% rename from document-parser/test_bbox.py rename to document-parser/tests/test_bbox.py index 2bfd5a7..d088f2d 100644 --- a/document-parser/test_bbox.py +++ b/document-parser/tests/test_bbox.py @@ -3,7 +3,7 @@ import pytest from docling_core.types.doc.base import BoundingBox, CoordOrigin -from bbox import to_topleft_list +from domain.bbox import to_topleft_list class TestToTopleftList: diff --git a/frontend/nginx.conf b/frontend/nginx.conf index ca50c32..515b230 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -9,9 +9,11 @@ server { } location /api/ { - proxy_pass http://backend:8081; + proxy_pass http://document-parser:8000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 600s; + proxy_send_timeout 600s; client_max_body_size 50M; } } diff --git a/frontend/src/features/settings/store.js b/frontend/src/features/settings/store.js index 2cb9a10..011cc4f 100644 --- a/frontend/src/features/settings/store.js +++ b/frontend/src/features/settings/store.js @@ -2,8 +2,7 @@ import { defineStore } from 'pinia' import { ref } from 'vue' export const useSettingsStore = defineStore('settings', () => { - const parserUrl = ref('http://localhost:8000') - const backendUrl = ref('http://localhost:8081') + const apiUrl = ref('http://localhost:8000') - return { parserUrl, backendUrl } + return { apiUrl } }) diff --git a/frontend/src/features/settings/ui/SettingsPanel.vue b/frontend/src/features/settings/ui/SettingsPanel.vue index 5f0c428..02d561b 100644 --- a/frontend/src/features/settings/ui/SettingsPanel.vue +++ b/frontend/src/features/settings/ui/SettingsPanel.vue @@ -1,12 +1,8 @@