Mke some fixes and refacto

This commit is contained in:
pjmalandrino 2026-03-20 12:36:26 +01:00
parent 6ee54fafe9
commit ad1f1a81d3
20 changed files with 189 additions and 107 deletions

View file

@ -87,6 +87,6 @@ async def preview(
return Response(content=png_bytes, media_type="image/png")
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
except Exception as e:
except Exception:
logger.exception("Failed to generate preview")
raise HTTPException(status_code=422, detail=f"Failed to generate preview: {e}")
raise HTTPException(status_code=422, detail="Failed to generate preview")

View file

@ -8,7 +8,7 @@ from __future__ import annotations
from datetime import datetime
from pydantic import BaseModel, ConfigDict
from pydantic import BaseModel, ConfigDict, field_validator
def _to_camel(name: str) -> str:
@ -61,6 +61,20 @@ class PipelineOptionsRequest(BaseModel):
generate_page_images: bool = False
images_scale: float = 1.0
@field_validator("table_mode")
@classmethod
def validate_table_mode(cls, v: str) -> str:
if v not in ("accurate", "fast"):
raise ValueError('table_mode must be "accurate" or "fast"')
return v
@field_validator("images_scale")
@classmethod
def validate_images_scale(cls, v: float) -> float:
if v <= 0 or v > 10:
raise ValueError("images_scale must be between 0 (exclusive) and 10")
return v
class CreateAnalysisRequest(BaseModel):
documentId: str # camelCase to match existing frontend contract

View file

@ -255,23 +255,23 @@ def convert_document(
and images_scale == 1.0
)
if is_default:
conv = get_default_converter()
else:
conv = build_converter(
do_ocr=do_ocr,
do_table_structure=do_table_structure,
table_mode=table_mode,
do_code_enrichment=do_code_enrichment,
do_formula_enrichment=do_formula_enrichment,
do_picture_classification=do_picture_classification,
do_picture_description=do_picture_description,
generate_picture_images=generate_picture_images,
generate_page_images=generate_page_images,
images_scale=images_scale,
)
with _converter_lock:
if is_default:
conv = get_default_converter()
else:
conv = build_converter(
do_ocr=do_ocr,
do_table_structure=do_table_structure,
table_mode=table_mode,
do_code_enrichment=do_code_enrichment,
do_formula_enrichment=do_formula_enrichment,
do_picture_classification=do_picture_classification,
do_picture_description=do_picture_description,
generate_picture_images=generate_picture_images,
generate_page_images=generate_page_images,
images_scale=images_scale,
)
result = conv.convert(file_path)
doc = result.document

View file

@ -3,7 +3,7 @@
from __future__ import annotations
from domain.models import AnalysisJob, AnalysisStatus
from persistence.database import get_db
from persistence.database import get_connection
def _row_to_job(row) -> AnalysisJob:
@ -30,45 +30,36 @@ _SELECT_WITH_DOC = """
async def insert(job: AnalysisJob) -> None:
db = await get_db()
try:
async with get_connection() as db:
await db.execute(
"""INSERT INTO analysis_jobs (id, document_id, status, created_at)
VALUES (?, ?, ?, ?)""",
(job.id, job.document_id, job.status.value, str(job.created_at)),
)
await db.commit()
finally:
await db.close()
async def find_all() -> list[AnalysisJob]:
db = await get_db()
try:
async def find_all(*, limit: int = 200, offset: int = 0) -> list[AnalysisJob]:
async with get_connection() as db:
cursor = await db.execute(
f"{_SELECT_WITH_DOC} ORDER BY aj.created_at DESC"
f"{_SELECT_WITH_DOC} ORDER BY aj.created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
)
rows = await cursor.fetchall()
return [_row_to_job(r) for r in rows]
finally:
await db.close()
async def find_by_id(job_id: str) -> AnalysisJob | None:
db = await get_db()
try:
async with get_connection() as db:
cursor = await db.execute(
f"{_SELECT_WITH_DOC} WHERE aj.id = ?", (job_id,)
)
row = await cursor.fetchone()
return _row_to_job(row) if row else None
finally:
await db.close()
async def update_status(job: AnalysisJob) -> None:
db = await get_db()
try:
async with get_connection() as db:
await db.execute(
"""UPDATE analysis_jobs
SET status = ?, content_markdown = ?, content_html = ?,
@ -81,28 +72,20 @@ async def update_status(job: AnalysisJob) -> None:
job.id),
)
await db.commit()
finally:
await db.close()
async def delete(job_id: str) -> bool:
db = await get_db()
try:
async with get_connection() as db:
cursor = await db.execute("DELETE FROM analysis_jobs WHERE id = ?", (job_id,))
await db.commit()
return cursor.rowcount > 0
finally:
await db.close()
async def delete_by_document(document_id: str) -> int:
"""Delete all analysis jobs for a given document. Returns count deleted."""
db = await get_db()
try:
async with get_connection() as db:
cursor = await db.execute(
"DELETE FROM analysis_jobs WHERE document_id = ?", (document_id,)
)
await db.commit()
return cursor.rowcount
finally:
await db.close()

View file

@ -4,6 +4,7 @@ from __future__ import annotations
import logging
import os
from contextlib import asynccontextmanager
import aiosqlite
@ -24,7 +25,7 @@ CREATE TABLE IF NOT EXISTS documents (
CREATE TABLE IF NOT EXISTS analysis_jobs (
id TEXT PRIMARY KEY,
document_id TEXT NOT NULL REFERENCES documents(id),
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
status TEXT NOT NULL DEFAULT 'PENDING',
content_markdown TEXT,
content_html TEXT,
@ -52,3 +53,13 @@ async def get_db() -> aiosqlite.Connection:
db.row_factory = aiosqlite.Row
await db.execute("PRAGMA foreign_keys = ON")
return db
@asynccontextmanager
async def get_connection():
"""Context manager that opens and auto-closes a database connection."""
db = await get_db()
try:
yield db
finally:
await db.close()

View file

@ -3,7 +3,7 @@
from __future__ import annotations
from domain.models import Document
from persistence.database import get_db
from persistence.database import get_connection
def _row_to_document(row) -> Document:
@ -19,8 +19,7 @@ def _row_to_document(row) -> Document:
async def insert(doc: Document) -> None:
db = await get_db()
try:
async with get_connection() as db:
await db.execute(
"""INSERT INTO documents (id, filename, content_type, file_size, page_count, storage_path, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
@ -28,49 +27,36 @@ async def insert(doc: Document) -> None:
doc.page_count, doc.storage_path, str(doc.created_at)),
)
await db.commit()
finally:
await db.close()
async def find_all() -> list[Document]:
db = await get_db()
try:
async def find_all(*, limit: int = 200, offset: int = 0) -> list[Document]:
async with get_connection() as db:
cursor = await db.execute(
"SELECT * FROM documents ORDER BY created_at DESC"
"SELECT * FROM documents ORDER BY created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
)
rows = await cursor.fetchall()
return [_row_to_document(r) for r in rows]
finally:
await db.close()
async def find_by_id(doc_id: str) -> Document | None:
db = await get_db()
try:
async with get_connection() as db:
cursor = await db.execute("SELECT * FROM documents WHERE id = ?", (doc_id,))
row = await cursor.fetchone()
return _row_to_document(row) if row else None
finally:
await db.close()
async def update_page_count(doc_id: str, page_count: int) -> None:
db = await get_db()
try:
async with get_connection() as db:
await db.execute(
"UPDATE documents SET page_count = ? WHERE id = ?",
(page_count, doc_id),
)
await db.commit()
finally:
await db.close()
async def delete(doc_id: str) -> bool:
db = await get_db()
try:
async with get_connection() as db:
cursor = await db.execute("DELETE FROM documents WHERE id = ?", (doc_id,))
await db.commit()
return cursor.rowcount > 0
finally:
await db.close()

View file

@ -13,6 +13,9 @@ from persistence import analysis_repo, document_repo
logger = logging.getLogger(__name__)
# Maximum time (seconds) allowed for a single document conversion.
CONVERSION_TIMEOUT = int(__import__("os").environ.get("CONVERSION_TIMEOUT", "600"))
async def create(document_id: str, *, pipeline_options: dict | None = None) -> AnalysisJob:
"""Create a new analysis job and launch background processing."""
@ -24,12 +27,25 @@ async def create(document_id: str, *, pipeline_options: dict | None = None) -> A
job.document_filename = doc.filename
await analysis_repo.insert(job)
# Fire-and-forget background task
asyncio.create_task(_run_analysis(job.id, doc.storage_path, doc.filename, pipeline_options))
# Fire background task with error logging callback
task = asyncio.create_task(
_run_analysis(job.id, doc.storage_path, doc.filename, pipeline_options)
)
task.add_done_callback(_on_task_done)
return job
def _on_task_done(task: asyncio.Task) -> None:
"""Log unhandled exceptions from background analysis tasks."""
if task.cancelled():
logger.warning("Analysis task was cancelled")
return
exc = task.exception()
if exc:
logger.error("Unhandled exception in analysis task: %s", exc, exc_info=exc)
async def find_all() -> list[AnalysisJob]:
return await analysis_repo.find_all()
@ -46,22 +62,23 @@ async def _run_analysis(
job_id: str, file_path: str, filename: str, pipeline_options: dict | None = None,
) -> None:
"""Background task: run Docling conversion and update job status."""
job = await analysis_repo.find_by_id(job_id)
if not job:
logger.error("Analysis job %s not found", job_id)
return
job.mark_running()
await analysis_repo.update_status(job)
logger.info("Analysis started: %s (file: %s)", job_id, filename)
try:
job = await analysis_repo.find_by_id(job_id)
if not job:
logger.error("Analysis job %s not found", job_id)
return
job.mark_running()
await analysis_repo.update_status(job)
logger.info("Analysis started: %s (file: %s)", job_id, filename)
# Build kwargs from pipeline options
convert_kwargs = pipeline_options or {}
# Run blocking Docling conversion in a thread
result: ConversionResult = await asyncio.to_thread(
convert_document, file_path, **convert_kwargs,
# Run blocking Docling conversion in a thread with timeout
result: ConversionResult = await asyncio.wait_for(
asyncio.to_thread(convert_document, file_path, **convert_kwargs),
timeout=CONVERSION_TIMEOUT,
)
pages_json = json.dumps([asdict(p) for p in result.pages])
@ -79,7 +96,21 @@ async def _run_analysis(
logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count)
except asyncio.TimeoutError:
logger.error("Analysis timed out after %ds: %s", CONVERSION_TIMEOUT, job_id)
await _mark_failed(job_id, f"Conversion timed out after {CONVERSION_TIMEOUT}s")
except Exception as e:
logger.exception("Analysis failed: %s", job_id)
job.mark_failed(str(e))
await analysis_repo.update_status(job)
await _mark_failed(job_id, str(e))
async def _mark_failed(job_id: str, error: str) -> None:
"""Safely mark a job as failed, handling DB errors gracefully."""
try:
job = await analysis_repo.find_by_id(job_id)
if job:
job.mark_failed(error)
await analysis_repo.update_status(job)
except Exception:
logger.exception("Could not mark job %s as failed", job_id)

View file

@ -18,11 +18,18 @@ UPLOAD_DIR = os.environ.get("UPLOAD_DIR", "./uploads")
MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB
# PDF magic bytes: %PDF
_PDF_MAGIC = b"%PDF"
async def upload(filename: str, content_type: str, file_content: bytes) -> Document:
"""Save uploaded file to disk and persist metadata."""
if len(file_content) > MAX_FILE_SIZE:
raise ValueError("File too large (max 50 MB)")
if not file_content[:4].startswith(_PDF_MAGIC):
raise ValueError("Invalid file: not a PDF document")
os.makedirs(UPLOAD_DIR, exist_ok=True)
safe_name = f"{uuid.uuid4()}_{filename}"

View file

@ -1,7 +1,9 @@
"""Tests for API schemas — camelCase serialization."""
"""Tests for API schemas — camelCase serialization and validation."""
from datetime import datetime
import pytest
from api.schemas import (
AnalysisResponse,
CreateAnalysisRequest,
@ -109,6 +111,28 @@ class TestPipelineOptionsRequest:
assert data["do_ocr"] is False
assert data["do_table_structure"] is True # default preserved
def test_invalid_table_mode_rejected(self):
with pytest.raises(ValueError, match='table_mode must be "accurate" or "fast"'):
PipelineOptionsRequest(table_mode="invalid")
def test_negative_images_scale_rejected(self):
with pytest.raises(ValueError, match="images_scale must be between"):
PipelineOptionsRequest(images_scale=-1.0)
def test_zero_images_scale_rejected(self):
with pytest.raises(ValueError, match="images_scale must be between"):
PipelineOptionsRequest(images_scale=0)
def test_excessive_images_scale_rejected(self):
with pytest.raises(ValueError, match="images_scale must be between"):
PipelineOptionsRequest(images_scale=11.0)
def test_boundary_images_scale_accepted(self):
opts = PipelineOptionsRequest(images_scale=0.1)
assert opts.images_scale == 0.1
opts2 = PipelineOptionsRequest(images_scale=10.0)
assert opts2.images_scale == 10.0
class TestCreateAnalysisRequest:
def test_parses_document_id(self):

View file

@ -8,6 +8,7 @@
"name": "docling-studio",
"version": "0.1.0",
"dependencies": {
"dompurify": "^3.3.3",
"marked": "^17.0.4",
"pinia": "^2.3.0",
"vue": "^3.4.0",
@ -765,6 +766,12 @@
"integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==",
"dev": true
},
"node_modules/@types/trusted-types": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz",
"integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==",
"optional": true
},
"node_modules/@vitejs/plugin-vue": {
"version": "5.2.4",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.2.4.tgz",
@ -1064,6 +1071,14 @@
"node": ">=6"
}
},
"node_modules/dompurify": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.3.tgz",
"integrity": "sha512-Oj6pzI2+RqBfFG+qOaOLbFXLQ90ARpcGG6UePL82bJLtdsa6CYJD7nmiU8MW9nQNOtCHV3lZ/Bzq1X0QYbBZCA==",
"optionalDependencies": {
"@types/trusted-types": "^2.0.7"
}
},
"node_modules/entities": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",

View file

@ -11,10 +11,11 @@
"test:run": "vitest run"
},
"dependencies": {
"vue": "^3.4.0",
"vue-router": "^4.6.4",
"dompurify": "^3.3.3",
"marked": "^17.0.4",
"pinia": "^2.3.0",
"marked": "^17.0.4"
"vue": "^3.4.0",
"vue-router": "^4.6.4"
},
"devDependencies": {
"@vitejs/plugin-vue": "^5.2.0",

View file

@ -15,6 +15,11 @@ const routes = [
path: '/settings',
name: 'settings',
component: () => import('../../pages/SettingsPage.vue')
},
{
path: '/:pathMatch(.*)*',
name: 'not-found',
redirect: '/'
}
]

View file

@ -38,7 +38,7 @@ const props = defineProps({
const images = computed(() => {
const result = []
for (const page of props.pages) {
for (const el of page.elements) {
for (const el of (page.elements || [])) {
if (el.type === 'picture') {
result.push({ ...el, page: page.page_number })
}
@ -86,7 +86,7 @@ const images = computed(() => {
.card-type {
font-size: 12px;
font-weight: 600;
color: #22C55E;
color: var(--success);
text-transform: uppercase;
}

View file

@ -5,6 +5,7 @@
<script setup>
import { computed } from 'vue'
import { marked } from 'marked'
import DOMPurify from 'dompurify'
import { useI18n } from '../../../shared/i18n.js'
const props = defineProps({ content: String })
@ -12,7 +13,7 @@ const { t } = useI18n()
const rendered = computed(() => {
if (!props.content) return `<p class="empty">${t('results.noMarkdown')}</p>`
return marked.parse(props.content)
return DOMPurify.sanitize(marked.parse(props.content))
})
</script>

View file

@ -77,7 +77,7 @@ const pageMarkdown = computed(() => {
const page = currentPageData.value
if (!page) return ''
return page.elements
return (page.elements || [])
.map(el => formatElement(el))
.filter(Boolean)
.join('\n\n')

View file

@ -25,13 +25,8 @@
<script setup>
import { useDocumentStore } from '../store.js'
import { formatSize } from '../../../shared/format.js'
const store = useDocumentStore()
function formatSize(bytes) {
if (!bytes) return ''
const mb = bytes / (1024 * 1024)
return mb >= 1 ? `${mb.toFixed(1)} MB` : `${(bytes / 1024).toFixed(0)} KB`
}
</script>
<style scoped>

View file

@ -55,18 +55,13 @@ import { ref, onMounted } from 'vue'
import { HistoryList, useHistoryStore } from '../features/history/index.js'
import { useDocumentStore } from '../features/document/store.js'
import { useI18n } from '../shared/i18n.js'
import { formatSize } from '../shared/format.js'
const historyStore = useHistoryStore()
const docStore = useDocumentStore()
const { t } = useI18n()
const tab = ref('analyses')
function formatSize(bytes) {
if (!bytes) return ''
const mb = bytes / (1024 * 1024)
return mb >= 1 ? `${mb.toFixed(1)} MB` : `${(bytes / 1024).toFixed(0)} KB`
}
function formatDate(iso) {
if (!iso) return ''
return new Date(iso).toLocaleString()

View file

@ -268,7 +268,7 @@
</template>
<script setup>
import { ref, computed, watch, nextTick, onMounted, reactive } from 'vue'
import { ref, computed, watch, nextTick, onMounted, onBeforeUnmount, reactive } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { useDocumentStore } from '../features/document/store.js'
import { useAnalysisStore } from '../features/analysis/store.js'
@ -368,6 +368,10 @@ onMounted(async () => {
router.replace({ query: {} })
}
})
onBeforeUnmount(() => {
analysisStore.stopPolling()
})
</script>
<style scoped>

View file

@ -0,0 +1,10 @@
/**
* Format a byte count as a human-readable size string.
* @param {number|null|undefined} bytes
* @returns {string}
*/
export function formatSize(bytes) {
if (!bytes) return ''
const mb = bytes / (1024 * 1024)
return mb >= 1 ? `${mb.toFixed(1)} MB` : `${(bytes / 1024).toFixed(0)} KB`
}