danvics/docling-studio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #59 from scub-france/release/0.3.1

Browse source 
Release/0.3.1
This commit is contained in:
Pier-Jean Malandrino 2026-04-10 15:04:06 +02:00 committed by GitHub
commit dda2bbda12
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
136 changed files with 7397 additions and 880 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.dockerignore
View file

@ -18,3 +18,6 @@ document-parser/.mypy_cache/
document-parser/.ruff_cache/
document-parser/data/
document-parser/uploads/
# E2E tests (JVM/Maven — not needed in Docker images)
e2e/

3
.env.example
View file

@ -15,6 +15,9 @@
# Max parallel analysis jobs (default: 3)
# MAX_CONCURRENT_ANALYSES=3
# Max upload file size in MB (default: 50, 0 = unlimited)
# MAX_FILE_SIZE_MB=50
# Max pages per PDF (default: 0 = unlimited). Set to 20 for HF Spaces.
# MAX_PAGE_COUNT=0

32
.github/PULL_REQUEST_TEMPLATE.md vendored Normal file
View file

@ -0,0 +1,32 @@
## Type
- [ ] Feature (`feature/*`)
- [ ] Bug fix (`fix/*`)
- [ ] Hotfix (`hotfix/*`)
- [ ] Documentation
- [ ] Refactoring
- [ ] CI/CD
- [ ] Other: ___
## Summary
<!-- 1-3 sentences: what changed and why -->
## Related issues
<!-- Closes #123, Fixes #456 -->
## Checklist
- [ ] Branch follows naming convention (`feature/`, `fix/`, `hotfix/`)
- [ ] Commits follow [Conventional Commits](docs/git-workflow/commit-conventions.md)
- [ ] Tests added/updated for the change
- [ ] All tests pass (`pytest tests/ -v` + `npm run test:run`)
- [ ] Linting passes (`ruff check .` + `npx eslint src/`)
- [ ] `CHANGELOG.md` updated under `[Unreleased]`
- [ ] Documentation updated if behavior changed
- [ ] No secrets or credentials committed
## Screenshots / Evidence
<!-- If UI change: before/after screenshots. If API change: curl example or test output. -->

43
.github/workflows/auto-close-issues.yml vendored Normal file
View file

@ -0,0 +1,43 @@
name: Auto-close issues on release branch merge
on:
push:
branches:
- 'release/**'
permissions:
issues: write
jobs:
close-issues:
name: Close referenced issues
runs-on: ubuntu-latest
steps:
- name: Scan commits and close issues
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: ${{ github.repository }}
SHA: ${{ github.sha }}
run: |
# Collect all commit messages from the push event
commits='${{ toJSON(github.event.commits) }}'
# Extract issue numbers from Closes/Fixes patterns (case-insensitive)
issues=$(echo "$commits" \
| grep -ioE '(closes|fixes|close|fix|resolved|resolves)\s+#[0-9]+' \
| grep -oE '[0-9]+' \
| sort -u)
if [ -z "$issues" ]; then
echo "No issue references found in commit messages."
exit 0
fi
branch="${GITHUB_REF#refs/heads/}"
for issue in $issues; do
echo "Closing issue #$issue (referenced in $branch @ ${SHA:0:7})"
gh issue close "$issue" \
--repo "$REPO" \
--comment "Closed automatically — merged into \`$branch\` via ${SHA:0:7}." \
|| echo "Warning: could not close #$issue (may already be closed)"
done

134
.github/workflows/ci.yml vendored
View file

@ -4,13 +4,16 @@ on:
push:
branches: [main]
pull_request:
branches: [main]
branches: [main, 'release/**']
# Cancel previous runs on the same branch/PR
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
backend:
name: Backend tests
@ -73,3 +76,132 @@ jobs:
- name: Build
run: npm run build
e2e:
name: E2E API tests (Karate)
needs: [backend, frontend]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: actions/setup-java@v4
with:
java-version: "17"
distribution: temurin
cache: maven
- name: Generate test PDFs
run: |
pip install fpdf2 pypdfium2
python e2e/generate-test-data.py
- name: Start stack
run: docker compose up -d --wait --build
timeout-minutes: 10
env:
RATE_LIMIT_RPM: "0"
- name: Wait for health
run: |
for i in $(seq 1 30); do
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
echo "Backend healthy"
exit 0
fi
echo "Waiting for backend... ($i/30)"
sleep 5
done
echo "Backend failed to start"
docker compose logs
exit 1
- name: Run E2E tests
run: |
KARATE_TAGS="@smoke"
if [[ "${{ github.base_ref }}" == release/* ]] || [[ "${{ github.ref }}" == refs/heads/release/* ]]; then
KARATE_TAGS="@smoke,@regression,@e2e"
fi
mvn test -f e2e/api/pom.xml -DbaseUrl=http://localhost:3000 -Dkarate.options="--tags ${KARATE_TAGS}"
- name: Upload Karate reports
if: always()
uses: actions/upload-artifact@v4
with:
name: karate-api-reports
path: e2e/api/target/karate-reports/
- name: Tear down
if: always()
run: docker compose down
e2e-ui:
name: E2E UI tests (Karate UI)
if: github.ref == 'refs/heads/main'
needs: [backend, frontend]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: actions/setup-java@v4
with:
java-version: "17"
distribution: temurin
cache: maven
- name: Install Chrome
uses: browser-actions/setup-chrome@v1
with:
chrome-version: stable
- name: Generate test PDFs
run: |
pip install fpdf2 pypdfium2
python e2e/generate-test-data.py
- name: Start stack
run: docker compose up -d --wait --build
timeout-minutes: 10
env:
RATE_LIMIT_RPM: "0"
- name: Wait for health
run: |
for i in $(seq 1 30); do
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
echo "Backend healthy"
exit 0
fi
echo "Waiting for backend... ($i/30)"
sleep 5
done
echo "Backend failed to start"
docker compose logs
exit 1
- name: Run critical UI tests
run: >
mvn test -f e2e/ui/pom.xml
-DbaseUrl=http://localhost:3000
-DuiBaseUrl=http://localhost:3000
-Dkarate.options="--tags @critical"
- name: Upload Karate UI reports
if: always()
uses: actions/upload-artifact@v4
with:
name: karate-ui-reports
path: e2e/ui/target/karate-reports/
- name: Tear down
if: always()
run: docker compose down

3
.github/workflows/docling-compat.yml vendored
View file

@ -9,6 +9,9 @@ on:
- cron: "30 6 * * *" # Every day at 06:30 UTC
workflow_dispatch: # Manual trigger from GitHub UI
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
compat:
name: Test against latest Docling

3
.github/workflows/docs.yml vendored
View file

@ -16,6 +16,9 @@ concurrency:
group: docs
cancel-in-progress: true
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
deploy:
name: Build & deploy docs

782
.github/workflows/release-gate.yml vendored Normal file
View file

@ -0,0 +1,782 @@
name: Release Gate
on:
pull_request:
branches: [main]
types: [opened, synchronize, reopened]
concurrency:
group: release-gate-${{ github.ref }}
cancel-in-progress: true
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
# ──────────────────────────────────────────────────────────────────────────
# Phase 1 — Parallel validation (no Docker needed)
# ──────────────────────────────────────────────────────────────────────────
lint-typecheck:
name: Lint & type-check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: document-parser/requirements.txt
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Install backend deps
run: |
pip install --upgrade pip
pip install -r document-parser/requirements.txt
pip install ruff
- name: Install frontend deps
run: cd frontend && npm ci
- name: Backend lint
run: cd document-parser && ruff check .
- name: Frontend lint
run: cd frontend && npx eslint src/
- name: Frontend type-check
run: cd frontend && npm run type-check
unit-tests:
name: Unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: document-parser/requirements.txt
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Install system deps
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends poppler-utils
- name: Backend tests
run: |
cd document-parser
pip install --upgrade pip
pip install -r requirements.txt
pip install pytest pytest-asyncio httpx ruff
pytest tests/ -v --tb=short 2>&1 | tee /tmp/backend-tests.txt
- name: Frontend tests
run: |
cd frontend
npm ci
npm run test:run 2>&1 | tee /tmp/frontend-tests.txt
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4.6.2
with:
name: unit-test-results
path: /tmp/*-tests.txt
dep-audit:
name: Dependency audit
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: document-parser/requirements.txt
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Install deps
run: |
cd document-parser && pip install --upgrade pip && pip install -r requirements.txt && pip install pip-audit
cd ../frontend && npm ci
- name: Python audit
id: pip_audit
continue-on-error: true
run: |
cd document-parser
pip-audit --format=json --output=/tmp/pip-audit.json 2>&1 || true
pip-audit 2>&1 | tee /tmp/pip-audit.txt
# Check for critical vulnerabilities
if pip-audit --format=json 2>/dev/null | python3 -c "
import sys, json
data = json.load(sys.stdin)
vulns = data.get('dependencies', [])
crits = [v for dep in vulns for v in dep.get('vulns', []) if 'CRITICAL' in v.get('fix_versions', [''])]
sys.exit(1 if crits else 0)
" 2>/dev/null; then
echo "critical=false" >> "$GITHUB_OUTPUT"
else
echo "critical=true" >> "$GITHUB_OUTPUT"
fi
- name: Node audit
id: npm_audit
continue-on-error: true
run: |
cd frontend
npm audit --json > /tmp/npm-audit.json 2>&1 || true
npm audit 2>&1 | tee /tmp/npm-audit.txt
# Check for critical vulnerabilities
CRITICAL_COUNT=$(npm audit --json 2>/dev/null | python3 -c "
import sys, json
try:
data = json.load(sys.stdin)
print(data.get('metadata', {}).get('vulnerabilities', {}).get('critical', 0))
except: print(0)
" 2>/dev/null || echo "0")
if [ "$CRITICAL_COUNT" -gt 0 ]; then
echo "critical=true" >> "$GITHUB_OUTPUT"
else
echo "critical=false" >> "$GITHUB_OUTPUT"
fi
- name: Upload audit results
if: always()
uses: actions/upload-artifact@v4.6.2
with:
name: dep-audit-results
path: /tmp/*-audit.*
- name: Fail on critical vulnerabilities
if: steps.pip_audit.outputs.critical == 'true' || steps.npm_audit.outputs.critical == 'true'
run: |
echo "::error::Critical vulnerabilities found in dependencies"
exit 1
audit-checks:
name: Audit checks (commands.sh)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- name: Run automated audit checks
id: audit
run: |
bash profiles/fastapi-vue/commands.sh 2>&1 | tee /tmp/audit-checks.txt
continue-on-error: true
- name: Parse audit results
id: results
if: always()
run: |
# Strip ANSI codes before counting
CLEAN=$(sed 's/\x1b\[[0-9;]*m//g' /tmp/audit-checks.txt)
PASS=$(echo "$CLEAN" | grep -c "PASS" || true)
WARN=$(echo "$CLEAN" | grep -c "WARN" || true)
FAIL=$(echo "$CLEAN" | grep -c "FAIL" || true)
echo "pass=${PASS:-0}" >> "$GITHUB_OUTPUT"
echo "warn=${WARN:-0}" >> "$GITHUB_OUTPUT"
echo "fail=${FAIL:-0}" >> "$GITHUB_OUTPUT"
- name: Upload audit results
if: always()
uses: actions/upload-artifact@v4.6.2
with:
name: audit-checks-results
path: /tmp/audit-checks.txt
- name: Fail on audit failures
if: steps.audit.outcome == 'failure'
run: |
echo "::error::Audit checks failed — review profiles/fastapi-vue/commands.sh output"
exit 1
# ──────────────────────────────────────────────────────────────────────────
# Phase 2 — Docker build & validation
# ──────────────────────────────────────────────────────────────────────────
docker-build:
name: Docker build — ${{ matrix.target }}
runs-on: ubuntu-latest
strategy:
matrix:
target: [remote, local]
steps:
- uses: actions/checkout@v4.3.1
- uses: docker/setup-buildx-action@v3
- name: Extract version from branch
id: version
run: |
# release/1.2.3 → 1.2.3
BRANCH="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}"
VERSION=$(echo "$BRANCH" | sed 's|release/||')
echo "value=$VERSION" >> "$GITHUB_OUTPUT"
- name: Build image
uses: docker/build-push-action@v6
with:
context: .
target: ${{ matrix.target }}
load: true
tags: docling-studio:${{ matrix.target }}
build-args: APP_VERSION=${{ steps.version.outputs.value }}
cache-from: type=gha,scope=${{ matrix.target }}
cache-to: type=gha,mode=max,scope=${{ matrix.target }}
- name: Save image
run: |
docker save docling-studio:${{ matrix.target }} | gzip > /tmp/docling-studio-${{ matrix.target }}.tar.gz
- name: Upload image artifact
uses: actions/upload-artifact@v4.6.2
with:
name: docker-image-${{ matrix.target }}
path: /tmp/docling-studio-${{ matrix.target }}.tar.gz
retention-days: 1
- name: Record image size
run: |
SIZE_BYTES=$(docker image inspect docling-studio:${{ matrix.target }} --format='{{.Size}}')
SIZE_MB=$((SIZE_BYTES / 1024 / 1024))
echo "$SIZE_MB" > /tmp/image-size-${{ matrix.target }}.txt
echo "Image size (${{ matrix.target }}): ${SIZE_MB} MB"
- name: Upload size artifact
uses: actions/upload-artifact@v4.6.2
with:
name: image-size-${{ matrix.target }}
path: /tmp/image-size-${{ matrix.target }}.txt
docker-smoke:
name: Docker smoke test
needs: [docker-build]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- name: Download remote image
uses: actions/download-artifact@v4.3.0
with:
name: docker-image-remote
path: /tmp
- name: Load remote image
run: docker load < /tmp/docling-studio-remote.tar.gz
- name: Smoke test — remote
run: |
docker run -d --name smoke-remote -p 3000:3000 \
-e RATE_LIMIT_RPM=0 \
docling-studio:remote
echo "Waiting for container to start..."
for i in $(seq 1 30); do
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
echo "Remote image healthy!"
curl -s http://localhost:3000/api/health | python3 -m json.tool
break
fi
if [ "$i" -eq 30 ]; then
echo "::error::Remote image failed health check"
docker logs smoke-remote
exit 1
fi
sleep 2
done
# Verify response fields
HEALTH=$(curl -s http://localhost:3000/api/health)
ENGINE=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin)['engine'])")
STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
if [ "$STATUS" != "ok" ]; then
echo "::error::Health status is '$STATUS', expected 'ok'"
exit 1
fi
if [ "$ENGINE" != "remote" ]; then
echo "::error::Engine is '$ENGINE', expected 'remote'"
exit 1
fi
echo "Remote smoke test passed"
docker stop smoke-remote && docker rm smoke-remote
image-scan:
name: Security scan — ${{ matrix.target }}
needs: [docker-build]
runs-on: ubuntu-latest
strategy:
matrix:
target: [remote, local]
steps:
- uses: actions/checkout@v4.3.1
- name: Download image
uses: actions/download-artifact@v4.3.0
with:
name: docker-image-${{ matrix.target }}
path: /tmp
- name: Load image
run: docker load < /tmp/docling-studio-${{ matrix.target }}.tar.gz
- name: Run Trivy — CRITICAL (blocking)
uses: aquasecurity/trivy-action@v0.35.0
with:
image-ref: docling-studio:${{ matrix.target }}
format: table
exit-code: 1
severity: CRITICAL
output: /tmp/trivy-critical-${{ matrix.target }}.txt
- name: Run Trivy — HIGH (informational)
if: always()
uses: aquasecurity/trivy-action@v0.35.0
with:
image-ref: docling-studio:${{ matrix.target }}
format: table
exit-code: 0
severity: HIGH
output: /tmp/trivy-high-${{ matrix.target }}.txt
- name: Annotate HIGH vulnerabilities
if: always()
run: |
HIGH_COUNT=$(grep -c "HIGH" /tmp/trivy-high-${{ matrix.target }}.txt 2>/dev/null || echo "0")
if [ "$HIGH_COUNT" -gt 0 ]; then
echo "::warning::${{ matrix.target }} image has $HIGH_COUNT HIGH severity vulnerabilities — review Trivy report"
else
echo "No HIGH vulnerabilities found in ${{ matrix.target }} image"
fi
- name: Upload Trivy reports
if: always()
uses: actions/upload-artifact@v4.6.2
with:
name: trivy-${{ matrix.target }}
path: /tmp/trivy-*-${{ matrix.target }}.txt
image-size:
name: Image size check
needs: [docker-build]
runs-on: ubuntu-latest
permissions:
packages: read
steps:
- uses: actions/checkout@v4.3.1
- name: Download size artifacts
uses: actions/download-artifact@v4.3.0
with:
pattern: image-size-*
merge-multiple: true
path: /tmp/sizes
- name: Get previous release sizes
id: prev
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Find the latest release tag
PREV_TAG=$(gh release list --repo "${{ github.repository }}" --limit 1 --json tagName -q '.[0].tagName' 2>/dev/null || echo "")
if [ -z "$PREV_TAG" ]; then
echo "No previous release found — skipping delta check"
echo "remote=0" >> "$GITHUB_OUTPUT"
echo "local=0" >> "$GITHUB_OUTPUT"
echo "found=false" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "Previous release: $PREV_TAG"
echo "found=true" >> "$GITHUB_OUTPUT"
# Pull previous images and get sizes
for target in remote local; do
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${PREV_TAG#v}-${target}"
if docker pull "$IMAGE" 2>/dev/null; then
PREV_SIZE=$(docker image inspect "$IMAGE" --format='{{.Size}}')
PREV_MB=$((PREV_SIZE / 1024 / 1024))
echo "${target}=${PREV_MB}" >> "$GITHUB_OUTPUT"
echo "Previous $target size: ${PREV_MB} MB"
else
echo "Could not pull $IMAGE — skipping $target delta"
echo "${target}=0" >> "$GITHUB_OUTPUT"
fi
done
- name: Compare sizes
id: delta
run: |
for target in remote local; do
CURRENT=$(cat /tmp/sizes/image-size-${target}.txt 2>/dev/null || echo "0")
if [ "$target" = "remote" ]; then
PREV="${{ steps.prev.outputs.remote }}"
else
PREV="${{ steps.prev.outputs.local }}"
fi
if [ "$PREV" -gt 0 ] 2>/dev/null; then
DELTA=$((CURRENT - PREV))
DELTA_PCT=$((DELTA * 100 / PREV))
echo "${target}_current=${CURRENT}" >> "$GITHUB_OUTPUT"
echo "${target}_prev=${PREV}" >> "$GITHUB_OUTPUT"
echo "${target}_delta=${DELTA}" >> "$GITHUB_OUTPUT"
echo "${target}_delta_pct=${DELTA_PCT}" >> "$GITHUB_OUTPUT"
echo "$target: ${CURRENT}MB (was ${PREV}MB, delta: ${DELTA_PCT}%)"
if [ "$DELTA_PCT" -gt 10 ]; then
echo "::warning::$target image grew by ${DELTA_PCT}% (${PREV}MB → ${CURRENT}MB)"
elif [ "$DELTA_PCT" -lt -10 ]; then
echo "::notice::$target image shrank by ${DELTA_PCT#-}% (${PREV}MB → ${CURRENT}MB)"
fi
else
echo "${target}_current=${CURRENT}" >> "$GITHUB_OUTPUT"
echo "${target}_prev=n/a" >> "$GITHUB_OUTPUT"
echo "${target}_delta=n/a" >> "$GITHUB_OUTPUT"
echo "${target}_delta_pct=n/a" >> "$GITHUB_OUTPUT"
echo "$target: ${CURRENT}MB (no previous baseline)"
fi
done
- name: Save size report
run: |
cat > /tmp/image-size-report.txt <<EOF
Image Size Report
=================
remote: ${{ steps.delta.outputs.remote_current }}MB (prev: ${{ steps.delta.outputs.remote_prev }}MB, delta: ${{ steps.delta.outputs.remote_delta_pct }}%)
local: ${{ steps.delta.outputs.local_current }}MB (prev: ${{ steps.delta.outputs.local_prev }}MB, delta: ${{ steps.delta.outputs.local_delta_pct }}%)
EOF
cat /tmp/image-size-report.txt
- name: Upload size report
uses: actions/upload-artifact@v4.6.2
with:
name: image-size-report
path: /tmp/image-size-report.txt
# ──────────────────────────────────────────────────────────────────────────
# Phase 3 — E2E tests on built images
# ──────────────────────────────────────────────────────────────────────────
e2e-api:
name: E2E API tests (full scope)
needs: [docker-smoke]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: actions/setup-java@v4
with:
java-version: "17"
distribution: temurin
cache: maven
- name: Generate test PDFs
run: |
pip install fpdf2 pypdfium2
python e2e/generate-test-data.py
- name: Start stack
run: docker compose up -d --wait --build
timeout-minutes: 10
env:
RATE_LIMIT_RPM: "0"
- name: Wait for health
run: |
for i in $(seq 1 30); do
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
echo "Backend healthy"
exit 0
fi
echo "Waiting for backend... ($i/30)"
sleep 5
done
echo "Backend failed to start"
docker compose logs
exit 1
- name: Run E2E API tests (full scope)
run: |
mvn test -f e2e/api/pom.xml \
-DbaseUrl=http://localhost:3000 \
-Dkarate.options="--tags @smoke,@regression,@e2e"
- name: Upload Karate reports
if: always()
uses: actions/upload-artifact@v4.6.2
with:
name: karate-api-reports
path: e2e/api/target/karate-reports/
- name: Tear down
if: always()
run: docker compose down
e2e-ui:
name: E2E UI tests (@critical)
needs: [docker-smoke]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.3.1
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: actions/setup-java@v4
with:
java-version: "17"
distribution: temurin
cache: maven
- name: Install Chrome
uses: browser-actions/setup-chrome@v1
with:
chrome-version: stable
- name: Generate test PDFs
run: |
pip install fpdf2 pypdfium2
python e2e/generate-test-data.py
- name: Start stack
run: docker compose up -d --wait --build
timeout-minutes: 10
env:
RATE_LIMIT_RPM: "0"
- name: Wait for health
run: |
for i in $(seq 1 30); do
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
echo "Backend healthy"
exit 0
fi
echo "Waiting for backend... ($i/30)"
sleep 5
done
echo "Backend failed to start"
docker compose logs
exit 1
- name: Run critical UI tests
run: >
mvn test -f e2e/ui/pom.xml
-DbaseUrl=http://localhost:3000
-DuiBaseUrl=http://localhost:3000
-Dkarate.options="--tags @critical"
- name: Upload Karate UI reports
if: always()
uses: actions/upload-artifact@v4.6.2
with:
name: karate-ui-reports
path: e2e/ui/target/karate-reports/
- name: Tear down
if: always()
run: docker compose down
# ──────────────────────────────────────────────────────────────────────────
# Phase 4 — Release summary (comment on PR)
# ──────────────────────────────────────────────────────────────────────────
release-summary:
name: Release summary
if: always() && github.event_name == 'pull_request'
needs:
- lint-typecheck
- unit-tests
- dep-audit
- audit-checks
- docker-build
- docker-smoke
- image-scan
- image-size
- e2e-api
- e2e-ui
runs-on: ubuntu-latest
permissions:
pull-requests: write
actions: read
steps:
- uses: actions/checkout@v4.3.1
- name: Download image-size artifacts
uses: actions/download-artifact@v4.3.0
with:
pattern: image-size-*
path: /tmp/artifacts
- name: Download audit-checks artifact
uses: actions/download-artifact@v4.3.0
with:
name: audit-checks-results
path: /tmp/artifacts/audit-checks-results
- name: Build summary comment
id: summary
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Job results from needs context
LINT="${{ needs.lint-typecheck.result }}"
TESTS="${{ needs.unit-tests.result }}"
DEP="${{ needs.dep-audit.result }}"
AUDIT="${{ needs.audit-checks.result }}"
BUILD="${{ needs.docker-build.result }}"
SMOKE="${{ needs.docker-smoke.result }}"
SCAN="${{ needs.image-scan.result }}"
SIZE="${{ needs.image-size.result }}"
E2E_API="${{ needs.e2e-api.result }}"
E2E_UI="${{ needs.e2e-ui.result }}"
# Determine verdict
BLOCKING_FAILED="false"
for result in "$LINT" "$TESTS" "$BUILD" "$SMOKE" "$SCAN" "$E2E_API" "$E2E_UI"; do
if [ "$result" = "failure" ]; then
BLOCKING_FAILED="true"
fi
done
if [ "$BLOCKING_FAILED" = "true" ]; then
VERDICT="NO-GO"
VERDICT_ICON="🔴"
elif [ "$DEP" = "failure" ] || [ "$AUDIT" = "failure" ]; then
VERDICT="GO CONDITIONAL"
VERDICT_ICON="🟡"
else
VERDICT="GO"
VERDICT_ICON="🟢"
fi
# Status emoji helper
status_icon() {
case "$1" in
success) echo "✅" ;;
failure) echo "❌" ;;
skipped) echo "⏭️" ;;
cancelled) echo "🚫" ;;
*) echo "❓" ;;
esac
}
# Read image sizes
REMOTE_SIZE=$(cat /tmp/artifacts/image-size-remote/image-size-remote.txt 2>/dev/null || echo "?")
LOCAL_SIZE=$(cat /tmp/artifacts/image-size-local/image-size-local.txt 2>/dev/null || echo "?")
# Read size report
SIZE_REPORT=$(cat /tmp/artifacts/image-size-report/image-size-report.txt 2>/dev/null || echo "No size data available")
# Read audit summary
AUDIT_PASS=$(grep -c "PASS" /tmp/artifacts/audit-checks-results/audit-checks.txt 2>/dev/null || echo "?")
AUDIT_WARN=$(grep -c "WARN" /tmp/artifacts/audit-checks-results/audit-checks.txt 2>/dev/null || echo "?")
AUDIT_FAIL=$(grep -c "FAIL" /tmp/artifacts/audit-checks-results/audit-checks.txt 2>/dev/null || echo "?")
# Build the comment
cat > /tmp/summary.md << ENDOFCOMMENT
## ${VERDICT_ICON} Release Gate — ${VERDICT}
### Validation Results
| Check | Status | Blocking |
|-------|--------|----------|
| Lint & type-check | $(status_icon "$LINT") | Yes |
| Unit tests | $(status_icon "$TESTS") | Yes |
| Dependency audit | $(status_icon "$DEP") | Yes (CRITICAL) |
| Audit checks | $(status_icon "$AUDIT") | Yes |
| Docker build | $(status_icon "$BUILD") | Yes |
| Docker smoke test | $(status_icon "$SMOKE") | Yes |
| Image security scan | $(status_icon "$SCAN") | Yes (CRITICAL) |
| Image size check | $(status_icon "$SIZE") | No |
| E2E API (full scope) | $(status_icon "$E2E_API") | Yes |
| E2E UI (@critical) | $(status_icon "$E2E_UI") | Yes |
### Audit Checks Summary
| PASS | WARN | FAIL |
|------|------|------|
| ${AUDIT_PASS} | ${AUDIT_WARN} | ${AUDIT_FAIL} |
### Docker Image Sizes
| Target | Size |
|--------|------|
| remote | ${REMOTE_SIZE} MB |
| local | ${LOCAL_SIZE} MB |
<details>
<summary>Size delta vs previous release</summary>
\`\`\`
${SIZE_REPORT}
\`\`\`
</details>
---
<sub>Generated by <b>release-gate.yml</b> — see <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">workflow run</a> for full details</sub>
ENDOFCOMMENT
- name: Post or update PR comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
PR_NUMBER="${{ github.event.pull_request.number }}"
COMMENT_TAG="<!-- release-gate-summary -->"
# Check for existing comment
EXISTING_COMMENT_ID=$(gh api \
"repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
--jq ".[] | select(.body | contains(\"$COMMENT_TAG\")) | .id" \
2>/dev/null | head -1)
BODY=$(cat /tmp/summary.md)
BODY="${COMMENT_TAG}
${BODY}"
if [ -n "$EXISTING_COMMENT_ID" ]; then
gh api \
"repos/${{ github.repository }}/issues/comments/${EXISTING_COMMENT_ID}" \
-X PATCH \
-f body="$BODY"
echo "Updated existing comment #${EXISTING_COMMENT_ID}"
else
gh api \
"repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
-f body="$BODY"
echo "Posted new comment"
fi

1
.github/workflows/release.yml vendored
View file

@ -6,6 +6,7 @@ on:
- "v*"
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}

3
.gitignore vendored
View file

@ -43,3 +43,6 @@ hs_err_pid*
# Docker
docker-compose.override.yml
# E2E tests — Maven build outputs & Chrome user data
e2e/**/target/

15
CHANGELOG.md
View file

@ -4,6 +4,21 @@ All notable changes to Docling Studio will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/).
## [0.3.1] - 2026-04-09
### Added
- Batch conversion progress: segmented progress bar with ring indicator and per-batch visual feedback
- Inline mini progress bar in the top banner during analysis
- Informational notice in Prepare mode when chunking is unavailable (batch mode)
- `BATCH_PAGE_SIZE` environment variable forwarded in Docker Compose
### Fixed
- Batch progress reset to null on completion (progress_current/progress_total overwritten by stale in-memory job object)
- Regression test for batch progress preservation in `_run_analysis_inner` flow
- E2E assertion on final progress values in batch-progress feature
## [0.3.0] - 2026-04-07
### Added

43
CODE_OF_CONDUCT.md Normal file
View file

@ -0,0 +1,43 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment:
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
Examples of unacceptable behavior:
- The use of sexualized language or imagery, and sexual attention or advances of any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information without explicit permission
- Other conduct which could reasonably be considered inappropriate in a professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
## Scope
This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project maintainers at **[INSERT CONTACT EMAIL]**.
All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.

49
SECURITY.md Normal file
View file

@ -0,0 +1,49 @@
# Security Policy
## Supported Versions
| Version | Supported |
|---------|-----------|
| 0.3.x | Yes |
| < 0.3 | No |
## Reporting a Vulnerability
**Do NOT open a public GitHub issue for security vulnerabilities.**
Instead, please report them privately:
1. **Email**: Send a detailed report to **[INSERT SECURITY EMAIL]**
2. **GitHub Security Advisory**: Use [GitHub's private vulnerability reporting](https://github.com/scub-france/Docling-Studio/security/advisories/new)
### What to include
- Description of the vulnerability
- Steps to reproduce
- Affected component(s): `document-parser/`, `frontend/`, `docker-compose.yml`, etc.
- Impact assessment (data exposure, denial of service, privilege escalation, etc.)
- Suggested fix (if any)
### Response timeline
| Step | SLA |
|------|-----|
| Acknowledgment | < 48 hours |
| Initial assessment | < 7 days |
| Fix developed | < 14 days (critical), < 30 days (other) |
| Public disclosure | After fix is released |
### Process
1. We acknowledge your report and assign a severity level
2. We develop a fix in a **private branch** (never pushed publicly before the advisory)
3. We release the fix and publish a GitHub Security Advisory
4. We credit the reporter (unless they prefer anonymity)
## Security Best Practices (for contributors)
- Never commit secrets, API keys, or credentials
- Never disable CORS or security middleware without review
- Validate all user input at the API boundary
- Keep dependencies up to date (`pip audit`, `npm audit`)
- Follow the [OWASP Top 10](https://owasp.org/www-project-top-ten/) guidelines

3
docker-compose.yml
View file

@ -12,6 +12,9 @@ services:
CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173}
DOCLING_SERVE_URL: ${DOCLING_SERVE_URL:-}
DOCLING_SERVE_API_KEY: ${DOCLING_SERVE_API_KEY:-}
RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100}
MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50}
BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-0}
deploy:
resources:
limits:

179
docs/PROCESSES.md Normal file
View file

@ -0,0 +1,179 @@
# Available Processes
Index of all documented processes in Docling Studio. Each process is a structured, repeatable workflow with a clear trigger and deliverable.
---
## Development
| # | Process | Trigger / Command | Doc | Output |
|---|---------|-------------------|-----|--------|
| 1 | **Commit** | Every commit | [commit-conventions.md](git-workflow/commit-conventions.md) | Conventional Commit message |
| 2 | **Code Review** | Every PR | [code-review-checklist.md](git-workflow/code-review-checklist.md) | Reviewed PR with checklist |
| 3 | **Merge** | PR approved + CI green | [merge-policy.md](git-workflow/merge-policy.md) | Squash/merge commit on target branch |
| 4 | **ADR** | Architecture decision needed | [adr-guide.md](architecture/adr-guide.md) + [adr-template.md](architecture/adr-template.md) | `docs/architecture/adrs/ADR-NNN-*.md` |
### How to invoke
```
# Ask for a code review
Review cette PR avec docs/git-workflow/code-review-checklist.md
# Create an ADR
Crée un ADR pour [la décision à documenter]
```
---
## Quality & Audit
| # | Process | Trigger / Command | Doc | Output |
|---|---------|-------------------|-----|--------|
| 5 | **Full Release Audit** | Before merging `release/*` to `main` | [docs/audit/master.md](audit/master.md) | 12 audit reports + summary in `reports/release-X.Y.Z/` |
| 6 | **Single Audit** | Targeted check on one axis | [docs/audit/audits/*.md](audit/audits/) | Single audit report |
| 7 | **Re-Audit** | After fixing CRIT/MAJ findings | [docs/audit/master.md](audit/master.md) | Updated report |
| 8 | **Automated Checks** | Quick validation before audit | [profiles/fastapi-vue/commands.sh](../profiles/fastapi-vue/commands.sh) | PASS/WARN/FAIL per check |
### How to invoke
```
# Full audit
Audite la branche release/X.Y.Z en suivant docs/audit/master.md
# Single audit
Exécute l'audit docs/audit/audits/08-security.md sur la branche courante
# Re-audit after fixes
Re-audite les écarts CRIT et MAJ du rapport docs/audit/reports/release-X.Y.Z/summary.md
# Automated checks (shell)
bash profiles/fastapi-vue/commands.sh
```
---
## Release & Deploy
| # | Process | Trigger / Command | Doc | Output |
|---|---------|-------------------|-----|--------|
| 9 | **Release Gate** | Auto on push/PR `release/*` → `main` | [release-gate.yml](../.github/workflows/release-gate.yml) | GO / GO CONDITIONAL / NO-GO comment on PR |
| 10 | **Release** | Feature freeze on `release/*` | [CONTRIBUTING.md](../CONTRIBUTING.md#release-process) | Tag `vX.Y.Z`, Docker images on ghcr.io |
| 11 | **Deployment** | After release tag | [deployment-checklist.md](release/deployment-checklist.md) | Running instance at new version |
| 12 | **Rollback** | Post-deploy failure detected | [rollback-playbook.md](release/rollback-playbook.md) | Reverted to last known good version |
| 13 | **Hotfix** | Critical bug on released version | [CONTRIBUTING.md](../CONTRIBUTING.md#hotfix) | Patch release `vX.Y.Z+1` |
### Release Gate details
The release gate runs **automatically** on every push to `release/**` and on PRs targeting `main`. It validates 10 checks in 4 phases:
```
Phase 1 (parallel) Phase 2 (Docker) Phase 3 (E2E) Phase 4
┌──────────────┐ ┌──────────────┐ ┌─────────────┐ ┌─────────────┐
│ lint+typecheck│ │ docker build │──────▶│ e2e API │───▶│ release │
│ unit tests │ │ docker smoke │──────▶│ e2e UI │ │ summary │
│ dep audit │ │ image scan │ └─────────────┘ │ (PR comment)│
│ audit checks │ │ image size │ └─────────────┘
└──────────────┘ └──────────────┘
```
| Check | Blocks merge? | Details |
|-------|---------------|---------|
| Lint & type-check | Yes | ruff + ESLint + vue-tsc |
| Unit tests | Yes | pytest + Vitest |
| Dep audit | Yes (CRITICAL) | pip-audit + npm audit |
| Audit checks | Yes | `profiles/fastapi-vue/commands.sh` |
| Docker build | Yes | Both targets (remote + local) |
| Docker smoke | Yes | Start container, verify `/api/health` |
| Image scan (Trivy) | Yes (CRITICAL) | HIGH = warning annotation |
| Image size | No (warning) | Delta vs previous release, alert if > 10% |
| E2E API | Yes | `@smoke,@regression,@e2e` (full scope) |
| E2E UI | Yes | `@critical` |
**Verdict**: posted as a comment on the release PR:
- **GO** — all checks pass
- **GO CONDITIONAL** — blocking checks pass, dep audit or audit checks have warnings
- **NO-GO** — at least one blocking check failed
### How to invoke
```
# Release gate runs automatically — no manual trigger needed
# Just push to release/* or open a PR release/* → main
# Prepare a release
Prépare la release X.Y.Z en suivant CONTRIBUTING.md#release-process
# Deploy
Déploie en suivant docs/release/deployment-checklist.md
# Rollback
Rollback en suivant docs/release/rollback-playbook.md
```
---
## Operations
| # | Process | Trigger / Command | Doc | Output |
|---|---------|-------------------|-----|--------|
| 14 | **Incident Response** | Service down or degraded | [incident-response.md](operations/incident-response.md) | Mitigation + post-mortem |
| 15 | **Security Vulnerability** | Vuln reported | [security-response.md](operations/security-response.md) | Fix + advisory |
| 16 | **Monitoring Setup** | New deployment | [monitoring-checklist.md](operations/monitoring-checklist.md) | Monitoring configured |
### How to invoke
```
# Incident
Gère l'incident SEV-1 en suivant docs/operations/incident-response.md
# Security vulnerability
Traite la vulnérabilité signalée en suivant docs/operations/security-response.md
```
---
## Community & Governance
| # | Process | Trigger / Command | Doc | Output |
|---|---------|-------------------|-----|--------|
| 17 | **Onboarding** | New contributor | [onboarding-guide.md](community/onboarding-guide.md) | First PR merged |
| 18 | **Issue Triage** | New issue opened | [issue-triage-process.md](community/issue-triage-process.md) | Labeled + prioritized issue |
| 19 | **Roadmap Update** | Release cycle planning | [roadmap-template.md](community/roadmap-template.md) | Updated roadmap |
### How to invoke
```
# Triage issues
Trie les issues ouvertes en suivant docs/community/issue-triage-process.md
# Update roadmap
Mets à jour la roadmap en suivant docs/community/roadmap-template.md
```
---
## Quick Reference
| Category | Processes | Key doc |
|----------|-----------|---------|
| **Dev** | Commit, Review, Merge, ADR | `docs/git-workflow/` |
| **Quality** | Audit (full/single/re-audit), Auto-checks | `docs/audit/master.md` |
| **Release** | Release, Deploy, Rollback, Hotfix | `docs/release/` + `CONTRIBUTING.md` |
| **Ops** | Incident, Security, Monitoring | `docs/operations/` |
| **Community** | Onboarding, Triage, Roadmap | `docs/community/` |
---
## Standards & References
These are not processes but reference documents used by the processes above:
| Document | Purpose |
|----------|---------|
| [CONTRIBUTING.md](../CONTRIBUTING.md) | Dev setup, branching, release, versioning |
| [coding-standards.md](architecture/coding-standards.md) | Naming, style, architecture rules |
| [e2e/CONVENTIONS.md](../e2e/CONVENTIONS.md) | Karate / Karate UI test conventions |
| [architecture.md](architecture.md) | System architecture (backend + frontend) |
| [CODE_OF_CONDUCT.md](../CODE_OF_CONDUCT.md) | Community behavior standards |
| [SECURITY.md](../SECURITY.md) | Vulnerability reporting policy |
| [profiles/fastapi-vue/profile.md](../profiles/fastapi-vue/profile.md) | Stack layer mapping for audits |

52
docs/architecture/adr-guide.md Normal file
View file

@ -0,0 +1,52 @@
# Architecture Decision Records (ADR) — Guide
## What is an ADR?
An ADR is a short document that captures a significant architectural decision, along with the context and consequences. ADRs create a **decision log** so future contributors understand *why* the codebase looks the way it does.
## When to Write an ADR
Write an ADR when:
- Choosing or replacing a framework, library, or tool
- Changing the architecture (new layer, new pattern, new boundary)
- Making a trade-off that future developers might question
- Deciding NOT to do something (these are often the most valuable)
Do NOT write an ADR for:
- Implementation details that are obvious from the code
- Formatting or style choices (those go in [coding-standards.md](coding-standards.md))
- Bug fixes or minor refactors
## How to Write an ADR
1. Copy `adr-template.md` to `docs/architecture/adrs/ADR-NNN-short-title.md`
2. Number sequentially (ADR-001, ADR-002, ...)
3. Fill in all sections — especially **Context** (the *why*) and **Alternatives Considered**
4. Set status to `Proposed`
5. Open a PR for team review
6. Once merged, update status to `Accepted`
## Existing Decisions (captured retroactively)
These decisions were made before the ADR process was introduced. They are documented here for context:
| Decision | Rationale | Date |
|----------|-----------|------|
| Clean Architecture (hexagonal) for backend | Decouple domain from framework — enable converter swapping (local/remote) | 2025-01 |
| FastAPI over Django | Async-first, lightweight, Pydantic-native — better fit for a single-purpose API | 2025-01 |
| Vue 3 + Pinia over React | Composition API + built-in reactivity — smaller bundle for this use case | 2025-01 |
| SQLite over PostgreSQL | Single-file DB, zero ops — appropriate for a document processing tool | 2025-01 |
| Karate over Playwright for e2e | Team expertise, unified API+UI testing, JVM ecosystem | 2026-03 |
| Feature flags via `/api/health` | No external service needed — backend capabilities drive frontend UI | 2026-03 |
## Lifecycle
```
Proposed → Accepted → [Deprecated | Superseded by ADR-NNN]
```
- **Deprecated**: The decision is no longer relevant (feature removed, context changed)
- **Superseded**: A newer ADR replaces this one (link to the new ADR)
- Never delete an ADR — mark it as deprecated/superseded instead

39
docs/architecture/adr-template.md Normal file
View file

@ -0,0 +1,39 @@
# ADR-NNN: [Title]
**Date**: YYYY-MM-DD
**Status**: Proposed | Accepted | Deprecated | Superseded by ADR-NNN
**Deciders**: [names]
## Context
<!-- What is the issue we're seeing that is motivating this decision? What forces are at play? -->
## Decision
<!-- What is the change we're proposing and/or doing? -->
## Consequences
### Positive
- ...
### Negative
- ...
### Neutral
- ...
## Alternatives Considered
### Alternative 1: [Name]
- **Pros**: ...
- **Cons**: ...
- **Why rejected**: ...
## References
- [Link to issue, RFC, or discussion]

88
docs/architecture/coding-standards.md Normal file
View file

@ -0,0 +1,88 @@
# Coding Standards
Conventions for writing consistent, readable code across the Docling Studio codebase.
## Python (Backend — `document-parser/`)
### Tooling
| Tool | Purpose | Config |
|------|---------|--------|
| **Ruff** | Linting + formatting | `ruff.toml` / `pyproject.toml` |
| **pytest** | Testing | `pytest.ini` / `pyproject.toml` |
| **mypy** (optional) | Type checking | — |
### Naming
| Element | Convention | Example |
|---------|-----------|---------|
| Modules | `snake_case` | `analysis_repo.py` |
| Classes | `PascalCase` | `AnalysisJob`, `DocumentConverter` |
| Functions / methods | `snake_case` | `create_analysis()` |
| Constants | `UPPER_SNAKE_CASE` | `MAX_CONCURRENT_ANALYSES` |
| Private | `_leading_underscore` | `_build_converter()` |
### Style Rules
- Max function length: **30 lines** (soft limit — justify longer ones)
- Max file length: **300 lines** (split into modules if exceeded)
- Imports: standard library → third-party → local, separated by blank lines
- Type hints on all public functions
- Docstrings only on non-obvious public APIs (don't state the obvious)
- No `# type: ignore` without a comment explaining why
### Architecture Rules
- **Domain layer** (`domain/`): zero imports from `api/`, `persistence/`, `infra/`
- **Persistence layer** (`persistence/`): only imports from `domain/`
- **API layer** (`api/`): never imports from `persistence/` directly — goes through `services/`
- **Services** (`services/`): orchestrate, don't implement — delegate to domain and infra
## TypeScript / Vue (Frontend — `frontend/src/`)
### Tooling
| Tool | Purpose | Config |
|------|---------|--------|
| **ESLint** | Linting | `.eslintrc.*` |
| **Prettier** | Formatting | `.prettierrc` |
| **vue-tsc** | Type checking | `tsconfig.json` |
| **Vitest** | Testing | `vitest.config.ts` |
### Naming
| Element | Convention | Example |
|---------|-----------|---------|
| Components | `PascalCase.vue` | `BboxOverlay.vue` |
| Composables | `useCamelCase.ts` | `usePagination.ts` |
| Stores | `camelCase.ts` | `analysisStore.ts` |
| Types / Interfaces | `PascalCase` | `AnalysisJob`, `BboxRect` |
| Constants | `UPPER_SNAKE_CASE` | `DEFAULT_PAGE_SIZE` |
| CSS classes | `kebab-case` | `.bbox-overlay` |
| `data-e2e` attributes | `kebab-case` | `data-e2e="upload-zone"` |
### Style Rules
- **Composition API** only (`<script setup lang="ts">`) — no Options API
- One component per file
- Props defined with `defineProps<T>()` (type-based, not runtime)
- Emits defined with `defineEmits<T>()`
- Pinia stores: one per feature, in the feature directory
- No global state outside Pinia
- API calls only in `api.ts` files (never in components or stores directly)
### API Contract
- Frontend sends/receives **camelCase** (Pydantic `alias_generator`)
- Backend uses **snake_case** internally
- `pages_json` is an exception — contains raw snake_case from `dataclasses.asdict()`
## Karate (E2E — `e2e/`)
See [e2e/CONVENTIONS.md](../../e2e/CONVENTIONS.md) for detailed rules.
Key points:
- Use `data-e2e` selectors, never CSS classes
- Use `retry()`/`waitFor()`, never `Thread.sleep()` or `delay()`
- Setup via API, verify via UI, cleanup via API
- Tag tests: `@critical`, `@ui`, `@smoke`, `@regression`, `@e2e`

85
docs/community/issue-triage-process.md Normal file
View file

@ -0,0 +1,85 @@
# Issue Triage Process
How we classify, prioritize, and manage GitHub issues.
## Labels
### Type
| Label | Description |
|-------|-------------|
| `bug` | Something is broken |
| `feature` | New functionality |
| `enhancement` | Improvement to existing functionality |
| `docs` | Documentation only |
| `question` | Support / how-to question |
| `chore` | Tooling, CI, dependencies |
### Priority
| Label | Response SLA | Fix SLA | Description |
|-------|-------------|---------|-------------|
| `priority: P0` | Same day | < 3 days | Critical — service down, data loss, security |
| `priority: P1` | < 3 days | < 2 weeks | High — major feature broken, no workaround |
| `priority: P2` | < 1 week | Next release | Medium — feature degraded, workaround exists |
| `priority: P3` | < 2 weeks | Backlog | Low — minor, cosmetic, nice-to-have |
### Status
| Label | Description |
|-------|-------------|
| `needs-info` | Waiting for reporter to provide more details |
| `confirmed` | Bug reproduced or feature accepted |
| `good-first-issue` | Suitable for new contributors |
| `help-wanted` | Open for community contribution |
| `wont-fix` | Intentional behavior, out of scope, or won't be addressed |
| `duplicate` | Already tracked in another issue |
| `stale` | No activity for 30 days |
### Component
| Label | Maps to |
|-------|---------|
| `component: backend` | `document-parser/` |
| `component: frontend` | `frontend/` |
| `component: e2e` | `e2e/` |
| `component: docker` | Docker / docker-compose |
| `component: ci` | `.github/workflows/` |
## Triage Workflow
```
New issue
│
├─ Missing info? → label `needs-info`, comment asking for details
│ (auto-close after 14 days if no response)
│
├─ Duplicate? → label `duplicate`, link to original, close
│
├─ Out of scope? → label `wont-fix`, explain why, close
│
└─ Valid issue
│
├─ Add type label (bug / feature / enhancement / ...)
├─ Add component label
├─ Assess priority (P0 / P1 / P2 / P3)
├─ If simple → add `good-first-issue`
└─ Assign to milestone (if applicable)
```
## Stale Policy
| Condition | Action |
|-----------|--------|
| No activity for **30 days** | Bot labels `stale` + comment |
| No activity for **14 more days** | Bot closes the issue |
| Reporter responds | `stale` label removed, timer resets |
Issues labeled `priority: P0` or `priority: P1` are exempt from the stale policy.
## Response Expectations
- **Every issue gets a response** (even if it's "thanks, we'll look at this next week")
- Acknowledge within the SLA for the priority level
- If you can't fix it soon, say so — don't leave reporters hanging
- Close issues with a comment explaining the resolution

120
docs/community/onboarding-guide.md Normal file
View file

@ -0,0 +1,120 @@
# Onboarding Guide — First Contribution
Welcome! This guide helps you go from zero to your first merged PR.
## Prerequisites
| Tool | Version | Check |
|------|---------|-------|
| Python | 3.12+ | `python --version` |
| Node.js | 20+ | `node --version` |
| Docker & Docker Compose | latest | `docker compose version` |
| Git | 2.x | `git --version` |
| Java (for e2e) | 17+ | `java --version` |
| Maven (for e2e) | 3.9+ | `mvn --version` |
## Step 1 — Fork & Clone
```bash
# Fork on GitHub, then:
git clone https://github.com/<your-username>/Docling-Studio.git
cd Docling-Studio
git remote add upstream https://github.com/scub-france/Docling-Studio.git
```
## Step 2 — Run the Stack
The fastest way to see the app running:
```bash
docker compose up -d --wait
open http://localhost:3000
```
## Step 3 — Set Up for Development
### Backend
```bash
cd document-parser
python -m venv .venv && source .venv/bin/activate
pip install -r requirements.txt # remote mode (lightweight)
pip install ruff pytest pytest-asyncio httpx
uvicorn main:app --reload --port 8000
```
### Frontend
```bash
cd frontend
npm install
npm run dev # http://localhost:5173
```
## Step 4 — Pick Your First Issue
Look for issues labeled:
- `good-first-issue` — small, well-scoped, mentored
- `help-wanted` — we need help but it may be larger
- `docs` — documentation improvements (great starting point)
## Step 5 — Create a Branch
```bash
git checkout main && git pull upstream main
git checkout -b feature/my-change # or fix/my-fix
```
Follow the [branching strategy](../../CONTRIBUTING.md#branching-strategy).
## Step 6 — Code
- Read the [architecture docs](../architecture.md) to understand the codebase
- Follow the [coding standards](../architecture/coding-standards.md)
- Write tests for your changes
## Step 7 — Verify
```bash
# Backend
cd document-parser
ruff check . && ruff format --check .
pytest tests/ -v
# Frontend
cd frontend
npm run type-check
npx eslint src/
npm run test:run
```
## Step 8 — Commit & Push
Follow [commit conventions](../git-workflow/commit-conventions.md):
```bash
git add <files>
git commit -m "feat(scope): short description"
git push origin feature/my-change
```
## Step 9 — Open a PR
- Target: `main` (or `release/*` for pre-release fixes)
- Fill in the [PR template](../../.github/PULL_REQUEST_TEMPLATE.md)
- Add a line in `CHANGELOG.md` under `[Unreleased]`
- Wait for CI to pass, then request a review
## What to Expect
- A maintainer will review your PR within **3 business days**
- You may get feedback — this is normal and helpful
- Once approved, a maintainer will merge your PR
- Your contribution appears in the next release's changelog
## Need Help?
- Open a [Discussion](https://github.com/scub-france/Docling-Studio/discussions) on GitHub
- Check existing issues for similar questions
- Read the [CONTRIBUTING guide](../../CONTRIBUTING.md) for detailed rules

44
docs/community/roadmap-template.md Normal file
View file

@ -0,0 +1,44 @@
# Roadmap — Docling Studio
Last updated: YYYY-MM-DD
## Now (current release cycle)
<!-- Features actively being worked on -->
| Feature | Issue | Status |
|---------|-------|--------|
| ... | #NNN | In progress |
## Next (next release cycle)
<!-- Planned for the next release, prioritized -->
| Feature | Issue | Priority |
|---------|-------|----------|
| ... | #NNN | P1 |
## Later (future, no timeline)
<!-- Ideas accepted but not yet scheduled -->
| Feature | Issue |
|---------|-------|
| ... | #NNN |
## Not Planned
<!-- Explicitly declined or out of scope — explaining why avoids repeat requests -->
| Feature | Reason |
|---------|--------|
| ... | ... |
---
## How to Propose a Feature
1. Check [existing issues](https://github.com/scub-france/Docling-Studio/issues) — it may already be tracked
2. Open a new issue using the **Feature** template
3. If it's a significant change, consider writing an [ADR](../architecture/adr-guide.md)
4. The maintainers will triage and prioritize (see [issue triage](issue-triage-process.md))

28
docs/contributing.md
View file

@ -76,6 +76,34 @@ npx prettier --write src/ # auto-format
npm run test:run
```
=== "E2E API (Karate)"
```bash
# Generate test PDFs + start stack
python e2e/generate-test-data.py
docker compose up -d --wait
# Run all API tests
mvn test -f e2e/api/pom.xml
# Or by tag: @smoke, @regression, @e2e
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @smoke"
```
=== "E2E UI (Karate UI)"
```bash
# Generate test PDFs + start stack (if not already running)
python e2e/generate-test-data.py
docker compose up -d --wait
# Run critical UI tests (CI scope)
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @critical"
# Run all UI tests (local scope)
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @ui"
```
All tests must pass before submitting a PR.
## Pull Request Guidelines

54
docs/git-workflow/code-review-checklist.md Normal file
View file

@ -0,0 +1,54 @@
# Code Review Checklist
Use this checklist when reviewing a Pull Request. Not every item applies to every PR — skip what is irrelevant, but consciously skip it rather than miss it.
## Correctness
- [ ] The code does what the PR description says it does
- [ ] Edge cases are handled (empty input, missing data, concurrent access)
- [ ] Error paths return meaningful messages, not stack traces
- [ ] No regressions on existing behavior
## Architecture & Design
- [ ] Dependencies flow inward: `api → services → domain` (never reversed)
- [ ] Domain layer has no imports from `api/`, `persistence/`, `infra/`
- [ ] No business logic in API routes — delegated to services
- [ ] New abstractions are justified (no premature generalization)
- [ ] Pinia stores stay within their feature boundary
## Security
- [ ] User input is validated at the API boundary (Pydantic schemas)
- [ ] No secrets, API keys, or credentials in the code
- [ ] No `eval()`, `exec()`, or raw SQL
- [ ] File paths are sanitized (no path traversal)
- [ ] CORS configuration is unchanged or explicitly justified
## Tests
- [ ] New behavior has corresponding tests
- [ ] Tests are deterministic (no `sleep`, no random, no network)
- [ ] Test names describe the scenario, not the implementation
- [ ] E2E tests use `data-e2e` attributes, not CSS classes (see [e2e/CONVENTIONS.md](../../e2e/CONVENTIONS.md))
## Code Quality
- [ ] No dead code, no commented-out code
- [ ] No `TODO` or `FIXME` without a linked issue
- [ ] Functions are < 30 lines (or well-justified)
- [ ] Variable names are descriptive and consistent with existing code
- [ ] No duplicated logic that should be extracted
## Documentation
- [ ] `CHANGELOG.md` updated under `[Unreleased]` if user-facing change
- [ ] API changes reflected in Pydantic schemas (auto-documented)
- [ ] Breaking changes are flagged in the commit message
## Pragmatic Checks
- [ ] The PR does ONE thing (feature, fix, or refactor — not all at once)
- [ ] No unrelated formatting changes mixed in
- [ ] The diff is reviewable in < 15 minutes (split large PRs)
- [ ] CI passes (tests + lint + type-check + build)

73
docs/git-workflow/commit-conventions.md Normal file
View file

@ -0,0 +1,73 @@
# Commit Conventions
We follow [Conventional Commits](https://www.conventionalcommits.org/) to keep the git history readable and to enable automated changelog generation.
## Format
```
<type>(<scope>): <description>
[optional body]
[optional footer(s)]
```
## Types
| Type | When to use | Example |
|------|-------------|---------|
| `feat` | New feature | `feat(chunking): add hierarchical chunker support` |
| `fix` | Bug fix | `fix(upload): handle empty PDF gracefully` |
| `docs` | Documentation only | `docs: update architecture diagram` |
| `style` | Formatting, no logic change | `style(api): fix ruff formatting warnings` |
| `refactor` | Code restructuring, no behavior change | `refactor(persistence): extract repository base class` |
| `test` | Adding or updating tests | `test(analysis): add rechunk edge case tests` |
| `chore` | Tooling, CI, dependencies | `chore: bump docling to 2.31` |
| `perf` | Performance improvement | `perf(bbox): batch coordinate normalization` |
| `ci` | CI/CD pipeline changes | `ci: add multi-arch Docker build` |
## Scopes (optional)
Use the feature or component name:
| Scope | Maps to |
|-------|---------|
| `api` | `document-parser/api/` |
| `domain` | `document-parser/domain/` |
| `persistence` | `document-parser/persistence/` |
| `infra` | `document-parser/infra/` |
| `upload` | Upload feature (front + back) |
| `analysis` | Analysis feature (front + back) |
| `chunking` | Chunking feature (front + back) |
| `bbox` | Bounding box pipeline |
| `e2e` | `e2e/` tests |
| `docker` | Dockerfile, docker-compose |
| `ci` | `.github/workflows/` |
## Rules
1. **Subject line** — imperative mood, lowercase, no period, max 72 characters
2. **Body** — explain *why*, not *what* (the diff shows what)
3. **Breaking changes** — add `BREAKING CHANGE:` in the footer or `!` after the type: `feat(api)!: rename /analyses to /jobs`
4. **Issue references** — use `Closes #123` or `Fixes #456` in the footer
## Examples
```
feat(chunking): add page filtering in Prepare mode
Users can now select which pages to include in chunking.
The filter is persisted in the analysis job metadata.
Closes #87
```
```
fix(upload): reject files exceeding MAX_FILE_SIZE_MB
Previously, oversized files were accepted and failed silently
during Docling conversion. Now the API returns 413 with a
clear error message.
Fixes #102
```

53
docs/git-workflow/merge-policy.md Normal file
View file

@ -0,0 +1,53 @@
# Merge Policy
## Merge Requirements
Every PR must meet these conditions before merging:
1. **CI green** — all checks pass (tests, lint, type-check, build)
2. **At least 1 approval** from a maintainer
3. **No unresolved conversations** — all review comments addressed
4. **Branch up to date** with target branch (rebase or merge from target)
5. **CHANGELOG updated** — if the change is user-facing
## Merge Strategy
| Branch type | Merge method | Why |
|-------------|-------------|-----|
| `feature/*` → `main` | **Squash merge** | Clean history — one commit per feature |
| `fix/*` → `main` | **Squash merge** | Clean history — one commit per fix |
| `fix/*` → `release/*` | **Squash merge** | Same rationale |
| `release/*` → `main` | **Merge commit** | Preserves the release branch history |
| `hotfix/*` → `main` | **Squash merge** | One atomic fix |
### Squash merge commit message
When squashing, the final commit message should follow [Conventional Commits](commit-conventions.md):
```
feat(chunking): add page filtering in Prepare mode (#87)
```
The PR number is appended automatically by GitHub.
## Stale PR Policy
| Condition | Action |
|-----------|--------|
| No activity for **14 days** | Maintainer pings the author |
| No activity for **30 days** | PR labeled `stale` |
| No activity for **45 days** | PR closed with comment (can be reopened) |
Draft PRs are exempt from the stale policy.
## Conflict Resolution
- **Rebase preferred** over merge commits for feature/fix branches
- If conflicts are complex, merge from target branch into the PR branch
- Never force-push a branch that has active reviewers without warning them
## Branch Cleanup
- Feature and fix branches are **deleted after merge** (GitHub auto-delete enabled)
- Release branches are kept until the next minor release
- Tags are never deleted

93
docs/operations/incident-response.md Normal file
View file

@ -0,0 +1,93 @@
# Incident Response
## Severity Levels
| Level | Description | Examples | Response time |
|-------|-------------|----------|---------------|
| **SEV-1** | Service down, data loss, security breach | App unreachable, DB corrupted, credentials leaked | Immediate |
| **SEV-2** | Major feature broken, degraded for all users | Upload fails, analysis crashes, blank pages | < 2 hours |
| **SEV-3** | Minor feature broken, workaround exists | Bbox overlay misaligned, locale missing a key | Next business day |
## Response Steps
### 1. Detect
- Health endpoint returns error (`/api/health`)
- User report via GitHub issue
- CI/CD pipeline failure on `main`
- Docker container crash loop
### 2. Assess
- Determine severity (SEV-1/2/3)
- Identify affected component: backend, frontend, Docker, CI
- Check recent deployments: `git log --oneline -10 main`
### 3. Communicate
- **SEV-1**: Notify all maintainers immediately
- **SEV-2**: Open a GitHub issue with `priority: P0` label
- **SEV-3**: Open a GitHub issue with `priority: P1` label
### 4. Mitigate
**Rollback first, investigate later.**
- If the issue appeared after a deploy → [rollback](../release/rollback-playbook.md)
- If the issue is in a specific endpoint → disable the route or return a maintenance response
- If the issue is in Docling itself → switch to remote mode (`CONVERSION_ENGINE=remote`)
### 5. Fix
- Create a `hotfix/*` branch from the last stable tag
- Fix the root cause, add a regression test
- Run the [release audit](../audit/master.md) on the fix
- Deploy via the [deployment checklist](../release/deployment-checklist.md)
### 6. Post-Mortem
Write a post-mortem for SEV-1 and SEV-2 incidents using this template:
```markdown
# Post-Mortem: [Incident Title]
**Date**: YYYY-MM-DD
**Severity**: SEV-X
**Duration**: Xh Xm
**Author**: [name]
## Timeline
| Time | Event |
|------|-------|
| HH:MM | Incident detected |
| HH:MM | Severity assessed |
| HH:MM | Mitigation applied |
| HH:MM | Root cause identified |
| HH:MM | Fix deployed |
| HH:MM | Incident resolved |
## Root Cause
[What actually broke and why]
## Impact
[Who was affected, for how long, what data was at risk]
## What Went Well
- ...
## What Went Wrong
- ...
## Action Items
| Action | Owner | Due |
|--------|-------|-----|
| ... | ... | ... |
```
Store post-mortems in `docs/operations/post-mortems/YYYY-MM-DD-short-title.md`.

105
docs/operations/monitoring-checklist.md Normal file
View file

@ -0,0 +1,105 @@
# Monitoring Checklist
What to monitor in a Docling Studio deployment.
## Health Endpoint
The primary monitoring signal is the health endpoint:
```bash
curl -s http://localhost:3000/api/health
```
Expected response:
```json
{
"status": "ok",
"engine": "local",
"version": "0.3.0",
"deploymentMode": "self-hosted"
}
```
**Alert if**: status != "ok", endpoint unreachable, or response time > 5s.
## Four Golden Signals
### 1. Latency
| Endpoint | Expected | Alert threshold |
|----------|----------|-----------------|
| `GET /api/health` | < 100ms | > 1s |
| `POST /api/documents` (upload) | < 2s | > 10s |
| `POST /api/analyses` (create) | < 500ms (queuing only) | > 5s |
| `GET /api/analyses/:id` (results) | < 500ms | > 3s |
### 2. Traffic
| Metric | What to watch |
|--------|---------------|
| Requests per minute | Baseline for normal usage |
| Uploads per hour | Capacity planning |
| Concurrent analyses | Should stay <= `MAX_CONCURRENT_ANALYSES` |
### 3. Errors
| Signal | Alert threshold |
|--------|-----------------|
| HTTP 5xx rate | > 1% of requests |
| Analysis failure rate | > 10% of analyses |
| Rate limit hits (429) | Spike = possible abuse |
### 4. Saturation
| Resource | Check command | Alert threshold |
|----------|---------------|-----------------|
| CPU | `docker stats` | > 90% sustained |
| Memory | `docker stats` | > 85% (especially in local mode with PyTorch) |
| Disk (SQLite + uploads) | `du -sh data/` | > 80% of volume |
| Docker container restarts | `docker inspect --format='{{.RestartCount}}'` | > 0 |
## Docker Health Check
The `docker-compose.yml` includes a built-in health check:
```yaml
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000/api/health"]
interval: 30s
timeout: 10s
retries: 3
```
Docker will mark the container as `unhealthy` after 3 consecutive failures.
## Log Monitoring
### Backend logs (uvicorn)
```bash
docker compose logs -f backend
```
Watch for:
- `ERROR` or `CRITICAL` log levels
- `TimeoutError` from Docling processing
- `sqlite3.OperationalError` (DB issues)
- `429 Too Many Requests` spikes
### Frontend logs (nginx)
```bash
docker compose logs -f frontend
```
Watch for:
- `502 Bad Gateway` (backend down)
- `413 Request Entity Too Large` (file size limit)
## Recommended Setup
For production deployments, consider:
1. **Uptime monitor** — ping `/api/health` every 60s (UptimeRobot, Healthchecks.io)
2. **Log aggregation** — ship Docker logs to a central service
3. **Alerting** — notify on container restart, health check failure, or error spike

74
docs/operations/security-response.md Normal file
View file

@ -0,0 +1,74 @@
# Security Vulnerability Response
Process for handling reported security vulnerabilities. See also [SECURITY.md](../../SECURITY.md) for the public-facing policy.
## Response Timeline
| Step | SLA | Action |
|------|-----|--------|
| **Acknowledge** | < 48h | Confirm receipt to the reporter |
| **Assess** | < 7 days | Determine severity (Critical / High / Medium / Low) |
| **Fix** | < 14 days (Critical), < 30 days (other) | Develop and test the fix |
| **Release** | Same day as fix | Publish patched version |
| **Disclose** | After release | Publish GitHub Security Advisory |
## Severity Assessment
| Severity | Criteria | Example |
|----------|----------|---------|
| **Critical** | Remote exploitation, data breach, no auth required | SQL injection in upload endpoint |
| **High** | Significant impact, some conditions required | Path traversal on file download |
| **Medium** | Limited impact or requires authenticated access | XSS in analysis results display |
| **Low** | Minimal impact, theoretical only | Information disclosure in error messages |
## Fix Process
1. **Create a private branch** — never push vulnerability details to a public branch before the fix is released
2. **Develop the fix** — include a regression test
3. **Run the security audit** — `docs/audit/audits/08-security.md`
4. **Review** — at least one maintainer must review the fix
5. **Release** — tag, build, deploy (see [deployment checklist](../release/deployment-checklist.md))
6. **Publish advisory** — GitHub Security Advisory with CVE if applicable
## Advisory Template
```markdown
# Security Advisory: [Title]
**Severity**: Critical | High | Medium | Low
**Affected versions**: < X.Y.Z
**Fixed in**: X.Y.Z
**CVE**: CVE-YYYY-NNNNN (if assigned)
## Description
[What the vulnerability is, without providing exploitation details]
## Impact
[What an attacker could do]
## Mitigation
[Upgrade to X.Y.Z or apply workaround]
## Credit
[Reporter name, unless they prefer anonymity]
```
## Dependency Vulnerabilities
Run regular dependency audits:
```bash
# Backend
cd document-parser && pip audit
# Frontend
cd frontend && npm audit
```
For known vulnerabilities in dependencies:
- **Critical/High**: Update immediately, release a patch version
- **Medium/Low**: Include in the next planned release

70
docs/release/deployment-checklist.md Normal file
View file

@ -0,0 +1,70 @@
# Deployment Checklist
Checklist for deploying a new release of Docling Studio. Applies to both self-hosted and Hugging Face Space deployments.
## Pre-Deploy
- [ ] Release branch merged to `main` via PR
- [ ] Git tag `vX.Y.Z` created on `main`
- [ ] Release audit passed (score >= 80, 0 CRITICAL) — see [docs/audit/master.md](../audit/master.md)
- [ ] `CHANGELOG.md` section finalized with release date
- [ ] `frontend/package.json` version matches the tag
- [ ] All CI checks green on the tagged commit
- [ ] Docker images built and pushed to `ghcr.io`:
- `X.Y.Z-remote`, `X.Y.Z-local`
- `X.Y-remote`, `X.Y-local`
- `latest-remote`, `latest-local`
## Deploy — Self-Hosted (Docker Compose)
- [ ] Pull the new image:
```bash
docker compose pull
```
- [ ] Check environment variables (`.env` or `docker-compose.override.yml`):
- `CONVERSION_ENGINE` (local / remote)
- `RATE_LIMIT_RPM`
- `MAX_FILE_SIZE_MB`
- `MAX_CONCURRENT_ANALYSES`
- [ ] Start the stack:
```bash
docker compose up -d --wait
```
- [ ] Verify health endpoint:
```bash
curl -s http://localhost:3000/api/health | jq .
# Expected: {"status":"ok","engine":"...","version":"X.Y.Z","deploymentMode":"self-hosted"}
```
## Deploy — Hugging Face Space
- [ ] Upload to HF Space via `huggingface-cli`:
```bash
huggingface-cli upload <space-id> . . --repo-type space
```
- [ ] Set environment variables in HF Space settings
- [ ] Wait for build to complete in HF Space logs
- [ ] Verify the app loads and health endpoint returns correct version
## Post-Deploy Smoke Test
- [ ] Home page loads
- [ ] Upload a PDF — document appears in the list
- [ ] Run an analysis — completes without error
- [ ] View results — markdown, HTML, bbox overlays render correctly
- [ ] Download results
- [ ] If local mode: test chunking
- [ ] Check `/api/health` returns the new version
## Rollback Triggers
Rollback immediately if any of these occur:
| Trigger | Action |
|---------|--------|
| Health endpoint returns error or wrong version | Rollback |
| Upload or analysis fails on a previously working PDF | Rollback |
| Frontend shows blank page or JS errors | Rollback |
| Error rate > 5% in the first 15 minutes | Rollback |
For rollback procedure, see [rollback-playbook.md](rollback-playbook.md).

83
docs/release/rollback-playbook.md Normal file
View file

@ -0,0 +1,83 @@
# Rollback Playbook
## Decision: Rollback vs Hotfix?
| Situation | Strategy |
|-----------|----------|
| Broken deploy, previous version was stable | **Rollback** |
| Bug found but previous version also had issues | **Hotfix** |
| Data migration was applied (DB schema changed) | **Hotfix** (rollback may lose data) |
| Security vulnerability in the new release | **Rollback** + hotfix in parallel |
## Rollback Procedure — Docker Compose
### 1. Identify the last known good version
```bash
# List available tags
docker image ls ghcr.io/scub-france/docling-studio --format '{{.Tag}}' | sort -V
# Or check GitHub releases
gh release list --repo scub-france/Docling-Studio
```
### 2. Pin the image to the previous version
Edit `docker-compose.yml` or `docker-compose.override.yml`:
```yaml
services:
backend:
image: ghcr.io/scub-france/docling-studio:0.2.0-remote # pin to last good
frontend:
image: ghcr.io/scub-france/docling-studio:0.2.0-remote
```
### 3. Restart
```bash
docker compose down
docker compose up -d --wait
```
### 4. Verify
```bash
curl -s http://localhost:3000/api/health | jq .
# Confirm version matches the rolled-back version
```
### 5. Communicate
- Notify the team that a rollback was performed
- Open an issue describing the failure
- Link the failed release and the rollback commit
## Rollback Procedure — Hugging Face Space
1. Use `git revert` on the HF Space repo to revert to the previous commit
2. Or re-upload the previous version: `huggingface-cli upload <space-id> . . --repo-type space --revision <previous-commit>`
3. Verify the app loads correctly
## Database Considerations
Docling Studio uses **SQLite** with a file-based database. Key points:
- **No schema migrations** are applied automatically — the schema is created on first run
- If a new release adds columns, rolling back may cause "unknown column" errors
- **Before deploying a release that changes the DB schema**: back up the SQLite file
```bash
# Backup before deploy
cp data/docling-studio.db data/docling-studio.db.bak-$(date +%Y%m%d)
# Restore after rollback
cp data/docling-studio.db.bak-YYYYMMDD data/docling-studio.db
```
## Post-Rollback
1. Keep the rollback in place until the root cause is identified
2. Fix the issue on a `hotfix/*` branch
3. Re-run the [release audit](../audit/master.md) on the fix
4. Re-deploy following the [deployment checklist](deployment-checklist.md)

2
document-parser/api/analyses.py
View file

@ -39,6 +39,8 @@ def _to_response(job) -> AnalysisResponse:
chunks_json=job.chunks_json,
has_document_json=job.document_json is not None,
error_message=job.error_message,
progress_current=job.progress_current,
progress_total=job.progress_total,
started_at=str(job.started_at) if job.started_at else None,
completed_at=str(job.completed_at) if job.completed_at else None,
created_at=str(job.created_at),

43
document-parser/api/documents.py
View file

@ -3,12 +3,13 @@
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, HTTPException, Query, UploadFile
from fastapi import APIRouter, Depends, HTTPException, Query, Request, UploadFile
from fastapi.responses import Response
from api.schemas import DocumentResponse
from services import document_service
from services.document_service import DocumentService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/documents", tags=["documents"])
@ -16,6 +17,13 @@ router = APIRouter(prefix="/api/documents", tags=["documents"])
_READ_CHUNK_SIZE = 64 * 1024 # 64 KB
def _get_service(request: Request) -> DocumentService:
return request.app.state.document_service
ServiceDep = Annotated[DocumentService, Depends(_get_service)]
def _to_response(doc) -> DocumentResponse:
return DocumentResponse(
id=doc.id,
@ -28,27 +36,29 @@ def _to_response(doc) -> DocumentResponse:
@router.post("/upload", response_model=DocumentResponse, status_code=200)
async def upload(file: UploadFile) -> DocumentResponse:
async def upload(file: UploadFile, service: ServiceDep) -> DocumentResponse:
"""Upload a PDF document."""
if not file.filename:
raise HTTPException(status_code=400, detail="No filename provided")
# Reject early if Content-Length exceeds limit (before reading body)
if file.size and file.size > document_service.MAX_FILE_SIZE:
raise HTTPException(status_code=413, detail="File too large (max 5 MB)")
_max = service.max_file_size
_detail = f"File too large (max {service.max_file_size_mb} MB)"
if _max > 0 and file.size and file.size > _max:
raise HTTPException(status_code=413, detail=_detail)
# Read in chunks to avoid holding the full upload in a single allocation
chunks: list[bytes] = []
total = 0
while chunk := await file.read(_READ_CHUNK_SIZE):
total += len(chunk)
if total > document_service.MAX_FILE_SIZE:
raise HTTPException(status_code=413, detail="File too large (max 5 MB)")
if _max > 0 and total > _max:
raise HTTPException(status_code=413, detail=_detail)
chunks.append(chunk)
content = b"".join(chunks)
try:
doc = await document_service.upload(
doc = await service.upload(
filename=file.filename,
content_type=file.content_type or "application/pdf",
file_content=content,
@ -60,25 +70,25 @@ async def upload(file: UploadFile) -> DocumentResponse:
@router.get("", response_model=list[DocumentResponse])
async def list_documents() -> list[DocumentResponse]:
async def list_documents(service: ServiceDep) -> list[DocumentResponse]:
"""List all documents."""
docs = await document_service.find_all()
docs = await service.find_all()
return [_to_response(d) for d in docs]
@router.get("/{doc_id}", response_model=DocumentResponse)
async def get_document(doc_id: str) -> DocumentResponse:
async def get_document(doc_id: str, service: ServiceDep) -> DocumentResponse:
"""Get a single document."""
doc = await document_service.find_by_id(doc_id)
doc = await service.find_by_id(doc_id)
if not doc:
raise HTTPException(status_code=404, detail="Document not found")
return _to_response(doc)
@router.delete("/{doc_id}", status_code=204)
async def delete_document(doc_id: str) -> None:
async def delete_document(doc_id: str, service: ServiceDep) -> None:
"""Delete a document and its file."""
deleted = await document_service.delete(doc_id)
deleted = await service.delete(doc_id)
if not deleted:
raise HTTPException(status_code=404, detail="Document not found")
@ -86,11 +96,12 @@ async def delete_document(doc_id: str) -> None:
@router.get("/{doc_id}/preview")
async def preview(
doc_id: str,
service: ServiceDep,
page: int = Query(1, ge=1),
dpi: int = Query(150, ge=72, le=300),
) -> Response:
"""Generate a PNG preview of a specific PDF page."""
doc = await document_service.find_by_id(doc_id)
doc = await service.find_by_id(doc_id)
if not doc:
raise HTTPException(status_code=404, detail="Document not found")
@ -103,7 +114,7 @@ async def preview(
try:
with open(doc.storage_path, "rb") as f:
file_content = f.read()
png_bytes = document_service.generate_preview(file_content, page=page, dpi=dpi)
png_bytes = DocumentService.generate_preview(file_content, page=page, dpi=dpi)
return Response(content=png_bytes, media_type="image/png")
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e)) from e

12
document-parser/api/schemas.py
View file

@ -26,6 +26,16 @@ class _CamelModel(BaseModel):
)
class HealthResponse(_CamelModel):
status: str
version: str
engine: str
deployment_mode: str
database: str
max_page_count: int | None = None
max_file_size_mb: int | None = None
class DocumentResponse(_CamelModel):
id: str
filename: str
@ -47,6 +57,8 @@ class AnalysisResponse(_CamelModel):
chunks_json: str | None = None
has_document_json: bool = False
error_message: str | None = None
progress_current: int | None = None
progress_total: int | None = None
started_at: str | datetime | None = None
completed_at: str | datetime | None = None
created_at: str | datetime

17
document-parser/domain/models.py
View file

@ -45,6 +45,8 @@ class AnalysisJob:
document_json: str | None = None
chunks_json: str | None = None
error_message: str | None = None
progress_current: int | None = None
progress_total: int | None = None
started_at: datetime | None = None
completed_at: datetime | None = None
created_at: datetime = field(default_factory=_utcnow)
@ -54,6 +56,8 @@ class AnalysisJob:
def mark_running(self) -> None:
"""Transition to RUNNING and record the start timestamp."""
if self.status != AnalysisStatus.PENDING:
raise ValueError(f"Cannot mark as RUNNING from {self.status} (expected PENDING)")
self.status = AnalysisStatus.RUNNING
self.started_at = _utcnow()
@ -66,6 +70,8 @@ class AnalysisJob:
chunks_json: str | None = None,
) -> None:
"""Transition to COMPLETED with conversion results."""
if self.status != AnalysisStatus.RUNNING:
raise ValueError(f"Cannot mark as COMPLETED from {self.status} (expected RUNNING)")
self.status = AnalysisStatus.COMPLETED
self.content_markdown = markdown
self.content_html = html
@ -74,8 +80,19 @@ class AnalysisJob:
self.chunks_json = chunks_json
self.completed_at = _utcnow()
def update_progress(self, current: int, total: int) -> None:
"""Update batch progress counters."""
if self.status != AnalysisStatus.RUNNING:
raise ValueError(f"Cannot update progress from {self.status} (expected RUNNING)")
self.progress_current = current
self.progress_total = total
def mark_failed(self, error: str) -> None:
"""Transition to FAILED with an error message."""
if self.status not in (AnalysisStatus.PENDING, AnalysisStatus.RUNNING):
raise ValueError(
f"Cannot mark as FAILED from {self.status} (expected PENDING or RUNNING)"
)
self.status = AnalysisStatus.FAILED
self.error_message = error
self.completed_at = _utcnow()

37
document-parser/domain/ports.py
View file

@ -9,6 +9,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Protocol
if TYPE_CHECKING:
from domain.models import AnalysisJob, Document
from domain.value_objects import (
ChunkingOptions,
ChunkResult,
@ -28,6 +29,8 @@ class DocumentConverter(Protocol):
self,
file_path: str,
options: ConversionOptions,
*,
page_range: tuple[int, int] | None = None,
) -> ConversionResult: ...
@ -42,3 +45,37 @@ class DocumentChunker(Protocol):
document_json: str,
options: ChunkingOptions,
) -> list[ChunkResult]: ...
class DocumentRepository(Protocol):
"""Port for document persistence."""
async def insert(self, doc: Document) -> None: ...
async def find_all(self, *, limit: int = 200, offset: int = 0) -> list[Document]: ...
async def find_by_id(self, doc_id: str) -> Document | None: ...
async def update_page_count(self, doc_id: str, page_count: int) -> None: ...
async def delete(self, doc_id: str) -> bool: ...
class AnalysisRepository(Protocol):
"""Port for analysis job persistence."""
async def insert(self, job: AnalysisJob) -> None: ...
async def find_all(self, *, limit: int = 200, offset: int = 0) -> list[AnalysisJob]: ...
async def find_by_id(self, job_id: str) -> AnalysisJob | None: ...
async def update_status(self, job: AnalysisJob) -> None: ...
async def update_progress(self, job_id: str, current: int, total: int) -> None: ...
async def update_chunks(self, job_id: str, chunks_json: str) -> bool: ...
async def delete(self, job_id: str) -> bool: ...
async def delete_by_document(self, document_id: str) -> int: ...

82
document-parser/domain/services.py Normal file
View file

@ -0,0 +1,82 @@
"""Domain services — pure business logic with no infrastructure dependencies."""
from __future__ import annotations
import re
from domain.value_objects import ConversionResult, PageDetail
# Regex to extract <body> content from Docling's well-formed HTML output.
_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", re.DOTALL | re.IGNORECASE)
def extract_html_body(html: str) -> str:
"""Extract content between <body> tags.
Docling produces well-formed HTML — regex is safe for this controlled output.
Returns raw html as fallback if no <body> tag is found.
"""
match = _BODY_RE.search(html)
return match.group(1).strip() if match else html
def merge_results(results: list[ConversionResult]) -> ConversionResult:
"""Merge multiple batch ConversionResults into a single consolidated result.
document_json is intentionally set to None: merging DoclingDocument's internal
tree structure across batches is error-prone. Re-chunking is disabled for
batched conversions (robustness decision for 0.3.1).
"""
if not results:
return ConversionResult(page_count=0, content_markdown="", content_html="", pages=[])
all_pages: list[PageDetail] = []
all_md: list[str] = []
html_bodies: list[str] = []
total_skipped = 0
for r in results:
all_pages.extend(r.pages)
all_md.append(r.content_markdown)
html_bodies.append(extract_html_body(r.content_html))
total_skipped += r.skipped_items
merged_body = "\n".join(html_bodies)
merged_html = (
f'<!DOCTYPE html><html><head><meta charset="utf-8"></head><body>{merged_body}</body></html>'
)
return ConversionResult(
page_count=sum(r.page_count for r in results),
content_markdown="\n\n".join(all_md),
content_html=merged_html,
pages=all_pages,
skipped_items=total_skipped,
document_json=None,
)
def classify_error(exc: Exception) -> str:
"""Return a user-friendly error message based on the exception type/content."""
msg = str(exc).lower()
if "invalidcxxcompiler" in msg or "no working c++ compiler" in msg:
return "Missing C++ compiler — set TORCHDYNAMO_DISABLE=1 to work around this"
if "out of memory" in msg or "oom" in msg:
return "Out of memory — try a smaller document or disable table structure analysis"
if "could not acquire converter lock" in msg:
return "Server busy — a previous conversion is still running. Please retry later"
if "pipeline" in msg and "failed" in msg:
return "Document processing failed — the document may be corrupted or unsupported"
if "timeout" in msg:
return "Processing took too long — try with fewer pages or simpler options"
# Fallback: truncate raw error to something reasonable
raw = str(exc)
if len(raw) > 200:
raw = raw[:200] + "…"
return raw

14
document-parser/domain/value_objects.py
View file

@ -9,7 +9,7 @@ from __future__ import annotations
from dataclasses import dataclass, field
@dataclass
@dataclass(frozen=True)
class PageElement:
type: str
bbox: list[float]
@ -17,7 +17,7 @@ class PageElement:
level: int = 0
@dataclass
@dataclass(frozen=True)
class PageDetail:
page_number: int
width: float
@ -25,7 +25,7 @@ class PageDetail:
elements: list[PageElement] = field(default_factory=list)
@dataclass
@dataclass(frozen=True)
class ConversionOptions:
do_ocr: bool = True
do_table_structure: bool = True
@ -43,7 +43,7 @@ class ConversionOptions:
return self == ConversionOptions()
@dataclass
@dataclass(frozen=True)
class ConversionResult:
page_count: int
content_markdown: str
@ -53,7 +53,7 @@ class ConversionResult:
document_json: str | None = None
@dataclass
@dataclass(frozen=True)
class ChunkingOptions:
chunker_type: str = "hybrid" # "hybrid", "hierarchical", "page"
max_tokens: int = 512
@ -65,13 +65,13 @@ class ChunkingOptions:
return self == ChunkingOptions()
@dataclass
@dataclass(frozen=True)
class ChunkBbox:
page: int
bbox: list[float] # [left, top, right, bottom] in TOPLEFT origin
@dataclass
@dataclass(frozen=True)
class ChunkResult:
text: str
headings: list[str] = field(default_factory=list)

44
document-parser/infra/local_converter.py
View file

@ -42,10 +42,12 @@ from domain.value_objects import (
PageElement,
)
from infra.bbox import to_topleft_list
from infra.settings import settings
logger = logging.getLogger(__name__)
# Thread lock — DoclingConverter is not thread-safe
# Thread lock — DoclingConverter is not thread-safe.
# Uses a timeout to prevent a frozen conversion from blocking all others.
_converter_lock = threading.Lock()
# US Letter page dimensions (points) — fallback when page size is unknown
@ -102,6 +104,7 @@ def _build_docling_converter(options: ConversionOptions) -> DoclingConverter:
generate_page_images=options.generate_page_images,
generate_picture_images=options.generate_picture_images,
images_scale=options.images_scale,
document_timeout=settings.document_timeout,
)
return DoclingConverter(
@ -111,16 +114,19 @@ def _build_docling_converter(options: ConversionOptions) -> DoclingConverter:
)
def _get_default_converter() -> DoclingConverter:
def _ensure_default_converter() -> DoclingConverter:
global _default_converter
if _default_converter is None:
_default_converter = _build_docling_converter(ConversionOptions())
try:
_default_converter = _build_docling_converter(ConversionOptions())
except Exception:
raise
return _default_converter
def _select_converter(options: ConversionOptions) -> DoclingConverter:
if options.is_default():
return _get_default_converter()
return _ensure_default_converter()
return _build_docling_converter(options)
@ -209,10 +215,30 @@ def _process_content_item(
# ---------------------------------------------------------------------------
def _convert_sync(file_path: str, options: ConversionOptions) -> ConversionResult:
with _converter_lock:
def _convert_sync(
file_path: str,
options: ConversionOptions,
*,
page_range: tuple[int, int] | None = None,
) -> ConversionResult:
acquired = _converter_lock.acquire(timeout=settings.lock_timeout)
if not acquired:
raise TimeoutError(
f"Could not acquire converter lock within {settings.lock_timeout}s — "
"a previous conversion may be frozen"
)
try:
conv = _select_converter(options)
result = conv.convert(file_path)
kwargs: dict = {}
if settings.max_page_count > 0:
kwargs["max_num_pages"] = settings.max_page_count
if settings.max_file_size > 0:
kwargs["max_file_size"] = settings.max_file_size
if page_range is not None:
kwargs["page_range"] = page_range
result = conv.convert(file_path, **kwargs)
finally:
_converter_lock.release()
doc = result.document
page_count = len(doc.pages)
@ -255,5 +281,7 @@ class LocalConverter:
self,
file_path: str,
options: ConversionOptions,
*,
page_range: tuple[int, int] | None = None,
) -> ConversionResult:
return await asyncio.to_thread(_convert_sync, file_path, options)
return await asyncio.to_thread(_convert_sync, file_path, options, page_range=page_range)

15
document-parser/infra/serve_converter.py
View file

@ -76,12 +76,14 @@ class ServeConverter:
self,
file_path: str,
options: ConversionOptions,
*,
page_range: tuple[int, int] | None = None,
) -> ConversionResult:
"""Convert a document by uploading it to Docling Serve."""
path = Path(file_path)
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
form_data = _build_form_data(options)
form_data = _build_form_data(options, page_range=page_range)
url = f"{self._base_url}{_API_PREFIX}/convert/file"
async with httpx.AsyncClient(timeout=self._timeout) as client:
@ -112,13 +114,17 @@ class ServeConverter:
return False
def _build_form_data(options: ConversionOptions) -> dict[str, str | list[str]]:
def _build_form_data(
options: ConversionOptions,
*,
page_range: tuple[int, int] | None = None,
) -> dict[str, str | list[str]]:
"""Build form fields matching Docling Serve's multipart form contract.
Array fields (to_formats) are sent as lists — httpx encodes them as
repeated form keys (to_formats=md&to_formats=html&to_formats=json).
"""
return {
data: dict[str, str | list[str]] = {
"to_formats": ["md", "html", "json"],
"do_ocr": str(options.do_ocr).lower(),
"do_table_structure": str(options.do_table_structure).lower(),
@ -131,6 +137,9 @@ def _build_form_data(options: ConversionOptions) -> dict[str, str | list[str]]:
"generate_page_images": str(options.generate_page_images).lower(),
"images_scale": str(options.images_scale),
}
if page_range is not None:
data["page_range"] = f"{page_range[0]}-{page_range[1]}"
return data
def _parse_response(data: dict) -> ConversionResult:

57
document-parser/infra/settings.py
View file

@ -14,14 +14,62 @@ class Settings:
docling_serve_url: str = "http://localhost:5001"
docling_serve_api_key: str | None = None
conversion_timeout: int = 900
document_timeout: float = 120.0 # Docling-level per-document timeout (seconds)
lock_timeout: int = 300 # converter lock acquisition timeout (seconds)
max_concurrent_analyses: int = 3
max_page_count: int = 0 # 0 = unlimited
default_table_mode: str = "accurate" # "accurate" or "fast"
max_page_count: int = 0 # 0 = unlimited (upload validation)
max_file_size: int = 0 # 0 = unlimited (Docling-level, bytes)
max_file_size_mb: int = 50 # upload limit in MB (0 = unlimited)
rate_limit_rpm: int = 100 # requests per minute per IP (0 = disabled)
batch_page_size: int = 0 # 0 = disabled, > 0 = pages per batch
upload_dir: str = "./uploads"
db_path: str = "./data/docling_studio.db"
cors_origins: list[str] = field(
default_factory=lambda: ["http://localhost:3000", "http://localhost:5173"]
)
def __post_init__(self) -> None:
errors: list[str] = []
if self.document_timeout <= 0:
errors.append(f"document_timeout must be > 0 (got {self.document_timeout})")
if self.conversion_timeout <= 0:
errors.append(f"conversion_timeout must be > 0 (got {self.conversion_timeout})")
if self.lock_timeout <= 0:
errors.append(f"lock_timeout must be > 0 (got {self.lock_timeout})")
if self.max_concurrent_analyses < 1:
errors.append(
f"max_concurrent_analyses must be >= 1 (got {self.max_concurrent_analyses})"
)
if self.max_page_count < 0:
errors.append(f"max_page_count must be >= 0 (got {self.max_page_count})")
if self.max_file_size < 0:
errors.append(f"max_file_size must be >= 0 (got {self.max_file_size})")
if self.max_file_size_mb < 0:
errors.append(f"max_file_size_mb must be >= 0 (got {self.max_file_size_mb})")
if self.rate_limit_rpm < 0:
errors.append(f"rate_limit_rpm must be >= 0 (got {self.rate_limit_rpm})")
if self.batch_page_size < 0:
errors.append(f"batch_page_size must be >= 0 (got {self.batch_page_size})")
if self.default_table_mode not in ("accurate", "fast"):
errors.append(
f"default_table_mode must be 'accurate' or 'fast' (got '{self.default_table_mode}')"
)
# Timeout cascade: document_timeout < lock_timeout < conversion_timeout
if self.document_timeout > 0 and self.lock_timeout > 0 and self.conversion_timeout > 0:
if self.document_timeout >= self.lock_timeout:
errors.append(
f"document_timeout ({self.document_timeout}s) must be "
f"< lock_timeout ({self.lock_timeout}s)"
)
if self.lock_timeout >= self.conversion_timeout:
errors.append(
f"lock_timeout ({self.lock_timeout}s) must be "
f"< conversion_timeout ({self.conversion_timeout}s)"
)
if errors:
raise ValueError("Invalid settings:\n " + "\n ".join(errors))
@classmethod
def from_env(cls) -> Settings:
"""Build a Settings instance from environment variables."""
@ -33,8 +81,15 @@ class Settings:
docling_serve_url=os.environ.get("DOCLING_SERVE_URL", "http://localhost:5001"),
docling_serve_api_key=os.environ.get("DOCLING_SERVE_API_KEY"),
conversion_timeout=int(os.environ.get("CONVERSION_TIMEOUT", "900")),
document_timeout=float(os.environ.get("DOCUMENT_TIMEOUT", "120.0")),
lock_timeout=int(os.environ.get("LOCK_TIMEOUT", "300")),
max_concurrent_analyses=int(os.environ.get("MAX_CONCURRENT_ANALYSES", "3")),
default_table_mode=os.environ.get("DEFAULT_TABLE_MODE", "accurate"),
max_page_count=int(os.environ.get("MAX_PAGE_COUNT", "0")),
max_file_size=int(os.environ.get("MAX_FILE_SIZE", "0")),
max_file_size_mb=int(os.environ.get("MAX_FILE_SIZE_MB", "50")),
rate_limit_rpm=int(os.environ.get("RATE_LIMIT_RPM", "100")),
batch_page_size=int(os.environ.get("BATCH_PAGE_SIZE", "0")),
upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"),
db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"),
cors_origins=[o.strip() for o in cors_raw.split(",")],

72
document-parser/main.py
View file

@ -20,10 +20,14 @@ from fastapi.middleware.cors import CORSMiddleware
from api.analyses import router as analyses_router
from api.documents import router as documents_router
from api.schemas import HealthResponse
from infra.rate_limiter import RateLimiterMiddleware
from infra.settings import settings
from persistence.analysis_repo import SqliteAnalysisRepository
from persistence.database import get_connection, init_db
from services.analysis_service import AnalysisService
from persistence.document_repo import SqliteDocumentRepository
from services.analysis_service import AnalysisConfig, AnalysisService
from services.document_service import DocumentConfig, DocumentService
logging.basicConfig(
level=logging.INFO,
@ -58,14 +62,44 @@ def _build_chunker():
return None
def _build_analysis_service() -> AnalysisService:
def _build_repos() -> tuple[SqliteDocumentRepository, SqliteAnalysisRepository]:
return SqliteDocumentRepository(), SqliteAnalysisRepository()
def _build_analysis_service(
document_repo: SqliteDocumentRepository,
analysis_repo: SqliteAnalysisRepository,
) -> AnalysisService:
converter = _build_converter()
chunker = _build_chunker()
config = AnalysisConfig(
default_table_mode=settings.default_table_mode,
batch_page_size=settings.batch_page_size,
)
return AnalysisService(
converter=converter,
analysis_repo=analysis_repo,
document_repo=document_repo,
chunker=chunker,
conversion_timeout=settings.conversion_timeout,
max_concurrent=settings.max_concurrent_analyses,
config=config,
)
def _build_document_service(
document_repo: SqliteDocumentRepository,
analysis_repo: SqliteAnalysisRepository,
) -> DocumentService:
config = DocumentConfig(
upload_dir=settings.upload_dir,
max_file_size_mb=settings.max_file_size_mb,
max_page_count=settings.max_page_count,
)
return DocumentService(
document_repo=document_repo,
analysis_repo=analysis_repo,
config=config,
)
@ -77,7 +111,9 @@ def _build_analysis_service() -> AnalysisService:
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
await init_db()
app.state.analysis_service = _build_analysis_service()
document_repo, analysis_repo = _build_repos()
app.state.analysis_service = _build_analysis_service(document_repo, analysis_repo)
app.state.document_service = _build_document_service(document_repo, analysis_repo)
logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine)
yield
@ -95,14 +131,19 @@ app.add_middleware(
allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
allow_headers=["Content-Type", "Authorization"],
)
app.add_middleware(RateLimiterMiddleware, requests_per_window=100, window_seconds=60)
if settings.rate_limit_rpm > 0:
app.add_middleware(
RateLimiterMiddleware,
requests_per_window=settings.rate_limit_rpm,
window_seconds=60,
)
app.include_router(documents_router)
app.include_router(analyses_router)
@app.get("/api/health")
async def health() -> dict[str, str | int]:
@app.get("/api/health", response_model=HealthResponse)
async def health() -> HealthResponse:
"""Health check endpoint — verifies database connectivity."""
db_status = "ok"
try:
@ -113,13 +154,12 @@ async def health() -> dict[str, str | int]:
logger.warning("Health check: database unreachable", exc_info=True)
status = "ok" if db_status == "ok" else "degraded"
result: dict[str, str | int] = {
"status": status,
"version": settings.app_version,
"engine": settings.conversion_engine,
"deploymentMode": settings.deployment_mode,
"database": db_status,
}
if settings.max_page_count > 0:
result["maxPageCount"] = settings.max_page_count
return result
return HealthResponse(
status=status,
version=settings.app_version,
engine=settings.conversion_engine,
deployment_mode=settings.deployment_mode,
database=db_status,
max_page_count=settings.max_page_count if settings.max_page_count > 0 else None,
max_file_size_mb=settings.max_file_size_mb if settings.max_file_size_mb > 0 else None,
)

157
document-parser/persistence/analysis_repo.py
View file

@ -27,6 +27,8 @@ def _row_to_job(row) -> AnalysisJob:
document_json=row["document_json"] if "document_json" in keys else None,
chunks_json=row["chunks_json"] if "chunks_json" in keys else None,
error_message=row["error_message"],
progress_current=row["progress_current"] if "progress_current" in keys else None,
progress_total=row["progress_total"] if "progress_total" in keys else None,
started_at=_parse_dt(row["started_at"]),
completed_at=_parse_dt(row["completed_at"]),
created_at=_parse_dt(row["created_at"]) or datetime.now(UTC),
@ -41,83 +43,94 @@ _SELECT_WITH_DOC = """
"""
async def insert(job: AnalysisJob) -> None:
"""Persist a new analysis job record."""
async with get_connection() as db:
await db.execute(
"""INSERT INTO analysis_jobs (id, document_id, status, created_at)
VALUES (?, ?, ?, ?)""",
(job.id, job.document_id, job.status.value, str(job.created_at)),
)
await db.commit()
class SqliteAnalysisRepository:
"""SQLite implementation of the AnalysisRepository port."""
async def insert(self, job: AnalysisJob) -> None:
"""Persist a new analysis job record."""
async with get_connection() as db:
await db.execute(
"""INSERT INTO analysis_jobs (id, document_id, status, created_at)
VALUES (?, ?, ?, ?)""",
(job.id, job.document_id, job.status.value, str(job.created_at)),
)
await db.commit()
async def find_all(*, limit: int = 200, offset: int = 0) -> list[AnalysisJob]:
"""Return analysis jobs with document info, newest first."""
async with get_connection() as db:
cursor = await db.execute(
f"{_SELECT_WITH_DOC} ORDER BY aj.created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
)
rows = await cursor.fetchall()
return [_row_to_job(r) for r in rows]
async def find_all(self, *, limit: int = 200, offset: int = 0) -> list[AnalysisJob]:
"""Return analysis jobs with document info, newest first."""
async with get_connection() as db:
cursor = await db.execute(
f"{_SELECT_WITH_DOC} ORDER BY aj.created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
)
rows = await cursor.fetchall()
return [_row_to_job(r) for r in rows]
async def find_by_id(self, job_id: str) -> AnalysisJob | None:
"""Find an analysis job by ID (with document filename), or return None."""
async with get_connection() as db:
cursor = await db.execute(f"{_SELECT_WITH_DOC} WHERE aj.id = ?", (job_id,))
row = await cursor.fetchone()
return _row_to_job(row) if row else None
async def find_by_id(job_id: str) -> AnalysisJob | None:
"""Find an analysis job by ID (with document filename), or return None."""
async with get_connection() as db:
cursor = await db.execute(f"{_SELECT_WITH_DOC} WHERE aj.id = ?", (job_id,))
row = await cursor.fetchone()
return _row_to_job(row) if row else None
async def update_status(self, job: AnalysisJob) -> None:
"""Persist all mutable fields of an analysis job (status, results, timestamps)."""
async with get_connection() as db:
await db.execute(
"""UPDATE analysis_jobs
SET status = ?, content_markdown = ?, content_html = ?,
pages_json = ?, document_json = ?, chunks_json = ?,
error_message = ?, progress_current = ?, progress_total = ?,
started_at = ?, completed_at = ?
WHERE id = ?""",
(
job.status.value,
job.content_markdown,
job.content_html,
job.pages_json,
job.document_json,
job.chunks_json,
job.error_message,
job.progress_current,
job.progress_total,
str(job.started_at) if job.started_at else None,
str(job.completed_at) if job.completed_at else None,
job.id,
),
)
await db.commit()
async def update_progress(self, job_id: str, current: int, total: int) -> None:
"""Update only the progress columns for a running analysis."""
async with get_connection() as db:
await db.execute(
"UPDATE analysis_jobs SET progress_current = ?, progress_total = ? WHERE id = ?",
(current, total, job_id),
)
await db.commit()
async def update_status(job: AnalysisJob) -> None:
"""Persist all mutable fields of an analysis job (status, results, timestamps)."""
async with get_connection() as db:
await db.execute(
"""UPDATE analysis_jobs
SET status = ?, content_markdown = ?, content_html = ?,
pages_json = ?, document_json = ?, chunks_json = ?,
error_message = ?, started_at = ?, completed_at = ?
WHERE id = ?""",
(
job.status.value,
job.content_markdown,
job.content_html,
job.pages_json,
job.document_json,
job.chunks_json,
job.error_message,
str(job.started_at) if job.started_at else None,
str(job.completed_at) if job.completed_at else None,
job.id,
),
)
await db.commit()
async def update_chunks(self, job_id: str, chunks_json: str) -> bool:
"""Update only the chunks_json column for a completed analysis."""
async with get_connection() as db:
cursor = await db.execute(
"UPDATE analysis_jobs SET chunks_json = ? WHERE id = ?",
(chunks_json, job_id),
)
await db.commit()
return cursor.rowcount > 0
async def delete(self, job_id: str) -> bool:
"""Delete an analysis job by ID. Returns True if a row was removed."""
async with get_connection() as db:
cursor = await db.execute("DELETE FROM analysis_jobs WHERE id = ?", (job_id,))
await db.commit()
return cursor.rowcount > 0
async def update_chunks(job_id: str, chunks_json: str) -> bool:
"""Update only the chunks_json column for a completed analysis."""
async with get_connection() as db:
cursor = await db.execute(
"UPDATE analysis_jobs SET chunks_json = ? WHERE id = ?",
(chunks_json, job_id),
)
await db.commit()
return cursor.rowcount > 0
async def delete(job_id: str) -> bool:
"""Delete an analysis job by ID. Returns True if a row was removed."""
async with get_connection() as db:
cursor = await db.execute("DELETE FROM analysis_jobs WHERE id = ?", (job_id,))
await db.commit()
return cursor.rowcount > 0
async def delete_by_document(document_id: str) -> int:
"""Delete all analysis jobs for a given document. Returns count deleted."""
async with get_connection() as db:
cursor = await db.execute("DELETE FROM analysis_jobs WHERE document_id = ?", (document_id,))
await db.commit()
return cursor.rowcount
async def delete_by_document(self, document_id: str) -> int:
"""Delete all analysis jobs for a given document. Returns count deleted."""
async with get_connection() as db:
cursor = await db.execute(
"DELETE FROM analysis_jobs WHERE document_id = ?", (document_id,)
)
await db.commit()
return cursor.rowcount

2
document-parser/persistence/database.py
View file

@ -50,6 +50,8 @@ CREATE INDEX IF NOT EXISTS idx_documents_created_at ON documents(created_at);
_MIGRATIONS = [
("document_json", "ALTER TABLE analysis_jobs ADD COLUMN document_json TEXT"),
("chunks_json", "ALTER TABLE analysis_jobs ADD COLUMN chunks_json TEXT"),
("progress_current", "ALTER TABLE analysis_jobs ADD COLUMN progress_current INTEGER"),
("progress_total", "ALTER TABLE analysis_jobs ADD COLUMN progress_total INTEGER"),
]

97
document-parser/persistence/document_repo.py
View file

@ -25,57 +25,56 @@ def _row_to_document(row) -> Document:
)
async def insert(doc: Document) -> None:
"""Persist a new document record."""
async with get_connection() as db:
await db.execute(
"""INSERT INTO documents (id, filename, content_type, file_size, page_count, storage_path, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
doc.id,
doc.filename,
doc.content_type,
doc.file_size,
doc.page_count,
doc.storage_path,
str(doc.created_at),
),
)
await db.commit()
class SqliteDocumentRepository:
"""SQLite implementation of the DocumentRepository port."""
async def insert(self, doc: Document) -> None:
"""Persist a new document record."""
async with get_connection() as db:
await db.execute(
"""INSERT INTO documents (id, filename, content_type, file_size, page_count, storage_path, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(
doc.id,
doc.filename,
doc.content_type,
doc.file_size,
doc.page_count,
doc.storage_path,
str(doc.created_at),
),
)
await db.commit()
async def find_all(*, limit: int = 200, offset: int = 0) -> list[Document]:
"""Return documents ordered by creation date (newest first)."""
async with get_connection() as db:
cursor = await db.execute(
"SELECT * FROM documents ORDER BY created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
)
rows = await cursor.fetchall()
return [_row_to_document(r) for r in rows]
async def find_all(self, *, limit: int = 200, offset: int = 0) -> list[Document]:
"""Return documents ordered by creation date (newest first)."""
async with get_connection() as db:
cursor = await db.execute(
"SELECT * FROM documents ORDER BY created_at DESC LIMIT ? OFFSET ?",
(limit, offset),
)
rows = await cursor.fetchall()
return [_row_to_document(r) for r in rows]
async def find_by_id(self, doc_id: str) -> Document | None:
"""Find a document by its ID, or return None."""
async with get_connection() as db:
cursor = await db.execute("SELECT * FROM documents WHERE id = ?", (doc_id,))
row = await cursor.fetchone()
return _row_to_document(row) if row else None
async def find_by_id(doc_id: str) -> Document | None:
"""Find a document by its ID, or return None."""
async with get_connection() as db:
cursor = await db.execute("SELECT * FROM documents WHERE id = ?", (doc_id,))
row = await cursor.fetchone()
return _row_to_document(row) if row else None
async def update_page_count(self, doc_id: str, page_count: int) -> None:
"""Update the page count after conversion has determined it."""
async with get_connection() as db:
await db.execute(
"UPDATE documents SET page_count = ? WHERE id = ?",
(page_count, doc_id),
)
await db.commit()
async def update_page_count(doc_id: str, page_count: int) -> None:
"""Update the page count after conversion has determined it."""
async with get_connection() as db:
await db.execute(
"UPDATE documents SET page_count = ? WHERE id = ?",
(page_count, doc_id),
)
await db.commit()
async def delete(doc_id: str) -> bool:
"""Delete a document by ID. Returns True if a row was removed."""
async with get_connection() as db:
cursor = await db.execute("DELETE FROM documents WHERE id = ?", (doc_id,))
await db.commit()
return cursor.rowcount > 0
async def delete(self, doc_id: str) -> bool:
"""Delete a document by ID. Returns True if a row was removed."""
async with get_connection() as db:
cursor = await db.execute("DELETE FROM documents WHERE id = ?", (doc_id,))
await db.commit()
return cursor.rowcount > 0

1
document-parser/requirements.txt
View file

@ -6,3 +6,4 @@ pdf2image>=1.17.0,<2.0.0
pillow>=10.0.0,<11.0.0
aiosqlite>=0.20.0,<1.0.0
httpx>=0.27.0,<1.0.0
pypdfium2>=4.0.0,<5.0.0

307
document-parser/services/analysis_service.py
View file

@ -1,7 +1,7 @@
"""Analysis service — async document parsing orchestration.
Uses an injected DocumentConverter (port) so the service is decoupled
from the conversion implementation (local Docling lib vs remote Docling Serve).
Uses injected ports (converter, chunker, repositories) so the service is
decoupled from infrastructure implementations.
"""
from __future__ import annotations
@ -10,15 +10,28 @@ import asyncio
import functools
import json
import logging
from dataclasses import asdict
import math
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING
import pypdfium2 as pdfium
from domain.models import AnalysisJob, AnalysisStatus
from domain.value_objects import ChunkingOptions, ChunkResult, ConversionOptions, ConversionResult
from domain.services import classify_error, merge_results
from domain.value_objects import (
ChunkingOptions,
ChunkResult,
ConversionOptions,
ConversionResult,
)
if TYPE_CHECKING:
from domain.ports import DocumentChunker, DocumentConverter
from persistence import analysis_repo, document_repo
from domain.ports import (
AnalysisRepository,
DocumentChunker,
DocumentConverter,
DocumentRepository,
)
logger = logging.getLogger(__name__)
@ -38,20 +51,48 @@ def _chunk_to_dict(c: ChunkResult) -> dict:
_DEFAULT_MAX_CONCURRENT = 3
def _count_pdf_pages(file_path: str) -> int:
"""Count pages in a PDF. Returns 0 if the file is not a valid PDF."""
try:
pdf = pdfium.PdfDocument(file_path)
count = len(pdf)
pdf.close()
return count
except Exception:
logger.debug("Cannot open %s as PDF, batching disabled", file_path)
return 0
@dataclass
class AnalysisConfig:
"""Configuration values needed by AnalysisService, extracted from settings."""
default_table_mode: str = "accurate"
batch_page_size: int = 0
class AnalysisService:
"""Orchestrates document analysis using an injected converter."""
"""Orchestrates document analysis using injected ports."""
def __init__(
self,
converter: DocumentConverter,
analysis_repo: AnalysisRepository,
document_repo: DocumentRepository,
chunker: DocumentChunker | None = None,
conversion_timeout: int = 600,
max_concurrent: int = _DEFAULT_MAX_CONCURRENT,
config: AnalysisConfig | None = None,
):
self._converter = converter
self._chunker = chunker
self._analysis_repo = analysis_repo
self._document_repo = document_repo
self._conversion_timeout = conversion_timeout
self._semaphore = asyncio.Semaphore(max_concurrent)
self._running_tasks: dict[str, asyncio.Task] = {}
self._background_tasks: set[asyncio.Task] = set()
self._config = config or AnalysisConfig()
async def create(
self,
@ -61,13 +102,13 @@ class AnalysisService:
chunking_options: dict | None = None,
) -> AnalysisJob:
"""Create a new analysis job and launch background processing."""
doc = await document_repo.find_by_id(document_id)
doc = await self._document_repo.find_by_id(document_id)
if not doc:
raise ValueError(f"Document not found: {document_id}")
job = AnalysisJob(document_id=document_id)
job.document_filename = doc.filename
await analysis_repo.insert(job)
await self._analysis_repo.insert(job)
task = asyncio.create_task(
self._run_analysis(
@ -78,25 +119,30 @@ class AnalysisService:
chunking_options,
)
)
task.add_done_callback(functools.partial(_on_task_done, job_id=job.id))
self._running_tasks[job.id] = task
task.add_done_callback(functools.partial(self._on_task_done, job_id=job.id))
return job
async def find_all(self) -> list[AnalysisJob]:
"""Return all analysis jobs, newest first."""
return await analysis_repo.find_all()
return await self._analysis_repo.find_all()
async def find_by_id(self, job_id: str) -> AnalysisJob | None:
"""Find an analysis job by ID, or return None."""
return await analysis_repo.find_by_id(job_id)
return await self._analysis_repo.find_by_id(job_id)
async def delete(self, job_id: str) -> bool:
"""Delete an analysis job. Returns True if it existed."""
return await analysis_repo.delete(job_id)
"""Delete an analysis job, cancelling any running task. Returns True if it existed."""
task = self._running_tasks.pop(job_id, None)
if task and not task.done():
task.cancel()
logger.info("Cancelled running task for job %s", job_id)
return await self._analysis_repo.delete(job_id)
async def rechunk(self, job_id: str, chunking_options: dict) -> list[ChunkResult]:
"""Re-chunk an existing completed analysis with new options."""
job = await analysis_repo.find_by_id(job_id)
job = await self._analysis_repo.find_by_id(job_id)
if not job:
raise ValueError(f"Analysis not found: {job_id}")
if job.status != AnalysisStatus.COMPLETED:
@ -110,10 +156,100 @@ class AnalysisService:
chunks = await self._chunker.chunk(job.document_json, options)
chunks_json = json.dumps([_chunk_to_dict(c) for c in chunks])
await analysis_repo.update_chunks(job_id, chunks_json)
await self._analysis_repo.update_chunks(job_id, chunks_json)
return chunks
async def _run_batched_conversion(
self,
job_id: str,
file_path: str,
options: ConversionOptions,
total_pages: int,
batch_size: int,
) -> ConversionResult | None:
"""Convert a document in batches using page_range.
Returns None if the job was deleted mid-batch (caller should abort).
Raises on batch failure (fail-fast: entire job fails).
"""
num_batches = math.ceil(total_pages / batch_size)
await self._analysis_repo.update_progress(job_id, 0, total_pages)
logger.info(
"Batched conversion: %d pages in %d batches of %d for job %s",
total_pages,
num_batches,
batch_size,
job_id,
)
results: list[ConversionResult] = []
for batch_idx in range(num_batches):
start = batch_idx * batch_size + 1
end = min(start + batch_size - 1, total_pages)
if not await self._analysis_repo.find_by_id(job_id):
logger.info(
"Job %s deleted during batch %d/%d, aborting",
job_id,
batch_idx + 1,
num_batches,
)
return None
try:
batch_result = await asyncio.wait_for(
self._converter.convert(file_path, options, page_range=(start, end)),
timeout=self._conversion_timeout,
)
except Exception as exc:
raise RuntimeError(
f"Batch {batch_idx + 1}/{num_batches} (pages {start}-{end}) failed: {exc}"
) from exc
results.append(batch_result)
await self._analysis_repo.update_progress(job_id, end, total_pages)
logger.info(
"Batch %d/%d done (pages %d-%d) for job %s",
batch_idx + 1,
num_batches,
start,
end,
job_id,
)
return merge_results(results)
def _on_task_done(self, task: asyncio.Task, *, job_id: str) -> None:
"""Cleanup running tasks and handle failures."""
self._running_tasks.pop(job_id, None)
if task.cancelled():
logger.warning("Analysis task was cancelled: %s", job_id)
self._schedule_mark_failed(job_id, "Task was cancelled")
return
exc = task.exception()
if exc:
logger.error("Unhandled exception in analysis task %s: %s", job_id, exc, exc_info=True)
self._schedule_mark_failed(job_id, classify_error(exc))
def _schedule_mark_failed(self, job_id: str, error: str) -> None:
"""Schedule _mark_failed as a tracked background task."""
t = asyncio.ensure_future(self._mark_failed(job_id, error))
self._background_tasks.add(t)
t.add_done_callback(self._background_tasks.discard)
async def _mark_failed(self, job_id: str, error: str) -> None:
"""Safely mark a job as failed, handling DB errors gracefully."""
try:
job = await self._analysis_repo.find_by_id(job_id)
if job:
job.mark_failed(error)
await self._analysis_repo.update_status(job)
except OSError:
logger.exception("Database I/O error marking job %s as failed", job_id)
except Exception:
logger.exception("Unexpected error marking job %s as failed", job_id)
async def _run_analysis(
self,
job_id: str,
@ -132,6 +268,67 @@ class AnalysisService:
job_id, file_path, filename, pipeline_options, chunking_options
)
def _build_conversion_options(self, pipeline_options: dict | None) -> ConversionOptions:
"""Build ConversionOptions, applying default table_mode if not specified."""
opts_dict = pipeline_options or {}
if "table_mode" not in opts_dict:
opts_dict = {**opts_dict, "table_mode": self._config.default_table_mode}
return ConversionOptions(**opts_dict)
async def _run_conversion(
self,
job_id: str,
file_path: str,
options: ConversionOptions,
) -> ConversionResult | None:
"""Run batched or single conversion. Returns None if the job was deleted mid-batch."""
total_pages = _count_pdf_pages(file_path)
batch_size = self._config.batch_page_size
if batch_size > 0 and total_pages > batch_size:
return await self._run_batched_conversion(
job_id, file_path, options, total_pages, batch_size
)
return await asyncio.wait_for(
self._converter.convert(file_path, options),
timeout=self._conversion_timeout,
)
async def _finalize_analysis(
self,
job_id: str,
result: ConversionResult,
chunking_options: dict | None,
) -> None:
"""Serialize results, optionally chunk, mark job completed, update page count."""
pages_json = json.dumps([asdict(p) for p in result.pages])
chunks_json = None
if chunking_options and self._chunker and result.document_json:
chunk_opts = ChunkingOptions(**chunking_options)
chunks = await self._chunker.chunk(result.document_json, chunk_opts)
chunks_json = json.dumps([_chunk_to_dict(c) for c in chunks])
logger.info("Chunking produced %d chunks for job %s", len(chunks), job_id)
# Re-read the job so we don't lose progress_current/progress_total
# written to the DB during batched conversion.
job = await self._analysis_repo.find_by_id(job_id)
if not job:
return
job.mark_completed(
markdown=result.content_markdown,
html=result.content_html,
pages_json=pages_json,
document_json=result.document_json,
chunks_json=chunks_json,
)
await self._analysis_repo.update_status(job)
if result.page_count:
await self._document_repo.update_page_count(job.document_id, result.page_count)
logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count)
async def _run_analysis_inner(
self,
job_id: str,
@ -142,84 +339,28 @@ class AnalysisService:
) -> None:
"""Inner analysis logic — called under the concurrency semaphore."""
try:
job = await analysis_repo.find_by_id(job_id)
job = await self._analysis_repo.find_by_id(job_id)
if not job:
logger.error("Analysis job %s not found", job_id)
return
job.mark_running()
await analysis_repo.update_status(job)
await self._analysis_repo.update_status(job)
logger.info("Analysis started: %s (file: %s)", job_id, filename)
options = ConversionOptions(**(pipeline_options or {}))
options = self._build_conversion_options(pipeline_options)
result = await self._run_conversion(job_id, file_path, options)
if result is None:
return # job was deleted mid-batch
result: ConversionResult = await asyncio.wait_for(
self._converter.convert(file_path, options),
timeout=self._conversion_timeout,
)
pages_json = json.dumps([asdict(p) for p in result.pages])
chunks_json = None
if chunking_options and self._chunker and result.document_json:
chunk_opts = ChunkingOptions(**chunking_options)
chunks = await self._chunker.chunk(result.document_json, chunk_opts)
chunks_json = json.dumps([_chunk_to_dict(c) for c in chunks])
logger.info("Chunking produced %d chunks for job %s", len(chunks), job_id)
job.mark_completed(
markdown=result.content_markdown,
html=result.content_html,
pages_json=pages_json,
document_json=result.document_json,
chunks_json=chunks_json,
)
await analysis_repo.update_status(job)
if result.page_count:
await document_repo.update_page_count(job.document_id, result.page_count)
logger.info("Analysis completed: %s (%d pages)", job_id, result.page_count)
await self._finalize_analysis(job_id, result, chunking_options)
except TimeoutError:
logger.error("Analysis timed out after %ds: %s", self._conversion_timeout, job_id)
await _mark_failed(job_id, f"Conversion timed out after {self._conversion_timeout}s")
await self._mark_failed(
job_id, f"Conversion timed out after {self._conversion_timeout}s"
)
except Exception as e:
logger.exception("Analysis failed: %s", job_id)
await _mark_failed(job_id, str(e))
_background_tasks: set[asyncio.Task] = set()
def _on_task_done(task: asyncio.Task, *, job_id: str) -> None:
"""Log unhandled exceptions from background analysis tasks and mark job as FAILED."""
if task.cancelled():
logger.warning("Analysis task was cancelled: %s", job_id)
_schedule_mark_failed(job_id, "Task was cancelled")
return
exc = task.exception()
if exc:
logger.error("Unhandled exception in analysis task %s: %s", job_id, exc, exc_info=True)
_schedule_mark_failed(job_id, str(exc))
def _schedule_mark_failed(job_id: str, error: str) -> None:
"""Schedule _mark_failed as a tracked background task."""
t = asyncio.ensure_future(_mark_failed(job_id, error))
_background_tasks.add(t)
t.add_done_callback(_background_tasks.discard)
async def _mark_failed(job_id: str, error: str) -> None:
"""Safely mark a job as failed, handling DB errors gracefully."""
try:
job = await analysis_repo.find_by_id(job_id)
if job:
job.mark_failed(error)
await analysis_repo.update_status(job)
except OSError:
logger.exception("Database I/O error marking job %s as failed", job_id)
except Exception:
logger.exception("Unexpected error marking job %s as failed", job_id)
await self._mark_failed(job_id, classify_error(e))

196
document-parser/services/document_service.py
View file

@ -6,113 +6,149 @@ import io
import logging
import os
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING
from pdf2image import convert_from_bytes, pdfinfo_from_bytes
from domain.models import Document
from infra.settings import settings
from persistence import analysis_repo, document_repo
if TYPE_CHECKING:
from domain.ports import AnalysisRepository, DocumentRepository
logger = logging.getLogger(__name__)
UPLOAD_DIR = settings.upload_dir
MAX_FILE_SIZE = 5 * 1024 * 1024 # 5 MB
MAX_PAGE_COUNT = settings.max_page_count # 0 = unlimited
# PDF magic bytes: %PDF
_PDF_MAGIC = b"%PDF"
_UPLOAD_CHUNK_SIZE = 64 * 1024 # 64 KB chunks for streaming writes
async def upload(filename: str, content_type: str, file_content: bytes) -> Document:
"""Save uploaded file to disk and persist metadata.
@dataclass
class DocumentConfig:
"""Configuration values needed by DocumentService, extracted from settings."""
Writes the file in fixed-size chunks to keep peak memory usage low.
"""
if len(file_content) > MAX_FILE_SIZE:
raise ValueError("File too large (max 5 MB)")
if not file_content[:4].startswith(_PDF_MAGIC):
raise ValueError("Invalid file: not a PDF document")
os.makedirs(UPLOAD_DIR, exist_ok=True)
ext = ".pdf" # Content already validated as PDF
safe_name = f"{uuid.uuid4()}{ext}"
file_path = os.path.join(UPLOAD_DIR, safe_name)
# Write in chunks to avoid doubling memory usage for large files
with open(file_path, "wb") as f:
for offset in range(0, len(file_content), _UPLOAD_CHUNK_SIZE):
f.write(file_content[offset : offset + _UPLOAD_CHUNK_SIZE])
# Count PDF pages
page_count = _count_pages(file_content)
if MAX_PAGE_COUNT > 0 and page_count is not None and page_count > MAX_PAGE_COUNT:
os.unlink(file_path)
raise ValueError(f"Too many pages ({page_count}). Maximum allowed: {MAX_PAGE_COUNT}")
doc = Document(
filename=filename,
content_type=content_type,
file_size=len(file_content),
page_count=page_count,
storage_path=os.path.abspath(file_path),
)
await document_repo.insert(doc)
return doc
upload_dir: str = "uploads"
max_file_size_mb: int = 0
max_page_count: int = 0
async def find_all() -> list[Document]:
"""Return all documents, newest first."""
return await document_repo.find_all()
class DocumentService:
"""Orchestrates document upload, storage, and preview."""
def __init__(
self,
document_repo: DocumentRepository,
analysis_repo: AnalysisRepository,
config: DocumentConfig,
):
self._document_repo = document_repo
self._analysis_repo = analysis_repo
self._config = config
self._upload_dir = config.upload_dir
self._max_file_size = (
config.max_file_size_mb * 1024 * 1024 if config.max_file_size_mb > 0 else 0
)
self._max_page_count = config.max_page_count
async def find_by_id(doc_id: str) -> Document | None:
"""Find a document by its ID, or return None."""
return await document_repo.find_by_id(doc_id)
@property
def max_file_size(self) -> int:
return self._max_file_size
@property
def max_file_size_mb(self) -> int:
return self._config.max_file_size_mb
async def delete(doc_id: str) -> bool:
"""Delete document file, associated analyses, and database record."""
doc = await document_repo.find_by_id(doc_id)
if not doc:
return False
async def upload(self, filename: str, content_type: str, file_content: bytes) -> Document:
"""Save uploaded file to disk and persist metadata.
# Delete associated analyses first (cascade)
await analysis_repo.delete_by_document(doc_id)
Writes the file in fixed-size chunks to keep peak memory usage low.
"""
if self._max_file_size > 0 and len(file_content) > self._max_file_size:
raise ValueError(f"File too large (max {self._config.max_file_size_mb} MB)")
# Delete file from disk (only if inside UPLOAD_DIR)
try:
real_path = os.path.realpath(doc.storage_path)
real_upload_dir = os.path.realpath(UPLOAD_DIR)
if real_path.startswith(real_upload_dir + os.sep) and os.path.exists(real_path):
os.unlink(real_path)
elif os.path.exists(doc.storage_path):
logger.warning("Refused to delete file outside upload dir: %s", doc.storage_path)
except FileNotFoundError:
logger.info("File already removed: %s", doc.storage_path)
except PermissionError:
logger.error("Permission denied deleting file: %s", doc.storage_path)
except OSError:
logger.warning("Could not delete file: %s", doc.storage_path, exc_info=True)
if not file_content[:4].startswith(_PDF_MAGIC):
raise ValueError("Invalid file: not a PDF document")
return await document_repo.delete(doc_id)
os.makedirs(self._upload_dir, exist_ok=True)
ext = ".pdf" # Content already validated as PDF
safe_name = f"{uuid.uuid4()}{ext}"
file_path = os.path.join(self._upload_dir, safe_name)
def generate_preview(file_content: bytes, page: int = 1, dpi: int = 150) -> bytes:
"""Generate a PNG preview of a specific PDF page."""
images = convert_from_bytes(file_content, first_page=page, last_page=page, dpi=dpi)
if not images:
raise ValueError(f"Page {page} not found")
# Write in chunks to avoid doubling memory usage for large files
with open(file_path, "wb") as f:
for offset in range(0, len(file_content), _UPLOAD_CHUNK_SIZE):
f.write(file_content[offset : offset + _UPLOAD_CHUNK_SIZE])
buf = io.BytesIO()
images[0].save(buf, format="PNG")
return buf.getvalue()
# Count PDF pages
page_count = _count_pages(file_content)
if (
self._max_page_count > 0
and page_count is not None
and page_count > self._max_page_count
):
os.unlink(file_path)
raise ValueError(
f"Too many pages ({page_count}). Maximum allowed: {self._max_page_count}"
)
doc = Document(
filename=filename,
content_type=content_type,
file_size=len(file_content),
page_count=page_count,
storage_path=os.path.abspath(file_path),
)
await self._document_repo.insert(doc)
return doc
async def find_all(self) -> list[Document]:
"""Return all documents, newest first."""
return await self._document_repo.find_all()
async def find_by_id(self, doc_id: str) -> Document | None:
"""Find a document by its ID, or return None."""
return await self._document_repo.find_by_id(doc_id)
async def delete(self, doc_id: str) -> bool:
"""Delete document file, associated analyses, and database record."""
doc = await self._document_repo.find_by_id(doc_id)
if not doc:
return False
# Delete associated analyses first (cascade)
await self._analysis_repo.delete_by_document(doc_id)
# Delete file from disk (only if inside upload dir)
try:
real_path = os.path.realpath(doc.storage_path)
real_upload_dir = os.path.realpath(self._upload_dir)
if real_path.startswith(real_upload_dir + os.sep) and os.path.exists(real_path):
os.unlink(real_path)
elif os.path.exists(doc.storage_path):
logger.warning("Refused to delete file outside upload dir: %s", doc.storage_path)
except FileNotFoundError:
logger.info("File already removed: %s", doc.storage_path)
except PermissionError:
logger.error("Permission denied deleting file: %s", doc.storage_path)
except OSError:
logger.warning("Could not delete file: %s", doc.storage_path, exc_info=True)
return await self._document_repo.delete(doc_id)
@staticmethod
def generate_preview(file_content: bytes, page: int = 1, dpi: int = 150) -> bytes:
"""Generate a PNG preview of a specific PDF page."""
images = convert_from_bytes(file_content, first_page=page, last_page=page, dpi=dpi)
if not images:
raise ValueError(f"Page {page} not found")
buf = io.BytesIO()
images[0].save(buf, format="PNG")
return buf.getvalue()
def _count_pages(file_content: bytes) -> int | None:

441
document-parser/tests/test_analysis_service.py
View file

@ -1,13 +1,27 @@
"""Tests for AnalysisService — callbacks, concurrency, and orchestration."""
"""Tests for AnalysisService — callbacks, concurrency, orchestration, and batching."""
from __future__ import annotations
import asyncio
import functools
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from services.analysis_service import AnalysisService, _on_task_done
from domain.services import extract_html_body, merge_results
from domain.value_objects import ConversionResult, PageDetail
from services.analysis_service import AnalysisConfig, AnalysisService, _count_pdf_pages
def _make_service(**kwargs) -> AnalysisService:
"""Create an AnalysisService with mock repos for testing."""
defaults = {
"converter": MagicMock(),
"analysis_repo": MagicMock(),
"document_repo": MagicMock(),
}
defaults.update(kwargs)
return AnalysisService(**defaults)
class TestOnTaskDone:
@ -17,25 +31,45 @@ class TestOnTaskDone:
async def test_exception_marks_job_failed(self):
"""When a background task raises, the job should be marked FAILED."""
job_id = "job-123"
service = _make_service()
# Create a task that raises
async def failing_task():
raise RuntimeError("boom")
raise RuntimeError("unexpected failure")
task = asyncio.create_task(failing_task())
await asyncio.sleep(0) # let the task fail
await asyncio.sleep(0)
with patch("services.analysis_service._mark_failed", new_callable=AsyncMock) as mock_mark:
_on_task_done(task, job_id=job_id)
# ensure_future schedules it; give the event loop a tick
with patch.object(service, "_mark_failed", new_callable=AsyncMock) as mock_mark:
service._on_task_done(task, job_id=job_id)
await asyncio.sleep(0)
mock_mark.assert_called_once_with(job_id, "boom")
mock_mark.assert_called_once_with(job_id, "unexpected failure")
@pytest.mark.asyncio
async def test_exception_uses_classify_error(self):
"""_on_task_done should route exceptions through classify_error."""
job_id = "job-classify"
service = _make_service()
async def timeout_task():
raise TimeoutError("timeout exceeded while processing")
task = asyncio.create_task(timeout_task())
await asyncio.sleep(0)
with patch.object(service, "_mark_failed", new_callable=AsyncMock) as mock_mark:
service._on_task_done(task, job_id=job_id)
await asyncio.sleep(0)
mock_mark.assert_called_once_with(
job_id, "Processing took too long — try with fewer pages or simpler options"
)
@pytest.mark.asyncio
async def test_cancelled_task_marks_job_failed(self):
"""When a background task is cancelled, the job should be marked FAILED."""
job_id = "job-456"
service = _make_service()
async def slow_task():
await asyncio.sleep(999)
@ -47,8 +81,8 @@ class TestOnTaskDone:
with contextlib.suppress(asyncio.CancelledError):
await task
with patch("services.analysis_service._mark_failed", new_callable=AsyncMock) as mock_mark:
_on_task_done(task, job_id=job_id)
with patch.object(service, "_mark_failed", new_callable=AsyncMock) as mock_mark:
service._on_task_done(task, job_id=job_id)
await asyncio.sleep(0)
mock_mark.assert_called_once_with(job_id, "Task was cancelled")
@ -57,6 +91,7 @@ class TestOnTaskDone:
async def test_successful_task_does_not_mark_failed(self):
"""When a background task succeeds, _mark_failed should not be called."""
job_id = "job-789"
service = _make_service()
async def ok_task():
return "done"
@ -64,24 +99,73 @@ class TestOnTaskDone:
task = asyncio.create_task(ok_task())
await task
with patch("services.analysis_service._mark_failed", new_callable=AsyncMock) as mock_mark:
_on_task_done(task, job_id=job_id)
with patch.object(service, "_mark_failed", new_callable=AsyncMock) as mock_mark:
service._on_task_done(task, job_id=job_id)
await asyncio.sleep(0)
mock_mark.assert_not_called()
class TestAnalysisServiceCancellation:
"""Verify delete cancels running tasks."""
@pytest.mark.asyncio
async def test_delete_cancels_running_task(self):
"""Deleting a job while running should cancel its task."""
mock_analysis_repo = MagicMock()
mock_analysis_repo.delete = AsyncMock(return_value=True)
service = _make_service(analysis_repo=mock_analysis_repo)
blocker = asyncio.Event()
async def slow_analysis():
await blocker.wait()
task = asyncio.create_task(slow_analysis())
service._running_tasks["j1"] = task
result = await service.delete("j1")
assert result is True
assert task.cancelling() or task.cancelled()
assert "j1" not in service._running_tasks
@pytest.mark.asyncio
async def test_delete_completed_job_no_error(self):
"""Deleting a completed job should not raise even if no task tracked."""
mock_analysis_repo = MagicMock()
mock_analysis_repo.delete = AsyncMock(return_value=True)
service = _make_service(analysis_repo=mock_analysis_repo)
result = await service.delete("j-gone")
assert result is True
@pytest.mark.asyncio
async def test_task_cleaned_from_running_on_completion(self):
"""After a task completes, it should be removed from _running_tasks."""
service = _make_service()
async def instant():
pass
task = asyncio.create_task(instant())
service._running_tasks["j1"] = task
task.add_done_callback(functools.partial(service._on_task_done, job_id="j1"))
await asyncio.gather(task)
assert "j1" not in service._running_tasks
class TestAnalysisServiceConcurrency:
"""Verify that the semaphore limits concurrent analysis jobs."""
def test_semaphore_initialized_with_max_concurrent(self):
converter = MagicMock()
service = AnalysisService(converter=converter, max_concurrent=5)
service = _make_service(max_concurrent=5)
assert service._semaphore._value == 5
def test_default_max_concurrent(self):
converter = MagicMock()
service = AnalysisService(converter=converter)
service = _make_service()
assert service._semaphore._value == 3
@pytest.mark.asyncio
@ -90,8 +174,7 @@ class TestAnalysisServiceConcurrency:
call_order: list[str] = []
blocker = asyncio.Event()
converter = MagicMock()
service = AnalysisService(converter=converter, max_concurrent=1)
service = _make_service(max_concurrent=1)
async def fake_inner(self, *args, **kwargs):
call_order.append("start")
@ -112,3 +195,323 @@ class TestAnalysisServiceConcurrency:
# Both should have completed
assert call_order.count("start") == 2
assert call_order.count("end") == 2
# ---------------------------------------------------------------------------
# Batch helper tests
# ---------------------------------------------------------------------------
class TestCountPdfPages:
def test_valid_pdf(self, tmp_path):
"""A real (minimal) PDF should return its page count."""
import pypdfium2 as pdfium
pdf = pdfium.PdfDocument.new()
pdf.new_page(612, 792)
path = tmp_path / "test.pdf"
pdf.save(str(path))
pdf.close()
assert _count_pdf_pages(str(path)) == 1
def test_non_pdf_file(self, tmp_path):
"""A non-PDF file should return 0."""
path = tmp_path / "test.docx"
path.write_bytes(b"PK\x03\x04 not a pdf")
assert _count_pdf_pages(str(path)) == 0
def test_nonexistent_file(self):
"""A nonexistent file should return 0."""
assert _count_pdf_pages("/no/such/file.pdf") == 0
def test_empty_file(self, tmp_path):
"""An empty file should return 0."""
path = tmp_path / "empty.pdf"
path.write_bytes(b"")
assert _count_pdf_pages(str(path)) == 0
class TestExtractHtmlBody:
def test_extracts_body(self):
html = '<html><head></head><body class="x"><p>Hello</p></body></html>'
assert extract_html_body(html) == "<p>Hello</p>"
def test_no_body_tag_returns_raw(self):
html = "<p>No body tag</p>"
assert extract_html_body(html) == html
def test_empty_body(self):
html = "<html><body></body></html>"
assert extract_html_body(html) == ""
class TestMergeResults:
def test_empty_list(self):
result = merge_results([])
assert result.page_count == 0
assert result.content_markdown == ""
assert result.pages == []
assert result.document_json is None
def test_single_result_passthrough(self):
r = ConversionResult(
page_count=3,
content_markdown="# Page 1",
content_html="<html><body><p>Page 1</p></body></html>",
pages=[PageDetail(page_number=1, width=612, height=792)],
document_json='{"pages": {}}',
)
merged = merge_results([r])
assert merged.page_count == 3
assert merged.content_markdown == "# Page 1"
assert merged.pages == [PageDetail(page_number=1, width=612, height=792)]
assert merged.document_json is None # intentionally dropped
def test_merges_multiple_results(self):
r1 = ConversionResult(
page_count=2,
content_markdown="# Batch 1",
content_html="<html><body><p>B1</p></body></html>",
pages=[
PageDetail(page_number=1, width=612, height=792),
PageDetail(page_number=2, width=612, height=792),
],
skipped_items=1,
)
r2 = ConversionResult(
page_count=2,
content_markdown="# Batch 2",
content_html="<html><body><p>B2</p></body></html>",
pages=[
PageDetail(page_number=3, width=612, height=792),
PageDetail(page_number=4, width=612, height=792),
],
skipped_items=2,
)
merged = merge_results([r1, r2])
assert merged.page_count == 4
assert merged.content_markdown == "# Batch 1\n\n# Batch 2"
assert len(merged.pages) == 4
assert merged.pages[0].page_number == 1
assert merged.pages[3].page_number == 4
assert merged.skipped_items == 3
assert merged.document_json is None
assert "<p>B1</p>" in merged.content_html
assert "<p>B2</p>" in merged.content_html
# Valid HTML structure
assert merged.content_html.startswith("<!DOCTYPE html>")
assert "</body></html>" in merged.content_html
# ---------------------------------------------------------------------------
# Batched conversion integration tests
# ---------------------------------------------------------------------------
class TestBatchedConversion:
@pytest.mark.asyncio
async def test_batched_conversion_produces_merged_result(self):
"""When batch_page_size is set and document exceeds it, results are merged."""
converter = AsyncMock()
# Simulate 2 batches: pages 1-5 and 6-8
converter.convert.side_effect = [
ConversionResult(
page_count=5,
content_markdown="# Batch 1",
content_html="<html><body><p>B1</p></body></html>",
pages=[PageDetail(page_number=i, width=612, height=792) for i in range(1, 6)],
),
ConversionResult(
page_count=3,
content_markdown="# Batch 2",
content_html="<html><body><p>B2</p></body></html>",
pages=[PageDetail(page_number=i, width=612, height=792) for i in range(6, 9)],
),
]
mock_analysis_repo = MagicMock()
mock_analysis_repo.find_by_id = AsyncMock(return_value=MagicMock())
mock_analysis_repo.update_progress = AsyncMock()
service = _make_service(
converter=converter,
analysis_repo=mock_analysis_repo,
conversion_timeout=60,
)
result = await service._run_batched_conversion(
"job-1",
"/fake.pdf",
MagicMock(),
total_pages=8,
batch_size=5,
)
assert result is not None
assert result.page_count == 8
assert len(result.pages) == 8
assert result.document_json is None
assert converter.convert.call_count == 2
# Verify page_range was passed correctly
call1_kwargs = converter.convert.call_args_list[0].kwargs
call2_kwargs = converter.convert.call_args_list[1].kwargs
assert call1_kwargs["page_range"] == (1, 5)
assert call2_kwargs["page_range"] == (6, 8)
@pytest.mark.asyncio
async def test_batch_failure_raises_with_enriched_message(self):
"""If a batch fails, RuntimeError is raised with batch info."""
converter = AsyncMock()
converter.convert.side_effect = [
ConversionResult(
page_count=5,
content_markdown="ok",
content_html="<html><body>ok</body></html>",
pages=[PageDetail(page_number=i, width=612, height=792) for i in range(1, 6)],
),
RuntimeError("OOM"),
]
mock_analysis_repo = MagicMock()
mock_analysis_repo.find_by_id = AsyncMock(return_value=MagicMock())
mock_analysis_repo.update_progress = AsyncMock()
service = _make_service(
converter=converter,
analysis_repo=mock_analysis_repo,
conversion_timeout=60,
)
with pytest.raises(RuntimeError, match=r"Batch 2/2 \(pages 6-8\) failed: OOM"):
await service._run_batched_conversion(
"job-fail",
"/fake.pdf",
MagicMock(),
total_pages=8,
batch_size=5,
)
@pytest.mark.asyncio
async def test_progress_preserved_through_full_analysis_flow(self):
"""Progress written during batches must survive the final update_status.
Regression: _run_analysis_inner used to re-read the job from DB at the
start, then call update_status(job) at the end — overwriting
progress_current/progress_total with None because the in-memory object
was stale. The fix re-reads the job before mark_completed.
"""
from domain.models import AnalysisJob, AnalysisStatus
converter = AsyncMock()
converter.convert.side_effect = [
ConversionResult(
page_count=5,
content_markdown="# B1",
content_html="<html><body><p>B1</p></body></html>",
pages=[PageDetail(page_number=i, width=612, height=792) for i in range(1, 6)],
),
ConversionResult(
page_count=3,
content_markdown="# B2",
content_html="<html><body><p>B2</p></body></html>",
pages=[PageDetail(page_number=i, width=612, height=792) for i in range(6, 9)],
),
]
# Simulated DB state: find_by_id is called 4 times:
# 1) start of _run_analysis_inner (initial load)
# 2) batch 1 mid-flight deletion check
# 3) batch 2 mid-flight deletion check
# 4) re-read before mark_completed (must carry progress)
initial_job = AnalysisJob(
id="job-progress",
document_id="doc-1",
status=AnalysisStatus.PENDING,
)
batch_check_job = AnalysisJob(
id="job-progress",
document_id="doc-1",
status=AnalysisStatus.RUNNING,
)
refreshed_job = AnalysisJob(
id="job-progress",
document_id="doc-1",
status=AnalysisStatus.RUNNING,
progress_current=8,
progress_total=8,
)
saved_jobs: list[AnalysisJob] = []
async def capture_update_status(job):
saved_jobs.append(job)
mock_analysis_repo = MagicMock()
mock_analysis_repo.find_by_id = AsyncMock(
side_effect=[initial_job, batch_check_job, batch_check_job, refreshed_job]
)
mock_analysis_repo.update_status = AsyncMock(side_effect=capture_update_status)
mock_analysis_repo.update_progress = AsyncMock()
mock_document_repo = MagicMock()
mock_document_repo.update_page_count = AsyncMock()
config = AnalysisConfig(default_table_mode="accurate", batch_page_size=5)
service = AnalysisService(
converter=converter,
analysis_repo=mock_analysis_repo,
document_repo=mock_document_repo,
conversion_timeout=60,
config=config,
)
with patch("services.analysis_service._count_pdf_pages", return_value=8):
await service._run_analysis_inner("job-progress", "/fake.pdf", "fake.pdf")
# The last update_status call is the COMPLETED one
completed_job = saved_jobs[-1]
assert completed_job.status == AnalysisStatus.COMPLETED
assert completed_job.progress_current == 8, (
"progress_current must be preserved (was the bug: got None)"
)
assert completed_job.progress_total == 8, (
"progress_total must be preserved (was the bug: got None)"
)
@pytest.mark.asyncio
async def test_job_deleted_mid_batch_returns_none(self):
"""If the job is deleted between batches, conversion aborts with None."""
converter = AsyncMock()
converter.convert.return_value = ConversionResult(
page_count=5,
content_markdown="ok",
content_html="<html><body>ok</body></html>",
pages=[PageDetail(page_number=i, width=612, height=792) for i in range(1, 6)],
)
mock_analysis_repo = MagicMock()
# First check returns job, second returns None (deleted)
mock_analysis_repo.find_by_id = AsyncMock(side_effect=[MagicMock(), None])
mock_analysis_repo.update_progress = AsyncMock()
service = _make_service(
converter=converter,
analysis_repo=mock_analysis_repo,
conversion_timeout=60,
)
result = await service._run_batched_conversion(
"job-del",
"/fake.pdf",
MagicMock(),
total_pages=10,
batch_size=5,
)
assert result is None
# Only first batch should have been converted
assert converter.convert.call_count == 1

106
document-parser/tests/test_api_endpoints.py
View file

@ -1,6 +1,6 @@
"""Tests for FastAPI API endpoints using TestClient."""
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi.testclient import TestClient
@ -24,6 +24,18 @@ def mock_analysis_service(client):
app.state.analysis_service = original
@pytest.fixture
def mock_document_service(client):
"""Inject a mock DocumentService into app.state for the duration of the test."""
mock_svc = MagicMock()
mock_svc.max_file_size = 50 * 1024 * 1024
mock_svc.max_file_size_mb = 50
original = getattr(app.state, "document_service", None)
app.state.document_service = mock_svc
yield mock_svc
app.state.document_service = original
class TestHealthEndpoint:
def test_health(self, client):
resp = client.get("/api/health")
@ -33,14 +45,21 @@ class TestHealthEndpoint:
assert "engine" in data
assert "database" in data
def test_health_exposes_max_file_size_mb(self, client):
resp = client.get("/api/health")
data = resp.json()
assert "maxFileSizeMb" in data
assert data["maxFileSizeMb"] == 50
class TestDocumentEndpoints:
@patch("services.document_service.find_all", new_callable=AsyncMock)
def test_list_documents(self, mock_find_all, client):
mock_find_all.return_value = [
Document(id="d1", filename="a.pdf", storage_path="/tmp/a"),
Document(id="d2", filename="b.pdf", storage_path="/tmp/b"),
]
def test_list_documents(self, client, mock_document_service):
mock_document_service.find_all = AsyncMock(
return_value=[
Document(id="d1", filename="a.pdf", storage_path="/tmp/a"),
Document(id="d2", filename="b.pdf", storage_path="/tmp/b"),
]
)
resp = client.get("/api/documents")
assert resp.status_code == 200
@ -51,15 +70,16 @@ class TestDocumentEndpoints:
# Verify camelCase
assert "createdAt" in data[0]
@patch("services.document_service.find_by_id", new_callable=AsyncMock)
def test_get_document(self, mock_find, client):
mock_find.return_value = Document(
id="d1",
filename="test.pdf",
content_type="application/pdf",
file_size=2048,
page_count=3,
storage_path="/tmp/test",
def test_get_document(self, client, mock_document_service):
mock_document_service.find_by_id = AsyncMock(
return_value=Document(
id="d1",
filename="test.pdf",
content_type="application/pdf",
file_size=2048,
page_count=3,
storage_path="/tmp/test",
)
)
resp = client.get("/api/documents/d1")
@ -69,21 +89,21 @@ class TestDocumentEndpoints:
assert data["fileSize"] == 2048
assert data["pageCount"] == 3
@patch("services.document_service.find_by_id", new_callable=AsyncMock)
def test_get_document_not_found(self, mock_find, client):
mock_find.return_value = None
def test_get_document_not_found(self, client, mock_document_service):
mock_document_service.find_by_id = AsyncMock(return_value=None)
resp = client.get("/api/documents/missing")
assert resp.status_code == 404
@patch("services.document_service.upload", new_callable=AsyncMock)
def test_upload_document(self, mock_upload, client):
mock_upload.return_value = Document(
id="new-1",
filename="uploaded.pdf",
content_type="application/pdf",
file_size=512,
storage_path="/tmp/uploaded",
def test_upload_document(self, client, mock_document_service):
mock_document_service.upload = AsyncMock(
return_value=Document(
id="new-1",
filename="uploaded.pdf",
content_type="application/pdf",
file_size=512,
storage_path="/tmp/uploaded",
)
)
resp = client.post(
@ -95,9 +115,10 @@ class TestDocumentEndpoints:
assert data["id"] == "new-1"
assert data["filename"] == "uploaded.pdf"
@patch("services.document_service.upload", new_callable=AsyncMock)
def test_upload_too_large(self, mock_upload, client):
mock_upload.side_effect = ValueError("File too large (max 5 MB)")
def test_upload_too_large(self, client, mock_document_service):
mock_document_service.upload = AsyncMock(
side_effect=ValueError("File too large (max 5 MB)")
)
resp = client.post(
"/api/documents/upload",
@ -105,29 +126,28 @@ class TestDocumentEndpoints:
)
assert resp.status_code == 400
@patch("services.document_service.find_by_id", new_callable=AsyncMock)
def test_preview_page_out_of_range(self, mock_find, client):
mock_find.return_value = Document(
id="d1",
filename="test.pdf",
page_count=3,
storage_path="/tmp/test.pdf",
def test_preview_page_out_of_range(self, client, mock_document_service):
mock_document_service.find_by_id = AsyncMock(
return_value=Document(
id="d1",
filename="test.pdf",
page_count=3,
storage_path="/tmp/test.pdf",
)
)
resp = client.get("/api/documents/d1/preview?page=10")
assert resp.status_code == 400
assert "out of range" in resp.json()["detail"]
@patch("services.document_service.delete", new_callable=AsyncMock)
def test_delete_document(self, mock_delete, client):
mock_delete.return_value = True
def test_delete_document(self, client, mock_document_service):
mock_document_service.delete = AsyncMock(return_value=True)
resp = client.delete("/api/documents/d1")
assert resp.status_code == 204
@patch("services.document_service.delete", new_callable=AsyncMock)
def test_delete_document_not_found(self, mock_delete, client):
mock_delete.return_value = False
def test_delete_document_not_found(self, client, mock_document_service):
mock_document_service.delete = AsyncMock(return_value=False)
resp = client.delete("/api/documents/missing")
assert resp.status_code == 404

131
document-parser/tests/test_document_service.py
View file

@ -1,4 +1,4 @@
"""Tests for document_service — upload, preview, page counting, and deletion."""
"""Tests for DocumentService — upload, preview, page counting, and deletion."""
from __future__ import annotations
@ -8,82 +8,87 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from domain.models import Document
from services import document_service
from services.document_service import DocumentConfig, DocumentService, _count_pages
def _make_service(
upload_dir: str = "/tmp/uploads",
max_file_size_mb: int = 50,
max_page_count: int = 0,
) -> DocumentService:
"""Create a DocumentService with mock repos for testing."""
config = DocumentConfig(
upload_dir=upload_dir,
max_file_size_mb=max_file_size_mb,
max_page_count=max_page_count,
)
return DocumentService(
document_repo=AsyncMock(),
analysis_repo=AsyncMock(),
config=config,
)
class TestUploadValidation:
@pytest.mark.asyncio
async def test_rejects_oversized_file(self):
content = b"x" * (document_service.MAX_FILE_SIZE + 1)
service = _make_service(max_file_size_mb=1)
content = b"x" * (1 * 1024 * 1024 + 1)
with pytest.raises(ValueError, match="File too large"):
await document_service.upload("big.pdf", "application/pdf", content)
await service.upload("big.pdf", "application/pdf", content)
@pytest.mark.asyncio
async def test_rejects_non_pdf(self):
service = _make_service()
content = b"NOT-A-PDF-FILE"
with pytest.raises(ValueError, match="not a PDF"):
await document_service.upload("fake.pdf", "application/pdf", content)
await service.upload("fake.pdf", "application/pdf", content)
@pytest.mark.asyncio
async def test_rejects_too_many_pages(self, tmp_path, monkeypatch):
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
monkeypatch.setattr(document_service, "MAX_PAGE_COUNT", 20)
async def test_rejects_too_many_pages(self, tmp_path):
service = _make_service(upload_dir=str(tmp_path), max_page_count=20)
with patch.object(document_service, "_count_pages", return_value=40):
with patch("services.document_service._count_pages", return_value=40):
content = b"%PDF-1.4 fake pdf content"
with pytest.raises(ValueError, match="Too many pages"):
await document_service.upload("big.pdf", "application/pdf", content)
await service.upload("big.pdf", "application/pdf", content)
# Verify temp file was cleaned up
assert len(os.listdir(tmp_path)) == 0
@pytest.mark.asyncio
async def test_allows_pdf_under_page_limit(self, tmp_path, monkeypatch):
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
monkeypatch.setattr(document_service, "MAX_PAGE_COUNT", 20)
async def test_allows_pdf_under_page_limit(self, tmp_path):
service = _make_service(upload_dir=str(tmp_path), max_page_count=20)
mock_insert = AsyncMock()
with (
patch("persistence.document_repo.insert", mock_insert),
patch.object(document_service, "_count_pages", return_value=15),
):
with patch("services.document_service._count_pages", return_value=15):
content = b"%PDF-1.4 fake pdf content"
doc = await document_service.upload("ok.pdf", "application/pdf", content)
doc = await service.upload("ok.pdf", "application/pdf", content)
assert doc.page_count == 15
mock_insert.assert_called_once()
service._document_repo.insert.assert_called_once()
@pytest.mark.asyncio
async def test_unlimited_pages_when_zero(self, tmp_path, monkeypatch):
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
monkeypatch.setattr(document_service, "MAX_PAGE_COUNT", 0)
async def test_unlimited_pages_when_zero(self, tmp_path):
service = _make_service(upload_dir=str(tmp_path), max_page_count=0)
mock_insert = AsyncMock()
with (
patch("persistence.document_repo.insert", mock_insert),
patch.object(document_service, "_count_pages", return_value=100),
):
with patch("services.document_service._count_pages", return_value=100):
content = b"%PDF-1.4 fake pdf content"
doc = await document_service.upload("big.pdf", "application/pdf", content)
doc = await service.upload("big.pdf", "application/pdf", content)
assert doc.page_count == 100
@pytest.mark.asyncio
async def test_accepts_valid_pdf(self, tmp_path, monkeypatch):
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
async def test_accepts_valid_pdf(self, tmp_path):
service = _make_service(upload_dir=str(tmp_path))
mock_insert = AsyncMock()
with (
patch("persistence.document_repo.insert", mock_insert),
patch.object(document_service, "_count_pages", return_value=5),
):
with patch("services.document_service._count_pages", return_value=5):
content = b"%PDF-1.4 fake pdf content"
doc = await document_service.upload("test.pdf", "application/pdf", content)
doc = await service.upload("test.pdf", "application/pdf", content)
assert doc.filename == "test.pdf"
assert doc.file_size == len(content)
assert doc.page_count == 5
mock_insert.assert_called_once()
service._document_repo.insert.assert_called_once()
# Verify file was actually written to disk
assert os.path.exists(doc.storage_path)
@ -98,7 +103,7 @@ class TestGeneratePreview:
patch("services.document_service.convert_from_bytes", return_value=[]),
pytest.raises(ValueError, match="Page 1 not found"),
):
document_service.generate_preview(b"%PDF-fake", page=1)
DocumentService.generate_preview(b"%PDF-fake", page=1)
def test_returns_png_bytes(self):
"""generate_preview should return PNG bytes from pdf2image."""
@ -106,7 +111,7 @@ class TestGeneratePreview:
mock_image.save = MagicMock(side_effect=lambda buf, format: buf.write(b"PNG-DATA"))
with patch("services.document_service.convert_from_bytes", return_value=[mock_image]):
result = document_service.generate_preview(b"%PDF-fake", page=1, dpi=72)
result = DocumentService.generate_preview(b"%PDF-fake", page=1, dpi=72)
assert result == b"PNG-DATA"
@ -117,21 +122,19 @@ class TestCountPages:
"services.document_service.pdfinfo_from_bytes",
return_value={"Pages": 42},
):
assert document_service._count_pages(b"pdf") == 42
assert _count_pages(b"pdf") == 42
def test_returns_none_on_error(self):
with patch(
"services.document_service.pdfinfo_from_bytes",
side_effect=FileNotFoundError("poppler not found"),
):
assert document_service._count_pages(b"pdf") is None
assert _count_pages(b"pdf") is None
class TestDelete:
@pytest.mark.asyncio
async def test_delete_removes_file_and_records(self, tmp_path, monkeypatch):
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path))
async def test_delete_removes_file_and_records(self, tmp_path):
# Create a fake file
fake_file = tmp_path / "test.pdf"
fake_file.write_bytes(b"content")
@ -142,21 +145,21 @@ class TestDelete:
storage_path=str(fake_file),
)
with (
patch("persistence.document_repo.find_by_id", AsyncMock(return_value=doc)),
patch("persistence.analysis_repo.delete_by_document", AsyncMock(return_value=2)),
patch("persistence.document_repo.delete", AsyncMock(return_value=True)),
):
result = await document_service.delete("doc-1")
service = _make_service(upload_dir=str(tmp_path))
service._document_repo.find_by_id = AsyncMock(return_value=doc)
service._document_repo.delete = AsyncMock(return_value=True)
service._analysis_repo.delete_by_document = AsyncMock(return_value=2)
result = await service.delete("doc-1")
assert result is True
assert not fake_file.exists()
@pytest.mark.asyncio
async def test_delete_refuses_file_outside_upload_dir(self, tmp_path, monkeypatch):
"""Files outside UPLOAD_DIR should not be deleted (path traversal protection)."""
monkeypatch.setattr(document_service, "UPLOAD_DIR", str(tmp_path / "uploads"))
os.makedirs(tmp_path / "uploads", exist_ok=True)
async def test_delete_refuses_file_outside_upload_dir(self, tmp_path):
"""Files outside upload dir should not be deleted (path traversal protection)."""
uploads = tmp_path / "uploads"
os.makedirs(uploads, exist_ok=True)
# File is outside the upload dir
outside_file = tmp_path / "secret.txt"
@ -164,18 +167,20 @@ class TestDelete:
doc = Document(id="doc-1", filename="x.pdf", storage_path=str(outside_file))
with (
patch("persistence.document_repo.find_by_id", AsyncMock(return_value=doc)),
patch("persistence.analysis_repo.delete_by_document", AsyncMock(return_value=0)),
patch("persistence.document_repo.delete", AsyncMock(return_value=True)),
):
await document_service.delete("doc-1")
service = _make_service(upload_dir=str(uploads))
service._document_repo.find_by_id = AsyncMock(return_value=doc)
service._document_repo.delete = AsyncMock(return_value=True)
service._analysis_repo.delete_by_document = AsyncMock(return_value=0)
await service.delete("doc-1")
# File should NOT have been deleted
assert outside_file.exists()
@pytest.mark.asyncio
async def test_delete_not_found_returns_false(self):
with patch("persistence.document_repo.find_by_id", AsyncMock(return_value=None)):
result = await document_service.delete("missing")
service = _make_service()
service._document_repo.find_by_id = AsyncMock(return_value=None)
result = await service.delete("missing")
assert result is False

66
document-parser/tests/test_models.py
View file

@ -2,6 +2,8 @@
from datetime import datetime
import pytest
from domain.models import AnalysisJob, AnalysisStatus, Document
@ -96,6 +98,70 @@ class TestAnalysisJob:
assert job.status == AnalysisStatus.COMPLETED
class TestAnalysisJobGuardClauses:
"""Guard clauses prevent invalid state transitions."""
def test_mark_running_from_running_raises(self):
job = AnalysisJob()
job.mark_running()
with pytest.raises(ValueError, match="Cannot mark as RUNNING"):
job.mark_running()
def test_mark_running_from_completed_raises(self):
job = AnalysisJob()
job.mark_running()
job.mark_completed(markdown="", html="", pages_json="[]")
with pytest.raises(ValueError, match="Cannot mark as RUNNING"):
job.mark_running()
def test_mark_running_from_failed_raises(self):
job = AnalysisJob()
job.mark_failed("err")
with pytest.raises(ValueError, match="Cannot mark as RUNNING"):
job.mark_running()
def test_mark_completed_from_pending_raises(self):
job = AnalysisJob()
with pytest.raises(ValueError, match="Cannot mark as COMPLETED"):
job.mark_completed(markdown="", html="", pages_json="[]")
def test_mark_completed_from_failed_raises(self):
job = AnalysisJob()
job.mark_failed("err")
with pytest.raises(ValueError, match="Cannot mark as COMPLETED"):
job.mark_completed(markdown="", html="", pages_json="[]")
def test_mark_failed_from_completed_raises(self):
job = AnalysisJob()
job.mark_running()
job.mark_completed(markdown="", html="", pages_json="[]")
with pytest.raises(ValueError, match="Cannot mark as FAILED"):
job.mark_failed("err")
def test_mark_failed_from_pending_allowed(self):
job = AnalysisJob()
job.mark_failed("err")
assert job.status == AnalysisStatus.FAILED
def test_mark_failed_from_running_allowed(self):
job = AnalysisJob()
job.mark_running()
job.mark_failed("err")
assert job.status == AnalysisStatus.FAILED
def test_update_progress_from_pending_raises(self):
job = AnalysisJob()
with pytest.raises(ValueError, match="Cannot update progress"):
job.update_progress(1, 10)
def test_update_progress_from_running_allowed(self):
job = AnalysisJob()
job.mark_running()
job.update_progress(5, 10)
assert job.progress_current == 5
assert job.progress_total == 10
class TestAnalysisStatus:
def test_values(self):
assert AnalysisStatus.PENDING == "PENDING"

126
document-parser/tests/test_pipeline_options.py
View file

@ -53,6 +53,7 @@ class TestBuildConverter:
assert opts.generate_page_images is False
assert opts.generate_picture_images is False
assert opts.images_scale == 1.0
assert opts.document_timeout is not None
def test_ocr_disabled(self):
conv = build_converter(ConversionOptions(do_ocr=False))
@ -145,7 +146,7 @@ class TestBuildConverter:
class TestConvertDocumentRouting:
"""Verify convert_document uses default converter for default opts, custom otherwise."""
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_default_converter_with_all_defaults(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -163,7 +164,7 @@ class TestConvertDocumentRouting:
mock_get_default.assert_called_once()
mock_build.assert_not_called()
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_ocr_disabled(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -181,7 +182,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once()
mock_get_default.assert_not_called()
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_table_mode_fast(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -199,7 +200,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once_with(opts)
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_code_enrichment_on(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -217,7 +218,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once_with(opts)
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_formula_enrichment_on(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -234,7 +235,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once()
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_picture_options_on(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -251,7 +252,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once()
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_generate_images_on(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -268,7 +269,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once()
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_uses_custom_converter_when_images_scale_changed(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -286,7 +287,7 @@ class TestConvertDocumentRouting:
mock_build.assert_called_once_with(opts)
@patch("infra.local_converter._get_default_converter")
@patch("infra.local_converter._ensure_default_converter")
@patch("infra.local_converter._build_docling_converter")
def test_forwards_all_options_to_build_converter(self, mock_build, mock_get_default):
mock_conv = MagicMock()
@ -324,6 +325,22 @@ class TestConvertDocumentRouting:
class TestServiceForwardsPipelineOptions:
"""Verify analysis_service.create and _run_analysis forward pipeline options."""
def _make_service(self, converter):
from services.analysis_service import AnalysisService
mock_analysis_repo = MagicMock()
mock_analysis_repo.find_by_id = AsyncMock()
mock_analysis_repo.insert = AsyncMock()
mock_analysis_repo.update_status = AsyncMock()
mock_document_repo = MagicMock()
mock_document_repo.find_by_id = AsyncMock()
mock_document_repo.update_page_count = AsyncMock()
return AnalysisService(
converter=converter,
analysis_repo=mock_analysis_repo,
document_repo=mock_document_repo,
)
@pytest.fixture
def mock_doc(self):
from domain.models import Document
@ -336,22 +353,11 @@ class TestServiceForwardsPipelineOptions:
return AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf")
@patch("services.analysis_service.document_repo")
@patch("services.analysis_service.analysis_repo")
@pytest.mark.asyncio
async def test_create_passes_pipeline_options_to_run(
self,
mock_analysis_repo,
mock_doc_repo,
mock_doc,
):
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
mock_analysis_repo.insert = AsyncMock()
async def test_create_passes_pipeline_options_to_run(self, mock_doc):
mock_converter = AsyncMock()
from services.analysis_service import AnalysisService
svc = AnalysisService(converter=mock_converter)
svc = self._make_service(mock_converter)
svc._document_repo.find_by_id = AsyncMock(return_value=mock_doc)
opts = {"do_ocr": False, "table_mode": "fast"}
@ -359,42 +365,20 @@ class TestServiceForwardsPipelineOptions:
await svc.create("d1", pipeline_options=opts)
mock_task.assert_called_once()
@patch("services.analysis_service.document_repo")
@patch("services.analysis_service.analysis_repo")
@pytest.mark.asyncio
async def test_create_passes_none_when_no_options(
self,
mock_analysis_repo,
mock_doc_repo,
mock_doc,
):
mock_doc_repo.find_by_id = AsyncMock(return_value=mock_doc)
mock_analysis_repo.insert = AsyncMock()
async def test_create_passes_none_when_no_options(self, mock_doc):
mock_converter = AsyncMock()
from services.analysis_service import AnalysisService
svc = AnalysisService(converter=mock_converter)
svc = self._make_service(mock_converter)
svc._document_repo.find_by_id = AsyncMock(return_value=mock_doc)
with patch("services.analysis_service.asyncio.create_task") as mock_task:
await svc.create("d1")
mock_task.assert_called_once()
@patch("services.analysis_service.analysis_repo")
@patch("services.analysis_service.document_repo")
@pytest.mark.asyncio
async def test_run_analysis_forwards_options_to_convert(
self,
mock_doc_repo,
mock_analysis_repo,
mock_job,
):
async def test_run_analysis_forwards_options_to_convert(self, mock_job):
from domain.value_objects import ConversionResult, PageDetail
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
mock_analysis_repo.update_status = AsyncMock()
mock_doc_repo.update_page_count = AsyncMock()
mock_converter = AsyncMock()
mock_converter.convert.return_value = ConversionResult(
page_count=1,
@ -403,9 +387,8 @@ class TestServiceForwardsPipelineOptions:
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
)
from services.analysis_service import AnalysisService
svc = AnalysisService(converter=mock_converter)
svc = self._make_service(mock_converter)
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
opts = {
"do_ocr": False,
@ -431,21 +414,10 @@ class TestServiceForwardsPipelineOptions:
assert conv_opts.generate_picture_images is True
assert conv_opts.images_scale == 2.0
@patch("services.analysis_service.analysis_repo")
@patch("services.analysis_service.document_repo")
@pytest.mark.asyncio
async def test_run_analysis_uses_defaults_when_no_options(
self,
mock_doc_repo,
mock_analysis_repo,
mock_job,
):
async def test_run_analysis_uses_defaults_when_no_options(self, mock_job):
from domain.value_objects import ConversionResult, PageDetail
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
mock_analysis_repo.update_status = AsyncMock()
mock_doc_repo.update_page_count = AsyncMock()
mock_converter = AsyncMock()
mock_converter.convert.return_value = ConversionResult(
page_count=1,
@ -454,9 +426,8 @@ class TestServiceForwardsPipelineOptions:
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
)
from services.analysis_service import AnalysisService
svc = AnalysisService(converter=mock_converter)
svc = self._make_service(mock_converter)
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", None)
@ -465,30 +436,19 @@ class TestServiceForwardsPipelineOptions:
assert call_args[0][0] == "/tmp/test.pdf"
assert call_args[0][1] == ConversionOptions()
@patch("services.analysis_service.analysis_repo")
@patch("services.analysis_service.document_repo")
@pytest.mark.asyncio
async def test_run_analysis_marks_failed_on_error(
self,
mock_doc_repo,
mock_analysis_repo,
mock_job,
):
mock_analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
mock_analysis_repo.update_status = AsyncMock()
async def test_run_analysis_marks_failed_on_error(self, mock_job):
mock_converter = AsyncMock()
mock_converter.convert.side_effect = RuntimeError("Docling crashed")
from services.analysis_service import AnalysisService
svc = AnalysisService(converter=mock_converter)
svc = self._make_service(mock_converter)
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", {"do_ocr": False})
# Should have called update_status twice: RUNNING then FAILED
assert mock_analysis_repo.update_status.call_count == 2
last_job = mock_analysis_repo.update_status.call_args_list[-1][0][0]
assert svc._analysis_repo.update_status.call_count == 2
last_job = svc._analysis_repo.update_status.call_args_list[-1][0][0]
assert last_job.status.value == "FAILED"
assert "Docling crashed" in last_job.error_message

55
document-parser/tests/test_repos.py
View file

@ -3,8 +3,9 @@
import pytest
from domain.models import AnalysisJob, AnalysisStatus, Document
from persistence import analysis_repo, document_repo
from persistence.analysis_repo import SqliteAnalysisRepository
from persistence.database import init_db
from persistence.document_repo import SqliteDocumentRepository
@pytest.fixture(autouse=True)
@ -16,8 +17,18 @@ async def setup_db(monkeypatch, tmp_path):
yield
@pytest.fixture
def document_repo():
return SqliteDocumentRepository()
@pytest.fixture
def analysis_repo():
return SqliteAnalysisRepository()
class TestDocumentRepo:
async def test_insert_and_find_by_id(self):
async def test_insert_and_find_by_id(self, document_repo):
doc = Document(
id="doc-1",
filename="test.pdf",
@ -33,11 +44,11 @@ class TestDocumentRepo:
assert found.filename == "test.pdf"
assert found.file_size == 1024
async def test_find_by_id_not_found(self):
async def test_find_by_id_not_found(self, document_repo):
found = await document_repo.find_by_id("nonexistent")
assert found is None
async def test_find_all(self):
async def test_find_all(self, document_repo):
for i in range(3):
doc = Document(id=f"doc-{i}", filename=f"file{i}.pdf", storage_path=f"/tmp/{i}")
await document_repo.insert(doc)
@ -45,7 +56,7 @@ class TestDocumentRepo:
all_docs = await document_repo.find_all()
assert len(all_docs) == 3
async def test_update_page_count(self):
async def test_update_page_count(self, document_repo):
doc = Document(id="doc-1", filename="test.pdf", storage_path="/tmp/test.pdf")
await document_repo.insert(doc)
@ -54,7 +65,7 @@ class TestDocumentRepo:
updated = await document_repo.find_by_id("doc-1")
assert updated.page_count == 10
async def test_delete(self):
async def test_delete(self, document_repo):
doc = Document(id="doc-1", filename="test.pdf", storage_path="/tmp/test.pdf")
await document_repo.insert(doc)
@ -64,19 +75,19 @@ class TestDocumentRepo:
found = await document_repo.find_by_id("doc-1")
assert found is None
async def test_delete_nonexistent(self):
async def test_delete_nonexistent(self, document_repo):
deleted = await document_repo.delete("nonexistent")
assert deleted is False
class TestAnalysisRepo:
async def _insert_doc(self):
async def _insert_doc(self, document_repo):
doc = Document(id="doc-1", filename="test.pdf", storage_path="/tmp/test.pdf")
await document_repo.insert(doc)
return doc
async def test_insert_and_find_by_id(self):
await self._insert_doc()
async def test_insert_and_find_by_id(self, document_repo, analysis_repo):
await self._insert_doc(document_repo)
job = AnalysisJob(id="job-1", document_id="doc-1")
await analysis_repo.insert(job)
@ -87,12 +98,12 @@ class TestAnalysisRepo:
assert found.status == AnalysisStatus.PENDING
assert found.document_filename == "test.pdf"
async def test_find_by_id_not_found(self):
async def test_find_by_id_not_found(self, analysis_repo):
found = await analysis_repo.find_by_id("nonexistent")
assert found is None
async def test_find_all(self):
await self._insert_doc()
async def test_find_all(self, document_repo, analysis_repo):
await self._insert_doc(document_repo)
for i in range(3):
job = AnalysisJob(id=f"job-{i}", document_id="doc-1")
await analysis_repo.insert(job)
@ -100,8 +111,8 @@ class TestAnalysisRepo:
all_jobs = await analysis_repo.find_all()
assert len(all_jobs) == 3
async def test_update_status(self):
await self._insert_doc()
async def test_update_status(self, document_repo, analysis_repo):
await self._insert_doc(document_repo)
job = AnalysisJob(id="job-1", document_id="doc-1")
await analysis_repo.insert(job)
@ -112,8 +123,8 @@ class TestAnalysisRepo:
assert found.status == AnalysisStatus.RUNNING
assert found.started_at is not None
async def test_update_status_completed(self):
await self._insert_doc()
async def test_update_status_completed(self, document_repo, analysis_repo):
await self._insert_doc(document_repo)
job = AnalysisJob(id="job-1", document_id="doc-1")
await analysis_repo.insert(job)
@ -127,8 +138,8 @@ class TestAnalysisRepo:
assert found.content_html == "<h1>Test</h1>"
assert found.pages_json == "[]"
async def test_delete(self):
await self._insert_doc()
async def test_delete(self, document_repo, analysis_repo):
await self._insert_doc(document_repo)
job = AnalysisJob(id="job-1", document_id="doc-1")
await analysis_repo.insert(job)
@ -138,12 +149,12 @@ class TestAnalysisRepo:
found = await analysis_repo.find_by_id("job-1")
assert found is None
async def test_delete_nonexistent(self):
async def test_delete_nonexistent(self, analysis_repo):
deleted = await analysis_repo.delete("nonexistent")
assert deleted is False
async def test_delete_by_document(self):
await self._insert_doc()
async def test_delete_by_document(self, document_repo, analysis_repo):
await self._insert_doc(document_repo)
for i in range(3):
job = AnalysisJob(id=f"job-{i}", document_id="doc-1")
await analysis_repo.insert(job)

359
document-parser/tests/test_robustness.py Normal file
View file

@ -0,0 +1,359 @@
"""Robustness tests — verify pipeline failure scenarios and protective mechanisms.
Tests cover: document_timeout (C1), lock timeout (C2), convert limits (C3),
default table_mode (H1), converter reset on failure (H5), error classification (M1).
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from domain.models import AnalysisJob
from domain.services import classify_error
from domain.value_objects import ConversionOptions, ConversionResult, PageDetail
from infra.settings import Settings
from services.analysis_service import AnalysisConfig, AnalysisService
# ---------------------------------------------------------------------------
# M1 — _classify_error: user-friendly error messages
# ---------------------------------------------------------------------------
class TestClassifyError:
"""Verify _classify_error maps exceptions to user-friendly messages."""
def test_cxx_compiler_error(self):
exc = RuntimeError("InvalidCxxCompiler: No working C++ compiler found")
assert "Missing C++ compiler" in classify_error(exc)
def test_no_working_compiler(self):
exc = RuntimeError("no working c++ compiler found in torch")
assert "Missing C++ compiler" in classify_error(exc)
def test_out_of_memory(self):
exc = MemoryError("Out of memory allocating tensor")
assert "Out of memory" in classify_error(exc)
def test_oom_shorthand(self):
exc = RuntimeError("OOM during inference on page 5")
assert "Out of memory" in classify_error(exc)
def test_lock_timeout(self):
exc = TimeoutError("Could not acquire converter lock within 300s")
assert "Server busy" in classify_error(exc)
def test_pipeline_failed(self):
exc = RuntimeError("Pipeline StandardPdfPipeline failed on page 3")
assert "Document processing failed" in classify_error(exc)
def test_timeout_generic(self):
exc = TimeoutError("timeout exceeded while processing")
assert "Processing took too long" in classify_error(exc)
def test_unknown_short_error(self):
exc = ValueError("something weird happened")
assert classify_error(exc) == "something weird happened"
def test_unknown_long_error_truncated(self):
long_msg = "x" * 300
exc = ValueError(long_msg)
result = classify_error(exc)
assert len(result) <= 201
assert result.endswith("…")
# ---------------------------------------------------------------------------
# H1 — default table_mode injection
# ---------------------------------------------------------------------------
class TestDefaultTableMode:
"""Verify _run_analysis_inner injects settings.default_table_mode."""
@pytest.fixture
def mock_job(self):
return AnalysisJob(id="j1", document_id="d1", document_filename="test.pdf")
def _make_result(self) -> ConversionResult:
return ConversionResult(
page_count=1,
content_markdown="# Test",
content_html="<h1>Test</h1>",
pages=[PageDetail(page_number=1, width=612.0, height=792.0)],
)
def _make_service(self, converter, *, default_table_mode="accurate"):
mock_analysis_repo = MagicMock()
mock_analysis_repo.find_by_id = AsyncMock()
mock_analysis_repo.update_status = AsyncMock()
mock_document_repo = MagicMock()
mock_document_repo.update_page_count = AsyncMock()
config = AnalysisConfig(default_table_mode=default_table_mode)
return AnalysisService(
converter=converter,
analysis_repo=mock_analysis_repo,
document_repo=mock_document_repo,
config=config,
)
@pytest.mark.asyncio
async def test_default_table_mode_injected_when_missing(self, mock_job):
mock_converter = AsyncMock()
mock_converter.convert.return_value = self._make_result()
svc = self._make_service(mock_converter)
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", {})
opts = mock_converter.convert.call_args[0][1]
assert opts.table_mode == "accurate"
@pytest.mark.asyncio
async def test_default_table_mode_injected_when_none(self, mock_job):
mock_converter = AsyncMock()
mock_converter.convert.return_value = self._make_result()
svc = self._make_service(mock_converter)
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", None)
opts = mock_converter.convert.call_args[0][1]
assert opts.table_mode == "accurate"
@pytest.mark.asyncio
async def test_user_table_mode_preserved(self, mock_job):
mock_converter = AsyncMock()
mock_converter.convert.return_value = self._make_result()
svc = self._make_service(mock_converter)
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", {"table_mode": "fast"})
opts = mock_converter.convert.call_args[0][1]
assert opts.table_mode == "fast"
@pytest.mark.asyncio
async def test_custom_default_from_settings(self, mock_job):
mock_converter = AsyncMock()
mock_converter.convert.return_value = self._make_result()
svc = self._make_service(mock_converter, default_table_mode="fast")
svc._analysis_repo.find_by_id = AsyncMock(return_value=mock_job)
await svc._run_analysis("j1", "/tmp/test.pdf", "test.pdf", {})
opts = mock_converter.convert.call_args[0][1]
assert opts.table_mode == "fast"
# ---------------------------------------------------------------------------
# Docling-dependent tests — skip if docling is not installed
# ---------------------------------------------------------------------------
docling = pytest.importorskip("docling", reason="docling library not installed")
from docling.datamodel.base_models import InputFormat # noqa: E402
import infra.local_converter as lc_mod # noqa: E402
build_converter = lc_mod._build_docling_converter
convert_sync = lc_mod._convert_sync
get_default_converter = lc_mod._ensure_default_converter
# ---------------------------------------------------------------------------
# C1 — document_timeout in PdfPipelineOptions
# ---------------------------------------------------------------------------
class TestDocumentTimeout:
"""Verify document_timeout from settings is wired into PdfPipelineOptions."""
def _get_pipeline_options(self, converter):
fmt_opt = converter.format_to_options[InputFormat.PDF]
return fmt_opt.pipeline_options
def test_document_timeout_from_settings(self):
conv = build_converter(ConversionOptions())
opts = self._get_pipeline_options(conv)
assert opts.document_timeout == 120.0
@patch("infra.local_converter.settings", Settings(document_timeout=45.0))
def test_custom_document_timeout(self):
conv = build_converter(ConversionOptions())
opts = self._get_pipeline_options(conv)
assert opts.document_timeout == 45.0
# ---------------------------------------------------------------------------
# C2 — converter lock timeout
# ---------------------------------------------------------------------------
def _mock_docling_result():
"""Create a minimal mock that satisfies _convert_sync's result processing."""
mock_result = MagicMock()
mock_result.document.pages = {}
mock_result.document.iterate_items.return_value = []
mock_result.document.export_to_markdown.return_value = ""
mock_result.document.export_to_html.return_value = ""
mock_result.document.export_to_dict.return_value = {}
return mock_result
class TestConverterLockTimeout:
"""Verify _convert_sync raises TimeoutError when lock cannot be acquired."""
@patch("infra.local_converter._converter_lock")
def test_lock_timeout_raises(self, mock_lock):
mock_lock.acquire.return_value = False
with pytest.raises(TimeoutError, match="Could not acquire converter lock"):
convert_sync("/tmp/test.pdf", ConversionOptions())
mock_lock.release.assert_not_called()
@patch("infra.local_converter._select_converter")
@patch("infra.local_converter._converter_lock")
def test_lock_released_on_success(self, mock_lock, mock_select):
mock_lock.acquire.return_value = True
mock_conv = MagicMock()
mock_conv.convert.return_value = _mock_docling_result()
mock_select.return_value = mock_conv
convert_sync("/tmp/test.pdf", ConversionOptions())
mock_lock.release.assert_called_once()
@patch("infra.local_converter._select_converter")
@patch("infra.local_converter._converter_lock")
def test_lock_released_on_error(self, mock_lock, mock_select):
mock_lock.acquire.return_value = True
mock_conv = MagicMock()
mock_conv.convert.side_effect = RuntimeError("boom")
mock_select.return_value = mock_conv
with pytest.raises(RuntimeError, match="boom"):
convert_sync("/tmp/test.pdf", ConversionOptions())
mock_lock.release.assert_called_once()
# ---------------------------------------------------------------------------
# C3 — max_num_pages and max_file_size forwarding
# ---------------------------------------------------------------------------
class TestConvertSyncLimits:
"""Verify _convert_sync forwards limit settings to conv.convert()."""
@patch("infra.local_converter._select_converter")
@patch("infra.local_converter._converter_lock")
@patch("infra.local_converter.settings", Settings(max_page_count=10, max_file_size=0))
def test_max_page_count_forwarded(self, mock_lock, mock_select):
mock_lock.acquire.return_value = True
mock_conv = MagicMock()
mock_conv.convert.return_value = _mock_docling_result()
mock_select.return_value = mock_conv
convert_sync("/tmp/test.pdf", ConversionOptions())
kwargs = mock_conv.convert.call_args.kwargs
assert kwargs["max_num_pages"] == 10
assert "max_file_size" not in kwargs
@patch("infra.local_converter._select_converter")
@patch("infra.local_converter._converter_lock")
@patch("infra.local_converter.settings", Settings(max_page_count=0, max_file_size=5_000_000))
def test_max_file_size_forwarded(self, mock_lock, mock_select):
mock_lock.acquire.return_value = True
mock_conv = MagicMock()
mock_conv.convert.return_value = _mock_docling_result()
mock_select.return_value = mock_conv
convert_sync("/tmp/test.pdf", ConversionOptions())
kwargs = mock_conv.convert.call_args.kwargs
assert kwargs["max_file_size"] == 5_000_000
assert "max_num_pages" not in kwargs
@patch("infra.local_converter._select_converter")
@patch("infra.local_converter._converter_lock")
@patch(
"infra.local_converter.settings",
Settings(max_page_count=20, max_file_size=10_000_000),
)
def test_both_limits_forwarded(self, mock_lock, mock_select):
mock_lock.acquire.return_value = True
mock_conv = MagicMock()
mock_conv.convert.return_value = _mock_docling_result()
mock_select.return_value = mock_conv
convert_sync("/tmp/test.pdf", ConversionOptions())
kwargs = mock_conv.convert.call_args.kwargs
assert kwargs["max_num_pages"] == 20
assert kwargs["max_file_size"] == 10_000_000
@patch("infra.local_converter._select_converter")
@patch("infra.local_converter._converter_lock")
@patch("infra.local_converter.settings", Settings(max_page_count=0, max_file_size=0))
def test_zero_limits_not_forwarded(self, mock_lock, mock_select):
mock_lock.acquire.return_value = True
mock_conv = MagicMock()
mock_conv.convert.return_value = _mock_docling_result()
mock_select.return_value = mock_conv
convert_sync("/tmp/test.pdf", ConversionOptions())
kwargs = mock_conv.convert.call_args.kwargs
assert "max_num_pages" not in kwargs
assert "max_file_size" not in kwargs
# ---------------------------------------------------------------------------
# H5 — _default_converter reset on init failure
# ---------------------------------------------------------------------------
class TestGetDefaultConverterReset:
"""Verify _ensure_default_converter resets on failure and retries on next call."""
@pytest.fixture(autouse=True)
def reset_default_converter(self):
original = lc_mod._default_converter
lc_mod._default_converter = None
yield
lc_mod._default_converter = original
@patch("infra.local_converter._build_docling_converter")
def test_init_failure_resets_to_none(self, mock_build):
mock_build.side_effect = RuntimeError("torch not found")
with pytest.raises(RuntimeError, match="torch not found"):
get_default_converter()
assert lc_mod._default_converter is None
@patch("infra.local_converter._build_docling_converter")
def test_retry_after_failure_succeeds(self, mock_build):
mock_conv = MagicMock()
mock_build.side_effect = [RuntimeError("torch not found"), mock_conv]
with pytest.raises(RuntimeError):
get_default_converter()
result = get_default_converter()
assert result is mock_conv
assert mock_build.call_count == 2
@patch("infra.local_converter._build_docling_converter")
def test_success_caches_converter(self, mock_build):
mock_conv = MagicMock()
mock_build.return_value = mock_conv
result1 = get_default_converter()
result2 = get_default_converter()
assert result1 is result2 is mock_conv
mock_build.assert_called_once()

8
document-parser/tests/test_serve_converter.py
View file

@ -56,6 +56,14 @@ class TestBuildFormData:
assert data["images_scale"] == "2.0"
assert data["include_images"] == "true"
def test_page_range_included_when_set(self):
data = _build_form_data(ConversionOptions(), page_range=(11, 20))
assert data["page_range"] == "11-20"
def test_page_range_absent_when_none(self):
data = _build_form_data(ConversionOptions())
assert "page_range" not in data
# ---------------------------------------------------------------------------
# Unit tests — response parsing

117
document-parser/tests/test_settings.py
View file

@ -14,7 +14,11 @@ class TestSettingsDefaults:
assert s.docling_serve_url == "http://localhost:5001"
assert s.docling_serve_api_key is None
assert s.conversion_timeout == 900
assert s.document_timeout == 120.0
assert s.lock_timeout == 300
assert s.max_page_count == 0
assert s.max_file_size_mb == 50
assert s.batch_page_size == 0
assert s.upload_dir == "./uploads"
assert s.db_path == "./data/docling_studio.db"
assert "http://localhost:3000" in s.cors_origins
@ -28,6 +32,107 @@ class TestSettingsDefaults:
s.upload_dir = "/other" # type: ignore[misc]
class TestSettingsValidation:
def test_negative_document_timeout_rejected(self):
import pytest
with pytest.raises(ValueError, match="document_timeout must be > 0"):
Settings(document_timeout=-1.0)
def test_zero_document_timeout_rejected(self):
import pytest
with pytest.raises(ValueError, match="document_timeout must be > 0"):
Settings(document_timeout=0)
def test_negative_conversion_timeout_rejected(self):
import pytest
with pytest.raises(ValueError, match="conversion_timeout must be > 0"):
Settings(conversion_timeout=-1)
def test_zero_max_concurrent_rejected(self):
import pytest
with pytest.raises(ValueError, match="max_concurrent_analyses must be >= 1"):
Settings(max_concurrent_analyses=0)
def test_negative_max_page_count_rejected(self):
import pytest
with pytest.raises(ValueError, match="max_page_count must be >= 0"):
Settings(max_page_count=-1)
def test_negative_max_file_size_rejected(self):
import pytest
with pytest.raises(ValueError, match="max_file_size must be >= 0"):
Settings(max_file_size=-1)
def test_negative_max_file_size_mb_rejected(self):
import pytest
with pytest.raises(ValueError, match="max_file_size_mb must be >= 0"):
Settings(max_file_size_mb=-1)
def test_zero_max_file_size_mb_accepted(self):
s = Settings(max_file_size_mb=0)
assert s.max_file_size_mb == 0
def test_positive_max_file_size_mb_accepted(self):
s = Settings(max_file_size_mb=100)
assert s.max_file_size_mb == 100
def test_negative_batch_page_size_rejected(self):
import pytest
with pytest.raises(ValueError, match="batch_page_size must be >= 0"):
Settings(batch_page_size=-1)
def test_zero_batch_page_size_accepted(self):
s = Settings(batch_page_size=0)
assert s.batch_page_size == 0
def test_positive_batch_page_size_accepted(self):
s = Settings(batch_page_size=10)
assert s.batch_page_size == 10
def test_zero_lock_timeout_rejected(self):
import pytest
with pytest.raises(ValueError, match="lock_timeout must be > 0"):
Settings(lock_timeout=0)
def test_invalid_table_mode_rejected(self):
import pytest
with pytest.raises(ValueError, match="default_table_mode must be"):
Settings(default_table_mode="turbo")
def test_cascade_document_ge_lock_rejected(self):
import pytest
with pytest.raises(ValueError, match=r"document_timeout.*< lock_timeout"):
Settings(document_timeout=400.0, lock_timeout=300, conversion_timeout=900)
def test_cascade_lock_ge_conversion_rejected(self):
import pytest
with pytest.raises(ValueError, match=r"lock_timeout.*< conversion_timeout"):
Settings(document_timeout=100.0, lock_timeout=900, conversion_timeout=900)
def test_cascade_valid_ordering_accepted(self):
s = Settings(document_timeout=60.0, lock_timeout=300, conversion_timeout=900)
assert s.document_timeout < s.lock_timeout < s.conversion_timeout
def test_multiple_errors_reported(self):
import pytest
with pytest.raises(ValueError, match="document_timeout") as exc_info:
Settings(document_timeout=-1, conversion_timeout=-1)
assert "conversion_timeout" in str(exc_info.value)
class TestSettingsFromEnv:
def test_reads_env_vars(self, monkeypatch):
monkeypatch.setenv("APP_VERSION", "1.2.3")
@ -35,8 +140,12 @@ class TestSettingsFromEnv:
monkeypatch.setenv("DEPLOYMENT_MODE", "huggingface")
monkeypatch.setenv("DOCLING_SERVE_URL", "http://serve:9000")
monkeypatch.setenv("DOCLING_SERVE_API_KEY", "secret-key")
monkeypatch.setenv("CONVERSION_TIMEOUT", "120")
monkeypatch.setenv("CONVERSION_TIMEOUT", "1200")
monkeypatch.setenv("DOCUMENT_TIMEOUT", "60.0")
monkeypatch.setenv("LOCK_TIMEOUT", "600")
monkeypatch.setenv("MAX_PAGE_COUNT", "20")
monkeypatch.setenv("MAX_FILE_SIZE_MB", "100")
monkeypatch.setenv("BATCH_PAGE_SIZE", "15")
monkeypatch.setenv("UPLOAD_DIR", "/data/uploads")
monkeypatch.setenv("DB_PATH", "/data/test.db")
monkeypatch.setenv("CORS_ORIGINS", "http://a.com, http://b.com")
@ -48,8 +157,12 @@ class TestSettingsFromEnv:
assert s.deployment_mode == "huggingface"
assert s.docling_serve_url == "http://serve:9000"
assert s.docling_serve_api_key == "secret-key"
assert s.conversion_timeout == 120
assert s.conversion_timeout == 1200
assert s.lock_timeout == 600
assert s.document_timeout == 60.0
assert s.max_page_count == 20
assert s.max_file_size_mb == 100
assert s.batch_page_size == 15
assert s.upload_dir == "/data/uploads"
assert s.db_path == "/data/test.db"
assert s.cors_origins == ["http://a.com", "http://b.com"]

212
e2e/CONVENTIONS.md Normal file
View file

@ -0,0 +1,212 @@
# E2E Test Conventions — Karate & Karate UI
Rules and patterns for writing reliable, maintainable e2e tests in this project.
## Architecture
```
e2e/
├── generate-test-data.py # Shared — generates PDFs for both suites
├── api/ # Karate API tests (HTTP-only, no browser)
│ ├── pom.xml
│ └── src/test/...
└── ui/ # Karate UI tests (browser, Chrome headless)
├── pom.xml
└── src/test/...
```
API and UI are **peer projects** — same level, each with its own `pom.xml`, runner, and `karate-config.js`. Never nest one inside the other.
## Golden rules
### 1. Never use `Thread.sleep()` — use `retry()` or `waitFor()`
```gherkin
# BAD — blocks the thread, ignores Karate retry mechanism
* def wait = function(){ java.lang.Thread.sleep(2000) }
* wait()
# GOOD — uses Karate's built-in retry with logging
* retry(30, 1000).waitFor('.chunk-card')
```
For "wait until one of several conditions":
```gherkin
# GOOD — retry().script() returns truthy when the expression matches
* retry(120, 1000).script("document.querySelector('.result-tabs') || document.querySelector('.error')")
```
### 2. Never use `delay()` — use `waitFor()` on the expected outcome
```gherkin
# BAD — arbitrary wait, flaky on slow CI
* driver.inputFile('input[type=file]', filePath)
* delay(2000)
* waitFor('.doc-item')
# GOOD — wait for the actual result
* driver.inputFile('input[type=file]', filePath)
* waitFor('.doc-item')
```
The **only acceptable `delay()`** is for CSS animations that have no observable DOM change (e.g., a 250ms sidebar transition). Even then, prefer `retry().script()` on the final state.
### 3. Use `karate.sizeOf()` for element counts — never raw `.length`
```gherkin
# BAD — .length may return #notpresent depending on Karate context
* def tabs = locateAll('.tab-btn')
* match tabs.length == 3
# BAD — bypasses Karate's auto-wait and retry mechanisms
* def count = script("document.querySelectorAll('.tab-btn').length")
# GOOD — idiomatic Karate
* match karate.sizeOf(locateAll('.tab-btn')) == 3
# GOOD — for assertions with >
* assert karate.sizeOf(locateAll('.element-card')) > 0
```
### 4. Use `data-e2e` attributes — never depend on CSS classes
```gherkin
# BAD — breaks when a dev renames a class for styling
* waitFor('.doc-item')
* click('.chunk-card')
# BAD — breaks when locale is FR
* click('{a}History')
# GOOD — decoupled from CSS and i18n
* waitFor('[data-e2e=doc-item]')
* click('[data-e2e=chunk-card]')
* click('[data-e2e=nav-history]')
```
CSS classes are for **styling**, `data-e2e` attributes are for **testing**. A frontend dev must be free to rename `.doc-item` to `.document-card` without breaking a single test. Add `data-e2e="xxx"` to every element the tests interact with.
Convention for `data-e2e` values:
- **kebab-case**, descriptive: `doc-item`, `upload-zone`, `run-btn`, `result-tabs`
- **Unique per role**, not per instance: use `doc-item` for all doc items (plural via `locateAll`), `tab-btn` for all tabs
- **Never** use CSS class names as `data-e2e` values — if they happen to match today, rename the `data-e2e` to avoid confusion
If you must match on text (e.g., i18n test), set the locale explicitly first:
```gherkin
* click('{button}EN')
* waitFor('.sidebar')
# Now it's safe to assert on English text
```
### 5. Setup via API, verify via UI, cleanup via API
```gherkin
# Setup — fast, deterministic
* def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def docId = result.docId
# Verify — the actual UI test
* driver uiBaseUrl + '/studio'
* waitFor('.doc-item')
* click('.doc-item')
...
# Cleanup — fast, doesn't depend on UI state
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
```
Exception: when the test IS about the UI action (e.g., upload.feature tests the upload UI itself).
### 6. Extract repeated patterns into callable helpers
```gherkin
# BAD — 5 lines of cleanup duplicated in every scenario
Given path '/api/documents'
When method GET
Then status 200
* def uploaded = karate.filter(response, function(d){ return d.filename == 'small.pdf' })
* def cleanupId = uploaded[0].id
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(cleanupId)' }
# GOOD — one-liner via helper
* call read('classpath:common/helpers/cleanup-by-name.feature') { filename: 'small.pdf' }
```
Helpers go in `common/helpers/` with the `@ignore` tag and a doc comment showing usage.
### 7. Use `optional()` for elements that may or may not appear
```gherkin
# GOOD — doesn't fail if the chevron is already open
* def chevronOpen = optional('.config-chevron.open')
* if (!chevronOpen.present) click('.config-toggle')
```
Never `waitFor()` an element that might not exist (e.g., a spinner that flashes too fast).
## Tag strategy
| Tag | Scope | Run when |
|-----|-------|----------|
| `@critical` | 5 core user journeys | CI — merge to `main` |
| `@ui` | All UI features | Local dev |
| `@smoke` | API health checks | Every PR |
| `@regression` | API full coverage | PR to `release/*` |
| `@e2e` | API cross-domain workflows | PR to `release/*` |
| `@ignore` | Callable helpers | Never run directly |
## Driver config (karate-config.js)
```javascript
karate.configure('driver', {
type: 'chrome',
headless: true,
showDriverLog: false,
addOptions: ['--no-sandbox', '--disable-gpu'], // required for CI Linux
screenshotOnFailure: true // auto-screenshot on failure
});
```
- `--no-sandbox` — mandatory in GitHub Actions / Docker containers
- `--disable-gpu` — avoids GPU-related headless issues
- `screenshotOnFailure` — auto-captures the browser state on failure for debugging
## File naming
| File | Naming convention |
|------|-------------------|
| Feature files | `kebab-case.feature` (e.g., `batch-progress.feature`) |
| Helpers | `kebab-case.feature` in `common/helpers/`, prefixed `ui-` for UI-specific |
| Runners | `PascalCase.java` (e.g., `UIRunner.java`, `E2ERunner.java`) |
| Config | `karate-config.js` (Karate convention) |
## Running tests
```bash
# Generate test PDFs (required once, or after clean)
python3 e2e/generate-test-data.py
# API tests
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @smoke"
# UI tests — critical (CI scope)
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @critical"
# UI tests — all (local scope)
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @ui"
```
## Common pitfalls
| Pitfall | Fix |
|---------|-----|
| `locateAll().length` returns `#notpresent` | Use `karate.sizeOf(locateAll(...))` |
| `match x > 0` fails with "no step-definition" | Use `assert x > 0` for numeric comparisons |
| `if (...) call read(...)` fails with JS eval error | Use `karate.call()` inside `if` expressions |
| `input()` on `input[type=file]` crashes | Use `driver.inputFile()` for file inputs |
| `waitFor('.spinner')` times out on fast ops | Use `optional()` or skip — wait for the result instead |
| Nav links break when locale changes | Use `data-e2e` attributes, not text matchers |
| Tests break when CSS class is renamed | Use `[data-e2e=xxx]` selectors, never `.class-name` |
| Tests fail on CI but pass locally | Add `--no-sandbox` to Chrome options |

70
e2e/api/README.md Normal file
View file

@ -0,0 +1,70 @@
# E2E API Tests — Karate
End-to-end API tests for Docling Studio using [Karate](https://karatelabs.github.io/karate/).
## Prerequisites
- **JDK 17+** (e.g. `brew install openjdk@17`)
- **Maven 3.9+** (e.g. `brew install maven`)
- **Python 3.12+** (for test data generation)
- **Docker** (for running the full stack)
## Quick start
```bash
# 1. Generate test PDFs (from repo root)
python e2e/generate-test-data.py
# 2. Start the stack
docker compose up -d --wait
# 3. Run all API tests
mvn test -f e2e/api/pom.xml
# 4. Tear down
docker compose down
```
## Run by tag
```bash
# Smoke only (~30s)
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @smoke"
# Regression (~2min)
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @regression"
# Full E2E workflows (~5min)
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @e2e"
```
## Custom base URL
```bash
mvn test -f e2e/api/pom.xml -DbaseUrl=http://your-host:8000
```
## Structure
```
e2e/api/
├── pom.xml # Maven + Karate dependency
├── src/test/java/
│ └── E2ERunner.java # JUnit5 Karate runner
└── src/test/resources/
├── karate-config.js # Base URL, timeouts, retry config
├── common/
│ ├── helpers/ # Callable features (upload, analyze, cleanup)
│ └── data/
│ ├── schemas/ # JSON match schemas
│ ├── test-cases/ # Data-driven JSON files
│ └── generated/ # Generated PDFs (gitignored)
├── health/ # @smoke
├── documents/ # @regression
├── analyses/ # @regression
└── workflows/ # @e2e (cross-domain)
```
## Reports
After a run, Karate HTML reports are in `e2e/api/target/karate-reports/`.

58
e2e/api/pom.xml Normal file
View file

@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.scub.docling-studio</groupId>
<artifactId>e2e-api-tests</artifactId>
<version>0.3.1</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<karate.version>1.5.0</karate.version>
</properties>
<dependencies>
<dependency>
<groupId>io.karatelabs</groupId>
<artifactId>karate-core</artifactId>
<version>${karate.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.karatelabs</groupId>
<artifactId>karate-junit5</artifactId>
<version>${karate.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version>
<configuration>
<argLine>-Dfile.encoding=UTF-8</argLine>
<includes>
<include>**/*Runner.java</include>
</includes>
<systemPropertyVariables>
<karate.options>${karate.options}</karate.options>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>

17
e2e/api/src/test/java/E2ERunner.java Normal file
View file

@ -0,0 +1,17 @@
import com.intuit.karate.junit5.Karate;
class E2ERunner {
@Karate.Test
Karate testAll() {
return Karate.run("classpath:health", "classpath:documents", "classpath:analyses", "classpath:workflows")
.relativeTo(getClass());
}
@Karate.Test
Karate testSmoke() {
return Karate.run("classpath:health")
.tags("@smoke")
.relativeTo(getClass());
}
}

53
e2e/api/src/test/resources/analyses/batch-progress.feature Normal file
View file

@ -0,0 +1,53 @@
@regression
Feature: Batched conversion with progress reporting
# Requires BATCH_PAGE_SIZE > 0 in the server configuration.
# If BATCH_PAGE_SIZE is 0 (disabled), the batch-specific assertions
# are skipped gracefully — the test still validates completion.
Background:
* url baseUrl
Scenario: Large PDF completes and reports page count
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'large.pdf' }
# Create analysis
Given path '/api/analyses'
And request { documentId: '#(uploaded.docId)' }
When method POST
Then status 200
* def jobId = response.id
# Poll until completed
Given path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
And match response.status == 'COMPLETED'
And match response.contentMarkdown == '#string'
And match response.pagesJson == '#string'
# When batched, progress must be preserved at completion (not reset to null)
# progressTotal > 0 proves batching was active
# progressCurrent == progressTotal proves the final update_status didn't erase them
* if (response.progressTotal > 0) karate.match('response.progressCurrent', response.progressTotal)
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Batched conversion disables document_json
# Only meaningful when BATCH_PAGE_SIZE > 0 and pages > BATCH_PAGE_SIZE
# Check health to see if batching is possible
Given path '/api/health'
When method GET
Then status 200
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'large.pdf' }
* def analysis = call read('classpath:common/helpers/analyze.feature') { docId: '#(uploaded.docId)' }
* match analysis.response.status == 'COMPLETED'
# If batched, hasDocumentJson should be false
# If not batched (BATCH_PAGE_SIZE=0), hasDocumentJson should be true
* match analysis.response.hasDocumentJson == '#boolean'
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }

40
e2e/api/src/test/resources/analyses/cancel.feature Normal file
View file

@ -0,0 +1,40 @@
@regression
Feature: Delete and cancel analysis jobs
Background:
* url baseUrl
Scenario: Delete completed analysis
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def analysis = call read('classpath:common/helpers/analyze.feature') { docId: '#(uploaded.docId)' }
# Delete the analysis
Given path '/api/analyses', analysis.jobId
When method DELETE
Then status 204
# Verify it's gone
Given path '/api/analyses', analysis.jobId
When method GET
Then status 404
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Delete non-existent analysis returns 404
Given path '/api/analyses', 'non-existent-id'
When method DELETE
Then status 404
Scenario: Delete document cascades to analyses
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def analysis = call read('classpath:common/helpers/analyze.feature') { docId: '#(uploaded.docId)' }
# Delete the document (should cascade)
Given path '/api/documents', uploaded.docId
When method DELETE
Then status 204
# Analysis should also be gone
Given path '/api/analyses', analysis.jobId
When method GET
Then status 404

66
e2e/api/src/test/resources/analyses/create-and-poll.feature Normal file
View file

@ -0,0 +1,66 @@
@regression
Feature: Create analysis and poll until completion
Background:
* url baseUrl
* def analysisSchema = read('classpath:common/data/schemas/analysis.json')
Scenario: Create analysis returns PENDING then completes
# Upload document
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
# Create analysis
Given path '/api/analyses'
And request { documentId: '#(uploaded.docId)' }
When method POST
Then status 200
And match response contains analysisSchema
And match response.status == 'PENDING'
And match response.documentId == uploaded.docId
* def jobId = response.id
# Poll until terminal state
Given path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
# Verify completed result
And match response.status == 'COMPLETED'
And match response.contentMarkdown == '#string'
And match response.contentHtml == '#string'
And match response.pagesJson == '#string'
And match response.startedAt == '#string'
And match response.completedAt == '#string'
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Create analysis for non-existent document returns 404
Given path '/api/analyses'
And request { documentId: 'non-existent-id' }
When method POST
Then status 404
Scenario: Get analysis by ID
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def analysis = call read('classpath:common/helpers/analyze.feature') { docId: '#(uploaded.docId)' }
Given path '/api/analyses', analysis.jobId
When method GET
Then status 200
And match response contains analysisSchema
And match response.id == analysis.jobId
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Get non-existent analysis returns 404
Given path '/api/analyses', 'non-existent-id'
When method GET
Then status 404
Scenario: List analyses returns array
Given path '/api/analyses'
When method GET
Then status 200
And match response == '#[]'

30
e2e/api/src/test/resources/analyses/pipeline-options.feature Normal file
View file

@ -0,0 +1,30 @@
@regression
Feature: Analysis with different pipeline options (data-driven)
Background:
* url baseUrl
Scenario Outline: Analysis completes with <description>
# Upload
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
# Create analysis with specific options
Given path '/api/analyses'
And request { documentId: '#(uploaded.docId)', pipelineOptions: #(options) }
When method POST
Then status 200
* def jobId = response.id
# Poll until terminal state
Given path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
And match response.status == 'COMPLETED'
And match response.contentMarkdown == '#string'
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Examples:
| read('classpath:common/data/test-cases/pipeline-options.json') |

49
e2e/api/src/test/resources/analyses/rechunk.feature Normal file
View file

@ -0,0 +1,49 @@
@regression
Feature: Re-chunking a completed analysis
Background:
* url baseUrl
Scenario: Rechunk with default options returns chunks
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'medium.pdf' }
* def analysis = call read('classpath:common/helpers/analyze.feature') { docId: '#(uploaded.docId)' }
* assert analysis.response.status == 'COMPLETED'
* assert analysis.response.hasDocumentJson == true
Given path '/api/analyses', analysis.jobId, 'rechunk'
And request { chunkingOptions: { chunkerType: 'hybrid', maxTokens: 512 } }
When method POST
Then status 200
And match response == '#[_ > 0]'
And match each response contains { text: '#string', tokenCount: '#number' }
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Rechunk with different max tokens changes results
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'medium.pdf' }
* def analysis = call read('classpath:common/helpers/analyze.feature') { docId: '#(uploaded.docId)' }
# Rechunk with small tokens
Given path '/api/analyses', analysis.jobId, 'rechunk'
And request { chunkingOptions: { chunkerType: 'hybrid', maxTokens: 64 } }
When method POST
Then status 200
* def smallChunks = response
# Rechunk with large tokens
Given path '/api/analyses', analysis.jobId, 'rechunk'
And request { chunkingOptions: { chunkerType: 'hybrid', maxTokens: 2048 } }
When method POST
Then status 200
* def largeChunks = response
# Smaller max tokens should produce more or equal chunks
* assert smallChunks.length >= largeChunks.length
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Rechunk non-existent analysis returns 400
Given path '/api/analyses', 'non-existent-id', 'rechunk'
And request { chunkingOptions: { chunkerType: 'hybrid', maxTokens: 512 } }
When method POST
Then status 400

3
e2e/api/src/test/resources/common/data/generated/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
# Generated test data — recreated by generate-test-data.py
*
!.gitignore

17
e2e/api/src/test/resources/common/data/schemas/analysis.json Normal file
View file

@ -0,0 +1,17 @@
{
"id": "#string",
"documentId": "#string",
"documentFilename": "#string",
"status": "#? _ == 'PENDING' || _ == 'RUNNING' || _ == 'COMPLETED' || _ == 'FAILED'",
"contentMarkdown": "##string",
"contentHtml": "##string",
"pagesJson": "##string",
"chunksJson": "##string",
"hasDocumentJson": "#boolean",
"errorMessage": "##string",
"progressCurrent": "##number",
"progressTotal": "##number",
"startedAt": "##string",
"completedAt": "##string",
"createdAt": "#string"
}

8
e2e/api/src/test/resources/common/data/schemas/document.json Normal file
View file

@ -0,0 +1,8 @@
{
"id": "#string",
"filename": "#string",
"contentType": "#string",
"fileSize": "#number",
"pageCount": "##number",
"createdAt": "#string"
}

7
e2e/api/src/test/resources/common/data/schemas/health.json Normal file
View file

@ -0,0 +1,7 @@
{
"status": "#? _ == 'ok' || _ == 'degraded'",
"version": "#string",
"engine": "#? _ == 'local' || _ == 'remote'",
"deploymentMode": "#? _ == 'self-hosted' || _ == 'huggingface'",
"database": "#? _ == 'ok' || _ == 'error'"
}

9
e2e/api/src/test/resources/common/data/test-cases/invalid-uploads.json Normal file
View file

@ -0,0 +1,9 @@
[
{
"description": "non-PDF file",
"file": "not-a-pdf.txt",
"contentType": "text/plain",
"expectedStatus": 400,
"expectedDetail": "PDF"
}
]

27
e2e/api/src/test/resources/common/data/test-cases/pipeline-options.json Normal file
View file

@ -0,0 +1,27 @@
[
{
"description": "default options",
"options": {}
},
{
"description": "table mode fast",
"options": { "tableMode": "fast" }
},
{
"description": "OCR disabled",
"options": { "doOcr": false }
},
{
"description": "table structure disabled",
"options": { "doTableStructure": false }
},
{
"description": "all enrichments enabled",
"options": {
"doOcr": true,
"doTableStructure": true,
"doCodeEnrichment": true,
"doFormulaEnrichment": true
}
}
]

26
e2e/api/src/test/resources/common/helpers/analyze.feature Normal file
View file

@ -0,0 +1,26 @@
@ignore
Feature: Helper — Create analysis and poll until terminal state
# Callable feature: returns { jobId, response }
# Usage: * def result = call read('classpath:common/helpers/analyze.feature') { docId: '#(docId)' }
# Optional: { docId: '...', pipelineOptions: { tableMode: 'fast' } }
Scenario:
* def opts = karate.get('pipelineOptions', null)
* def body = { documentId: '#(docId)' }
* if (opts != null) body.pipelineOptions = opts
# Create analysis
Given url baseUrl
And path '/api/analyses'
And request body
When method POST
Then status 200
* def jobId = response.id
# Poll until COMPLETED or FAILED
Given url baseUrl
And path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200

12
e2e/api/src/test/resources/common/helpers/cleanup.feature Normal file
View file

@ -0,0 +1,12 @@
@ignore
Feature: Helper — Delete a document and its analyses
# Callable feature: cleans up test data
# Usage: * call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
Scenario:
Given url baseUrl
And path '/api/documents', docId
When method DELETE
# Accept both 204 (deleted) and 404 (already gone)
Then assert responseStatus == 204 || responseStatus == 404

14
e2e/api/src/test/resources/common/helpers/upload.feature Normal file
View file

@ -0,0 +1,14 @@
@ignore
Feature: Helper — Upload a PDF document
# Callable feature: returns { docId, response }
# Usage: * def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
Scenario:
* def filePath = 'classpath:common/data/generated/' + file
Given url baseUrl
And path '/api/documents/upload'
And multipart file file = { read: '#(filePath)', filename: '#(file)', contentType: 'application/pdf' }
When method POST
Then status 200
* def docId = response.id

55
e2e/api/src/test/resources/documents/crud.feature Normal file
View file

@ -0,0 +1,55 @@
@regression
Feature: Document CRUD operations
Background:
* url baseUrl
* def docSchema = read('classpath:common/data/schemas/document.json')
Scenario: List documents includes uploaded document
# Upload
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
# List
Given path '/api/documents'
When method GET
Then status 200
And match response == '#[]'
And match response[*].id contains uploaded.docId
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Get document by ID
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
Given path '/api/documents', uploaded.docId
When method GET
Then status 200
And match response contains docSchema
And match response.id == uploaded.docId
And match response.filename == 'small.pdf'
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Get non-existent document returns 404
Given path '/api/documents', 'non-existent-id'
When method GET
Then status 404
Scenario: Delete document returns 204 then 404 on re-fetch
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
# Delete
Given path '/api/documents', uploaded.docId
When method DELETE
Then status 204
# Verify gone
Given path '/api/documents', uploaded.docId
When method GET
Then status 404
Scenario: Delete non-existent document returns 404
Given path '/api/documents', 'non-existent-id'
When method DELETE
Then status 404

45
e2e/api/src/test/resources/documents/preview.feature Normal file
View file

@ -0,0 +1,45 @@
@regression
Feature: Document page preview
Background:
* url baseUrl
Scenario: Preview page 1 returns PNG
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
Given path '/api/documents', uploaded.docId, 'preview'
And param page = 1
When method GET
Then status 200
And match header content-type contains 'image/png'
And match responseBytes == '#? _.length > 0'
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Preview with custom DPI
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
Given path '/api/documents', uploaded.docId, 'preview'
And param page = 1
And param dpi = 72
When method GET
Then status 200
And match header content-type contains 'image/png'
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Preview page out of range returns 400
* def uploaded = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
Given path '/api/documents', uploaded.docId, 'preview'
And param page = 999
When method GET
Then status 400
And match response.detail contains 'out of range'
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(uploaded.docId)' }
Scenario: Preview non-existent document returns 404
Given path '/api/documents', 'non-existent-id', 'preview'
When method GET
Then status 404

22
e2e/api/src/test/resources/documents/size-validation.feature Normal file
View file

@ -0,0 +1,22 @@
@regression
Feature: Document size validation
Background:
* url baseUrl
Scenario: File within size limit is accepted
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/small.pdf', filename: 'small.pdf', contentType: 'application/pdf' }
When method POST
Then status 200
And match response.id == '#string'
# Cleanup
* def docId = response.id
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
Scenario: Health endpoint exposes max file size
Given path '/api/health'
When method GET
Then status 200
# maxFileSizeMb is present when > 0
And match response.maxFileSizeMb == '#? _ == null || _ > 0'

41
e2e/api/src/test/resources/documents/upload.feature Normal file
View file

@ -0,0 +1,41 @@
@regression
Feature: Document upload validation
Background:
* url baseUrl
* def docSchema = read('classpath:common/data/schemas/document.json')
Scenario: Upload valid single-page PDF
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/small.pdf', filename: 'small.pdf', contentType: 'application/pdf' }
When method POST
Then status 200
And match response contains docSchema
And match response.filename == 'small.pdf'
And match response.pageCount == 1
And match response.fileSize == '#? _ > 0'
# Cleanup
* def docId = response.id
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
Scenario: Upload valid multi-page PDF
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/medium.pdf', filename: 'medium.pdf', contentType: 'application/pdf' }
When method POST
Then status 200
And match response.pageCount == 5
# Cleanup
* def docId = response.id
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
Scenario: Reject non-PDF file
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/not-a-pdf.txt', filename: 'not-a-pdf.txt', contentType: 'text/plain' }
When method POST
Then status 400
And match response.detail contains 'PDF'
Scenario: Reject upload without file
Given path '/api/documents/upload'
When method POST
Then status 422

22
e2e/api/src/test/resources/health/health.feature Normal file
View file

@ -0,0 +1,22 @@
@smoke
Feature: Health endpoint
Background:
* url baseUrl
* def healthSchema = read('classpath:common/data/schemas/health.json')
Scenario: Health returns OK with expected fields
Given path '/api/health'
When method GET
Then status 200
And match response contains healthSchema
And match response.status == 'ok'
And match response.database == 'ok'
Scenario: Health exposes configuration limits when set
Given path '/api/health'
When method GET
Then status 200
# maxFileSizeMb and maxPageCount are conditional (only if > 0)
And match response.engine == '#? _ == "local" || _ == "remote"'
And match response.version == '#string'

12
e2e/api/src/test/resources/karate-config.js Normal file
View file

@ -0,0 +1,12 @@
function fn() {
var config = {
baseUrl: karate.properties['baseUrl'] || 'http://localhost:8000',
pollInterval: 2000,
pollTimeout: 120000,
batchPollTimeout: 300000
};
karate.configure('connectTimeout', 10000);
karate.configure('readTimeout', 30000);
karate.configure('retry', { count: 60, interval: config.pollInterval });
return config;
}

59
e2e/api/src/test/resources/workflows/concurrent-analyses.feature Normal file
View file

@ -0,0 +1,59 @@
@e2e
Feature: Concurrent analysis execution
Background:
* url baseUrl
Scenario: Multiple analyses complete successfully in parallel
# Upload 3 documents
* def doc1 = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def doc2 = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def doc3 = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
# Create 3 analyses
Given path '/api/analyses'
And request { documentId: '#(doc1.docId)' }
When method POST
Then status 200
* def job1 = response.id
Given path '/api/analyses'
And request { documentId: '#(doc2.docId)' }
When method POST
Then status 200
* def job2 = response.id
Given path '/api/analyses'
And request { documentId: '#(doc3.docId)' }
When method POST
Then status 200
* def job3 = response.id
# Poll all three until terminal state
Given path '/api/analyses', job1
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
* def status1 = response.status
Given path '/api/analyses', job2
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
* def status2 = response.status
Given path '/api/analyses', job3
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
* def status3 = response.status
# All three should have completed
* match status1 == 'COMPLETED'
* match status2 == 'COMPLETED'
* match status3 == 'COMPLETED'
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(doc1.docId)' }
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(doc2.docId)' }
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(doc3.docId)' }

79
e2e/api/src/test/resources/workflows/full-happy-path.feature Normal file
View file

@ -0,0 +1,79 @@
@e2e
Feature: Full happy path — upload to cleanup
Background:
* url baseUrl
Scenario: Complete workflow — upload, analyze, verify, rechunk, cleanup
# Step 1: Upload a PDF
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/medium.pdf', filename: 'medium.pdf', contentType: 'application/pdf' }
When method POST
Then status 200
And match response.pageCount == 5
* def docId = response.id
# Step 2: Verify document appears in list
Given path '/api/documents'
When method GET
Then status 200
And match response[*].id contains docId
# Step 3: Get preview
Given path '/api/documents', docId, 'preview'
And param page = 1
When method GET
Then status 200
And match header content-type contains 'image/png'
# Step 4: Create analysis
Given path '/api/analyses'
And request { documentId: '#(docId)', pipelineOptions: { doOcr: true, tableMode: 'fast' } }
When method POST
Then status 200
And match response.status == 'PENDING'
* def jobId = response.id
# Step 5: Poll until completed
Given path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
And match response.status == 'COMPLETED'
And match response.contentMarkdown == '#string'
And match response.contentHtml == '#string'
And match response.pagesJson == '#string'
And match response.hasDocumentJson == true
# Step 6: Re-chunk the completed analysis
Given path '/api/analyses', jobId, 'rechunk'
And request { chunkingOptions: { chunkerType: 'hybrid', maxTokens: 256, mergePeers: true } }
When method POST
Then status 200
And match response == '#[_ > 0]'
And match each response contains { text: '#string', tokenCount: '#number' }
# Step 7: Verify analysis appears in list
Given path '/api/analyses'
When method GET
Then status 200
And match response[*].id contains jobId
# Step 8: Delete analysis
Given path '/api/analyses', jobId
When method DELETE
Then status 204
# Step 9: Delete document
Given path '/api/documents', docId
When method DELETE
Then status 204
# Step 10: Verify both are gone
Given path '/api/documents', docId
When method GET
Then status 404
Given path '/api/analyses', jobId
When method GET
Then status 404

80
e2e/generate-test-data.py Normal file
View file

@ -0,0 +1,80 @@
"""Generate deterministic test PDFs for E2E tests.
Uses fpdf2 to create valid PDFs with real text content so Docling
can extract and chunk them. No binary files committed to the repo.
Usage:
python e2e/generate-test-data.py
Dependencies:
pip install fpdf2
"""
from __future__ import annotations
import os
from fpdf import FPDF
OUTPUT_DIRS = [
os.path.join(os.path.dirname(__file__), "api", "src", "test", "resources", "common", "data", "generated"),
os.path.join(os.path.dirname(__file__), "ui", "src", "test", "resources", "common", "data", "generated"),
]
_PARAGRAPHS = [
"Document processing is a critical step in building retrieval-augmented generation systems.",
"Docling Studio provides tools for analyzing PDF documents and extracting structured content.",
"The conversion pipeline supports OCR, table detection, and formula enrichment features.",
"Chunking splits document content into semantically meaningful segments for vector indexing.",
"Each chunk preserves metadata such as page number, bounding boxes, and heading hierarchy.",
"The hybrid chunker combines hierarchical document structure with token-based splitting.",
"Vector stores like OpenSearch enable fast similarity search over embedded chunk vectors.",
"Quality control requires visual inspection of chunk boundaries and extracted text accuracy.",
"Batch processing of large documents uses page ranges to prevent memory exhaustion.",
"Progress reporting allows users to monitor long-running document conversion tasks.",
]
def _make_pdf(page_count: int, path: str) -> None:
"""Create a valid PDF with N pages containing text paragraphs."""
pdf = FPDF()
pdf.set_auto_page_break(auto=True, margin=25)
for page_num in range(page_count):
pdf.add_page()
pdf.set_font("Helvetica", "B", 16)
pdf.cell(0, 10, f"Page {page_num + 1} of {page_count}", new_x="LMARGIN", new_y="NEXT")
pdf.ln(5)
pdf.set_font("Helvetica", "", 11)
for i in range(5):
para = _PARAGRAPHS[(page_num * 5 + i) % len(_PARAGRAPHS)]
pdf.multi_cell(0, 6, para)
pdf.ln(3)
pdf.output(path)
size_kb = os.path.getsize(path) / 1024
print(f" {os.path.basename(path)}: {page_count} pages, {size_kb:.1f} KB")
def _make_non_pdf(path: str) -> None:
"""Create a non-PDF file for negative testing."""
with open(path, "wb") as f:
f.write(b"This is not a PDF file.\n")
print(f" {os.path.basename(path)}: not-a-pdf")
def main() -> None:
for output_dir in OUTPUT_DIRS:
os.makedirs(output_dir, exist_ok=True)
print(f"Generating test data in {output_dir}")
_make_pdf(1, os.path.join(output_dir, "small.pdf"))
_make_pdf(5, os.path.join(output_dir, "medium.pdf"))
_make_pdf(25, os.path.join(output_dir, "large.pdf"))
_make_non_pdf(os.path.join(output_dir, "not-a-pdf.txt"))
print("Done.")
if __name__ == "__main__":
main()

93
e2e/ui/README.md Normal file
View file

@ -0,0 +1,93 @@
# E2E UI Tests — Karate UI
Browser-based end-to-end tests for Docling Studio using [Karate UI](https://karatelabs.github.io/karate/karate-core/#ui-automation) (Chrome headless).
## Prerequisites
- **JDK 17+** (e.g. `brew install openjdk@17`)
- **Maven 3.9+** (e.g. `brew install maven`)
- **Google Chrome** (headless, auto-detected)
- **Python 3.12+** (for test data generation)
- **Docker** (for running the full stack)
## Quick start
```bash
# 1. Generate test PDFs (from repo root)
python e2e/generate-test-data.py
# 2. Start the stack
docker compose up -d --wait
# 3. Run all UI tests
mvn test -f e2e/ui/pom.xml
# 4. Tear down
docker compose down
```
## Run by tag
```bash
# Critical only — CI scope (~1min30)
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @critical"
# All UI tests — local scope (~3min)
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @ui"
```
## Custom URLs
```bash
mvn test -f e2e/ui/pom.xml -DbaseUrl=http://your-host:8000 -DuiBaseUrl=http://your-host:3000
```
## Structure
```
e2e/ui/
├── pom.xml # Maven + Karate dependency
├── src/test/java/
│ └── UIRunner.java # JUnit5 Karate runner
└── src/test/resources/
├── karate-config.js # URLs, timeouts, Chrome driver config
├── common/helpers/
│ ├── upload.feature # API helper — upload (setup)
│ ├── analyze.feature # API helper — analyze (setup)
│ ├── cleanup.feature # API helper — delete (teardown)
│ ├── ui-upload.feature # UI helper — upload via file input
│ └── ui-wait-analysis.feature # UI helper — poll for completion
├── documents/ # @critical @ui
│ ├── upload.feature # Upload + preview
│ ├── delete.feature # Delete via hover + click
│ └── error-states.feature # Non-PDF rejection, hints
├── analyses/ # @critical @ui
│ ├── analysis.feature # Run analysis, verify tabs
│ ├── batch-progress.feature # Progress bar on multi-page
│ ├── rechunk.feature # Prepare mode, rechunk
│ └── pipeline-options.feature # OCR, table mode toggles
├── navigation/ # @ui
│ ├── sidebar.feature # Sidebar navigation
│ └── i18n.feature # FR/EN language switch
└── workflows/ # @ui
└── full-ui-path.feature # Complete happy path via browser
```
## Tags
| Tag | Scope | When |
|-----|-------|------|
| `@critical` | 5 features, 6 scenarios | CI on `main` branch |
| `@ui` | All UI features | Local development |
## Design patterns
- **Setup via API, verify via UI** — fast setup with API helpers, then browser assertions
- **Cleanup via API** — `cleanup.feature` deletes test data after each scenario
- **Polling with `optional()`** — graceful handling of fast completions (no `waitFor` on spinners that may flash)
- **`data-e2e` selectors** — `[data-e2e=doc-item]` instead of `.doc-item` — decoupled from CSS, never breaks on style refactors
- **`karate.sizeOf()` for counts** — `karate.sizeOf(locateAll('[data-e2e=xxx]'))` instead of raw `.length` or `script()`
## Reports
After a run, Karate HTML reports are in `e2e/ui/target/karate-reports/`.

58
e2e/ui/pom.xml Normal file
View file

@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.scub.docling-studio</groupId>
<artifactId>e2e-ui-tests</artifactId>
<version>0.3.1</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<karate.version>1.5.0</karate.version>
</properties>
<dependencies>
<dependency>
<groupId>io.karatelabs</groupId>
<artifactId>karate-core</artifactId>
<version>${karate.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.karatelabs</groupId>
<artifactId>karate-junit5</artifactId>
<version>${karate.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version>
<configuration>
<argLine>-Dfile.encoding=UTF-8</argLine>
<includes>
<include>**/*Runner.java</include>
</includes>
<systemPropertyVariables>
<karate.options>${karate.options}</karate.options>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>

24
e2e/ui/src/test/java/UIRunner.java Normal file
View file

@ -0,0 +1,24 @@
import com.intuit.karate.junit5.Karate;
class UIRunner {
@Karate.Test
Karate testAll() {
return Karate.run("classpath:documents", "classpath:analyses", "classpath:navigation", "classpath:workflows")
.relativeTo(getClass());
}
@Karate.Test
Karate testCritical() {
return Karate.run("classpath:documents", "classpath:analyses")
.tags("@critical")
.relativeTo(getClass());
}
@Karate.Test
Karate testLocal() {
return Karate.run("classpath:documents", "classpath:analyses", "classpath:navigation", "classpath:workflows")
.tags("@ui")
.relativeTo(getClass());
}
}

50
e2e/ui/src/test/resources/analyses/analysis.feature Normal file
View file

@ -0,0 +1,50 @@
@critical @ui
Feature: UI — Launch an analysis and verify results
Background:
* url baseUrl
Scenario: Upload, analyze, and verify result tabs appear
# Setup via API — upload a document
* def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def docId = result.docId
# Open Studio page
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=doc-item]')
# Select the uploaded document
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
# Verify we are in Configure mode (first toggle button is active)
* waitFor('[data-e2e=toggle-btn].active')
# Click Run / Exécuter
* waitFor('[data-e2e=run-btn]')
* click('[data-e2e=run-btn]')
# Wait for analysis to complete
* call read('classpath:common/helpers/ui-wait-analysis.feature')
# Verify result tabs appear
* waitFor('[data-e2e=result-tabs]')
* waitFor('[data-e2e=tabs-header]')
* match karate.sizeOf(locateAll('[data-e2e=tab-btn]')) == 3
# Click on Markdown tab and verify content
* def tabs = locateAll('[data-e2e=tab-btn]')
* tabs[1].click()
* waitFor('[data-e2e=raw-markdown]')
* match text('[data-e2e=raw-content]') != ''
# Switch to Elements tab
* tabs[0].click()
* waitFor('[data-e2e=elements-list]')
* assert karate.sizeOf(locateAll('[data-e2e=element-card]')) > 0
# Verify page indicator
* waitFor('[data-e2e=page-indicator]')
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }

32
e2e/ui/src/test/resources/analyses/batch-progress.feature Normal file
View file

@ -0,0 +1,32 @@
@critical @ui
Feature: UI — Batch analysis progress bar
Background:
* url baseUrl
Scenario: Upload large PDF, analyze, and verify progress or completion
# Setup via API — upload a multi-page PDF for batch processing
* def result = call read('classpath:common/helpers/upload.feature') { file: 'medium.pdf' }
* def docId = result.docId
# Open Studio page
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=doc-item]')
# Select the document
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
# Click Run / Exécuter
* waitFor('[data-e2e=run-btn]')
* click('[data-e2e=run-btn]')
# Wait for analysis to complete (progress bar may or may not appear)
* call read('classpath:common/helpers/ui-wait-analysis.feature')
# Verify results loaded
* waitFor('[data-e2e=result-tabs]')
* match karate.sizeOf(locateAll('[data-e2e=tab-btn]')) == 3
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }

64
e2e/ui/src/test/resources/analyses/pipeline-options.feature Normal file
View file

@ -0,0 +1,64 @@
@ui
Feature: UI — Pipeline configuration options
Background:
* url baseUrl
Scenario: Toggle pipeline options in Configure mode
# Setup via API
* def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def docId = result.docId
# Open Studio
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=doc-item]')
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
# Verify config panel is visible in Configure mode
* waitFor('[data-e2e=config-panel]')
# Verify toggle switches exist (OCR, table structure, etc.)
* assert karate.sizeOf(locateAll('[data-e2e=toggle-switch]')) > 0
# Toggle OCR off then on
* def firstToggle = locateAll('[data-e2e=toggle-label]')[0]
* firstToggle.click()
* firstToggle.click()
# Verify select dropdown for table mode exists
* assert karate.sizeOf(locateAll('[data-e2e=config-select]')) > 0
# Change table mode
* select('[data-e2e=config-select]', 'fast')
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
Scenario: Verify pipeline options are preserved across mode switches
# Setup via API
* def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def docId = result.docId
# Open Studio and select document
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=doc-item]')
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
# Change table mode to fast
* waitFor('[data-e2e=config-select]')
* select('[data-e2e=config-select]', 'fast')
# Switch to Verify mode and back
* def toggleBtns = locateAll('[data-e2e=toggle-btn]')
* toggleBtns[1].click()
* waitFor('[data-e2e=toggle-btn].active')
* toggleBtns[0].click()
* waitFor('[data-e2e=config-select]')
# Verify table mode is still fast
* match value('[data-e2e=config-select]') == 'fast'
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }

62
e2e/ui/src/test/resources/analyses/rechunk.feature Normal file
View file

@ -0,0 +1,62 @@
@critical @ui
Feature: UI — Rechunk an analysis with different parameters
Background:
* url baseUrl
Scenario: Analyze a document via UI then rechunk via Prepare tab
# Setup via API — upload only (analysis will be done via UI)
* def uploadResult = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def docId = uploadResult.docId
# Open Studio page
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=doc-item]')
# Select the document
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
# Run analysis via UI (so the frontend store knows about it)
* waitFor('[data-e2e=run-btn]')
* click('[data-e2e=run-btn]')
# Wait for analysis to complete
* call read('classpath:common/helpers/ui-wait-analysis.feature')
* waitFor('[data-e2e=result-tabs]')
# Now Préparer toggle should be enabled — click the last one
* def toggleBtns = locateAll('[data-e2e=toggle-btn]')
* toggleBtns[karate.sizeOf(toggleBtns) - 1].click()
# Wait for chunk panel to load
* waitFor('[data-e2e=chunk-panel]')
# Expand config if collapsed
* def chevronOpen = optional('[data-e2e=config-chevron].open')
* if (!chevronOpen.present) click('[data-e2e=config-toggle]')
* waitFor('[data-e2e=config-body]')
# Modify max tokens — clear and set new value
* clear('[data-e2e=config-input]')
* input('[data-e2e=config-input]', '256')
# Click the Chunk / Run button
* waitFor('[data-e2e=chunk-btn]')
* click('[data-e2e=chunk-btn]')
# Wait for chunks to appear
* retry(30, 1000).waitFor('[data-e2e=chunk-card]')
# Verify chunk results
* waitFor('[data-e2e=chunk-results]')
* waitFor('[data-e2e=chunk-summary]')
* assert karate.sizeOf(locateAll('[data-e2e=chunk-card]')) > 0
# Verify each chunk has expected structure
* waitFor('[data-e2e=chunk-index]')
* waitFor('[data-e2e=chunk-tokens]')
* waitFor('[data-e2e=chunk-text]')
# Cleanup
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }

26
e2e/ui/src/test/resources/common/helpers/analyze.feature Normal file
View file

@ -0,0 +1,26 @@
@ignore
Feature: Helper — Create analysis and poll until terminal state
# Callable feature: returns { jobId, response }
# Usage: * def result = call read('classpath:common/helpers/analyze.feature') { docId: '#(docId)' }
# Optional: { docId: '...', pipelineOptions: { tableMode: 'fast' } }
Scenario:
* def opts = karate.get('pipelineOptions', null)
* def body = { documentId: '#(docId)' }
* if (opts != null) body.pipelineOptions = opts
# Create analysis
Given url baseUrl
And path '/api/analyses'
And request body
When method POST
Then status 200
* def jobId = response.id
# Poll until COMPLETED or FAILED
Given url baseUrl
And path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200

13
e2e/ui/src/test/resources/common/helpers/cleanup-by-name.feature Normal file
View file

@ -0,0 +1,13 @@
@ignore
Feature: Helper — Delete a document by filename
# Callable feature: finds a document by name via API and deletes it
# Usage: * call read('classpath:common/helpers/cleanup-by-name.feature') { filename: 'small.pdf' }
Scenario:
Given url baseUrl
And path '/api/documents'
When method GET
Then status 200
* def matches = karate.filter(response, function(d){ return d.filename == filename })
* if (karate.sizeOf(matches) > 0) karate.call('classpath:common/helpers/cleanup.feature', { docId: matches[0].id })

12
e2e/ui/src/test/resources/common/helpers/cleanup.feature Normal file
View file

@ -0,0 +1,12 @@
@ignore
Feature: Helper — Delete a document and its analyses
# Callable feature: cleans up test data
# Usage: * call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }
Scenario:
Given url baseUrl
And path '/api/documents', docId
When method DELETE
# Accept both 204 (deleted) and 404 (already gone)
Then assert responseStatus == 204 || responseStatus == 404

10
e2e/ui/src/test/resources/common/helpers/ui-upload.feature Normal file
View file

@ -0,0 +1,10 @@
@ignore
Feature: Helper — Upload a PDF via the UI
# Callable feature: uploads a file through the browser file input
# Usage: * call read('classpath:common/helpers/ui-upload.feature') { file: 'small.pdf' }
# Prerequisite: driver must already be on a page with .upload-zone (home or studio)
Scenario:
* def filePath = karate.toAbsolutePath('classpath:common/data/generated/' + file)
* driver.inputFile('input[type=file]', filePath)

10
e2e/ui/src/test/resources/common/helpers/ui-wait-analysis.feature Normal file
View file

@ -0,0 +1,10 @@
@ignore
Feature: Helper — Wait for analysis to complete via UI polling
# Callable feature: waits for analysis to finish — checks for result tabs or error state
# Usage: * call read('classpath:common/helpers/ui-wait-analysis.feature')
# Prerequisite: analysis must have been triggered, driver on the studio page
Scenario:
# Poll up to 2 minutes — result-tabs (COMPLETED) or error placeholder (FAILED)
* retry(120, 1000).script("document.querySelector('[data-e2e=result-tabs]') || document.querySelector('[data-e2e=result-error]')")

14
e2e/ui/src/test/resources/common/helpers/upload.feature Normal file
View file

@ -0,0 +1,14 @@
@ignore
Feature: Helper — Upload a PDF document
# Callable feature: returns { docId, response }
# Usage: * def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
Scenario:
* def filePath = 'classpath:common/data/generated/' + file
Given url baseUrl
And path '/api/documents/upload'
And multipart file file = { read: '#(filePath)', filename: '#(file)', contentType: 'application/pdf' }
When method POST
Then status 200
* def docId = response.id

30
e2e/ui/src/test/resources/documents/delete.feature Normal file
View file

@ -0,0 +1,30 @@
@critical @ui
Feature: UI — Delete a document
Background:
* url baseUrl
Scenario: Upload a document, then delete it via UI
# Setup via API — upload a doc
* def result = call read('classpath:common/helpers/upload.feature') { file: 'small.pdf' }
* def docId = result.docId
# Open Studio page
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=doc-item]')
# Count docs before deletion
* def countBefore = karate.sizeOf(locateAll('[data-e2e=doc-item]'))
# Select the first doc, hover to reveal delete, click it
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
* mouse('[data-e2e=doc-item].selected').go()
* waitFor('[data-e2e=doc-delete]')
* click('[data-e2e=doc-delete]')
# Wait for the doc count to decrease (UI confirmation)
* retry(10, 500).script("document.querySelectorAll('[data-e2e=doc-item]').length < " + countBefore)
# Cleanup — ensure our uploaded doc is gone regardless
* call read('classpath:common/helpers/cleanup.feature') { docId: '#(docId)' }

22
e2e/ui/src/test/resources/documents/error-states.feature Normal file
View file

@ -0,0 +1,22 @@
@ui
Feature: UI — Upload error states
Scenario: Reject non-PDF file and show error message
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=upload-zone]')
# Try to upload a non-PDF file
* def filePath = karate.toAbsolutePath('classpath:common/data/generated/not-a-pdf.txt')
* driver.inputFile('input[type=file]', filePath)
# Verify error message is displayed
* waitFor('[data-e2e=upload-error]')
* match text('[data-e2e=upload-error]') contains 'PDF'
Scenario: Upload hint displays max size info
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=upload-zone]')
# Verify upload hint is visible and not empty
* waitFor('[data-e2e=upload-hint]')
* match text('[data-e2e=upload-hint]') != ''

47
e2e/ui/src/test/resources/documents/upload.feature Normal file
View file

@ -0,0 +1,47 @@
@ui
Feature: UI — Upload a PDF and verify preview
Background:
* url baseUrl
Scenario: Upload a single-page PDF via UI and verify it appears with preview
# Open Studio page
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=upload-zone]')
# Upload via hidden file input
* def filePath = karate.toAbsolutePath('classpath:common/data/generated/small.pdf')
* driver.inputFile('input[type=file]', filePath)
# Verify document appears in the doc list sidebar
* waitFor('[data-e2e=doc-item]')
* match text('[data-e2e=doc-name]') contains 'small.pdf'
# Click the document to select it
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
# Verify PDF preview loads
* waitFor('[data-e2e=pdf-image]')
# Cleanup via API
* call read('classpath:common/helpers/cleanup-by-name.feature') { filename: 'small.pdf' }
Scenario: Upload a multi-page PDF and verify page navigation
# Open Studio page
* driver uiBaseUrl + '/studio'
* waitFor('[data-e2e=upload-zone]')
* def filePath = karate.toAbsolutePath('classpath:common/data/generated/medium.pdf')
* driver.inputFile('input[type=file]', filePath)
# Verify document appears
* waitFor('[data-e2e=doc-item]')
# Click to select and verify preview loads
* click('[data-e2e=doc-item]')
* waitFor('[data-e2e=doc-item].selected')
* waitFor('[data-e2e=pdf-image]')
# Cleanup
* call read('classpath:common/helpers/cleanup-by-name.feature') { filename: 'medium.pdf' }

23
e2e/ui/src/test/resources/karate-config.js Normal file
View file

@ -0,0 +1,23 @@
function fn() {
var config = {
baseUrl: karate.properties['baseUrl'] || 'http://localhost:8000',
uiBaseUrl: karate.properties['uiBaseUrl'] || 'http://localhost:3000',
pollInterval: 2000,
pollTimeout: 120000,
batchPollTimeout: 300000
};
karate.configure('connectTimeout', 10000);
karate.configure('readTimeout', 30000);
karate.configure('retry', { count: 60, interval: config.pollInterval });
// Karate UI — browser driver config (Chrome headless)
karate.configure('driver', {
type: 'chrome',
headless: true,
showDriverLog: false,
addOptions: ['--no-sandbox', '--disable-gpu'],
screenshotOnFailure: true
});
return config;
}

Some files were not shown because too many files have changed in this diff Show more