Compare commits
No commits in common. "main" and "v0.2.0" have entirely different histories.
362 changed files with 2723 additions and 38205 deletions
|
|
@ -18,6 +18,3 @@ document-parser/.mypy_cache/
|
|||
document-parser/.ruff_cache/
|
||||
document-parser/data/
|
||||
document-parser/uploads/
|
||||
|
||||
# E2E tests (JVM/Maven — not needed in Docker images)
|
||||
e2e/
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[*.py]
|
||||
indent_size = 4
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
51
.env.example
51
.env.example
|
|
@ -1,37 +1,3 @@
|
|||
# Docker Compose build target: "local" or "remote" (selects Dockerfile target)
|
||||
# CONVERSION_MODE=local
|
||||
|
||||
# Conversion engine: "local" (in-process Docling) or "remote" (Docling Serve)
|
||||
# Set automatically by the Docker target — override only for local dev
|
||||
# CONVERSION_ENGINE=local
|
||||
|
||||
# Docling Serve settings (remote mode only)
|
||||
# DOCLING_SERVE_URL=http://localhost:5001
|
||||
# DOCLING_SERVE_API_KEY=
|
||||
|
||||
# Max seconds per conversion (default: 900)
|
||||
# CONVERSION_TIMEOUT=900
|
||||
|
||||
# Max parallel analysis jobs (default: 3)
|
||||
# MAX_CONCURRENT_ANALYSES=3
|
||||
|
||||
# Max upload file size in MB (default: 50, 0 = unlimited)
|
||||
# MAX_FILE_SIZE_MB=50
|
||||
|
||||
# Nginx body size limit — nginx format (default: 200M, 0 = unlimited).
|
||||
# Must be >= MAX_FILE_SIZE_MB. Backend MAX_FILE_SIZE_MB is the effective arbiter.
|
||||
# NGINX_MAX_BODY_SIZE=200M
|
||||
|
||||
# Max pages per PDF (default: 0 = unlimited). Set to 20 for HF Spaces.
|
||||
# MAX_PAGE_COUNT=0
|
||||
|
||||
# Deployment mode: "self-hosted" (default) or "huggingface"
|
||||
# Shows disclaimer banner when set to "huggingface"
|
||||
# DEPLOYMENT_MODE=self-hosted
|
||||
|
||||
# Application version (set automatically by CI/Docker build)
|
||||
# APP_VERSION=dev
|
||||
|
||||
# CORS (comma-separated origins, only needed for custom deployments)
|
||||
# CORS_ORIGINS=http://localhost:3000,https://your-domain.com
|
||||
|
||||
|
|
@ -40,20 +6,3 @@
|
|||
|
||||
# Database path (inside container)
|
||||
# DB_PATH=./data/docling_studio.db
|
||||
|
||||
# OpenSearch URL (used by docker-compose.dev.yml, auto-set to service name)
|
||||
# OPENSEARCH_URL=http://opensearch:9200
|
||||
|
||||
# Embedding service URL (used by docker-compose.dev.yml, auto-set to service name)
|
||||
# EMBEDDING_URL=http://embedding:8001
|
||||
|
||||
# Embedding model (default: all-MiniLM-L6-v2, used by the embedding service)
|
||||
# EMBEDDING_MODEL=all-MiniLM-L6-v2
|
||||
|
||||
# Embedding vector dimension (default: 384 for Granite Embedding 30M / all-MiniLM-L6-v2)
|
||||
# EMBEDDING_DIMENSION=384
|
||||
|
||||
# Neo4j — graph-native document structure (used by docker-compose.dev.yml)
|
||||
# NEO4J_URI=bolt://neo4j:7687
|
||||
# NEO4J_USER=neo4j
|
||||
# NEO4J_PASSWORD=changeme
|
||||
|
|
|
|||
68
.github/ISSUE_TEMPLATE/bug.yml
vendored
68
.github/ISSUE_TEMPLATE/bug.yml
vendored
|
|
@ -1,68 +0,0 @@
|
|||
name: Bug report
|
||||
description: Report a bug or unexpected behavior
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting a bug!
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Description
|
||||
description: What happened?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: expected-behavior
|
||||
attributes:
|
||||
label: Expected behavior
|
||||
description: What should have happened instead?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: steps-to-reproduce
|
||||
attributes:
|
||||
label: Steps to reproduce
|
||||
description: How can we reproduce the issue?
|
||||
placeholder: |
|
||||
1. Go to ...
|
||||
2. Click on ...
|
||||
3. See error
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: scope
|
||||
attributes:
|
||||
label: Scope
|
||||
options:
|
||||
- Frontend
|
||||
- Backend
|
||||
- Infra / CI
|
||||
- Full-stack
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: severity
|
||||
attributes:
|
||||
label: Severity
|
||||
options:
|
||||
- Critical (app crash / data loss)
|
||||
- Major (feature broken)
|
||||
- Minor (cosmetic / workaround exists)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: Environment
|
||||
description: Browser, OS, deployment mode (Docker remote/local, dev), version
|
||||
placeholder: "Chrome 124, macOS, Docker remote, v0.3.0"
|
||||
validations:
|
||||
required: false
|
||||
48
.github/ISSUE_TEMPLATE/feature.yml
vendored
48
.github/ISSUE_TEMPLATE/feature.yml
vendored
|
|
@ -1,48 +0,0 @@
|
|||
name: Feature request
|
||||
description: Suggest a new feature or enhancement
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to suggest a feature!
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Description
|
||||
description: What do you want and why?
|
||||
placeholder: As a user, I want to ... so that ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: acceptance-criteria
|
||||
attributes:
|
||||
label: Acceptance criteria
|
||||
description: How do we know this is done?
|
||||
placeholder: |
|
||||
- [ ] ...
|
||||
- [ ] ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: technical-notes
|
||||
attributes:
|
||||
label: Technical notes
|
||||
description: Implementation hints, related files, architecture considerations (optional)
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: dropdown
|
||||
id: scope
|
||||
attributes:
|
||||
label: Scope
|
||||
options:
|
||||
- Frontend
|
||||
- Backend
|
||||
- Infra / CI
|
||||
- Full-stack
|
||||
validations:
|
||||
required: true
|
||||
32
.github/PULL_REQUEST_TEMPLATE.md
vendored
32
.github/PULL_REQUEST_TEMPLATE.md
vendored
|
|
@ -1,32 +0,0 @@
|
|||
## Type
|
||||
|
||||
- [ ] Feature (`feature/*`)
|
||||
- [ ] Bug fix (`fix/*`)
|
||||
- [ ] Hotfix (`hotfix/*`)
|
||||
- [ ] Documentation
|
||||
- [ ] Refactoring
|
||||
- [ ] CI/CD
|
||||
- [ ] Other: ___
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- 1-3 sentences: what changed and why -->
|
||||
|
||||
## Related issues
|
||||
|
||||
<!-- Closes #123, Fixes #456 -->
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Branch follows naming convention (`feature/`, `fix/`, `hotfix/`)
|
||||
- [ ] Commits follow [Conventional Commits](docs/git-workflow/commit-conventions.md)
|
||||
- [ ] Tests added/updated for the change
|
||||
- [ ] All tests pass (`pytest tests/ -v` + `npm run test:run`)
|
||||
- [ ] Linting passes (`ruff check .` + `npx eslint src/`)
|
||||
- [ ] `CHANGELOG.md` updated under `[Unreleased]`
|
||||
- [ ] Documentation updated if behavior changed
|
||||
- [ ] No secrets or credentials committed
|
||||
|
||||
## Screenshots / Evidence
|
||||
|
||||
<!-- If UI change: before/after screenshots. If API change: curl example or test output. -->
|
||||
43
.github/workflows/auto-close-issues.yml
vendored
43
.github/workflows/auto-close-issues.yml
vendored
|
|
@ -1,43 +0,0 @@
|
|||
name: Auto-close issues on release branch merge
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'release/**'
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
close-issues:
|
||||
name: Close referenced issues
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Scan commits and close issues
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
# Collect all commit messages from the push event
|
||||
commits='${{ toJSON(github.event.commits) }}'
|
||||
|
||||
# Extract issue numbers from Closes/Fixes patterns (case-insensitive)
|
||||
issues=$(echo "$commits" \
|
||||
| grep -ioE '(closes|fixes|close|fix|resolved|resolves)\s+#[0-9]+' \
|
||||
| grep -oE '[0-9]+' \
|
||||
| sort -u)
|
||||
|
||||
if [ -z "$issues" ]; then
|
||||
echo "No issue references found in commit messages."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
branch="${GITHUB_REF#refs/heads/}"
|
||||
for issue in $issues; do
|
||||
echo "Closing issue #$issue (referenced in $branch @ ${SHA:0:7})"
|
||||
gh issue close "$issue" \
|
||||
--repo "$REPO" \
|
||||
--comment "Closed automatically — merged into \`$branch\` via ${SHA:0:7}." \
|
||||
|| echo "Warning: could not close #$issue (may already be closed)"
|
||||
done
|
||||
151
.github/workflows/ci.yml
vendored
151
.github/workflows/ci.yml
vendored
|
|
@ -4,16 +4,13 @@ on:
|
|||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main, 'release/**']
|
||||
branches: [main]
|
||||
|
||||
# Cancel previous runs on the same branch/PR
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
name: Backend tests
|
||||
|
|
@ -29,7 +26,7 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: document-parser/requirements-test.txt
|
||||
cache-dependency-path: document-parser/requirements.txt
|
||||
|
||||
- name: Install system dependencies
|
||||
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends poppler-utils
|
||||
|
|
@ -37,11 +34,8 @@ jobs:
|
|||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements-test.txt
|
||||
pip install httpx ruff
|
||||
|
||||
- name: Lint
|
||||
run: ruff check .
|
||||
pip install -r requirements.txt
|
||||
pip install pytest pytest-asyncio httpx
|
||||
|
||||
- name: Run tests
|
||||
run: pytest tests/ -v
|
||||
|
|
@ -58,150 +52,15 @@ jobs:
|
|||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Lint
|
||||
run: npx eslint src/
|
||||
|
||||
- name: Type check
|
||||
run: npm run type-check
|
||||
|
||||
- name: Run tests
|
||||
run: npm run test:run
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
e2e:
|
||||
name: E2E API tests (Karate)
|
||||
needs: [backend, frontend]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: temurin
|
||||
cache: maven
|
||||
|
||||
- name: Generate test PDFs
|
||||
run: |
|
||||
pip install fpdf2 pypdfium2
|
||||
python e2e/generate-test-data.py
|
||||
|
||||
- name: Start stack
|
||||
run: docker compose up -d --wait --build
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
RATE_LIMIT_RPM: "0"
|
||||
|
||||
- name: Wait for health
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
|
||||
echo "Backend healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for backend... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
echo "Backend failed to start"
|
||||
docker compose logs
|
||||
exit 1
|
||||
|
||||
- name: Run E2E tests
|
||||
run: |
|
||||
KARATE_TAGS="@smoke"
|
||||
if [[ "${{ github.base_ref }}" == release/* ]] || [[ "${{ github.ref }}" == refs/heads/release/* ]]; then
|
||||
KARATE_TAGS="@smoke,@regression,@e2e"
|
||||
fi
|
||||
mvn test -f e2e/api/pom.xml -DbaseUrl=http://localhost:3000 -Dkarate.options="--tags ${KARATE_TAGS}"
|
||||
|
||||
- name: Upload Karate reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: karate-api-reports
|
||||
path: e2e/api/target/karate-reports/
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: docker compose down
|
||||
|
||||
e2e-ui:
|
||||
name: E2E UI tests (Karate UI)
|
||||
if: github.ref == 'refs/heads/main'
|
||||
needs: [backend, frontend]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: temurin
|
||||
cache: maven
|
||||
|
||||
- name: Install Chrome
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
with:
|
||||
chrome-version: stable
|
||||
|
||||
- name: Generate test PDFs
|
||||
run: |
|
||||
pip install fpdf2 pypdfium2
|
||||
python e2e/generate-test-data.py
|
||||
|
||||
- name: Start stack
|
||||
run: docker compose up -d --wait --build
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
RATE_LIMIT_RPM: "0"
|
||||
|
||||
- name: Wait for health
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
|
||||
echo "Backend healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for backend... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
echo "Backend failed to start"
|
||||
docker compose logs
|
||||
exit 1
|
||||
|
||||
- name: Run critical UI tests
|
||||
run: >
|
||||
mvn test -f e2e/ui/pom.xml
|
||||
-DbaseUrl=http://localhost:3000
|
||||
-DuiBaseUrl=http://localhost:3000
|
||||
-Dkarate.options="--tags @critical"
|
||||
|
||||
- name: Upload Karate UI reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: karate-ui-reports
|
||||
path: e2e/ui/target/karate-reports/
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: docker compose down
|
||||
|
|
|
|||
98
.github/workflows/docling-compat.yml
vendored
98
.github/workflows/docling-compat.yml
vendored
|
|
@ -1,98 +0,0 @@
|
|||
name: Docling compatibility
|
||||
|
||||
# Runs daily and on manual trigger.
|
||||
# Installs the LATEST docling + docling-core (ignoring version pins)
|
||||
# and runs the backend tests. Opens an issue if something breaks.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 6 * * *" # Every day at 06:30 UTC
|
||||
workflow_dispatch: # Manual trigger from GitHub UI
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
compat:
|
||||
name: Test against latest Docling
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: document-parser
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: document-parser/requirements-test.txt
|
||||
|
||||
- name: Install system dependencies
|
||||
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends poppler-utils
|
||||
|
||||
- name: Install pinned dependencies
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements-test.txt
|
||||
pip install httpx
|
||||
|
||||
- name: Upgrade docling to latest
|
||||
id: versions
|
||||
run: |
|
||||
pip install --upgrade docling docling-core
|
||||
echo "docling=$(pip show docling | grep Version | cut -d' ' -f2)" >> "$GITHUB_OUTPUT"
|
||||
echo "docling_core=$(pip show docling-core | grep Version | cut -d' ' -f2)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run tests
|
||||
run: pytest tests/ -v
|
||||
|
||||
- name: Open issue on failure
|
||||
if: failure()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const docling = '${{ steps.versions.outputs.docling }}';
|
||||
const doclingCore = '${{ steps.versions.outputs.docling_core }}';
|
||||
const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
// Don't open duplicate issues
|
||||
const { data: issues } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
state: 'open',
|
||||
labels: 'docling-compat',
|
||||
});
|
||||
|
||||
const title = `Docling compatibility break: docling ${docling} / docling-core ${doclingCore}`;
|
||||
|
||||
if (issues.some(i => i.title === title)) {
|
||||
console.log('Issue already exists, skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title,
|
||||
labels: ['docling-compat', 'bug'],
|
||||
body: [
|
||||
`## Docling compatibility failure`,
|
||||
``,
|
||||
`The daily compatibility check failed with the latest Docling releases:`,
|
||||
``,
|
||||
`| Package | Version |`,
|
||||
`|---------|---------|`,
|
||||
`| \`docling\` | \`${docling}\` |`,
|
||||
`| \`docling-core\` | \`${doclingCore}\` |`,
|
||||
``,
|
||||
`**Workflow run:** [View logs](${run})`,
|
||||
``,
|
||||
`### Next steps`,
|
||||
`1. Check the failing tests in the workflow logs`,
|
||||
`2. Identify breaking changes in the Docling changelog`,
|
||||
`3. Update the code and version pins in \`requirements.txt\``,
|
||||
`4. Close this issue once fixed`,
|
||||
].join('\n'),
|
||||
});
|
||||
48
.github/workflows/docs.yml
vendored
48
.github/workflows/docs.yml
vendored
|
|
@ -1,48 +0,0 @@
|
|||
name: Deploy docs
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "mkdocs.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: docs
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Build & deploy docs
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install MkDocs Material
|
||||
run: pip install mkdocs-material
|
||||
|
||||
- name: Build
|
||||
run: mkdocs build --strict
|
||||
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: site/
|
||||
|
||||
- id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
790
.github/workflows/release-gate.yml
vendored
790
.github/workflows/release-gate.yml
vendored
|
|
@ -1,790 +0,0 @@
|
|||
name: Release Gate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
concurrency:
|
||||
group: release-gate-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Phase 1 — Parallel validation (no Docker needed)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
lint-typecheck:
|
||||
name: Lint & type-check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: document-parser/requirements.txt
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install backend deps
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r document-parser/requirements.txt
|
||||
pip install ruff
|
||||
|
||||
- name: Install frontend deps
|
||||
run: cd frontend && npm ci
|
||||
|
||||
- name: Backend lint
|
||||
run: cd document-parser && ruff check .
|
||||
|
||||
- name: Frontend lint
|
||||
run: cd frontend && npx eslint src/
|
||||
|
||||
- name: Frontend type-check
|
||||
run: cd frontend && npm run type-check
|
||||
|
||||
unit-tests:
|
||||
name: Unit tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: document-parser/requirements.txt
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install system deps
|
||||
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends poppler-utils
|
||||
|
||||
- name: Backend tests
|
||||
run: |
|
||||
cd document-parser
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pytest pytest-asyncio httpx ruff
|
||||
pytest tests/ -v --tb=short 2>&1 | tee /tmp/backend-tests.txt
|
||||
|
||||
- name: Frontend tests
|
||||
run: |
|
||||
cd frontend
|
||||
npm ci
|
||||
npm run test:run 2>&1 | tee /tmp/frontend-tests.txt
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: unit-test-results
|
||||
path: /tmp/*-tests.txt
|
||||
|
||||
dep-audit:
|
||||
name: Dependency audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: document-parser/requirements.txt
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install deps
|
||||
run: |
|
||||
cd document-parser && pip install --upgrade pip && pip install -r requirements.txt && pip install pip-audit
|
||||
cd ../frontend && npm ci
|
||||
|
||||
- name: Python audit
|
||||
id: pip_audit
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd document-parser
|
||||
pip-audit --format=json --output=/tmp/pip-audit.json 2>&1 || true
|
||||
pip-audit 2>&1 | tee /tmp/pip-audit.txt
|
||||
# Check for critical vulnerabilities
|
||||
if pip-audit --format=json 2>/dev/null | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
vulns = data.get('dependencies', [])
|
||||
crits = [v for dep in vulns for v in dep.get('vulns', []) if 'CRITICAL' in v.get('fix_versions', [''])]
|
||||
sys.exit(1 if crits else 0)
|
||||
" 2>/dev/null; then
|
||||
echo "critical=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "critical=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Node audit
|
||||
id: npm_audit
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd frontend
|
||||
npm audit --json > /tmp/npm-audit.json 2>&1 || true
|
||||
npm audit 2>&1 | tee /tmp/npm-audit.txt
|
||||
# Check for critical vulnerabilities
|
||||
CRITICAL_COUNT=$(npm audit --json 2>/dev/null | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
print(data.get('metadata', {}).get('vulnerabilities', {}).get('critical', 0))
|
||||
except: print(0)
|
||||
" 2>/dev/null || echo "0")
|
||||
if [ "$CRITICAL_COUNT" -gt 0 ]; then
|
||||
echo "critical=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "critical=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Upload audit results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: dep-audit-results
|
||||
path: /tmp/*-audit.*
|
||||
|
||||
- name: Fail on critical vulnerabilities
|
||||
if: steps.pip_audit.outputs.critical == 'true' || steps.npm_audit.outputs.critical == 'true'
|
||||
run: |
|
||||
echo "::error::Critical vulnerabilities found in dependencies"
|
||||
exit 1
|
||||
|
||||
audit-checks:
|
||||
name: Audit checks (commands.sh)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- name: Run automated audit checks
|
||||
id: audit
|
||||
run: |
|
||||
bash profiles/fastapi-vue/commands.sh 2>&1 | tee /tmp/audit-checks.txt
|
||||
continue-on-error: true
|
||||
|
||||
- name: Parse audit results
|
||||
id: results
|
||||
if: always()
|
||||
run: |
|
||||
# Strip ANSI codes before counting
|
||||
CLEAN=$(sed 's/\x1b\[[0-9;]*m//g' /tmp/audit-checks.txt)
|
||||
PASS=$(echo "$CLEAN" | grep -c "PASS" || true)
|
||||
WARN=$(echo "$CLEAN" | grep -c "WARN" || true)
|
||||
FAIL=$(echo "$CLEAN" | grep -c "FAIL" || true)
|
||||
echo "pass=${PASS:-0}" >> "$GITHUB_OUTPUT"
|
||||
echo "warn=${WARN:-0}" >> "$GITHUB_OUTPUT"
|
||||
echo "fail=${FAIL:-0}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload audit results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: audit-checks-results
|
||||
path: /tmp/audit-checks.txt
|
||||
|
||||
- name: Fail on audit failures
|
||||
if: steps.audit.outcome == 'failure'
|
||||
run: |
|
||||
echo "::error::Audit checks failed — review profiles/fastapi-vue/commands.sh output"
|
||||
exit 1
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Phase 2 — Docker build & validation
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
docker-build:
|
||||
name: Docker build — ${{ matrix.target }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
target: [remote, local]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Extract version from branch
|
||||
id: version
|
||||
run: |
|
||||
# release/1.2.3 → 1.2.3
|
||||
BRANCH="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}"
|
||||
VERSION=$(echo "$BRANCH" | sed 's|release/||')
|
||||
echo "value=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
target: ${{ matrix.target }}
|
||||
load: true
|
||||
tags: docling-studio:${{ matrix.target }}
|
||||
build-args: APP_VERSION=${{ steps.version.outputs.value }}
|
||||
cache-from: type=gha,scope=${{ matrix.target }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.target }}
|
||||
|
||||
- name: Save image
|
||||
run: |
|
||||
docker save docling-studio:${{ matrix.target }} | gzip > /tmp/docling-studio-${{ matrix.target }}.tar.gz
|
||||
|
||||
- name: Upload image artifact
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: docker-image-${{ matrix.target }}
|
||||
path: /tmp/docling-studio-${{ matrix.target }}.tar.gz
|
||||
retention-days: 1
|
||||
|
||||
- name: Record image size
|
||||
run: |
|
||||
SIZE_BYTES=$(docker image inspect docling-studio:${{ matrix.target }} --format='{{.Size}}')
|
||||
SIZE_MB=$((SIZE_BYTES / 1024 / 1024))
|
||||
echo "$SIZE_MB" > /tmp/image-size-${{ matrix.target }}.txt
|
||||
echo "Image size (${{ matrix.target }}): ${SIZE_MB} MB"
|
||||
|
||||
- name: Upload size artifact
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: image-size-${{ matrix.target }}
|
||||
path: /tmp/image-size-${{ matrix.target }}.txt
|
||||
|
||||
docker-smoke:
|
||||
name: Docker smoke test
|
||||
needs: [docker-build]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- name: Download remote image
|
||||
uses: actions/download-artifact@v4.3.0
|
||||
with:
|
||||
name: docker-image-remote
|
||||
path: /tmp
|
||||
|
||||
- name: Load remote image
|
||||
run: docker load < /tmp/docling-studio-remote.tar.gz
|
||||
|
||||
- name: Smoke test — remote
|
||||
run: |
|
||||
docker run -d --name smoke-remote -p 3000:3000 \
|
||||
-e RATE_LIMIT_RPM=0 \
|
||||
docling-studio:remote
|
||||
|
||||
echo "Waiting for container to start..."
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
|
||||
echo "Remote image healthy!"
|
||||
curl -s http://localhost:3000/api/health | python3 -m json.tool
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo "::error::Remote image failed health check"
|
||||
docker logs smoke-remote
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Verify response fields
|
||||
HEALTH=$(curl -s http://localhost:3000/api/health)
|
||||
ENGINE=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin)['engine'])")
|
||||
STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
|
||||
|
||||
if [ "$STATUS" != "ok" ]; then
|
||||
echo "::error::Health status is '$STATUS', expected 'ok'"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$ENGINE" != "remote" ]; then
|
||||
echo "::error::Engine is '$ENGINE', expected 'remote'"
|
||||
exit 1
|
||||
fi
|
||||
echo "Remote smoke test passed"
|
||||
|
||||
docker stop smoke-remote && docker rm smoke-remote
|
||||
|
||||
image-scan:
|
||||
name: Security scan — ${{ matrix.target }}
|
||||
needs: [docker-build]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
target: [remote, local]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- name: Download image
|
||||
uses: actions/download-artifact@v4.3.0
|
||||
with:
|
||||
name: docker-image-${{ matrix.target }}
|
||||
path: /tmp
|
||||
|
||||
- name: Load image
|
||||
run: docker load < /tmp/docling-studio-${{ matrix.target }}.tar.gz
|
||||
|
||||
- name: Run Trivy — CRITICAL (blocking)
|
||||
uses: aquasecurity/trivy-action@v0.35.0
|
||||
with:
|
||||
image-ref: docling-studio:${{ matrix.target }}
|
||||
format: table
|
||||
exit-code: 1
|
||||
severity: CRITICAL
|
||||
output: /tmp/trivy-critical-${{ matrix.target }}.txt
|
||||
trivyignores: .trivyignore.yaml
|
||||
# `latest` instead of the action default — the previously hardcoded
|
||||
# v0.69.3 was yanked from GitHub releases mid-run (2026-04-29) and
|
||||
# broke the gate. Following Trivy stable is safer than chasing a
|
||||
# specific tag that can vanish.
|
||||
version: latest
|
||||
|
||||
- name: Run Trivy — HIGH (informational)
|
||||
if: always()
|
||||
uses: aquasecurity/trivy-action@v0.35.0
|
||||
with:
|
||||
image-ref: docling-studio:${{ matrix.target }}
|
||||
format: table
|
||||
exit-code: 0
|
||||
severity: HIGH
|
||||
output: /tmp/trivy-high-${{ matrix.target }}.txt
|
||||
trivyignores: .trivyignore.yaml
|
||||
version: latest
|
||||
|
||||
- name: Annotate HIGH vulnerabilities
|
||||
if: always()
|
||||
run: |
|
||||
HIGH_COUNT=$(grep -c "HIGH" /tmp/trivy-high-${{ matrix.target }}.txt 2>/dev/null || echo "0")
|
||||
if [ "$HIGH_COUNT" -gt 0 ]; then
|
||||
echo "::warning::${{ matrix.target }} image has $HIGH_COUNT HIGH severity vulnerabilities — review Trivy report"
|
||||
else
|
||||
echo "No HIGH vulnerabilities found in ${{ matrix.target }} image"
|
||||
fi
|
||||
|
||||
- name: Upload Trivy reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: trivy-${{ matrix.target }}
|
||||
path: /tmp/trivy-*-${{ matrix.target }}.txt
|
||||
|
||||
image-size:
|
||||
name: Image size check
|
||||
needs: [docker-build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- name: Download size artifacts
|
||||
uses: actions/download-artifact@v4.3.0
|
||||
with:
|
||||
pattern: image-size-*
|
||||
merge-multiple: true
|
||||
path: /tmp/sizes
|
||||
|
||||
- name: Get previous release sizes
|
||||
id: prev
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Find the latest release tag
|
||||
PREV_TAG=$(gh release list --repo "${{ github.repository }}" --limit 1 --json tagName -q '.[0].tagName' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$PREV_TAG" ]; then
|
||||
echo "No previous release found — skipping delta check"
|
||||
echo "remote=0" >> "$GITHUB_OUTPUT"
|
||||
echo "local=0" >> "$GITHUB_OUTPUT"
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Previous release: $PREV_TAG"
|
||||
echo "found=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Pull previous images and get sizes
|
||||
for target in remote local; do
|
||||
IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${PREV_TAG#v}-${target}"
|
||||
if docker pull "$IMAGE" 2>/dev/null; then
|
||||
PREV_SIZE=$(docker image inspect "$IMAGE" --format='{{.Size}}')
|
||||
PREV_MB=$((PREV_SIZE / 1024 / 1024))
|
||||
echo "${target}=${PREV_MB}" >> "$GITHUB_OUTPUT"
|
||||
echo "Previous $target size: ${PREV_MB} MB"
|
||||
else
|
||||
echo "Could not pull $IMAGE — skipping $target delta"
|
||||
echo "${target}=0" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Compare sizes
|
||||
id: delta
|
||||
run: |
|
||||
for target in remote local; do
|
||||
CURRENT=$(cat /tmp/sizes/image-size-${target}.txt 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$target" = "remote" ]; then
|
||||
PREV="${{ steps.prev.outputs.remote }}"
|
||||
else
|
||||
PREV="${{ steps.prev.outputs.local }}"
|
||||
fi
|
||||
|
||||
if [ "$PREV" -gt 0 ] 2>/dev/null; then
|
||||
DELTA=$((CURRENT - PREV))
|
||||
DELTA_PCT=$((DELTA * 100 / PREV))
|
||||
echo "${target}_current=${CURRENT}" >> "$GITHUB_OUTPUT"
|
||||
echo "${target}_prev=${PREV}" >> "$GITHUB_OUTPUT"
|
||||
echo "${target}_delta=${DELTA}" >> "$GITHUB_OUTPUT"
|
||||
echo "${target}_delta_pct=${DELTA_PCT}" >> "$GITHUB_OUTPUT"
|
||||
echo "$target: ${CURRENT}MB (was ${PREV}MB, delta: ${DELTA_PCT}%)"
|
||||
|
||||
if [ "$DELTA_PCT" -gt 10 ]; then
|
||||
echo "::warning::$target image grew by ${DELTA_PCT}% (${PREV}MB → ${CURRENT}MB)"
|
||||
elif [ "$DELTA_PCT" -lt -10 ]; then
|
||||
echo "::notice::$target image shrank by ${DELTA_PCT#-}% (${PREV}MB → ${CURRENT}MB)"
|
||||
fi
|
||||
else
|
||||
echo "${target}_current=${CURRENT}" >> "$GITHUB_OUTPUT"
|
||||
echo "${target}_prev=n/a" >> "$GITHUB_OUTPUT"
|
||||
echo "${target}_delta=n/a" >> "$GITHUB_OUTPUT"
|
||||
echo "${target}_delta_pct=n/a" >> "$GITHUB_OUTPUT"
|
||||
echo "$target: ${CURRENT}MB (no previous baseline)"
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Save size report
|
||||
run: |
|
||||
cat > /tmp/image-size-report.txt <<EOF
|
||||
Image Size Report
|
||||
=================
|
||||
remote: ${{ steps.delta.outputs.remote_current }}MB (prev: ${{ steps.delta.outputs.remote_prev }}MB, delta: ${{ steps.delta.outputs.remote_delta_pct }}%)
|
||||
local: ${{ steps.delta.outputs.local_current }}MB (prev: ${{ steps.delta.outputs.local_prev }}MB, delta: ${{ steps.delta.outputs.local_delta_pct }}%)
|
||||
EOF
|
||||
cat /tmp/image-size-report.txt
|
||||
|
||||
- name: Upload size report
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: image-size-report
|
||||
path: /tmp/image-size-report.txt
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Phase 3 — E2E tests on built images
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
e2e-api:
|
||||
name: E2E API tests (full scope)
|
||||
needs: [docker-smoke]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: temurin
|
||||
cache: maven
|
||||
|
||||
- name: Generate test PDFs
|
||||
run: |
|
||||
pip install fpdf2 pypdfium2
|
||||
python e2e/generate-test-data.py
|
||||
|
||||
- name: Start stack
|
||||
run: docker compose up -d --wait --build
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
RATE_LIMIT_RPM: "0"
|
||||
|
||||
- name: Wait for health
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
|
||||
echo "Backend healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for backend... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
echo "Backend failed to start"
|
||||
docker compose logs
|
||||
exit 1
|
||||
|
||||
- name: Run E2E API tests (full scope)
|
||||
run: |
|
||||
mvn test -f e2e/api/pom.xml \
|
||||
-DbaseUrl=http://localhost:3000 \
|
||||
-Dkarate.options="--tags @smoke,@regression,@e2e"
|
||||
|
||||
- name: Upload Karate reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: karate-api-reports
|
||||
path: e2e/api/target/karate-reports/
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: docker compose down
|
||||
|
||||
e2e-ui:
|
||||
name: E2E UI tests (@critical)
|
||||
needs: [docker-smoke]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: temurin
|
||||
cache: maven
|
||||
|
||||
- name: Install Chrome
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
with:
|
||||
chrome-version: stable
|
||||
|
||||
- name: Generate test PDFs
|
||||
run: |
|
||||
pip install fpdf2 pypdfium2
|
||||
python e2e/generate-test-data.py
|
||||
|
||||
- name: Start stack
|
||||
run: docker compose up -d --wait --build
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
RATE_LIMIT_RPM: "0"
|
||||
|
||||
- name: Wait for health
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:3000/api/health > /dev/null 2>&1; then
|
||||
echo "Backend healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for backend... ($i/30)"
|
||||
sleep 5
|
||||
done
|
||||
echo "Backend failed to start"
|
||||
docker compose logs
|
||||
exit 1
|
||||
|
||||
- name: Run critical UI tests
|
||||
run: >
|
||||
mvn test -f e2e/ui/pom.xml
|
||||
-DbaseUrl=http://localhost:3000
|
||||
-DuiBaseUrl=http://localhost:3000
|
||||
-Dkarate.options="--tags @critical"
|
||||
|
||||
- name: Upload Karate UI reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: karate-ui-reports
|
||||
path: e2e/ui/target/karate-reports/
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: docker compose down
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Phase 4 — Release summary (comment on PR)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
release-summary:
|
||||
name: Release summary
|
||||
if: always() && github.event_name == 'pull_request'
|
||||
needs:
|
||||
- lint-typecheck
|
||||
- unit-tests
|
||||
- dep-audit
|
||||
- audit-checks
|
||||
- docker-build
|
||||
- docker-smoke
|
||||
- image-scan
|
||||
- image-size
|
||||
- e2e-api
|
||||
- e2e-ui
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
actions: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4.3.1
|
||||
|
||||
- name: Download image-size artifacts
|
||||
uses: actions/download-artifact@v4.3.0
|
||||
with:
|
||||
pattern: image-size-*
|
||||
path: /tmp/artifacts
|
||||
|
||||
- name: Download audit-checks artifact
|
||||
uses: actions/download-artifact@v4.3.0
|
||||
with:
|
||||
name: audit-checks-results
|
||||
path: /tmp/artifacts/audit-checks-results
|
||||
|
||||
- name: Build summary comment
|
||||
id: summary
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Job results from needs context
|
||||
LINT="${{ needs.lint-typecheck.result }}"
|
||||
TESTS="${{ needs.unit-tests.result }}"
|
||||
DEP="${{ needs.dep-audit.result }}"
|
||||
AUDIT="${{ needs.audit-checks.result }}"
|
||||
BUILD="${{ needs.docker-build.result }}"
|
||||
SMOKE="${{ needs.docker-smoke.result }}"
|
||||
SCAN="${{ needs.image-scan.result }}"
|
||||
SIZE="${{ needs.image-size.result }}"
|
||||
E2E_API="${{ needs.e2e-api.result }}"
|
||||
E2E_UI="${{ needs.e2e-ui.result }}"
|
||||
|
||||
# Determine verdict
|
||||
BLOCKING_FAILED="false"
|
||||
for result in "$LINT" "$TESTS" "$BUILD" "$SMOKE" "$SCAN" "$E2E_API" "$E2E_UI"; do
|
||||
if [ "$result" = "failure" ]; then
|
||||
BLOCKING_FAILED="true"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$BLOCKING_FAILED" = "true" ]; then
|
||||
VERDICT="NO-GO"
|
||||
VERDICT_ICON="🔴"
|
||||
elif [ "$DEP" = "failure" ] || [ "$AUDIT" = "failure" ]; then
|
||||
VERDICT="GO CONDITIONAL"
|
||||
VERDICT_ICON="🟡"
|
||||
else
|
||||
VERDICT="GO"
|
||||
VERDICT_ICON="🟢"
|
||||
fi
|
||||
|
||||
# Status emoji helper
|
||||
status_icon() {
|
||||
case "$1" in
|
||||
success) echo "✅" ;;
|
||||
failure) echo "❌" ;;
|
||||
skipped) echo "⏭️" ;;
|
||||
cancelled) echo "🚫" ;;
|
||||
*) echo "❓" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Read image sizes
|
||||
REMOTE_SIZE=$(cat /tmp/artifacts/image-size-remote/image-size-remote.txt 2>/dev/null || echo "?")
|
||||
LOCAL_SIZE=$(cat /tmp/artifacts/image-size-local/image-size-local.txt 2>/dev/null || echo "?")
|
||||
|
||||
# Read size report
|
||||
SIZE_REPORT=$(cat /tmp/artifacts/image-size-report/image-size-report.txt 2>/dev/null || echo "No size data available")
|
||||
|
||||
# Read audit summary
|
||||
AUDIT_PASS=$(grep -c "PASS" /tmp/artifacts/audit-checks-results/audit-checks.txt 2>/dev/null || echo "?")
|
||||
AUDIT_WARN=$(grep -c "WARN" /tmp/artifacts/audit-checks-results/audit-checks.txt 2>/dev/null || echo "?")
|
||||
AUDIT_FAIL=$(grep -c "FAIL" /tmp/artifacts/audit-checks-results/audit-checks.txt 2>/dev/null || echo "?")
|
||||
|
||||
# Build the comment
|
||||
cat > /tmp/summary.md << ENDOFCOMMENT
|
||||
## ${VERDICT_ICON} Release Gate — ${VERDICT}
|
||||
|
||||
### Validation Results
|
||||
|
||||
| Check | Status | Blocking |
|
||||
|-------|--------|----------|
|
||||
| Lint & type-check | $(status_icon "$LINT") | Yes |
|
||||
| Unit tests | $(status_icon "$TESTS") | Yes |
|
||||
| Dependency audit | $(status_icon "$DEP") | Yes (CRITICAL) |
|
||||
| Audit checks | $(status_icon "$AUDIT") | Yes |
|
||||
| Docker build | $(status_icon "$BUILD") | Yes |
|
||||
| Docker smoke test | $(status_icon "$SMOKE") | Yes |
|
||||
| Image security scan | $(status_icon "$SCAN") | Yes (CRITICAL) |
|
||||
| Image size check | $(status_icon "$SIZE") | No |
|
||||
| E2E API (full scope) | $(status_icon "$E2E_API") | Yes |
|
||||
| E2E UI (@critical) | $(status_icon "$E2E_UI") | Yes |
|
||||
|
||||
### Audit Checks Summary
|
||||
|
||||
| PASS | WARN | FAIL |
|
||||
|------|------|------|
|
||||
| ${AUDIT_PASS} | ${AUDIT_WARN} | ${AUDIT_FAIL} |
|
||||
|
||||
### Docker Image Sizes
|
||||
|
||||
| Target | Size |
|
||||
|--------|------|
|
||||
| remote | ${REMOTE_SIZE} MB |
|
||||
| local | ${LOCAL_SIZE} MB |
|
||||
|
||||
<details>
|
||||
<summary>Size delta vs previous release</summary>
|
||||
|
||||
\`\`\`
|
||||
${SIZE_REPORT}
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<sub>Generated by <b>release-gate.yml</b> — see <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">workflow run</a> for full details</sub>
|
||||
ENDOFCOMMENT
|
||||
|
||||
- name: Post or update PR comment
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
PR_NUMBER="${{ github.event.pull_request.number }}"
|
||||
COMMENT_TAG="<!-- release-gate-summary -->"
|
||||
|
||||
# Check for existing comment
|
||||
EXISTING_COMMENT_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
|
||||
--jq ".[] | select(.body | contains(\"$COMMENT_TAG\")) | .id" \
|
||||
2>/dev/null | head -1)
|
||||
|
||||
BODY=$(cat /tmp/summary.md)
|
||||
BODY="${COMMENT_TAG}
|
||||
${BODY}"
|
||||
|
||||
if [ -n "$EXISTING_COMMENT_ID" ]; then
|
||||
gh api \
|
||||
"repos/${{ github.repository }}/issues/comments/${EXISTING_COMMENT_ID}" \
|
||||
-X PATCH \
|
||||
-f body="$BODY"
|
||||
echo "Updated existing comment #${EXISTING_COMMENT_ID}"
|
||||
else
|
||||
gh api \
|
||||
"repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
|
||||
-f body="$BODY"
|
||||
echo "Posted new comment"
|
||||
fi
|
||||
27
.github/workflows/release.yml
vendored
27
.github/workflows/release.yml
vendored
|
|
@ -1,4 +1,4 @@
|
|||
name: Release Docker Images
|
||||
name: Release Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
|
|
@ -6,22 +6,17 @@ on:
|
|||
- "v*"
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Build & push — ${{ matrix.target }}
|
||||
name: Build & push Docker image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
target: [remote, local]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
|
|
@ -33,28 +28,22 @@ jobs:
|
|||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Docker metadata
|
||||
- name: Extract version from tag
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}},suffix=-${{ matrix.target }}
|
||||
type=semver,pattern={{major}}.{{minor}},suffix=-${{ matrix.target }}
|
||||
type=raw,value=latest-${{ matrix.target }}
|
||||
|
||||
- name: Extract version from tag
|
||||
id: version
|
||||
run: echo "value=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
target: ${{ matrix.target }}
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build-args: APP_VERSION=${{ steps.version.outputs.value }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=${{ matrix.target }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.target }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
|
|
|||
12
.gitignore
vendored
12
.gitignore
vendored
|
|
@ -4,15 +4,12 @@ frontend/node_modules/
|
|||
|
||||
# Build outputs
|
||||
frontend/dist/
|
||||
site/
|
||||
|
||||
# IDE & local tooling
|
||||
.idea/
|
||||
.vscode/
|
||||
.run/
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
**/CLAUDE.md
|
||||
*.iml
|
||||
|
||||
# OS
|
||||
|
|
@ -34,18 +31,9 @@ __pycache__/
|
|||
.venv/
|
||||
*.egg-info/
|
||||
|
||||
# Excalidraw working files
|
||||
docs/excalidraw/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
hs_err_pid*
|
||||
|
||||
# Docker
|
||||
docker-compose.override.yml
|
||||
|
||||
# Audit profiles (internal tooling)
|
||||
profiles/
|
||||
|
||||
# E2E tests — Maven build outputs & Chrome user data
|
||||
e2e/**/target/
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
vulnerabilities:
|
||||
- id: CVE-2026-40393
|
||||
# No `paths:` constraint — Trivy reports OS-package CVEs against the
|
||||
# package name (libgbm1, libgl1-mesa-dri, libglx-mesa0, mesa-libgallium),
|
||||
# not against installed file paths. A `paths:` filter would silently
|
||||
# fail to match and the ignore would be a no-op (which it was).
|
||||
statement: >-
|
||||
Mesa OOB read (fix: Mesa 25.3.6 / 26.0.1). No Debian 13 backport available.
|
||||
Pulled transitively by libgl1 (required by OpenCV/RapidOCR). Tracked in #189
|
||||
with follow-up to drop libgl1 via opencv-python-headless.
|
||||
expired_at: 2026-06-30
|
||||
140
CHANGELOG.md
140
CHANGELOG.md
|
|
@ -1,140 +0,0 @@
|
|||
# Changelog
|
||||
|
||||
All notable changes to Docling Studio will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/).
|
||||
|
||||
## [0.5.1] - 2026-04-30
|
||||
|
||||
### Fixed
|
||||
|
||||
- Nginx upload body cap raised from 5 MB to 200 MB (`NGINX_MAX_BODY_SIZE`, default `200M`); uploads larger than 5 MB no longer returned 413 before reaching the backend.
|
||||
|
||||
## [0.5.0] - 2026-04-28
|
||||
|
||||
### Added
|
||||
|
||||
- Reasoning-trace viewer: SQLite-backed graph built from `document_json`, iteration-by-iteration overlay on the document outline (StructureViewer + GraphView), bidirectional PDF ↔ graph focus
|
||||
- Live reasoning runner via [docling-agent](https://github.com/docling-project/docling-agent) (Ollama backend): `POST /api/documents/:id/reasoning` returns answer + iteration trace + convergence flag (gated by `REASONING_ENABLED`, off by default)
|
||||
- `LLMProvider` port abstraction with `OllamaProvider` adapter — opens the door to alternate LLM backends once docling-agent supports them
|
||||
- Neo4j graph storage pipeline: `TreeWriter` + `ChunkWriter` adapters, schema bootstrap, graph fetch endpoint, `Maintain` step with cytoscape-based graph visualization
|
||||
- Architecture decision record (ADR-001): graph visualization library choice and 200-page endpoint cap
|
||||
- Remote chunking: enabled in Docling Serve mode (previously local-only)
|
||||
- Hexagonal architecture tests powered by `pytestarch` (CI-enforced)
|
||||
- Centralized magic numbers for page dimensions, limits, and timeouts
|
||||
- Paste image size/type limits (env vars `MAX_PASTE_IMAGE_SIZE_MB`, `PASTE_ALLOWED_IMAGE_TYPES`); surfaced in `/api/health`
|
||||
|
||||
### Changed
|
||||
|
||||
- **Breaking — `RAG_*` env vars renamed to `REASONING_*`**: `RAG_ENABLED` → `REASONING_ENABLED`, `RAG_MODEL_ID` → `REASONING_MODEL_ID`. Health response field `ragAvailable` → `reasoningAvailable`. New `LLM_PROVIDER_TYPE` env var (default `ollama`) materializes the LLM-provider abstraction.
|
||||
- **Breaking — reasoning endpoint renamed `/rag` → `/reasoning`**: `POST /api/documents/:id/rag` is now `POST /api/documents/:id/reasoning`. Aligns with frontend `reasoning` feature naming.
|
||||
- `api/reasoning.py` refactored to depend on `ReasoningRunnerPort`; concrete `docling-agent` integration moved to `infra/docling_agent_reasoning.py` (clean architecture)
|
||||
- Frontend `reasoning` feature flag now reads `reasoningAvailable` from `/api/health`; sidebar entry hides when the backend is not wired (instead of failing with 503 on click)
|
||||
- Documented that `docker-compose.yml` ships dev-only defaults (Neo4j `changeme` password, OpenSearch `DISABLE_SECURITY_PLUGIN=true`); operators must harden their own production deployments
|
||||
- Backend logs a loud warning at boot if Neo4j is wired (`NEO4J_URI` set) with the default `changeme` password, so prod operators can't silently inherit it
|
||||
- `DocumentConverter` port exposes `supports_page_batching: bool` so the analysis service no longer relies on `isinstance(converter, ServeConverter)` (LSP fix)
|
||||
- `VectorStore` port gains a `ping()` method; `IngestionService.ping()` now goes through the port instead of reaching into `_vector_store._client` (encapsulation)
|
||||
- API path parameters renamed `{job_id}` → `{analysis_id}` across `api/analyses.py` and `api/ingestion.py` to align the OpenAPI surface with the user-facing terminology (URL paths unchanged)
|
||||
- Centralised `localStorage` keys (`docling-theme`, `docling-locale`) into `frontend/src/shared/storage/keys.ts` (`STORAGE_KEYS`)
|
||||
- Removed the dead `apiUrl` ref from the settings store and its orphan `settings.apiUrl` i18n entries
|
||||
- Document-status string `"uploaded"` extracted to `DOCUMENT_STATUS_UPLOADED` in `api/schemas.py`
|
||||
- PDF preview endpoint (`GET /api/documents/{id}/preview`) now offloads the synchronous file read + rasterisation to a worker thread (`asyncio.to_thread`), unblocking the FastAPI event loop
|
||||
|
||||
### Fixed
|
||||
|
||||
- Graph: collapse Docling `InlineGroup` and `Picture` children to avoid empty leaf nodes (#197)
|
||||
- Neo4j: rewrite `fetch_graph` using `CALL` subqueries for proper relationship traversal
|
||||
- CI: install `pytestarch` in backend tests job (#177)
|
||||
- CI: ignore CVE-2026-40393 (Mesa) with expiry — Debian has no backport (#190)
|
||||
- Reasoning: re-scroll PDF when re-clicking the active iteration
|
||||
- `infra/docling_tree.py:101` migrated `isinstance(bbox, (list, tuple))` to PEP 604 union (Ruff UP038)
|
||||
- Cross-feature integration test moved out of `features/history/` into `src/__tests__/integration/` so feature folders stay self-contained
|
||||
- Tightened terminal `assert X is not None` checks in domain/repo/service tests to compare the value (e.g. `isinstance(.., datetime)` after `mark_running()`/`mark_completed()`)
|
||||
|
||||
## [0.4.0] - 2026-04-13
|
||||
|
||||
### Added
|
||||
|
||||
- Inline chunk text editing: double-click or edit button to modify chunk text, with save/cancel and "modified" badge
|
||||
- Docker Compose dev stack (`docker-compose.dev.yml`) with OpenSearch, Dashboards, hot-reload backend and Vite frontend
|
||||
- Soft-delete chunks: delete button with confirmation dialog, chunks hidden from UI but preserved in data
|
||||
- Vector index metadata schema: `IndexedChunk` domain model, OpenSearch mapping builder, configurable embedding dimension
|
||||
- `VectorStore` port (Protocol): `ensure_index`, `index_chunks`, `search_similar`, `get_chunks`, `delete_document`
|
||||
- OpenSearch adapter (`OpenSearchStore`): kNN vector search, full-text search, bulk indexing, document CRUD
|
||||
- Embedding microservice (`embedding-service/`): sentence-transformers REST API with batch processing and Dockerfile
|
||||
- `EmbeddingService` port and `EmbeddingClient` HTTP adapter for remote embedding generation
|
||||
- Orchestrated ingestion pipeline: Docling → chunking → embedding → OpenSearch indexing (idempotent)
|
||||
- Ingestion REST API: `POST /api/ingestion/{jobId}`, `DELETE /api/ingestion/{docId}`, `GET /api/ingestion/status`
|
||||
- Production docker-compose with OpenSearch and embedding service
|
||||
- E2E Karate test for full ingestion workflow (PDF → chunks in OpenSearch)
|
||||
- My Documents screen: search, filter (all/indexed/not indexed), sort (name/date), ingestion status badges
|
||||
- Ingest button in Studio: one-click ingestion from completed analysis with progress feedback
|
||||
|
||||
### Fixed
|
||||
|
||||
### Changed
|
||||
|
||||
## [0.3.1] - 2026-04-09
|
||||
|
||||
### Added
|
||||
|
||||
- Batch conversion progress: segmented progress bar with ring indicator and per-batch visual feedback
|
||||
- Inline mini progress bar in the top banner during analysis
|
||||
- Informational notice in Prepare mode when chunking is unavailable (batch mode)
|
||||
- `BATCH_PAGE_SIZE` environment variable forwarded in Docker Compose
|
||||
|
||||
### Fixed
|
||||
|
||||
- Batch progress reset to null on completion (progress_current/progress_total overwritten by stale in-memory job object)
|
||||
- Regression test for batch progress preservation in `_run_analysis_inner` flow
|
||||
- E2E assertion on final progress values in batch-progress feature
|
||||
|
||||
## [0.3.0] - 2026-04-07
|
||||
|
||||
### Added
|
||||
|
||||
- Chunking support: domain objects, persistence, API endpoints, and frontend Prepare mode
|
||||
- Chunk-to-bbox hover highlighting in Prepare mode
|
||||
- Page filtering and collapsible config in Prepare mode
|
||||
- Feature flipping mechanism
|
||||
- Reusable pagination composable and PaginationBar component
|
||||
- Version display in sidebar, settings page, and health endpoint
|
||||
|
||||
### Fixed
|
||||
|
||||
- Feature flag health check blocked by CORS
|
||||
- Zombie jobs and unprotected JSON parse
|
||||
- Upload error not displayed in DocumentUpload component
|
||||
- Serve API contract: send `to_formats` as repeated form fields
|
||||
- Audit findings: security, robustness, dead code, domain-infra violation
|
||||
|
||||
### Changed
|
||||
|
||||
- Refactored backend to hexagonal architecture for converter extensibility
|
||||
- Added ServeConverter adapter for remote Docling Serve integration
|
||||
- Moved `@vitest/mocker` from dependencies to devDependencies
|
||||
|
||||
## [0.2.0] - 2025-05-14
|
||||
|
||||
### Added
|
||||
|
||||
- Multi-arch Docker image release pipeline (GitHub Actions)
|
||||
- Docker image published to `ghcr.io/scub-france/Docling-Studio`
|
||||
|
||||
## [0.1.0] - 2025-01-01
|
||||
|
||||
### Added
|
||||
|
||||
- Initial release of Docling Studio
|
||||
- PDF upload and document management
|
||||
- Configurable Docling pipeline (OCR, tables, code, formulas, images)
|
||||
- Bounding box visualization with color-coded overlays
|
||||
- Per-page results synchronized with PDF viewer
|
||||
- Markdown and HTML export
|
||||
- Analysis history with re-visit capability
|
||||
- Dark/Light theme support
|
||||
- French and English localization
|
||||
- Docker and Docker Compose deployment
|
||||
- CI/CD with GitHub Actions (tests + multi-arch Docker build)
|
||||
- Health check endpoint (`/health`)
|
||||
- SQLite-backed persistence
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment:
|
||||
|
||||
- Using welcoming and inclusive language
|
||||
- Being respectful of differing viewpoints and experiences
|
||||
- Gracefully accepting constructive criticism
|
||||
- Focusing on what is best for the community
|
||||
- Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior:
|
||||
|
||||
- The use of sexualized language or imagery, and sexual attention or advances of any kind
|
||||
- Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
- Public or private harassment
|
||||
- Publishing others' private information without explicit permission
|
||||
- Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project maintainers at **[INSERT CONTACT EMAIL]**.
|
||||
|
||||
All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.
|
||||
234
CONTRIBUTING.md
234
CONTRIBUTING.md
|
|
@ -1,234 +0,0 @@
|
|||
# Contributing to Docling Studio
|
||||
|
||||
Thank you for your interest in contributing to Docling Studio! This guide will help you get started.
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Fork** the repository on GitHub
|
||||
2. **Clone** your fork locally:
|
||||
```bash
|
||||
git clone https://github.com/<your-username>/Docling-Studio.git
|
||||
cd Docling-Studio
|
||||
```
|
||||
3. **Create a branch** for your work:
|
||||
```bash
|
||||
git checkout -b feature/my-feature
|
||||
```
|
||||
|
||||
## Development Setup
|
||||
|
||||
### Docker Dev Stack (recommended)
|
||||
|
||||
The fastest way to get the full stack running (backend + frontend + OpenSearch):
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.dev.yml up
|
||||
```
|
||||
|
||||
This starts:
|
||||
|
||||
| Service | URL | Notes |
|
||||
|---------|-----|-------|
|
||||
| Frontend (Vite) | http://localhost:3000 | HMR enabled |
|
||||
| Backend (FastAPI) | http://localhost:8000 | Auto-reload on file changes |
|
||||
| OpenSearch | http://localhost:9200 | Single-node, security disabled |
|
||||
| OpenSearch Dashboards | http://localhost:5601 | Index inspection UI |
|
||||
|
||||
Source code is bind-mounted — edits on your host are reflected immediately.
|
||||
|
||||
To use remote conversion mode instead of local:
|
||||
|
||||
```bash
|
||||
CONVERSION_MODE=remote docker compose -f docker-compose.dev.yml up
|
||||
```
|
||||
|
||||
### Manual Setup
|
||||
|
||||
If you prefer running services directly on your machine:
|
||||
|
||||
### Backend (Python 3.12+)
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
|
||||
# Remote mode (lightweight — delegates to Docling Serve)
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Local mode (full — runs Docling in-process)
|
||||
pip install -r requirements-local.txt
|
||||
|
||||
pip install ruff pytest pytest-asyncio httpx # dev tools
|
||||
uvicorn main:app --reload --port 8000
|
||||
```
|
||||
|
||||
### Frontend (Node 20+)
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## Code Quality
|
||||
|
||||
### Backend — Ruff
|
||||
|
||||
We use [Ruff](https://docs.astral.sh/ruff/) for linting and formatting Python code.
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
ruff check . # lint
|
||||
ruff check . --fix # lint with auto-fix
|
||||
ruff format . # format
|
||||
```
|
||||
|
||||
### Frontend — TypeScript + ESLint + Prettier
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm run type-check # type check (vue-tsc)
|
||||
npx eslint src/ # lint
|
||||
npx prettier --check src/ # check formatting
|
||||
npx prettier --write src/ # auto-format
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# Backend (377 tests)
|
||||
cd document-parser
|
||||
pytest tests/ -v
|
||||
|
||||
# Frontend (156 tests)
|
||||
cd frontend
|
||||
npm run test:run
|
||||
```
|
||||
|
||||
### E2E API (Karate)
|
||||
|
||||
```bash
|
||||
# Generate test PDFs + start stack
|
||||
python e2e/generate-test-data.py
|
||||
docker compose up -d --wait
|
||||
|
||||
# Run all API tests
|
||||
mvn test -f e2e/api/pom.xml
|
||||
|
||||
# Or by tag: @smoke, @regression, @e2e
|
||||
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @smoke"
|
||||
```
|
||||
|
||||
### E2E UI (Karate UI)
|
||||
|
||||
```bash
|
||||
# Generate test PDFs + start stack (if not already running)
|
||||
python e2e/generate-test-data.py
|
||||
docker compose up -d --wait
|
||||
|
||||
# Run critical UI tests (CI scope)
|
||||
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @critical"
|
||||
|
||||
# Run all UI tests (local scope)
|
||||
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @ui"
|
||||
```
|
||||
|
||||
All tests must pass before submitting a PR.
|
||||
|
||||
## Submitting Changes
|
||||
|
||||
1. **Commit** with clear, descriptive messages
|
||||
2. **Push** your branch to your fork
|
||||
3. Open a **Pull Request** against `main`
|
||||
4. Describe **what** changed and **why** in the PR description
|
||||
5. Ensure CI passes (tests + build)
|
||||
|
||||
## Branching Strategy
|
||||
|
||||
We follow a simplified Git Flow:
|
||||
|
||||
| Branch | Purpose |
|
||||
|--------|---------|
|
||||
| `main` | Always stable — latest release merged back |
|
||||
| `release/X.Y.Z` | Release preparation (freeze, bugfixes, changelog) |
|
||||
| `feature/*` | New features — PR to `main` |
|
||||
| `fix/*` | Bug fixes — PR to `main` (or `release/*` for pre-release fixes) |
|
||||
| `hotfix/X.Y.Z` | Urgent fix on a released version — PR to `main` |
|
||||
|
||||
Rules:
|
||||
- All PRs target `main` (never stack branches on other feature branches)
|
||||
- `release/*` branches are created from `main` when preparing a release
|
||||
- `hotfix/*` branches are created from the release tag
|
||||
|
||||
## Versioning
|
||||
|
||||
We use [Semantic Versioning](https://semver.org/): `MAJOR.MINOR.PATCH`.
|
||||
|
||||
- **Source of truth**: the git tag (`vX.Y.Z`)
|
||||
- `package.json` version should match the current release branch
|
||||
- The build injects the version automatically (Vite `__APP_VERSION__` for frontend, `APP_VERSION` env var for backend)
|
||||
|
||||
## Release Process
|
||||
|
||||
1. **Create the release branch** from `main`:
|
||||
```bash
|
||||
git checkout main && git pull
|
||||
git checkout -b release/X.Y.Z
|
||||
```
|
||||
|
||||
2. **On the release branch**, only:
|
||||
- Bug fixes
|
||||
- Move `[Unreleased]` to `[X.Y.Z] - YYYY-MM-DD` in `CHANGELOG.md`
|
||||
- Update `version` in `frontend/package.json`
|
||||
|
||||
3. **Merge into `main`** via PR, then **tag on `main`**:
|
||||
```bash
|
||||
git checkout main && git pull
|
||||
git tag vX.Y.Z
|
||||
git push origin vX.Y.Z
|
||||
```
|
||||
|
||||
4. The tag triggers the **release workflow** which builds and pushes the Docker image to `ghcr.io`.
|
||||
|
||||
### Docker Image Tags
|
||||
|
||||
Each release produces two image variants:
|
||||
|
||||
| Tag | Description |
|
||||
|-----|-------------|
|
||||
| `X.Y.Z-remote` | Exact version — lightweight (Docling Serve) |
|
||||
| `X.Y.Z-local` | Exact version — full (in-process Docling) |
|
||||
| `X.Y-remote` | Latest patch of this minor — lightweight |
|
||||
| `X.Y-local` | Latest patch of this minor — full |
|
||||
| `latest-remote` | Latest stable — lightweight |
|
||||
| `latest-local` | Latest stable — full |
|
||||
|
||||
### Hotfix
|
||||
|
||||
```bash
|
||||
git checkout vX.Y.Z # from the release tag
|
||||
git checkout -b hotfix/X.Y.Z+1
|
||||
# fix, commit, PR to main
|
||||
git tag vX.Y.Z+1 # tag on main after merge
|
||||
```
|
||||
|
||||
### Changelog
|
||||
|
||||
We follow [Keep a Changelog](https://keepachangelog.com/). Every PR should add a line under `[Unreleased]` in `CHANGELOG.md`. The release branch moves `[Unreleased]` to the versioned section.
|
||||
|
||||
## Pull Request Guidelines
|
||||
|
||||
- Keep PRs focused — one feature or fix per PR
|
||||
- Add tests for new functionality
|
||||
- Update documentation if behavior changes
|
||||
- Follow existing code style and conventions
|
||||
|
||||
## Reporting Issues
|
||||
|
||||
- Use GitHub Issues to report bugs or request features
|
||||
- Include steps to reproduce for bugs
|
||||
- Mention your OS, Python/Node version, and Docker version if relevant
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the [MIT License](LICENSE).
|
||||
57
Dockerfile
57
Dockerfile
|
|
@ -1,37 +1,33 @@
|
|||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# Docling Studio — single-image build (frontend + backend, multi-target)
|
||||
# Docling Studio — single-image build (frontend + backend)
|
||||
#
|
||||
# Usage:
|
||||
# docker build --target remote -t docling-studio:remote .
|
||||
# docker build --target local -t docling-studio:local .
|
||||
# docker build -t docling-studio .
|
||||
# docker run -p 3000:3000 docling-studio
|
||||
# =============================================================================
|
||||
|
||||
# --- Stage 1: Build frontend assets ---
|
||||
FROM node:20-alpine AS frontend-build
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
|
||||
WORKDIR /build
|
||||
COPY frontend/package*.json ./
|
||||
RUN npm ci
|
||||
COPY frontend/ .
|
||||
RUN VITE_APP_VERSION=${APP_VERSION} npm run build
|
||||
RUN npm run build
|
||||
|
||||
# --- Stage 2: Base runtime (Python + Nginx) ---
|
||||
FROM python:3.12-slim AS base
|
||||
# --- Stage 2: Runtime (Python + Nginx) ---
|
||||
FROM python:3.12-slim
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
# System deps: poppler (pdf2image), nginx, gettext-base (envsubst for nginx template)
|
||||
# System deps: poppler (pdf2image), nginx, and OpenCV runtime libs
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
poppler-utils \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
nginx \
|
||||
gettext-base \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Python deps (common)
|
||||
# Python deps
|
||||
WORKDIR /app
|
||||
COPY document-parser/requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
|
@ -42,39 +38,16 @@ COPY document-parser/ .
|
|||
# Frontend static files
|
||||
COPY --from=frontend-build /build/dist /usr/share/nginx/html
|
||||
|
||||
# Nginx config (template stored outside sites-enabled to avoid nginx loading it raw)
|
||||
COPY nginx.conf.template /etc/nginx/default.template
|
||||
|
||||
# Non-root user
|
||||
RUN useradd --create-home --shell /bin/bash appuser
|
||||
# Nginx config
|
||||
COPY nginx.conf /etc/nginx/sites-enabled/default
|
||||
|
||||
# Data directories
|
||||
RUN mkdir -p /app/uploads /app/data && chown -R appuser:appuser /app
|
||||
RUN mkdir -p /app/uploads /app/data
|
||||
|
||||
ENV UPLOAD_DIR=/app/uploads
|
||||
ENV DB_PATH=/app/data/docling_studio.db
|
||||
ENV NGINX_MAX_BODY_SIZE=200M
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["sh", "-c", "envsubst '${NGINX_MAX_BODY_SIZE}' < /etc/nginx/default.template > /etc/nginx/sites-enabled/default && nginx && exec su appuser -c 'uvicorn main:app --host 127.0.0.1 --port 8000'"]
|
||||
|
||||
# --- Remote: lightweight, delegates to Docling Serve ---
|
||||
FROM base AS remote
|
||||
ENV CONVERSION_ENGINE=remote
|
||||
|
||||
# --- Local: full Docling in-process ---
|
||||
FROM base AS local
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY document-parser/requirements-local.txt .
|
||||
RUN pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu \
|
||||
&& pip install --no-cache-dir -r requirements-local.txt
|
||||
|
||||
RUN chown -R appuser:appuser /app \
|
||||
&& chown -R appuser:appuser /usr/local/lib/python3.12/site-packages/rapidocr/models
|
||||
ENV CONVERSION_ENGINE=local
|
||||
# Start both nginx and uvicorn
|
||||
CMD ["sh", "-c", "nginx && exec uvicorn main:app --host 127.0.0.1 --port 8000"]
|
||||
|
|
|
|||
257
README.md
257
README.md
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
# Docling Studio
|
||||
|
||||
|
||||
|
|
@ -6,23 +5,11 @@
|
|||
|
||||
|
||||
|
||||
[](https://github.com/scub-france/Docling-Studio)
|
||||
|
||||
A visual document analysis studio powered by [Docling](https://github.com/DS4SD/docling).
|
||||
Upload a PDF, configure the extraction pipeline, and visualize the results — text, tables, images, formulas, bounding boxes — all from your browser.
|
||||
|
||||
|
||||
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://www.star-history.com/?repos=scub-france%2FDocling-Studio&type=timeline&legend=top-left">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=scub-france/Docling-Studio&type=timeline&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=scub-france/Docling-Studio&type=timeline&legend=top-left" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/chart?repos=scub-france/Docling-Studio&type=timeline&legend=top-left" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
|
||||
## Features
|
||||
|
||||
|
|
@ -31,43 +18,45 @@ Upload a PDF, configure the extraction pipeline, and visualize the results — t
|
|||
- **Configurable Docling pipeline** — OCR, table extraction, code/formula enrichment, picture classification & description, image generation
|
||||
- **Bounding box visualization** — color-coded element overlay directly on the PDF
|
||||
- **Per-page results** — right panel syncs with the current PDF page
|
||||
- **Chunking** — split extracted content into semantic chunks (hierarchical, hybrid, or page-based) with configurable token limits and inline editing
|
||||
- **Ingestion pipeline** — Docling → chunking → embedding → OpenSearch vector indexing (one-click from Studio)
|
||||
- **Graph storage (Neo4j)** — full DoclingDocument tree (sections, paragraphs, tables, pages, chunks) mirrored as a graph with `PARENT_OF`, `NEXT`, `ON_PAGE`, `HAS_CHUNK`, `DERIVED_FROM` relations, with an in-app graph view powered by Cytoscape.js
|
||||
- **Markdown & HTML export** of extracted content
|
||||
- **Document management** — upload, list, delete, search, filter by indexing status
|
||||
- **Document management** — upload, list, delete
|
||||
- **Analysis history** — re-visit and open past analyses
|
||||
- **Upload limits** — configurable max file size and max page count per document
|
||||
- **Rate limiting** — configurable requests per minute per IP
|
||||
- **Dark / Light theme** and **FR / EN** localization
|
||||
|
||||
<details>
|
||||
<summary>More screenshots</summary>
|
||||
|
||||
| Import | Configure | Results |
|
||||
|--------|-----------|---------|
|
||||
|  |  |  |
|
||||
|
||||
</details>
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌────────────┐ ┌──────────────────────┐
|
||||
│ Frontend │────────▶│ Document Parser │
|
||||
│ Vue 3 │ /api/* │ FastAPI + Docling │
|
||||
│ port 3000 │ │ SQLite + file storage│
|
||||
└────────────┘ │ port 8000 │
|
||||
└──────────────────────┘
|
||||
┌────────────┐ ┌───────────────────────┐
|
||||
│ Frontend │────────▶│ Document Parser │
|
||||
│ Vue 3 │ /api/* │ FastAPI + Docling │
|
||||
│ port 3000 │ │ SQLite + file storage │
|
||||
└────────────┘ │ port 8000 │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
| Service | Stack | Role |
|
||||
|---------|-------|------|
|
||||
| **frontend** | Vue 3, TypeScript, Vite, Pinia | UI, PDF viewer, results display |
|
||||
| **frontend** | Vue 3, Vite, Pinia | UI, PDF viewer, results display |
|
||||
| **document-parser** | FastAPI, Docling, SQLite, pdf2image | REST API, document parsing, storage |
|
||||
|
||||
### Backend structure (hexagonal architecture — ports & adapters)
|
||||
### Backend structure (clean architecture)
|
||||
|
||||
```
|
||||
document-parser/
|
||||
├── main.py # FastAPI app, CORS, lifespan
|
||||
├── domain/ # Pure domain — no HTTP, no DB
|
||||
│ ├── models.py # Document, AnalysisJob dataclasses
|
||||
│ ├── ports.py # Abstract protocols (converter, chunker)
|
||||
│ └── value_objects.py # ConversionResult, PageDetail, ChunkResult
|
||||
│ ├── parsing.py # Docling conversion & page extraction
|
||||
│ └── bbox.py # Bounding box coordinate normalization
|
||||
├── api/ # HTTP layer (FastAPI routers)
|
||||
│ ├── schemas.py # Pydantic DTOs (camelCase serialization)
|
||||
│ ├── documents.py # /api/documents endpoints
|
||||
|
|
@ -79,7 +68,7 @@ document-parser/
|
|||
├── services/ # Use case orchestration
|
||||
│ ├── document_service.py # Upload, delete, preview
|
||||
│ └── analysis_service.py # Async Docling processing
|
||||
└── tests/ # 377 tests (pytest)
|
||||
└── tests/ # 99 tests (pytest)
|
||||
```
|
||||
|
||||
### Frontend structure (feature-based)
|
||||
|
|
@ -98,47 +87,27 @@ frontend/src/
|
|||
│ ├── document/ # Document store, API, upload, list
|
||||
│ ├── history/ # History store, API, navigation
|
||||
│ └── settings/ # Settings store
|
||||
└── shared/ # Shared utilities (types, i18n, http, format)
|
||||
└── shared/ # Shared utilities (i18n, http, format)
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
One command, nothing else to install:
|
||||
### Docker (fastest)
|
||||
|
||||
```bash
|
||||
docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local
|
||||
docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000), upload a PDF, and get results. That's it.
|
||||
Open [http://localhost:3000](http://localhost:3000)
|
||||
|
||||
> **Note:** The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast.
|
||||
|
||||
### Image variants
|
||||
|
||||
| Variant | Image tag | Size | Description |
|
||||
|---------|-----------|------|-------------|
|
||||
| **local** | `latest-local` | ~1.9 GB | Full — runs Docling in-process, CPU-only |
|
||||
| **remote** | `latest-remote` | ~270 MB | Lightweight — delegates to an external [Docling Serve](https://github.com/DS4SD/docling-serve) instance |
|
||||
|
||||
For remote mode:
|
||||
|
||||
```bash
|
||||
docker run -p 3000:3000 \
|
||||
-e DOCLING_SERVE_URL=http://your-docling-serve:5001 \
|
||||
ghcr.io/scub-france/docling-studio:latest-remote
|
||||
```
|
||||
|
||||
### Docker Compose
|
||||
### Docker Compose (for development)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/scub-france/Docling-Studio.git
|
||||
cd Docling-Studio
|
||||
|
||||
# Simple mode (backend + frontend only)
|
||||
docker compose up --build
|
||||
|
||||
# With ingestion pipeline (OpenSearch + embeddings)
|
||||
docker compose --profile ingestion -f docker-compose.yml -f docker-compose.ingestion.yml up --build
|
||||
```
|
||||
|
||||
### Local Development
|
||||
|
|
@ -147,13 +116,7 @@ docker compose --profile ingestion -f docker-compose.yml -f docker-compose.inges
|
|||
```bash
|
||||
cd document-parser
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
|
||||
# Remote mode (lightweight)
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Local mode (with Docling)
|
||||
pip install -r requirements-local.txt
|
||||
|
||||
uvicorn main:app --reload --port 8000
|
||||
```
|
||||
|
||||
|
|
@ -167,12 +130,12 @@ npm run dev
|
|||
### Running Tests
|
||||
|
||||
```bash
|
||||
# Backend (377 tests)
|
||||
# Backend (99 tests)
|
||||
cd document-parser
|
||||
pip install pytest pytest-asyncio httpx
|
||||
pytest tests/ -v
|
||||
|
||||
# Frontend (156 tests)
|
||||
# Frontend (87 tests)
|
||||
cd frontend
|
||||
npm run test:run
|
||||
```
|
||||
|
|
@ -200,152 +163,10 @@ All configuration is done via environment variables. See [`.env.example`](.env.e
|
|||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `CONVERSION_ENGINE` | `local` | `local` (in-process Docling) or `remote` (Docling Serve) |
|
||||
| `DOCLING_SERVE_URL` | `http://localhost:5001` | Docling Serve endpoint (remote mode only) |
|
||||
| `DOCLING_SERVE_API_KEY` | — | API key for Docling Serve (optional) |
|
||||
| `CORS_ORIGINS` | `http://localhost:3000,...` | CORS allowed origins (comma-separated) |
|
||||
| `UPLOAD_DIR` | `./uploads` | File storage directory |
|
||||
| `DB_PATH` | `./data/docling_studio.db` | SQLite database path |
|
||||
| `CONVERSION_TIMEOUT` | `600` | Max seconds for a single Docling conversion |
|
||||
| `BATCH_PAGE_SIZE` | `10` | Pages per batch (`0` = process all at once) |
|
||||
| `MAX_FILE_SIZE_MB` | `50` | Maximum upload file size in MB (`0` = unlimited) |
|
||||
| `MAX_PAGE_COUNT` | `0` | Maximum number of pages per document (`0` = unlimited) |
|
||||
| `NGINX_MAX_BODY_SIZE` | `200M` | Nginx request body limit — nginx format (`200M`, `0` = unlimited). Must be ≥ `MAX_FILE_SIZE_MB`. |
|
||||
| `RATE_LIMIT_RPM` | `100` | Max requests per minute per IP (`0` = disabled) |
|
||||
|
||||
## Upload Limits
|
||||
|
||||
Docling Studio enforces configurable limits on uploaded documents to protect the server against oversized files and long-running analyses:
|
||||
|
||||
- **`MAX_FILE_SIZE_MB`** (default `50`) — rejects uploads exceeding this size. Validated at two levels: early `Content-Length` check and streaming byte count.
|
||||
- **`MAX_PAGE_COUNT`** (default `0` = unlimited) — rejects documents with more pages than allowed. Useful on shared instances or Hugging Face Spaces to cap processing time.
|
||||
- **`NGINX_MAX_BODY_SIZE`** (default `200M`) — nginx-level body cap, applied before the request reaches the backend. Defaults to `200M` so `MAX_FILE_SIZE_MB` is always the effective limit. Use nginx format (`50M`, `1G`, `0` for unlimited).
|
||||
|
||||
Both application limits are exposed in the `/api/health` endpoint so the frontend can display them to the user before upload. Set either to `0` to disable the corresponding check.
|
||||
|
||||
## Ingestion Pipeline (opt-in)
|
||||
|
||||
Docling Studio can optionally index extracted chunks into [OpenSearch](https://opensearch.org/) for vector and full-text search. This requires two additional services (OpenSearch + embedding) and is **disabled by default**.
|
||||
|
||||
To enable ingestion with Docker Compose:
|
||||
|
||||
```bash
|
||||
docker compose --profile ingestion \
|
||||
-f docker-compose.yml -f docker-compose.ingestion.yml \
|
||||
up --build
|
||||
```
|
||||
|
||||
When ingestion is enabled, the UI shows:
|
||||
- An **Ingest** button in Studio to push chunks to OpenSearch
|
||||
- An **OpenSearch** connection status badge in the sidebar
|
||||
- **Indexed / Not indexed** filters on the Documents page
|
||||
- A **Search** page for full-text and vector search across indexed documents
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `OPENSEARCH_URL` | — | OpenSearch endpoint (empty = ingestion disabled) |
|
||||
| `EMBEDDING_URL` | — | Embedding service endpoint (empty = ingestion disabled) |
|
||||
| `EMBEDDING_DIMENSION` | `384` | Vector dimension (must match embedding model) |
|
||||
|
||||
## Graph storage with Neo4j (opt-in)
|
||||
|
||||
Docling Studio can mirror the full **DoclingDocument tree** into a [Neo4j](https://neo4j.com/) graph: sections, paragraphs, tables, figures, pages, and chunks all become first-class nodes connected by `HAS_ROOT`, `PARENT_OF`, `NEXT`, `ON_PAGE`, `HAS_CHUNK`, and `DERIVED_FROM` edges. This enables queries that are impossible with a flat chunk store — navigating a document's outline, finding all tables under a given section, or tracing a chunk back to its source elements.
|
||||
|
||||
Enable Neo4j with the ingestion profile (it ships alongside OpenSearch):
|
||||
|
||||
```bash
|
||||
docker compose --profile ingestion \
|
||||
-f docker-compose.yml -f docker-compose.ingestion.yml \
|
||||
up --build
|
||||
```
|
||||
|
||||
The Neo4j Browser is available at <http://localhost:7474> (user `neo4j`, password `changeme` by default).
|
||||
|
||||
### Schema at a glance
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
D[Document] -->|HAS_ROOT| SH[SectionHeader]
|
||||
D -->|HAS_CHUNK| C[Chunk]
|
||||
SH -->|PARENT_OF| P[Paragraph]
|
||||
SH -->|PARENT_OF| T[Table]
|
||||
P -->|NEXT| T
|
||||
P -->|ON_PAGE| PG[Page]
|
||||
T -->|ON_PAGE| PG
|
||||
C -->|DERIVED_FROM| P
|
||||
C -->|DERIVED_FROM| T
|
||||
```
|
||||
|
||||
### Example Cypher queries
|
||||
|
||||
Find all "Methods" sections across documents (impossible in vector-only stores):
|
||||
|
||||
```cypher
|
||||
MATCH (d:Document)-[:HAS_ROOT]->(:Element)-[:PARENT_OF*]->(s:SectionHeader)
|
||||
WHERE toLower(s.text) CONTAINS 'method'
|
||||
RETURN d.title, s.text, s.level
|
||||
```
|
||||
|
||||
Get the parent section and sibling elements of a chunk (context for RAG):
|
||||
|
||||
```cypher
|
||||
MATCH (c:Chunk {id: $chunk_id})-[:DERIVED_FROM]->(e:Element)
|
||||
MATCH (e)<-[:PARENT_OF]-(parent:Element)-[:PARENT_OF]->(sibling:Element)
|
||||
RETURN parent, collect(sibling) AS siblings
|
||||
```
|
||||
|
||||
List all tables from documents ingested from an `invoices/` path:
|
||||
|
||||
```cypher
|
||||
MATCH (d:Document)-[:HAS_ROOT]->(:Element)-[:PARENT_OF*]->(t:Table)
|
||||
WHERE d.source_uri CONTAINS 'invoices/'
|
||||
RETURN d.title, t.caption, t.cells_json
|
||||
```
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `NEO4J_URI` | — | Neo4j Bolt endpoint (empty = graph storage disabled) |
|
||||
| `NEO4J_USER` | `neo4j` | Neo4j username |
|
||||
| `NEO4J_PASSWORD` | `changeme` | Neo4j password |
|
||||
|
||||
The in-app **Graph** tab (under *Results*) renders the per-document graph with [Cytoscape.js](https://js.cytoscape.org/) (see [ADR-001](docs/architecture/adrs/ADR-001-graph-visualization-library.md) for the library choice). Documents with more than **200 pages** return `HTTP 413` from `GET /api/documents/{id}/graph`; pagination ships in v0.6.
|
||||
|
||||
## Live Reasoning (opt-in, R&D)
|
||||
|
||||
Docling Studio can run [docling-agent](https://github.com/docling-project/docling-agent)'s Chunkless RAG loop against an analyzed document and return a full **reasoning trace** — the path the agent walked through the document outline, with the section reference / rationale / answer for each iteration. The trace is overlaid on the document graph so you can *see* how the agent navigated the structure.
|
||||
|
||||
Disabled by default — pulls heavy deps (`docling-agent`, `mellea`, ~60 MB) and needs a reachable Ollama instance with the target model already pulled.
|
||||
|
||||
### Enable
|
||||
|
||||
```bash
|
||||
export REASONING_ENABLED=true
|
||||
export OLLAMA_HOST=http://localhost:11434 # default
|
||||
export REASONING_MODEL_ID=gpt-oss:20b # any model already pulled in Ollama
|
||||
# Optional, future-proof — only "ollama" is realizable today (see Architecture below):
|
||||
export LLM_PROVIDER_TYPE=ollama
|
||||
```
|
||||
|
||||
Then `pip install docling-agent mellea` (or use the `local` Docker image which bundles them) and restart the backend. The frontend reads `reasoningAvailable` from `/api/health` and hides the **Reasoning** sidebar entry when the runner isn't wired — so users never click through to a 503.
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `REASONING_ENABLED` | `false` | Master switch — `true` to enable the live runner |
|
||||
| `OLLAMA_HOST` | `http://localhost:11434` | Ollama daemon URL |
|
||||
| `REASONING_MODEL_ID` | `gpt-oss:20b` | Default model id (per-call override allowed via the API) |
|
||||
| `LLM_PROVIDER_TYPE` | `ollama` | LLM backend selector — only `ollama` is supported today |
|
||||
|
||||
### Architecture
|
||||
|
||||
The reasoning subsystem is wired through a `ReasoningRunner` port (`document-parser/domain/ports.py`) and an `LLMProvider` abstraction:
|
||||
|
||||
- `domain/ports.py` defines `ReasoningRunner`, `LLMProvider`, `ReasoningParseError` (no third-party imports)
|
||||
- `domain/value_objects.py` defines `LLMProviderType`, `ReasoningResult`, `ReasoningIteration`
|
||||
- `infra/llm/ollama_provider.py` implements `LLMProvider` for Ollama
|
||||
- `infra/docling_agent_reasoning.py` implements `ReasoningRunner` using docling-agent + mellea — all upstream coupling is here, including the `_rag_loop` workaround tracked at [docling-agent#26](https://github.com/docling-project/docling-agent/issues/26)
|
||||
- `api/reasoning.py` consumes `app.state.reasoning_runner` — zero coupling to docling-agent
|
||||
|
||||
This makes alternate LLM backends a question of adding new `LLMProvider` adapters once docling-agent (or a replacement) supports them upstream.
|
||||
|
||||
## CI / Release
|
||||
|
||||
|
|
@ -353,11 +174,14 @@ GitHub Actions pipelines (see [`.github/workflows/`](.github/workflows/)):
|
|||
|
||||
| Workflow | Trigger | What it does |
|
||||
|----------|---------|--------------|
|
||||
| **CI** | push to `main`, pull requests | Lint + type check + Backend tests + Frontend tests + build |
|
||||
| **Release** | push tag `v*` | Build & push **two** multi-arch Docker images (`remote` + `local`) to `ghcr.io` |
|
||||
| **Docs** | push to `main` (docs changes) | Build & deploy MkDocs to GitHub Pages |
|
||||
| **CI** | push to `main`, pull requests | Backend tests (99) + Frontend tests (87) + build |
|
||||
| **Release** | push tag `v*` | Build & push multi-arch Docker image to `ghcr.io` |
|
||||
|
||||
We follow [Semantic Versioning](https://semver.org/) with a simplified Git Flow. See [CONTRIBUTING.md](CONTRIBUTING.md) for the full release process.
|
||||
To publish a new version:
|
||||
```bash
|
||||
git tag v0.2.0
|
||||
git push origin v0.2.0
|
||||
```
|
||||
|
||||
## Performance & System Requirements
|
||||
|
||||
|
|
@ -369,11 +193,12 @@ We follow [Semantic Versioning](https://semver.org/) with a simplified Git Flow.
|
|||
|
||||
### Docker Desktop settings
|
||||
|
||||
| | Remote image | Local image |
|
||||
|---|---|---|
|
||||
| **Image size** | ~270 MB | ~1.9 GB |
|
||||
| **Memory** | 2 GB | 6 GB (recommended 8 GB+) |
|
||||
| **CPUs** | 2 | 4 (recommended 8+) |
|
||||
The document parser needs **at least 4 GB of RAM**:
|
||||
|
||||
| Resource | Minimum | Recommended |
|
||||
|----------|---------|-------------|
|
||||
| Memory | 6 GB | 8 GB+ |
|
||||
| CPUs | 4 | 8+ |
|
||||
|
||||
### Platform support
|
||||
|
||||
|
|
@ -381,7 +206,7 @@ All Docker images are multi-arch (linux/amd64 + linux/arm64). No GPU required.
|
|||
|
||||
## Tech Stack
|
||||
|
||||
- **Frontend**: Vue 3, TypeScript, Vite, Pinia, DOMPurify
|
||||
- **Frontend**: Vue 3, Vite, Pinia, DOMPurify
|
||||
- **Backend**: FastAPI, Docling 2.x, SQLite (aiosqlite), pdf2image
|
||||
- **CI**: GitHub Actions
|
||||
- **Infra**: Docker Compose + Nginx
|
||||
|
|
|
|||
49
SECURITY.md
49
SECURITY.md
|
|
@ -1,49 +0,0 @@
|
|||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Version | Supported |
|
||||
|---------|-----------|
|
||||
| 0.3.x | Yes |
|
||||
| < 0.3 | No |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
**Do NOT open a public GitHub issue for security vulnerabilities.**
|
||||
|
||||
Instead, please report them privately:
|
||||
|
||||
1. **Email**: Send a detailed report to **[INSERT SECURITY EMAIL]**
|
||||
2. **GitHub Security Advisory**: Use [GitHub's private vulnerability reporting](https://github.com/scub-france/Docling-Studio/security/advisories/new)
|
||||
|
||||
### What to include
|
||||
|
||||
- Description of the vulnerability
|
||||
- Steps to reproduce
|
||||
- Affected component(s): `document-parser/`, `frontend/`, `docker-compose.yml`, etc.
|
||||
- Impact assessment (data exposure, denial of service, privilege escalation, etc.)
|
||||
- Suggested fix (if any)
|
||||
|
||||
### Response timeline
|
||||
|
||||
| Step | SLA |
|
||||
|------|-----|
|
||||
| Acknowledgment | < 48 hours |
|
||||
| Initial assessment | < 7 days |
|
||||
| Fix developed | < 14 days (critical), < 30 days (other) |
|
||||
| Public disclosure | After fix is released |
|
||||
|
||||
### Process
|
||||
|
||||
1. We acknowledge your report and assign a severity level
|
||||
2. We develop a fix in a **private branch** (never pushed publicly before the advisory)
|
||||
3. We release the fix and publish a GitHub Security Advisory
|
||||
4. We credit the reporter (unless they prefer anonymity)
|
||||
|
||||
## Security Best Practices (for contributors)
|
||||
|
||||
- Never commit secrets, API keys, or credentials
|
||||
- Never disable CORS or security middleware without review
|
||||
- Validate all user input at the API boundary
|
||||
- Keep dependencies up to date (`pip audit`, `npm audit`)
|
||||
- Follow the [OWASP Top 10](https://owasp.org/www-project-top-ten/) guidelines
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
# =============================================================================
|
||||
# Docling Studio — Development stack
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f docker-compose.dev.yml up
|
||||
#
|
||||
# Includes OpenSearch single-node + Dashboards for search/sync features.
|
||||
# Frontend runs Vite dev server with HMR, backend runs with --reload.
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
# --- Neo4j (graph-native document structure) ---
|
||||
neo4j:
|
||||
image: neo4j:5.15-community
|
||||
environment:
|
||||
NEO4J_AUTH: ${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-changeme}
|
||||
NEO4J_server_memory_heap_initial__size: 512m
|
||||
NEO4J_server_memory_heap_max__size: 1g
|
||||
ports:
|
||||
- "7474:7474"
|
||||
- "7687:7687"
|
||||
volumes:
|
||||
- neo4j_data:/data
|
||||
- neo4j_logs:/logs
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "cypher-shell -u $${NEO4J_USER:-neo4j} -p $${NEO4J_PASSWORD:-changeme} 'RETURN 1' || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
|
||||
# --- OpenSearch (single-node, security disabled for local dev) ---
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:2
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
|
||||
ports:
|
||||
- "9200:9200"
|
||||
volumes:
|
||||
- opensearch_data:/usr/share/opensearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# --- OpenSearch Dashboards (index inspection UI) ---
|
||||
opensearch-dashboards:
|
||||
image: opensearchproject/opensearch-dashboards:2
|
||||
environment:
|
||||
OPENSEARCH_HOSTS: '["http://opensearch:9200"]'
|
||||
DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
|
||||
ports:
|
||||
- "5601:5601"
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
|
||||
# --- Embedding service (sentence-transformers) ---
|
||||
embedding:
|
||||
build:
|
||||
context: ./embedding-service
|
||||
ports:
|
||||
- "8001:8001"
|
||||
environment:
|
||||
EMBEDDING_MODEL: ${EMBEDDING_MODEL:-all-MiniLM-L6-v2}
|
||||
EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:8001/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2g
|
||||
|
||||
# --- Backend (FastAPI with hot-reload) ---
|
||||
document-parser:
|
||||
build:
|
||||
context: ./document-parser
|
||||
target: ${CONVERSION_MODE:-local}
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./document-parser:/app
|
||||
- uploads_data:/app/uploads
|
||||
- db_data:/app/data
|
||||
environment:
|
||||
CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173}
|
||||
DOCLING_SERVE_URL: ${DOCLING_SERVE_URL:-}
|
||||
DOCLING_SERVE_API_KEY: ${DOCLING_SERVE_API_KEY:-}
|
||||
RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100}
|
||||
MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50}
|
||||
BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-10}
|
||||
OPENSEARCH_URL: http://opensearch:9200
|
||||
EMBEDDING_URL: http://embedding:8001
|
||||
NEO4J_URI: bolt://neo4j:7687
|
||||
NEO4J_USER: ${NEO4J_USER:-neo4j}
|
||||
NEO4J_PASSWORD: ${NEO4J_PASSWORD:-changeme}
|
||||
command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
embedding:
|
||||
condition: service_healthy
|
||||
neo4j:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4g
|
||||
|
||||
# --- Frontend (Vite dev server with HMR) ---
|
||||
frontend:
|
||||
image: node:20-alpine
|
||||
working_dir: /app
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./frontend:/app
|
||||
- frontend_node_modules:/app/node_modules
|
||||
environment:
|
||||
VITE_APP_VERSION: dev
|
||||
command: ["sh", "-c", "npm install && npm run dev -- --host"]
|
||||
depends_on:
|
||||
- document-parser
|
||||
|
||||
volumes:
|
||||
opensearch_data:
|
||||
neo4j_data:
|
||||
neo4j_logs:
|
||||
uploads_data:
|
||||
db_data:
|
||||
frontend_node_modules:
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Override to enable the ingestion pipeline (OpenSearch + embeddings).
|
||||
#
|
||||
# Usage:
|
||||
# docker compose --profile ingestion -f docker-compose.yml -f docker-compose.ingestion.yml up --build
|
||||
#
|
||||
# This wires the backend to the OpenSearch and embedding services started
|
||||
# by the "ingestion" profile and ensures they are healthy before the
|
||||
# backend starts.
|
||||
|
||||
services:
|
||||
document-parser:
|
||||
environment:
|
||||
OPENSEARCH_URL: http://opensearch:9200
|
||||
EMBEDDING_URL: http://embedding:8001
|
||||
depends_on:
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
embedding:
|
||||
condition: service_healthy
|
||||
|
|
@ -1,130 +1,27 @@
|
|||
# =============================================================================
|
||||
# Docling Studio — local development / quick-start compose.
|
||||
#
|
||||
# DEV DEFAULTS — NOT PRODUCTION-READY.
|
||||
# This file ships sensible defaults for `docker compose up` to "just work" on
|
||||
# a developer laptop. It is NOT a production deployment template:
|
||||
# - Neo4j boots with `NEO4J_PASSWORD=changeme` if the env var is unset.
|
||||
# - OpenSearch runs single-node with `DISABLE_SECURITY_PLUGIN=true`
|
||||
# (no TLS, no auth). Fine for local indexing experiments, never for
|
||||
# anything reachable from outside `localhost`.
|
||||
# - No HTTPS termination, no reverse-proxy hardening, no rate limit on
|
||||
# the embedding/OpenSearch services, ports bound to all interfaces.
|
||||
#
|
||||
# Operators running their own Docling Studio instance MUST:
|
||||
# 1. Override `NEO4J_PASSWORD` (and rotate it) — see `.env.example`.
|
||||
# 2. Re-enable the OpenSearch security plugin and configure TLS/users —
|
||||
# see https://opensearch.org/docs/latest/security/.
|
||||
# 3. Bind sensitive services to internal networks only, terminate TLS at
|
||||
# a reverse proxy, and align CORS_ORIGINS / RATE_LIMIT_RPM with the
|
||||
# deployment surface.
|
||||
#
|
||||
# The audit pipeline expects this file to flag the dev defaults as such —
|
||||
# don't silently strengthen them here without also updating the dev DX.
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
# --- Neo4j (graph-native document structure) ---
|
||||
# Dev-only auth: NEO4J_PASSWORD defaults to "changeme". Override in
|
||||
# `.env` (or any orchestrator secret store) before exposing this stack.
|
||||
neo4j:
|
||||
profiles: ["ingestion"]
|
||||
image: neo4j:5.15-community
|
||||
environment:
|
||||
NEO4J_AUTH: ${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-changeme}
|
||||
NEO4J_server_memory_heap_initial__size: 512m
|
||||
NEO4J_server_memory_heap_max__size: 1g
|
||||
volumes:
|
||||
- neo4j_data:/data
|
||||
- neo4j_logs:/logs
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "cypher-shell -u $${NEO4J_USER:-neo4j} -p $${NEO4J_PASSWORD:-changeme} 'RETURN 1' || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
|
||||
# --- OpenSearch (single-node, security DISABLED — DEV ONLY) ---
|
||||
# `DISABLE_SECURITY_PLUGIN: "true"` removes auth, TLS, and audit logging.
|
||||
# This is fine for a local dev cluster bound to 127.0.0.1; it is NOT safe
|
||||
# for anything reachable from another host. For production, drop this
|
||||
# var, switch to a hardened image, and configure users + TLS:
|
||||
# https://opensearch.org/docs/latest/security/configuration/
|
||||
opensearch:
|
||||
profiles: ["ingestion"]
|
||||
image: opensearchproject/opensearch:2
|
||||
environment:
|
||||
discovery.type: single-node
|
||||
DISABLE_SECURITY_PLUGIN: "true"
|
||||
OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
|
||||
volumes:
|
||||
- opensearch_data:/usr/share/opensearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# --- Embedding service (sentence-transformers) ---
|
||||
embedding:
|
||||
profiles: ["ingestion"]
|
||||
build:
|
||||
context: ./embedding-service
|
||||
environment:
|
||||
EMBEDDING_MODEL: ${EMBEDDING_MODEL:-all-MiniLM-L6-v2}
|
||||
EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:8001/health || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
start_period: 120s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2g
|
||||
|
||||
# --- Backend (FastAPI) ---
|
||||
document-parser:
|
||||
build:
|
||||
context: ./document-parser
|
||||
target: ${CONVERSION_MODE:-local}
|
||||
expose:
|
||||
- "8000"
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- uploads_data:/app/uploads
|
||||
- db_data:/app/data
|
||||
environment:
|
||||
CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173}
|
||||
DOCLING_SERVE_URL: ${DOCLING_SERVE_URL:-}
|
||||
DOCLING_SERVE_API_KEY: ${DOCLING_SERVE_API_KEY:-}
|
||||
RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100}
|
||||
MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50}
|
||||
BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-10}
|
||||
OPENSEARCH_URL: ${OPENSEARCH_URL:-}
|
||||
EMBEDDING_URL: ${EMBEDDING_URL:-}
|
||||
NEO4J_URI: ${NEO4J_URI:-}
|
||||
NEO4J_USER: ${NEO4J_USER:-neo4j}
|
||||
NEO4J_PASSWORD: ${NEO4J_PASSWORD:-changeme}
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4g
|
||||
|
||||
# --- Frontend (nginx) ---
|
||||
frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
ports:
|
||||
- "3000:80"
|
||||
environment:
|
||||
NGINX_MAX_BODY_SIZE: ${NGINX_MAX_BODY_SIZE:-200M}
|
||||
depends_on:
|
||||
- document-parser
|
||||
|
||||
volumes:
|
||||
opensearch_data:
|
||||
neo4j_data:
|
||||
neo4j_logs:
|
||||
uploads_data:
|
||||
db_data:
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
****# Available Processes
|
||||
|
||||
Index of all documented processes in Docling Studio. Each process is a structured, repeatable workflow with a clear trigger and deliverable.
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
| # | Process | Trigger / Command | Doc | Output |
|
||||
|---|---------|-------------------|-----|--------|
|
||||
| 1 | **Commit** | Every commit | [commit-conventions.md](git-workflow/commit-conventions.md) | Conventional Commit message |
|
||||
| 2 | **Code Review** | Every PR | [code-review-checklist.md](git-workflow/code-review-checklist.md) | Reviewed PR with checklist |
|
||||
| 3 | **Merge** | PR approved + CI green | [merge-policy.md](git-workflow/merge-policy.md) | Squash/merge commit on target branch |
|
||||
| 4 | **ADR** | Architecture decision needed | [adr-guide.md](architecture/adr-guide.md) + [adr-template.md](architecture/adr-template.md) | `docs/architecture/adrs/ADR-NNN-*.md` |
|
||||
|
||||
### How to invoke
|
||||
|
||||
```
|
||||
# Ask for a code review
|
||||
Review cette PR avec docs/git-workflow/code-review-checklist.md
|
||||
|
||||
# Create an ADR
|
||||
Crée un ADR pour [la décision à documenter]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quality & Audit
|
||||
|
||||
| # | Process | Trigger / Command | Doc | Output |
|
||||
|---|---------|-------------------|-----|--------|
|
||||
| 5 | **Full Release Audit** | Before merging `release/*` to `main` | [docs/audit/master.md](audit/master.md) | 12 audit reports + summary in `reports/release-X.Y.Z/` |
|
||||
| 6 | **Single Audit** | Targeted check on one axis | [docs/audit/audits/*.md](audit/audits/01-clean-architecture.md) | Single audit report |
|
||||
| 7 | **Re-Audit** | After fixing CRIT/MAJ findings | [docs/audit/master.md](audit/master.md) | Updated report |
|
||||
| 8 | **Automated Checks** | Quick validation before audit | [profiles/fastapi-vue/commands.sh](https://github.com/scub-france/Docling-Studio/blob/main/profiles/fastapi-vue/commands.sh) | PASS/WARN/FAIL per check |
|
||||
|
||||
### How to invoke
|
||||
|
||||
```
|
||||
# Full audit
|
||||
Audite la branche release/X.Y.Z en suivant docs/audit/master.md
|
||||
|
||||
# Single audit
|
||||
Exécute l'audit docs/audit/audits/08-security.md sur la branche courante
|
||||
|
||||
# Re-audit after fixes
|
||||
Re-audite les écarts CRIT et MAJ du rapport docs/audit/reports/release-X.Y.Z/summary.md
|
||||
|
||||
# Automated checks (shell)
|
||||
bash profiles/fastapi-vue/commands.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Release & Deploy
|
||||
|
||||
| # | Process | Trigger / Command | Doc | Output |
|
||||
|---|---------|-------------------|-----|--------|
|
||||
| 9 | **Release Gate** | Auto on push/PR `release/*` → `main` | [release-gate.yml](https://github.com/scub-france/Docling-Studio/blob/main/.github/workflows/release-gate.yml) | GO / GO CONDITIONAL / NO-GO comment on PR |
|
||||
| 10 | **Release** | Feature freeze on `release/*` | [CONTRIBUTING.md](https://github.com/scub-france/Docling-Studio/blob/main/CONTRIBUTING.md#release-process) | Tag `vX.Y.Z`, Docker images on ghcr.io |
|
||||
| 11 | **Deployment** | After release tag | [deployment-checklist.md](release/deployment-checklist.md) | Running instance at new version |
|
||||
| 12 | **Rollback** | Post-deploy failure detected | [rollback-playbook.md](release/rollback-playbook.md) | Reverted to last known good version |
|
||||
| 13 | **Hotfix** | Critical bug on released version | [CONTRIBUTING.md](https://github.com/scub-france/Docling-Studio/blob/main/CONTRIBUTING.md#hotfix) | Patch release `vX.Y.Z+1` |
|
||||
|
||||
### Release Gate details
|
||||
|
||||
The release gate runs **automatically** on every push to `release/**` and on PRs targeting `main`. It validates 10 checks in 4 phases:
|
||||
|
||||
```
|
||||
Phase 1 (parallel) Phase 2 (Docker) Phase 3 (E2E) Phase 4
|
||||
┌──────────────┐ ┌──────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ lint+typecheck│ │ docker build │──────▶│ e2e API │───▶│ release │
|
||||
│ unit tests │ │ docker smoke │──────▶│ e2e UI │ │ summary │
|
||||
│ dep audit │ │ image scan │ └─────────────┘ │ (PR comment)│
|
||||
│ audit checks │ │ image size │ └─────────────┘
|
||||
└──────────────┘ └──────────────┘
|
||||
```
|
||||
|
||||
| Check | Blocks merge? | Details |
|
||||
|-------|---------------|---------|
|
||||
| Lint & type-check | Yes | ruff + ESLint + vue-tsc |
|
||||
| Unit tests | Yes | pytest + Vitest |
|
||||
| Dep audit | Yes (CRITICAL) | pip-audit + npm audit |
|
||||
| Audit checks | Yes | `profiles/fastapi-vue/commands.sh` |
|
||||
| Docker build | Yes | Both targets (remote + local) |
|
||||
| Docker smoke | Yes | Start container, verify `/api/health` |
|
||||
| Image scan (Trivy) | Yes (CRITICAL) | HIGH = warning annotation |
|
||||
| Image size | No (warning) | Delta vs previous release, alert if > 10% |
|
||||
| E2E API | Yes | `@smoke,@regression,@e2e` (full scope) |
|
||||
| E2E UI | Yes | `@critical` |
|
||||
|
||||
**Verdict**: posted as a comment on the release PR:
|
||||
- **GO** — all checks pass
|
||||
- **GO CONDITIONAL** — blocking checks pass, dep audit or audit checks have warnings
|
||||
- **NO-GO** — at least one blocking check failed
|
||||
|
||||
### How to invoke
|
||||
|
||||
```
|
||||
# Release gate runs automatically — no manual trigger needed
|
||||
# Just push to release/* or open a PR release/* → main
|
||||
|
||||
# Prepare a release
|
||||
Prépare la release X.Y.Z en suivant CONTRIBUTING.md#release-process
|
||||
|
||||
# Deploy
|
||||
Déploie en suivant docs/release/deployment-checklist.md
|
||||
|
||||
# Rollback
|
||||
Rollback en suivant docs/release/rollback-playbook.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Operations
|
||||
|
||||
| # | Process | Trigger / Command | Doc | Output |
|
||||
|---|---------|-------------------|-----|--------|
|
||||
| 14 | **Incident Response** | Service down or degraded | [incident-response.md](operations/incident-response.md) | Mitigation + post-mortem |
|
||||
| 15 | **Security Vulnerability** | Vuln reported | [security-response.md](operations/security-response.md) | Fix + advisory |
|
||||
| 16 | **Monitoring Setup** | New deployment | [monitoring-checklist.md](operations/monitoring-checklist.md) | Monitoring configured |
|
||||
|
||||
### How to invoke
|
||||
|
||||
```
|
||||
# Incident
|
||||
Gère l'incident SEV-1 en suivant docs/operations/incident-response.md
|
||||
|
||||
# Security vulnerability
|
||||
Traite la vulnérabilité signalée en suivant docs/operations/security-response.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Community & Governance
|
||||
|
||||
| # | Process | Trigger / Command | Doc | Output |
|
||||
|---|---------|-------------------|-----|--------|
|
||||
| 17 | **Onboarding** | New contributor | [onboarding-guide.md](community/onboarding-guide.md) | First PR merged |
|
||||
| 18 | **Issue Triage** | New issue opened | [issue-triage-process.md](community/issue-triage-process.md) | Labeled + prioritized issue |
|
||||
| 19 | **Roadmap Update** | Release cycle planning | [roadmap-template.md](community/roadmap-template.md) | Updated roadmap |
|
||||
|
||||
### How to invoke
|
||||
|
||||
```
|
||||
# Triage issues
|
||||
Trie les issues ouvertes en suivant docs/community/issue-triage-process.md
|
||||
|
||||
# Update roadmap
|
||||
Mets à jour la roadmap en suivant docs/community/roadmap-template.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Category | Processes | Key doc |
|
||||
|----------|-----------|---------|
|
||||
| **Dev** | Commit, Review, Merge, ADR | `docs/git-workflow/` |
|
||||
| **Quality** | Audit (full/single/re-audit), Auto-checks | `docs/audit/master.md` |
|
||||
| **Release** | Release, Deploy, Rollback, Hotfix | `docs/release/` + `CONTRIBUTING.md` |
|
||||
| **Ops** | Incident, Security, Monitoring | `docs/operations/` |
|
||||
| **Community** | Onboarding, Triage, Roadmap | `docs/community/` |
|
||||
|
||||
---
|
||||
|
||||
## Standards & References
|
||||
|
||||
These are not processes but reference documents used by the processes above:
|
||||
|
||||
| Document | Purpose |
|
||||
|----------|---------|
|
||||
| [CONTRIBUTING.md](https://github.com/scub-france/Docling-Studio/blob/main/CONTRIBUTING.md) | Dev setup, branching, release, versioning |
|
||||
| [coding-standards.md](architecture/coding-standards.md) | Naming, style, architecture rules |
|
||||
| [e2e/CONVENTIONS.md](https://github.com/scub-france/Docling-Studio/blob/main/e2e/CONVENTIONS.md) | Karate / Karate UI test conventions |
|
||||
| [architecture.md](architecture.md) | System architecture (backend + frontend) |
|
||||
| [CODE_OF_CONDUCT.md](https://github.com/scub-france/Docling-Studio/blob/main/CODE_OF_CONDUCT.md) | Community behavior standards |
|
||||
| [SECURITY.md](https://github.com/scub-france/Docling-Studio/blob/main/SECURITY.md) | Vulnerability reporting policy |
|
||||
| [profiles/fastapi-vue/profile.md](https://github.com/scub-france/Docling-Studio/blob/main/profiles/fastapi-vue/profile.md) | Stack layer mapping for audits |
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 55 KiB |
|
|
@ -1,210 +0,0 @@
|
|||
# Architecture
|
||||
|
||||
## Overview
|
||||
|
||||
{ width="700" }
|
||||
|
||||
Two services communicating via REST. The frontend is a Vue 3 SPA served by Nginx in production. The backend is a FastAPI app that wraps Docling's document conversion engine.
|
||||
|
||||
### Zooming into the backend
|
||||
|
||||
The schema above shows the macro view. Inside the backend, the code follows a **Hexagonal Architecture** (ports & adapters) with strict layer boundaries:
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ Backend │
|
||||
│ │
|
||||
│ ┌──────────┐ │
|
||||
│ │ api/ │ ← HTTP (FastAPI routes, Pydantic) │
|
||||
│ └────┬─────┘ │
|
||||
│ │ calls │
|
||||
│ ┌────▼─────┐ │
|
||||
│ │services/ │ ← Use case orchestration │
|
||||
│ └──┬────┬──┘ │
|
||||
│ │ │ │
|
||||
│ ┌───▼──┐ ┌▼───────────┐ │
|
||||
│ │domain│ │persistence/ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │bbox │ │ SQLite CRUD │ ← Storage (your blue box) │
|
||||
│ │parse │ │ file store │ │
|
||||
│ └──────┘ └─────────────┘ │
|
||||
│ ↑ pure Python, no deps ↑ aiosqlite │
|
||||
└──────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Dependencies flow **inward**: `api → services → domain`. The domain layer has zero knowledge of HTTP or database.
|
||||
|
||||
## Backend — Hexagonal Architecture (ports & adapters)
|
||||
|
||||
The backend follows the hexagonal / ports-and-adapters pattern. The domain layer defines **ports** (abstract protocols in `domain/ports.py`); `infra/` provides **adapters** that implement them. Dependencies flow inward: API → Services → Domain. The domain layer has zero knowledge of HTTP, database, or any framework.
|
||||
|
||||
```
|
||||
document-parser/
|
||||
├── main.py # FastAPI app, CORS, lifespan, health endpoint
|
||||
│
|
||||
├── domain/ # Pure domain — no HTTP, no DB
|
||||
│ ├── models.py # Document, AnalysisJob dataclasses
|
||||
│ ├── ports.py # Abstract protocols (DocumentConverter, DocumentChunker)
|
||||
│ ├── value_objects.py # ConversionResult, ChunkingOptions, ChunkResult
|
||||
│ └── bbox.py # Bounding box coordinate normalization
|
||||
│
|
||||
├── api/ # HTTP layer (FastAPI routers)
|
||||
│ ├── schemas.py # Pydantic DTOs (camelCase serialization)
|
||||
│ ├── documents.py # /api/documents endpoints
|
||||
│ └── analyses.py # /api/analyses endpoints (create, rechunk, delete)
|
||||
│
|
||||
├── persistence/ # Data layer (SQLite via aiosqlite)
|
||||
│ ├── database.py # Connection management, schema init
|
||||
│ ├── document_repo.py # Document CRUD
|
||||
│ └── analysis_repo.py # AnalysisJob CRUD
|
||||
│
|
||||
├── infra/ # Infrastructure adapters
|
||||
│ ├── settings.py # Environment-based configuration
|
||||
│ ├── local_converter.py # In-process Docling converter (local mode)
|
||||
│ ├── serve_converter.py # HTTP client for Docling Serve (remote mode)
|
||||
│ ├── local_chunker.py # In-process chunking (HierarchicalChunker, HybridChunker)
|
||||
│ ├── rate_limiter.py # Sliding-window rate limiting middleware
|
||||
│ └── bbox.py # Bbox coordinate normalization helpers
|
||||
│
|
||||
├── services/ # Use case orchestration
|
||||
│ ├── document_service.py # Upload, delete, preview
|
||||
│ └── analysis_service.py # Async Docling processing + chunking
|
||||
│
|
||||
└── tests/ # pytest (199 tests)
|
||||
```
|
||||
|
||||
### Layer responsibilities
|
||||
|
||||
| Layer | Role | Depends on |
|
||||
|-------|------|------------|
|
||||
| **domain** | Dataclasses, value objects, abstract ports | Nothing (pure Python) |
|
||||
| **persistence** | SQLite CRUD, aiosqlite | domain (models) |
|
||||
| **infra** | Adapters: converters, chunker, rate limiter, settings | domain (ports, value objects) |
|
||||
| **services** | Orchestrate use cases, call converters/chunkers | domain + persistence + infra |
|
||||
| **api** | HTTP endpoints, Pydantic DTOs, error handling | services |
|
||||
|
||||
### API contract
|
||||
|
||||
The API uses **camelCase** serialization (via Pydantic `alias_generator`), while the backend uses **snake_case** internally. The `pages_json` field contains raw `dataclasses.asdict()` output, so page data uses **snake_case** (`page_number`, not `pageNumber`).
|
||||
|
||||
## Frontend — Feature-Based
|
||||
|
||||
The frontend is organized by feature, each with its own store, API client, and UI components.
|
||||
|
||||
```
|
||||
frontend/src/
|
||||
├── app/ # App shell, router, global styles
|
||||
├── pages/ # Route-level pages
|
||||
│ ├── HomePage.vue
|
||||
│ ├── StudioPage.vue # PDF viewer + config + results
|
||||
│ ├── DocumentsPage.vue
|
||||
│ ├── HistoryPage.vue
|
||||
│ └── SettingsPage.vue
|
||||
│
|
||||
├── features/ # Feature modules
|
||||
│ ├── analysis/ # Analysis store, API, bbox scaling, UI
|
||||
│ │ ├── store.ts
|
||||
│ │ ├── api.ts
|
||||
│ │ ├── bboxScaling.ts # Pure math: page coords → pixel coords
|
||||
│ │ └── ui/
|
||||
│ │ ├── BboxOverlay.vue
|
||||
│ │ ├── AnalysisPanel.vue
|
||||
│ │ ├── StructureViewer.vue
|
||||
│ │ └── ...
|
||||
│ ├── chunking/ # Chunk panel UI + rechunk action
|
||||
│ ├── document/ # Document store, API, upload
|
||||
│ ├── feature-flags/ # Feature flag store (reads /api/health)
|
||||
│ ├── history/ # History store, navigation
|
||||
│ └── settings/ # Theme, locale, API URL
|
||||
│
|
||||
└── shared/ # Cross-feature utilities
|
||||
├── types.ts # All shared TypeScript interfaces
|
||||
├── i18n.ts # FR/EN translations
|
||||
├── format.ts # Date/size formatters
|
||||
└── api/http.ts # HTTP client (fetch wrapper)
|
||||
```
|
||||
|
||||
### Data flow
|
||||
|
||||
```
|
||||
User action → Pinia store action → API client (fetch) → Backend REST endpoint
|
||||
│
|
||||
Backend response → Pinia store state → Vue reactivity → UI update
|
||||
```
|
||||
|
||||
### Key design decisions
|
||||
|
||||
- **Pinia stores** per feature, not global. Each feature owns its state.
|
||||
- **TypeScript strict mode** with shared interfaces in `shared/types.ts`.
|
||||
- **No component library** — custom CSS with CSS variables for theming.
|
||||
- **vue-tsc** in CI to catch type errors before merge.
|
||||
|
||||
## Feature Flags
|
||||
|
||||
The frontend adapts its UI based on the backend's capabilities. On startup, the feature flag store fetches `/api/health` and reads the `engine` and `deploymentMode` fields.
|
||||
|
||||
| Flag | Condition | Effect |
|
||||
|------|-----------|--------|
|
||||
| `chunking` | `engine === 'local'` | Shows chunking options in the analysis panel |
|
||||
| `disclaimer` | `deploymentMode === 'huggingface'` | Shows a disclaimer banner at the top of the app |
|
||||
|
||||
This allows the same frontend build to work with both local and remote backends without conditional compilation.
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
The backend applies a sliding-window rate limiter as middleware:
|
||||
|
||||
- **60 requests** per **60 seconds** per client IP
|
||||
- The `/api/health` endpoint is excluded
|
||||
- When the limit is exceeded, the API returns `429 Too Many Requests` with a `Retry-After` header
|
||||
|
||||
## Analysis Lifecycle
|
||||
|
||||
An analysis job follows this state machine:
|
||||
|
||||
```
|
||||
PENDING → RUNNING → COMPLETED
|
||||
→ FAILED
|
||||
```
|
||||
|
||||
| Status | Description |
|
||||
|--------|-------------|
|
||||
| `PENDING` | Job created, waiting for a processing slot |
|
||||
| `RUNNING` | Docling conversion in progress |
|
||||
| `COMPLETED` | Conversion finished — results available (markdown, HTML, pages, chunks) |
|
||||
| `FAILED` | Conversion error — `error_message` contains details |
|
||||
|
||||
The backend limits parallel jobs via `MAX_CONCURRENT_ANALYSES` (default: 3) to avoid overloading the CPU during Docling processing.
|
||||
|
||||
## Local vs Remote Mode
|
||||
|
||||
The backend supports two conversion engines, selected via the `CONVERSION_ENGINE` environment variable:
|
||||
|
||||
| | Local | Remote |
|
||||
|---|---|---|
|
||||
| **Engine** | In-process Docling (PyTorch) | HTTP client to [Docling Serve](https://github.com/DS4SD/docling-serve) |
|
||||
| **Chunking** | Available (in-process) | Not available |
|
||||
| **Docker image** | `latest-local` (~1.9 GB) | `latest-remote` (~270 MB) |
|
||||
| **ML models** | Downloaded on first run (~400 MB) | Managed by Docling Serve |
|
||||
| **CPU/RAM** | 4+ CPUs, 6+ GB RAM | 2 CPUs, 2 GB RAM |
|
||||
|
||||
The converter is selected at startup in `main.py` via `_build_converter()`. The chunker (`_build_chunker()`) is only instantiated in local mode — in remote mode, the chunking feature flag is disabled and the UI hides the chunking panel.
|
||||
|
||||
## Health Endpoint
|
||||
|
||||
`GET /api/health` returns the backend status:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"engine": "local",
|
||||
"version": "0.3.0",
|
||||
"deploymentMode": "self-hosted"
|
||||
}
|
||||
```
|
||||
|
||||
The frontend uses this response to:
|
||||
|
||||
1. Verify the backend is reachable
|
||||
2. Evaluate feature flags (chunking, disclaimer)
|
||||
3. Display the app version
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
# Architecture Decision Records (ADR) — Guide
|
||||
|
||||
## What is an ADR?
|
||||
|
||||
An ADR is a short document that captures a significant architectural decision, along with the context and consequences. ADRs create a **decision log** so future contributors understand *why* the codebase looks the way it does.
|
||||
|
||||
## When to Write an ADR
|
||||
|
||||
Write an ADR when:
|
||||
|
||||
- Choosing or replacing a framework, library, or tool
|
||||
- Changing the architecture (new layer, new pattern, new boundary)
|
||||
- Making a trade-off that future developers might question
|
||||
- Deciding NOT to do something (these are often the most valuable)
|
||||
|
||||
Do NOT write an ADR for:
|
||||
|
||||
- Implementation details that are obvious from the code
|
||||
- Formatting or style choices (those go in [coding-standards.md](coding-standards.md))
|
||||
- Bug fixes or minor refactors
|
||||
|
||||
## How to Write an ADR
|
||||
|
||||
1. Copy `adr-template.md` to `docs/architecture/adrs/ADR-NNN-short-title.md`
|
||||
2. Number sequentially (ADR-001, ADR-002, ...)
|
||||
3. Fill in all sections — especially **Context** (the *why*) and **Alternatives Considered**
|
||||
4. Set status to `Proposed`
|
||||
5. Open a PR for team review
|
||||
6. Once merged, update status to `Accepted`
|
||||
|
||||
## Existing Decisions (captured retroactively)
|
||||
|
||||
These decisions were made before the ADR process was introduced. They are documented here for context:
|
||||
|
||||
| Decision | Rationale | Date |
|
||||
|----------|-----------|------|
|
||||
| Hexagonal Architecture (ports & adapters) for backend | Decouple domain from framework — enable converter swapping (local/remote) via ports | 2025-01 |
|
||||
| FastAPI over Django | Async-first, lightweight, Pydantic-native — better fit for a single-purpose API | 2025-01 |
|
||||
| Vue 3 + Pinia over React | Composition API + built-in reactivity — smaller bundle for this use case | 2025-01 |
|
||||
| SQLite over PostgreSQL | Single-file DB, zero ops — appropriate for a document processing tool | 2025-01 |
|
||||
| Karate over Playwright for e2e | Team expertise, unified API+UI testing, JVM ecosystem | 2026-03 |
|
||||
| Feature flags via `/api/health` | No external service needed — backend capabilities drive frontend UI | 2026-03 |
|
||||
|
||||
## Lifecycle
|
||||
|
||||
```
|
||||
Proposed → Accepted → [Deprecated | Superseded by ADR-NNN]
|
||||
```
|
||||
|
||||
- **Deprecated**: The decision is no longer relevant (feature removed, context changed)
|
||||
- **Superseded**: A newer ADR replaces this one (link to the new ADR)
|
||||
- Never delete an ADR — mark it as deprecated/superseded instead
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# ADR-NNN: [Title]
|
||||
|
||||
**Date**: YYYY-MM-DD
|
||||
**Status**: Proposed | Accepted | Deprecated | Superseded by ADR-NNN
|
||||
**Deciders**: [names]
|
||||
|
||||
## Context
|
||||
|
||||
<!-- What is the issue we're seeing that is motivating this decision? What forces are at play? -->
|
||||
|
||||
## Decision
|
||||
|
||||
<!-- What is the change we're proposing and/or doing? -->
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- ...
|
||||
|
||||
### Negative
|
||||
|
||||
- ...
|
||||
|
||||
### Neutral
|
||||
|
||||
- ...
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### Alternative 1: [Name]
|
||||
|
||||
- **Pros**: ...
|
||||
- **Cons**: ...
|
||||
- **Why rejected**: ...
|
||||
|
||||
## References
|
||||
|
||||
- [Link to issue, RFC, or discussion]
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
# ADR-001: Graph visualization library for the Neo4j graph view
|
||||
|
||||
**Date**: 2026-04-17
|
||||
**Status**: Proposed
|
||||
**Deciders**: Pier-Jean Malandrino
|
||||
|
||||
## Context
|
||||
|
||||
v0.5.0 introduces Neo4j as a graph-native storage layer for parsed documents
|
||||
(see [docs/design/neo4j-integration.md](../../design/neo4j-integration.md)
|
||||
and [#186](https://github.com/scub-france/Docling-Studio/issues/186)). We need
|
||||
an in-app visualization of that graph: the `DoclingDocument` tree as rendered
|
||||
in Neo4j, with nodes colored by element type (`SectionHeader`, `Paragraph`,
|
||||
`Table`, `Figure`, `ListItem`, `Formula`) and edges (`PARENT_OF`, `NEXT`,
|
||||
`ON_PAGE`, `HAS_CHUNK`, `DERIVED_FROM`).
|
||||
|
||||
The view lives in the existing Vue 3 debug panel. It is the **primary demo
|
||||
artifact** for the Hackernoon hackathon (Neo4j partner), so polish matters as
|
||||
much as correctness.
|
||||
|
||||
### Constraints
|
||||
|
||||
- Vue 3 + Vite frontend, no framework change
|
||||
- Must render the full tree of a 200-page document (worst case ≈ a few
|
||||
thousand nodes; see graph endpoint cap in the design doc §8.4)
|
||||
- Needs a **clean hierarchical layout** — documents are trees, not arbitrary
|
||||
graphs; a good tree layout is the single biggest UX lever
|
||||
- Needs per-node styling (shape + color by label), click, hover, zoom, pan
|
||||
- Must be installable without Java/Python-side changes
|
||||
- License compatible with the repo (MIT-ish preferred)
|
||||
|
||||
### Non-goals for v0.5.0
|
||||
|
||||
- 3D rendering
|
||||
- Force-directed simulation as the primary layout (we have a tree)
|
||||
- Editing nodes in place (read-only view)
|
||||
- Rendering millions of nodes
|
||||
|
||||
## Decision
|
||||
|
||||
Use **Cytoscape.js** via a thin Vue wrapper (`vue-cytoscape` or a bespoke
|
||||
`GraphView.vue` that imports `cytoscape` directly and uses the
|
||||
`dagre`/`breadthfirst` layouts).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Battle-tested library (13k+ GitHub stars, maintained since 2013, used by
|
||||
Neo4j's own "Bloom"-style visualizations in the community)
|
||||
- First-class support for hierarchical layouts via `cytoscape-dagre` (hub-and-
|
||||
spoke / tree) and built-in `breadthfirst` — both map naturally to our
|
||||
`PARENT_OF` structure
|
||||
- CSS-like selector syntax for styling (`node[label = "Table"] { ... }`),
|
||||
which is pleasant to evolve as we add node types
|
||||
- Permissive licensing (MIT)
|
||||
- Headless mode available, so it can be tested outside a DOM (Jest + jsdom
|
||||
works cleanly)
|
||||
- Active ecosystem: `cytoscape-cola`, `cytoscape-klay`, `cytoscape-popper` for
|
||||
tooltips, all maintained
|
||||
- Bundle size is reasonable for a demo: ~300 KB min+gz for core + dagre, well
|
||||
below our current frontend budget
|
||||
|
||||
### Negative
|
||||
|
||||
- Styling DSL is powerful but has its own syntax to learn; not plain CSS
|
||||
- Large graphs (>10k nodes) benefit from canvas+WebGL libraries
|
||||
(sigma.js, reagraph) — we are explicitly not in that regime for v0.5, but
|
||||
we would need to swap if we later visualize the cross-document graph
|
||||
- No Vue 3 component library that is both maintained and popular — we wrap it
|
||||
ourselves in `GraphView.vue` (the wrapper is ~50 LOC, so this is minor)
|
||||
|
||||
### Neutral
|
||||
|
||||
- Not "Neo4j-branded": we do not use Neovis.js, which is a thin Cytoscape
|
||||
wrapper around the Bolt protocol. Our graph API already returns shaped
|
||||
JSON, so the Neovis convenience is not worth the lock-in
|
||||
- We take on one runtime dependency (`cytoscape` + `cytoscape-dagre`)
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### Alternative 1: vis-network (vis.js)
|
||||
|
||||
- **Pros**: Very easy to get started, built-in physics, shipped by Neo4j
|
||||
Browser historically
|
||||
- **Cons**: Maintenance has been rocky (original vis.js split into several
|
||||
forks; `vis-network` is the maintained branch but releases are sparse);
|
||||
hierarchical layout is OK but less configurable than dagre; styling API is
|
||||
less expressive; TypeScript types lag behind the JS API
|
||||
- **Why rejected**: Hierarchical layout quality is the single most important
|
||||
criterion for a document tree, and vis-network is clearly a notch below
|
||||
Cytoscape + dagre here. Maintenance trajectory is also a concern for a
|
||||
release we want to keep shipping on
|
||||
|
||||
### Alternative 2: Neovis.js
|
||||
|
||||
- **Pros**: Built by Neo4j Labs, connects directly to a Bolt endpoint, nice
|
||||
out-of-the-box "Neo4j look"
|
||||
- **Cons**: Wraps Cytoscape anyway, so everything it can do we can do with
|
||||
Cytoscape directly; expects the browser to talk Bolt, which forces us to
|
||||
expose Neo4j creds in the frontend OR to proxy Bolt through the backend
|
||||
(both worse than our current "backend returns JSON" design); limited
|
||||
customization compared to raw Cytoscape
|
||||
- **Why rejected**: The auth story is a non-starter for a hackathon demo we
|
||||
want to show publicly, and we lose nothing vs. Cytoscape by going one
|
||||
layer lower
|
||||
|
||||
### Alternative 3: D3 (d3-hierarchy + d3-force)
|
||||
|
||||
- **Pros**: Maximum flexibility; beautiful, publication-grade output; full
|
||||
SVG control
|
||||
- **Cons**: Much more code for the same result — layout, zoom, pan, hover,
|
||||
selection all hand-rolled; steeper learning curve for future contributors
|
||||
to the project; no built-in graph data model
|
||||
- **Why rejected**: We're building a product feature, not a data-viz
|
||||
artefact. The time budget (1 day of Day 3) doesn't fit a D3 build-your-own
|
||||
|
||||
### Alternative 4: Reagraph / react-force-graph / sigma.js (WebGL)
|
||||
|
||||
- **Pros**: Scales to tens of thousands of nodes at 60 FPS; good for future
|
||||
cross-document visualization
|
||||
- **Cons**: Optimized for force-directed layouts, weaker hierarchical
|
||||
support; Reagraph is React-only (requires a React island inside Vue);
|
||||
sigma.js's tree layout is immature
|
||||
- **Why rejected**: Wrong regime for a single-document tree. Worth
|
||||
reconsidering if/when we visualize the full corpus graph in a later release
|
||||
|
||||
### Alternative 5: Mermaid
|
||||
|
||||
- **Pros**: Trivial to embed, already used in docs
|
||||
- **Cons**: Static rendering, no interactivity, not designed for thousands of
|
||||
nodes, no per-node click/hover
|
||||
- **Why rejected**: A viewer, not a visualizer. We need interactivity
|
||||
|
||||
## References
|
||||
|
||||
- [Neo4j integration design doc](../../design/neo4j-integration.md) §8.3
|
||||
- [Issue #186 — Neo4j integration](https://github.com/scub-france/Docling-Studio/issues/186)
|
||||
- [Cytoscape.js](https://js.cytoscape.org/)
|
||||
- [cytoscape-dagre](https://github.com/cytoscape/cytoscape.js-dagre)
|
||||
- [vis-network](https://visjs.github.io/vis-network/docs/network/)
|
||||
- [Neovis.js](https://github.com/neo4j-contrib/neovis.js)
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
# Coding Standards
|
||||
|
||||
Conventions for writing consistent, readable code across the Docling Studio codebase.
|
||||
|
||||
## Python (Backend — `document-parser/`)
|
||||
|
||||
### Tooling
|
||||
|
||||
| Tool | Purpose | Config |
|
||||
|------|---------|--------|
|
||||
| **Ruff** | Linting + formatting | `ruff.toml` / `pyproject.toml` |
|
||||
| **pytest** | Testing | `pytest.ini` / `pyproject.toml` |
|
||||
| **mypy** (optional) | Type checking | — |
|
||||
|
||||
### Naming
|
||||
|
||||
| Element | Convention | Example |
|
||||
|---------|-----------|---------|
|
||||
| Modules | `snake_case` | `analysis_repo.py` |
|
||||
| Classes | `PascalCase` | `AnalysisJob`, `DocumentConverter` |
|
||||
| Functions / methods | `snake_case` | `create_analysis()` |
|
||||
| Constants | `UPPER_SNAKE_CASE` | `MAX_CONCURRENT_ANALYSES` |
|
||||
| Private | `_leading_underscore` | `_build_converter()` |
|
||||
|
||||
### Style Rules
|
||||
|
||||
- Max function length: **30 lines** (soft limit — justify longer ones)
|
||||
- Max file length: **300 lines** (split into modules if exceeded)
|
||||
- Imports: standard library → third-party → local, separated by blank lines
|
||||
- Type hints on all public functions
|
||||
- Docstrings only on non-obvious public APIs (don't state the obvious)
|
||||
- No `# type: ignore` without a comment explaining why
|
||||
|
||||
### Architecture Rules
|
||||
|
||||
- **Domain layer** (`domain/`): zero imports from `api/`, `persistence/`, `infra/`
|
||||
- **Persistence layer** (`persistence/`): only imports from `domain/`
|
||||
- **API layer** (`api/`): never imports from `persistence/` directly — goes through `services/`
|
||||
- **Services** (`services/`): orchestrate, don't implement — delegate to domain and infra
|
||||
|
||||
## TypeScript / Vue (Frontend — `frontend/src/`)
|
||||
|
||||
### Tooling
|
||||
|
||||
| Tool | Purpose | Config |
|
||||
|------|---------|--------|
|
||||
| **ESLint** | Linting | `.eslintrc.*` |
|
||||
| **Prettier** | Formatting | `.prettierrc` |
|
||||
| **vue-tsc** | Type checking | `tsconfig.json` |
|
||||
| **Vitest** | Testing | `vitest.config.ts` |
|
||||
|
||||
### Naming
|
||||
|
||||
| Element | Convention | Example |
|
||||
|---------|-----------|---------|
|
||||
| Components | `PascalCase.vue` | `BboxOverlay.vue` |
|
||||
| Composables | `useCamelCase.ts` | `usePagination.ts` |
|
||||
| Stores | `camelCase.ts` | `analysisStore.ts` |
|
||||
| Types / Interfaces | `PascalCase` | `AnalysisJob`, `BboxRect` |
|
||||
| Constants | `UPPER_SNAKE_CASE` | `DEFAULT_PAGE_SIZE` |
|
||||
| CSS classes | `kebab-case` | `.bbox-overlay` |
|
||||
| `data-e2e` attributes | `kebab-case` | `data-e2e="upload-zone"` |
|
||||
|
||||
### Style Rules
|
||||
|
||||
- **Composition API** only (`<script setup lang="ts">`) — no Options API
|
||||
- One component per file
|
||||
- Props defined with `defineProps<T>()` (type-based, not runtime)
|
||||
- Emits defined with `defineEmits<T>()`
|
||||
- Pinia stores: one per feature, in the feature directory
|
||||
- No global state outside Pinia
|
||||
- API calls only in `api.ts` files (never in components or stores directly)
|
||||
|
||||
### API Contract
|
||||
|
||||
- Frontend sends/receives **camelCase** (Pydantic `alias_generator`)
|
||||
- Backend uses **snake_case** internally
|
||||
- `pages_json` is an exception — contains raw snake_case from `dataclasses.asdict()`
|
||||
|
||||
## Karate (E2E — `e2e/`)
|
||||
|
||||
See [e2e/CONVENTIONS.md](https://github.com/scub-france/Docling-Studio/blob/main/e2e/CONVENTIONS.md) for detailed rules.
|
||||
|
||||
Key points:
|
||||
- Use `data-e2e` selectors, never CSS classes
|
||||
- Use `retry()`/`waitFor()`, never `Thread.sleep()` or `delay()`
|
||||
- Setup via API, verify via UI, cleanup via API
|
||||
- Tag tests: `@critical`, `@ui`, `@smoke`, `@regression`, `@e2e`
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
# Audit 01 — Hexagonal Architecture (ports & adapters)
|
||||
|
||||
**Objectif** : verifier que le backend respecte le pattern hexagonal (ports dans `domain/ports.py`, adaptateurs dans `infra/`), le flux de dependances strict `api -> services -> domain`, et que chaque couche a une responsabilite claire.
|
||||
|
||||
**Cible** : `document-parser/` (hors `.venv/`, `__pycache__/`, `tests/`)
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 1.1 Domain (couche pure)
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 1.1.1 | `domain/` n'importe ni FastAPI, ni aiosqlite, ni aucune lib infra | 3 |
|
||||
| 1.1.2 | Les modeles, value objects et ports ne font aucun I/O (file, HTTP, DB) | 3 |
|
||||
| 1.1.3 | Toute interaction avec l'exterieur passe par un protocole dans `domain/ports.py` | 3 |
|
||||
| 1.1.4 | Pas de Pydantic dans domain — le domain utilise des dataclasses | 2 |
|
||||
|
||||
### 1.2 Services (orchestration)
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 1.2.1 | `services/` n'importe jamais `fastapi`, `Request`, `Response`, `Depends` | 3 |
|
||||
| 1.2.2 | Les services appellent les repos, jamais de requetes SQL directes | 3 |
|
||||
| 1.2.3 | Les regles metier vivent dans `domain/`, pas dans les services | 2 |
|
||||
| 1.2.4 | Les services recoivent leurs dependances par injection, pas par import direct de concretions | 2 |
|
||||
|
||||
### 1.3 API (couche HTTP)
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 1.3.1 | Les routes n'importent pas `persistence/` directement | 3 |
|
||||
| 1.3.2 | Les transformations camelCase/snake_case restent dans `api/schemas.py` | 1 |
|
||||
| 1.3.3 | Les endpoints delegent toute la logique aux services | 2 |
|
||||
|
||||
### 1.4 Infra (adaptateurs)
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 1.4.1 | Chaque adaptateur dans `infra/` implemente un protocole de `domain/ports.py` | 3 |
|
||||
| 1.4.2 | Les valeurs de config viennent de `infra/settings.py`, pas de constantes en dur | 2 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 1.1.1 — Domain ne doit importer aucune lib infra
|
||||
grep -rn "from fastapi\|from aiosqlite\|from pydantic\|import fastapi\|import aiosqlite" document-parser/domain/
|
||||
|
||||
# 1.2.1 — Services ne doivent pas importer FastAPI
|
||||
grep -rn "from fastapi\|import fastapi" document-parser/services/
|
||||
|
||||
# 1.3.1 — API ne doit pas importer persistence
|
||||
grep -rn "from persistence\|import persistence" document-parser/api/
|
||||
|
||||
# 1.4.2 — Constantes en dur dans infra (hors settings)
|
||||
grep -rn "= ['\"]http\|= [0-9]\{4,\}" document-parser/infra/ --include="*.py" | grep -v settings.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
# Audit 02 — Domain-Driven Design (DDD)
|
||||
|
||||
**Objectif** : verifier que le code respecte les principes DDD — bounded contexts clairs, entites et value objects bien definis, ubiquitous language coherent, et separation des responsabilites metier.
|
||||
|
||||
**Cible** : `document-parser/domain/`, `document-parser/services/`, `frontend/src/features/`, `frontend/src/shared/types.ts`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 2.1 Bounded Contexts
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 2.1.1 | Les contextes metier sont clairement identifies et isoles (document, analysis, chunking) | 3 |
|
||||
| 2.1.2 | Chaque contexte a ses propres modeles — pas de modele "god object" partage entre contextes | 3 |
|
||||
| 2.1.3 | Les frontieres entre contextes sont explicites — la communication passe par des contrats definis (DTOs, events), pas par des imports directs de modeles internes d'un autre contexte | 2 |
|
||||
| 2.1.4 | Le frontend respecte les memes bounded contexts (features = contextes) | 2 |
|
||||
|
||||
### 2.2 Entites et Value Objects
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 2.2.1 | Les entites ont une identite unique (`id`) et un cycle de vie (Document, AnalysisJob) | 2 |
|
||||
| 2.2.2 | Les value objects sont immutables et definis par leurs attributs, pas par une identite (ConversionResult, ChunkingOptions, BoundingBox) | 2 |
|
||||
| 2.2.3 | Les value objects ne contiennent pas de logique de persistence (pas de `save()`, `update()`) | 3 |
|
||||
| 2.2.4 | Les entites ne sont pas de simples "sacs de donnees" — elles portent du comportement metier quand c'est pertinent | 1 |
|
||||
|
||||
### 2.3 Ubiquitous Language
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 2.3.1 | Le vocabulaire metier est coherent entre domain, services, API et frontend (ex: "analysis" partout, pas "job" d'un cote et "analysis" de l'autre) | 2 |
|
||||
| 2.3.2 | Les noms de classes/fonctions/variables refletent le langage du domaine, pas des termes techniques generiques (pas de `DataProcessor`, `Handler`, `Manager` sans contexte) | 1 |
|
||||
| 2.3.3 | Les statuts metier utilisent un vocabulaire explicite (PENDING, RUNNING, COMPLETED, FAILED) | 1 |
|
||||
|
||||
### 2.4 Agregats et invariants
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 2.4.1 | Chaque agregat a une racine claire (Document est la racine de son agregat, AnalysisJob de son agregat) | 2 |
|
||||
| 2.4.2 | Les invariants metier sont proteges dans le domaine — pas de creation d'etats invalides depuis l'exterieur | 3 |
|
||||
| 2.4.3 | Les modifications d'un agregat passent par sa racine, pas par manipulation directe de ses composants internes | 2 |
|
||||
|
||||
### 2.5 Repositories et anti-corruption
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 2.5.1 | Les repositories (`persistence/`) manipulent des entites du domaine, pas des dictionnaires bruts ou des Row objects | 2 |
|
||||
| 2.5.2 | La couche anti-corruption (schemas Pydantic) transforme les donnees externes (HTTP) en objets du domaine | 2 |
|
||||
| 2.5.3 | Les adaptateurs infra (`infra/`) ne leakent pas leurs types internes vers les services (pas de types Docling exposes aux services) | 3 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 2.1.2 — Chercher un modele omniscient
|
||||
wc -l document-parser/domain/models.py
|
||||
|
||||
# 2.2.2 — Value objects mutables (setters, attributs reassignes)
|
||||
grep -rn "\..*=" document-parser/domain/value_objects.py | grep -v "self\." | grep -v "__"
|
||||
|
||||
# 2.3.1 — Incoherences de vocabulaire (job vs analysis)
|
||||
grep -rni "\bjob\b" document-parser/api/ document-parser/services/ --include="*.py" | grep -v "AnalysisJob"
|
||||
grep -rni "\bjob\b" frontend/src/ --include="*.ts" --include="*.vue" | grep -v "node_modules"
|
||||
|
||||
# 2.5.3 — Types Docling qui leakent vers services
|
||||
grep -rn "from docling\|import docling" document-parser/services/ --include="*.py"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
# Audit 03 — Clean Code
|
||||
|
||||
**Objectif** : verifier la lisibilite, la clarte et la maintenabilite du code.
|
||||
|
||||
**Cible** : `document-parser/` (hors `.venv/`, `__pycache__/`), `frontend/src/`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 3.1 Nommage
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 3.1.1 | Les fonctions sont nommees avec des verbes d'action (`create_analysis`, `upload_document`) | 1 |
|
||||
| 3.1.2 | Les variables expriment l'intention (`remaining_pages` et non `rp`) | 1 |
|
||||
| 3.1.3 | Tout le code est en anglais — les traductions i18n sont dans `shared/i18n.ts` | 2 |
|
||||
| 3.1.4 | Pas d'abbreviations ambigues sauf conventions etablies (`dto`, `bbox`, `id`, `url`) | 1 |
|
||||
|
||||
### 3.2 Fonctions
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 3.2.1 | Chaque fonction fait une seule chose (Single Responsibility) | 2 |
|
||||
| 3.2.2 | Aucune fonction ne depasse 30 lignes (hors boilerplate inevitable) | 1 |
|
||||
| 3.2.3 | Aucune fonction n'a plus de 4 parametres | 1 |
|
||||
| 3.2.4 | Pas de flag arguments (booleen qui change le comportement) | 1 |
|
||||
| 3.2.5 | Une fonction `get_*` ne modifie pas d'etat (pas de side-effects caches) | 2 |
|
||||
|
||||
### 3.3 Fichiers et structure
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 3.3.1 | Aucun fichier source ne depasse 300 lignes | 1 |
|
||||
| 3.3.2 | Un seul concept par fichier — pas de fichier fourre-tout | 2 |
|
||||
| 3.3.3 | Imports ordonnes : stdlib, deps externes, imports internes | 1 |
|
||||
|
||||
### 3.4 Commentaires
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 3.4.1 | Le code est auto-documentant — les commentaires expliquent le "pourquoi", pas le "quoi" | 1 |
|
||||
| 3.4.2 | Pas de code commente laisse en place (dead code) | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 3.3.1 — Fichiers Python > 300 lignes
|
||||
find document-parser -name "*.py" -not -path "*/.venv/*" -not -path "*/__pycache__/*" -not -path "*/tests/*" | xargs wc -l | sort -rn | head -20
|
||||
|
||||
# 3.3.1 — Fichiers Vue/TS > 300 lignes
|
||||
find frontend/src -name "*.vue" -o -name "*.ts" | xargs wc -l | sort -rn | head -20
|
||||
|
||||
# 3.2.3 — Fonctions avec > 4 parametres (Python)
|
||||
grep -rn "def .*,.*,.*,.*,.*," document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=__pycache__ --exclude-dir=tests
|
||||
|
||||
# 3.4.2 — Code commente
|
||||
grep -rn "^[[:space:]]*#.*=\|^[[:space:]]*#.*def \|^[[:space:]]*#.*return\|^[[:space:]]*//.*=\|^[[:space:]]*//.*function" document-parser --include="*.py" --exclude-dir=.venv frontend/src --include="*.ts" --include="*.vue"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
# Audit 04 — KISS (Keep It Simple, Stupid)
|
||||
|
||||
**Objectif** : verifier que le code reste simple et ne contient pas de sur-ingenierie.
|
||||
|
||||
**Cible** : `document-parser/` (hors `.venv/`, `__pycache__/`), `frontend/src/`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 4.1 | Pas de design pattern complexe la ou un simple `if` ou une fonction suffit (factory, strategy, observer superflus) | 2 |
|
||||
| 4.2 | Le code resout le probleme actuel, pas un probleme hypothetique futur (pas de genericite prematuree) | 2 |
|
||||
| 4.3 | Pas de fonction wrapper qui ne fait qu'appeler une autre fonction sans valeur ajoutee | 1 |
|
||||
| 4.4 | Utilisation des outils standard (Python stdlib, Vue composables natifs) avant de creer des solutions maison | 1 |
|
||||
| 4.5 | Configuration simple — pas de systeme de config complexe la ou une variable d'env suffit | 1 |
|
||||
| 4.6 | Pas d'indirection inutile — le chemin d'execution d'une requete ne traverse pas plus de couches que necessaire | 2 |
|
||||
| 4.7 | Pas de meta-programmation ou de magie (decorateurs complexes, metaclasses) sauf necessite avere | 2 |
|
||||
| 4.8 | Les structures de donnees utilisees sont les plus simples possibles (liste plutot que arbre si la liste suffit) | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 4.1 — Patterns potentiellement superflus
|
||||
grep -rn "class.*Factory\|class.*Strategy\|class.*Observer\|class.*Builder\|class.*Singleton" document-parser --include="*.py" --exclude-dir=.venv
|
||||
|
||||
# 4.7 — Meta-programmation
|
||||
grep -rn "__metaclass__\|type(.*,.*,.*)\|__init_subclass__\|__class_getitem__" document-parser --include="*.py" --exclude-dir=.venv
|
||||
|
||||
# 4.3 — Fonctions tres courtes (potentiels wrappers inutiles, < 3 lignes)
|
||||
# Verification manuelle recommandee sur les fonctions identifiees
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
# Audit 05 — DRY (Don't Repeat Yourself)
|
||||
|
||||
**Objectif** : verifier l'absence de duplication significative dans le code.
|
||||
|
||||
**Cible** : `document-parser/` (hors `.venv/`, `__pycache__/`), `frontend/src/`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 5.1 | Aucun bloc de code identique ou quasi-identique n'apparait 3+ fois sans etre factorise | 2 |
|
||||
| 5.2 | Les interfaces/types partages sont centralises dans `shared/types.ts` (frontend) et `domain/models.py` (backend) | 2 |
|
||||
| 5.3 | Pas de magic numbers ou magic strings eparpilles — les constantes sont nommees et centralisees | 2 |
|
||||
| 5.4 | La logique reactive partagee est dans `shared/composables/` (frontend) | 1 |
|
||||
| 5.5 | Les appels API ne dupliquent pas la config HTTP (base URL, headers) — centralises dans `shared/api/http.ts` | 2 |
|
||||
| 5.6 | Les schemas Pydantic ne dupliquent pas les modeles du domain — ils transforment, ils ne redefinissent pas | 2 |
|
||||
| 5.7 | Les regles de validation ne sont definies qu'a un seul endroit (schema Pydantic OU frontend, pas les deux en desaccord) | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 5.3 — Magic numbers (backend)
|
||||
grep -rn "[^a-zA-Z_\"'][0-9]\{3,\}[^a-zA-Z_\"']" document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=tests --exclude-dir=__pycache__
|
||||
|
||||
# 5.3 — Magic strings repetees (backend)
|
||||
grep -rohn '"[a-z_]\{5,\}"' document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=tests | sort | uniq -c | sort -rn | head -20
|
||||
|
||||
# 5.5 — Appels fetch en dehors du client HTTP centralise
|
||||
grep -rn "fetch(" frontend/src/ --include="*.ts" --include="*.vue" | grep -v "http.ts\|api.ts\|node_modules"
|
||||
|
||||
# 5.6 — Champs dupliques entre schemas et models
|
||||
diff <(grep -o "[a-z_]*:" document-parser/api/schemas.py | sort -u) <(grep -o "[a-z_]*:" document-parser/domain/models.py | sort -u)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
# Audit 06 — SOLID
|
||||
|
||||
**Objectif** : verifier le respect des 5 principes SOLID.
|
||||
|
||||
**Cible** : `document-parser/` (hors `.venv/`, `__pycache__/`), `frontend/src/`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 6.1 S — Single Responsibility Principle
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 6.1.1 | Chaque service a une responsabilite unique (`document_service` = documents, `analysis_service` = analyses) | 2 |
|
||||
| 6.1.2 | Chaque store Pinia gere un seul feature | 2 |
|
||||
| 6.1.3 | Les routes API sont groupees par ressource (`documents.py`, `analyses.py`) | 1 |
|
||||
| 6.1.4 | Aucune classe ou module ne cumule des responsabilites heterogenes (ex: un service qui fait du parsing ET de la persistence) | 2 |
|
||||
|
||||
### 6.2 O — Open/Closed Principle
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 6.2.1 | Les ports (`domain/ports.py`) permettent d'ajouter de nouveaux adaptateurs sans modifier le code existant | 2 |
|
||||
| 6.2.2 | Le systeme local/remote est extensible via `_build_converter()` sans modifier les services | 2 |
|
||||
| 6.2.3 | L'ajout d'un nouveau format d'export ne necessite pas de modifier les endpoints existants | 1 |
|
||||
|
||||
### 6.3 L — Liskov Substitution Principle
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 6.3.1 | `LocalConverter` et `ServeConverter` sont interchangeables (meme protocole, meme contrat de retour) | 3 |
|
||||
| 6.3.2 | Les implementations de ports ne lancent pas d'exceptions non prevues par le contrat | 2 |
|
||||
| 6.3.3 | Pas de `isinstance()` ou `type()` check pour differencier les implementations | 2 |
|
||||
|
||||
### 6.4 I — Interface Segregation Principle
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 6.4.1 | `DocumentConverter` et `DocumentChunker` sont des ports separes (pas une "god interface") | 2 |
|
||||
| 6.4.2 | Aucun port ne force une implementation a definir des methodes qu'elle n'utilise pas | 2 |
|
||||
|
||||
### 6.5 D — Dependency Inversion Principle
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 6.5.1 | Les services dependent de protocoles abstraits (ports), pas d'implementations concretes | 3 |
|
||||
| 6.5.2 | L'injection se fait dans `main.py` (composition root) | 2 |
|
||||
| 6.5.3 | Pas d'instanciation directe d'adaptateurs dans les services (`LocalConverter()` dans un service = violation) | 3 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 6.3.3 — isinstance checks sur les adaptateurs
|
||||
grep -rn "isinstance\|type(" document-parser/services/ --include="*.py"
|
||||
|
||||
# 6.5.3 — Instanciation directe d'adaptateurs dans services
|
||||
grep -rn "LocalConverter\|ServeConverter\|LocalChunker" document-parser/services/ --include="*.py"
|
||||
|
||||
# 6.5.1 — Imports directs d'infra dans services
|
||||
grep -rn "from infra\.\|import infra\." document-parser/services/ --include="*.py"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
# Audit 07 — Decouplage
|
||||
|
||||
**Objectif** : verifier le decouplage entre frontend et backend, entre features, et la clarte des contrats d'interface.
|
||||
|
||||
**Cible** : `document-parser/`, `frontend/src/`, `docker-compose.yml`, `nginx.conf`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 7.1 Decouplage Frontend / Backend
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 7.1.1 | Le frontend communique avec le backend uniquement via l'API REST — pas de couplage par fichier partage, DB partagee, ou import croise | 3 |
|
||||
| 7.1.2 | Le contrat API est stable — les types TypeScript frontend correspondent aux schemas Pydantic backend | 3 |
|
||||
| 7.1.3 | Le frontend peut tourner avec un mock du backend (les appels API sont isoles dans des fichiers `api.ts` par feature) | 2 |
|
||||
| 7.1.4 | Le backend peut etre teste sans le frontend (endpoints testables via `httpx` / `TestClient`) | 2 |
|
||||
| 7.1.5 | Pas de logique metier dupliquee entre front et back (ex: validation faite cote back ET reinventee cote front) | 2 |
|
||||
|
||||
### 7.2 Decouplage inter-features (Frontend)
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 7.2.1 | Chaque feature (`features/analysis`, `features/document`, ...) a son propre store, API client et composants UI | 2 |
|
||||
| 7.2.2 | Les features ne s'importent pas mutuellement — la communication passe par `shared/` ou par les props/events Vue | 3 |
|
||||
| 7.2.3 | Les types partages entre features sont dans `shared/types.ts`, pas dans une feature specifique | 2 |
|
||||
| 7.2.4 | Un store Pinia n'accede pas directement au state d'un autre store (sauf via des getters exposes) | 2 |
|
||||
|
||||
### 7.3 Decouplage inter-couches (Backend)
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 7.3.1 | Les repos (`persistence/`) retournent des objets du domaine, pas des dicts ou des Row SQLite | 2 |
|
||||
| 7.3.2 | Les adaptateurs infra n'exposent pas les types de leurs libs internes aux services (pas de types `docling.*` dans les signatures de services) | 3 |
|
||||
| 7.3.3 | Le changement de base de donnees (SQLite -> PostgreSQL) ne necessite de modifier que `persistence/` | 2 |
|
||||
| 7.3.4 | Le changement de framework HTTP (FastAPI -> autre) ne necessite de modifier que `api/` et `main.py` | 2 |
|
||||
|
||||
### 7.4 Contrats et interfaces
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 7.4.1 | Les ports dans `domain/ports.py` definissent des signatures claires avec des types du domaine | 2 |
|
||||
| 7.4.2 | Les schemas Pydantic (`api/schemas.py`) documentent le contrat HTTP — pas de `dict` ou `Any` dans les responses | 2 |
|
||||
| 7.4.3 | Les reponses API ont un format coherent (enveloppe, codes d'erreur normalises) | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 7.2.2 — Imports croises entre features
|
||||
grep -rn "from.*features/" frontend/src/features/ --include="*.ts" --include="*.vue" | grep -v "node_modules" | grep -v "__tests__"
|
||||
|
||||
# 7.2.4 — Store qui accede au state d'un autre store
|
||||
grep -rn "useDocumentStore\|useAnalysisStore\|useChunkingStore\|useHistoryStore\|useSettingsStore" frontend/src/features/ --include="*.ts" | grep -v "index.ts"
|
||||
|
||||
# 7.3.2 — Types docling qui leakent
|
||||
grep -rn "from docling\|import docling" document-parser/services/ --include="*.py"
|
||||
|
||||
# 7.4.2 — dict ou Any dans les reponses API
|
||||
grep -rn "-> dict\|-> Any\|Dict\[str, Any\]" document-parser/api/ --include="*.py"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
# Audit 08 — Securite
|
||||
|
||||
**Objectif** : verifier l'absence de vulnerabilites courantes (OWASP Top 10) et le respect des bonnes pratiques de securite.
|
||||
|
||||
**Cible** : tout le projet
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 8.1 Secrets et credentials
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 8.1.1 | Aucune cle API, token, ou mot de passe en dur dans le code source | 3 |
|
||||
| 8.1.2 | Les fichiers `.env` sont dans `.gitignore` | 3 |
|
||||
| 8.1.3 | Les secrets Docker sont passes par variables d'environnement, pas en build args | 2 |
|
||||
|
||||
### 8.2 Validation des entrees
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 8.2.1 | Toutes les entrees utilisateur sont validees par des schemas Pydantic | 3 |
|
||||
| 8.2.2 | `MAX_FILE_SIZE_MB` est configuree et appliquee a l'upload | 3 |
|
||||
| 8.2.3 | Les types de fichiers acceptes sont valides (pas d'upload de `.exe`, `.sh`, etc.) | 2 |
|
||||
|
||||
### 8.3 Injection
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 8.3.1 | Les requetes SQL utilisent des parametres lies (`?`), jamais de string formatting/f-strings | 3 |
|
||||
| 8.3.2 | Pas de `eval()`, `exec()`, ou `os.system()` avec des entrees utilisateur | 3 |
|
||||
| 8.3.3 | Le frontend utilise DOMPurify pour tout rendu de contenu HTML/Markdown | 3 |
|
||||
|
||||
### 8.4 CORS et reseau
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 8.4.1 | Les origines CORS autorisees sont configurees explicitement, pas de `*` en production | 3 |
|
||||
| 8.4.2 | Le rate limiter est actif sur tous les endpoints sauf `/api/health` | 2 |
|
||||
| 8.4.3 | Nginx ne sert que les fichiers statiques prevus — pas de directory listing | 2 |
|
||||
|
||||
### 8.5 Dependances
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 8.5.1 | Pas de dependance avec des CVE critiques connues | 3 |
|
||||
| 8.5.2 | Les versions des dependances sont epinglees (pas de `>=` sans borne superieure) | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 8.1.1 — Secrets potentiels dans le code
|
||||
grep -rni "password\s*=\|secret\s*=\|api_key\s*=\|token\s*=" document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=tests
|
||||
grep -rni "password\|secret\|api.key\|token" frontend/src/ --include="*.ts" --include="*.vue"
|
||||
|
||||
# 8.1.2 — .env dans gitignore
|
||||
grep "\.env" .gitignore
|
||||
|
||||
# 8.3.1 — SQL injection (f-strings dans les requetes)
|
||||
grep -rn 'f".*SELECT\|f".*INSERT\|f".*UPDATE\|f".*DELETE\|f".*DROP' document-parser --include="*.py" --exclude-dir=.venv
|
||||
|
||||
# 8.3.2 — eval/exec/os.system
|
||||
grep -rn "eval(\|exec(\|os\.system(\|subprocess\.call(" document-parser --include="*.py" --exclude-dir=.venv
|
||||
|
||||
# 8.3.3 — DOMPurify usage
|
||||
grep -rn "DOMPurify\|v-html\|innerHTML" frontend/src/ --include="*.vue" --include="*.ts"
|
||||
|
||||
# 8.4.1 — CORS wildcard
|
||||
grep -rn 'allow_origins.*\*\|"*"' document-parser --include="*.py" --exclude-dir=.venv
|
||||
|
||||
# 8.5.1 — Audit npm
|
||||
cd frontend && npm audit --production 2>&1 | tail -10
|
||||
|
||||
# 8.5.1 — Audit pip (si pip-audit installe)
|
||||
cd document-parser && pip-audit 2>&1 | tail -10
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
# Audit 09 — Tests
|
||||
|
||||
**Objectif** : verifier la couverture, la qualite et la fiabilite de la suite de tests.
|
||||
|
||||
**Cible** : `document-parser/tests/`, `frontend/src/**/*.test.*`, `e2e/`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 9.1 Execution
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 9.1.1 | Tous les tests backend passent (`pytest tests/ -v`) | 3 |
|
||||
| 9.1.2 | Tous les tests frontend passent (`npm run test:run`) | 3 |
|
||||
| 9.1.3 | Les tests e2e Karate UI passent | 2 |
|
||||
|
||||
### 9.2 Couverture
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 9.2.1 | Chaque endpoint API a au moins un test (happy path) | 2 |
|
||||
| 9.2.2 | Les cas d'erreur des endpoints sont testes (400, 404, 413, 429) | 2 |
|
||||
| 9.2.3 | Les services ont des tests unitaires couvrant la logique d'orchestration | 2 |
|
||||
| 9.2.4 | Les fonctions du domain (bbox, value objects) sont testees | 1 |
|
||||
| 9.2.5 | Les composants Vue critiques ont des tests (stores, composables) | 2 |
|
||||
|
||||
### 9.3 Qualite des tests
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 9.3.1 | Pas de `.only` ou `fdescribe` ou `fit` laisse par accident | 3 |
|
||||
| 9.3.2 | Pas de `@pytest.mark.skip` ou `.skip()` sans justification en commentaire | 1 |
|
||||
| 9.3.3 | Les tests sont deterministes — pas de dependance a l'heure, au reseau, ou a l'ordre d'execution | 2 |
|
||||
| 9.3.4 | Les tests d'integration testent le flux reel, pas un mock complet | 2 |
|
||||
| 9.3.5 | Les assertions sont specifiques (pas juste `assert result is not None`) | 1 |
|
||||
| 9.3.6 | Chaque test a un nom explicite qui decrit le comportement teste | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 9.1.1 — Tests backend
|
||||
cd document-parser && python -m pytest tests/ -v --tb=short 2>&1 | tail -30
|
||||
|
||||
# 9.1.2 — Tests frontend
|
||||
cd frontend && npm run test:run 2>&1 | tail -30
|
||||
|
||||
# 9.3.1 — .only / fdescribe / fit
|
||||
grep -rn "\.only\|fdescribe\|fit(" frontend/src/ --include="*.test.*"
|
||||
|
||||
# 9.3.2 — Skip sans commentaire
|
||||
grep -rn -B1 "@pytest.mark.skip\|\.skip(" document-parser/tests/ frontend/src/
|
||||
|
||||
# 9.3.5 — Assertions vagues
|
||||
grep -rn "assert.*is not None$\|assert.*!= None$\|expect.*toBeTruthy()$" document-parser/tests/ frontend/src/ --include="*.test.*" --include="*.py"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
# Audit 10 — CI / Build
|
||||
|
||||
**Objectif** : verifier que la pipeline CI est verte, que le build Docker fonctionne, et que les outils de qualite sont actifs.
|
||||
|
||||
**Cible** : `.github/`, `Dockerfile`, `docker-compose.yml`, `nginx.conf`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 10.1 Pipeline CI
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 10.1.1 | Toutes les GitHub Actions passent sur la branche de release | 3 |
|
||||
| 10.1.2 | Les warnings ESLint sont resolus (0 warning) | 1 |
|
||||
| 10.1.3 | Les warnings Ruff sont resolus (0 warning) | 1 |
|
||||
| 10.1.4 | Le type-check frontend passe (`vue-tsc --noEmit`) | 2 |
|
||||
| 10.1.5 | Le formatting est conforme (`ruff format --check`, `prettier --check`) | 1 |
|
||||
|
||||
### 10.2 Build Docker
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 10.2.1 | `docker compose build` reussit sans erreur | 3 |
|
||||
| 10.2.2 | Le container demarre et repond sur `/api/health` | 3 |
|
||||
| 10.2.3 | Les deux variantes (local/remote) buildent correctement | 2 |
|
||||
| 10.2.4 | Pas de fichier inutile dans l'image (node_modules frontend, .venv, .git) — `.dockerignore` est a jour | 1 |
|
||||
|
||||
### 10.3 Configuration
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 10.3.1 | Nginx route correctement `/api/*` vers le backend et sert le frontend sur `/` | 2 |
|
||||
| 10.3.2 | Les variables d'environnement sont documentees et ont des valeurs par defaut coherentes | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 10.1.2 + 10.1.3 — Lint
|
||||
cd document-parser && ruff check . 2>&1 | tail -5
|
||||
cd frontend && npx eslint src/ 2>&1 | tail -5
|
||||
|
||||
# 10.1.4 — Type check
|
||||
cd frontend && npx vue-tsc --noEmit 2>&1 | tail -5
|
||||
|
||||
# 10.1.5 — Formatting
|
||||
cd document-parser && ruff format --check . 2>&1 | tail -5
|
||||
cd frontend && npx prettier --check src/ 2>&1 | tail -5
|
||||
|
||||
# 10.2.1 — Build Docker
|
||||
docker compose build 2>&1 | tail -10
|
||||
|
||||
# 10.2.4 — .dockerignore
|
||||
cat .dockerignore 2>/dev/null || echo "ABSENT"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
# Audit 11 — Documentation & Changelog
|
||||
|
||||
**Objectif** : verifier que la release est correctement documentee et versionnee.
|
||||
|
||||
**Cible** : `CHANGELOG.md`, `frontend/package.json`, `docs/`, code source
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 11.1 Changelog
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 11.1.1 | La section `[Unreleased]` a ete renommee en `[X.Y.Z] - YYYY-MM-DD` | 3 |
|
||||
| 11.1.2 | Toutes les modifications significatives de la release sont listees dans le changelog | 2 |
|
||||
| 11.1.3 | Les breaking changes sont clairement identifies | 3 |
|
||||
| 11.1.4 | Le format respecte [Keep a Changelog](https://keepachangelog.com/) | 1 |
|
||||
|
||||
### 11.2 Versioning
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 11.2.1 | `frontend/package.json` contient la bonne version X.Y.Z | 2 |
|
||||
| 11.2.2 | La version suit le Semantic Versioning | 2 |
|
||||
|
||||
### 11.3 Code propre
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 11.3.1 | Les `TODO` et `FIXME` restants sont volontaires et documentes (pas de TODO orphelin) | 1 |
|
||||
| 11.3.2 | Pas de `console.log` de debug laisse dans le code frontend | 2 |
|
||||
| 11.3.3 | Pas de `print()` de debug laisse dans le code backend (hors logging structure) | 2 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 11.1.1 — Section Unreleased encore presente
|
||||
grep -n "Unreleased" CHANGELOG.md
|
||||
|
||||
# 11.2.1 — Version dans package.json
|
||||
grep '"version"' frontend/package.json
|
||||
|
||||
# 11.3.1 — TODOs restants
|
||||
grep -rn "TODO\|FIXME\|HACK\|XXX" document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=__pycache__
|
||||
grep -rn "TODO\|FIXME\|HACK\|XXX" frontend/src --include="*.ts" --include="*.vue"
|
||||
|
||||
# 11.3.2 — console.log de debug
|
||||
grep -rn "console\.log\|console\.debug\|console\.warn" frontend/src/ --include="*.ts" --include="*.vue" | grep -v "node_modules"
|
||||
|
||||
# 11.3.3 — print() de debug
|
||||
grep -rn "^\s*print(" document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=tests --exclude-dir=__pycache__
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# Audit 12 — Performance & Ressources
|
||||
|
||||
**Objectif** : verifier l'absence de problemes de performance evidents et la bonne gestion des ressources.
|
||||
|
||||
**Cible** : `document-parser/` (hors `.venv/`), `frontend/src/`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### 12.1 Backend
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 12.1.1 | Pas de requete N+1 — les acces DB sont optimises (pas de boucle avec requete unitaire) | 2 |
|
||||
| 12.1.2 | Les fichiers uploades temporaires sont supprimes apres traitement | 2 |
|
||||
| 12.1.3 | `MAX_CONCURRENT_ANALYSES` est configuree et respectee (semaphore) | 2 |
|
||||
| 12.1.4 | Les operations longues (conversion Docling) sont asynchrones et ne bloquent pas l'event loop | 3 |
|
||||
| 12.1.5 | Pas de chargement de fichier entier en memoire sans necesssite (streaming si possible) | 2 |
|
||||
|
||||
### 12.2 Frontend
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 12.2.1 | Les watchers Vue ont leur cleanup (pas de memory leak) | 2 |
|
||||
| 12.2.2 | Les event listeners sont supprimes dans `onUnmounted` | 2 |
|
||||
| 12.2.3 | Les requetes API ont un mecanisme d'annulation ou de debounce quand pertinent | 1 |
|
||||
| 12.2.4 | Pas de re-render excessif — les computed sont utilises plutot que des fonctions dans le template | 1 |
|
||||
| 12.2.5 | Les assets lourds (images, fonts) sont optimises | 1 |
|
||||
|
||||
### 12.3 Infrastructure
|
||||
|
||||
| # | Item | Poids |
|
||||
|---|------|-------|
|
||||
| 12.3.1 | Nginx a une configuration de cache pour les fichiers statiques | 1 |
|
||||
| 12.3.2 | Les reponses API sont de taille raisonnable (pas d'envoi de donnees inutiles) | 1 |
|
||||
| 12.3.3 | Le health check est leger et ne charge pas le systeme | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Commandes de verification
|
||||
|
||||
```bash
|
||||
# 12.1.1 — Pattern N+1 (boucle avec requete DB)
|
||||
grep -rn -A5 "for.*in.*:" document-parser/persistence/ --include="*.py" | grep -i "select\|fetch\|execute"
|
||||
|
||||
# 12.1.4 — Appels bloquants dans du code async
|
||||
grep -rn "time\.sleep\|open(" document-parser/services/ --include="*.py"
|
||||
|
||||
# 12.2.1 + 12.2.2 — Watchers et listeners sans cleanup
|
||||
grep -rn "addEventListener\|watch(\|watchEffect(" frontend/src/ --include="*.vue" --include="*.ts" | grep -v "node_modules"
|
||||
grep -rn "onUnmounted\|onBeforeUnmount\|removeEventListener" frontend/src/ --include="*.vue" | grep -v "node_modules"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Regles de notation
|
||||
|
||||
- Tout item de poids 3 non conforme = ecart `[CRIT]`
|
||||
- Tout item de poids 2 non conforme = ecart `[MAJ]`
|
||||
- Tout item de poids 1 non conforme = ecart `[MIN]`
|
||||
|
|
@ -1,206 +0,0 @@
|
|||
# Audit Master — Release Branch Quality Gate
|
||||
|
||||
Referentiel central d'audit qualite pour les branches `release/X.Y.Z` de Docling Studio.
|
||||
|
||||
Ce document est le **chef d'orchestre** : il definit les regles, commandite les audits unitaires, et normalise les rapports.
|
||||
|
||||
---
|
||||
|
||||
## 1. Perimetre
|
||||
|
||||
L'audit s'execute sur une branche `release/X.Y.Z` **apres feature freeze**, avant le merge dans `main`.
|
||||
|
||||
**Cibles :**
|
||||
|
||||
| Cible | Chemin |
|
||||
|-------|--------|
|
||||
| Backend (Python/FastAPI) | `document-parser/` (hors `.venv/`, `__pycache__/`, `tests/`) |
|
||||
| Frontend (Vue 3/TypeScript) | `frontend/src/` |
|
||||
| Tests backend | `document-parser/tests/` |
|
||||
| Tests frontend | `frontend/src/**/*.test.*` |
|
||||
| Tests e2e | `e2e/` |
|
||||
| Infrastructure | `Dockerfile`, `docker-compose.yml`, `nginx.conf`, `.github/` |
|
||||
|
||||
---
|
||||
|
||||
## 2. Niveaux de criticite
|
||||
|
||||
Chaque ecart trouve lors d'un audit est classe selon ces 4 niveaux :
|
||||
|
||||
| Niveau | Tag | Description | Impact sur le GO/NO-GO |
|
||||
|--------|-----|-------------|------------------------|
|
||||
| **CRITICAL** | `[CRIT]` | Violation bloquante — faille de securite, corruption de donnees, violation d'architecture majeure | **Bloquant** — le release ne peut pas partir |
|
||||
| **MAJOR** | `[MAJ]` | Violation significative — couplage fort, dette technique importante, test manquant sur un chemin critique | Bloquant si > 3 ecarts MAJOR non resolus |
|
||||
| **MINOR** | `[MIN]` | Ecart de qualite — nommage, taille de fichier, duplication legere | Non bloquant — a corriger dans le prochain cycle |
|
||||
| **INFO** | `[INFO]` | Observation, suggestion d'amelioration, bonne pratique non respectee mais sans risque | Non bloquant — informatif |
|
||||
|
||||
---
|
||||
|
||||
## 3. Bareme de compliance
|
||||
|
||||
Chaque audit unitaire produit une **note de compliance sur 100**.
|
||||
|
||||
### Calcul
|
||||
|
||||
Chaque item de checklist a un **poids** defini dans la fiche d'audit :
|
||||
|
||||
| Poids | Signification |
|
||||
|-------|---------------|
|
||||
| 3 | Critique — violation = ecart CRITICAL |
|
||||
| 2 | Important — violation = ecart MAJOR |
|
||||
| 1 | Standard — violation = ecart MINOR |
|
||||
|
||||
**Formule :**
|
||||
|
||||
```
|
||||
score = (somme des poids des items conformes / somme totale des poids) * 100
|
||||
```
|
||||
|
||||
### Seuils de decision
|
||||
|
||||
| Score | Verdict | Action |
|
||||
|-------|---------|--------|
|
||||
| >= 80 | **GO** | Release autorisee |
|
||||
| 60 - 79 | **GO CONDITIONNEL** | Release autorisee si 0 CRITICAL, plan de remediation pour les MAJOR |
|
||||
| < 60 | **NO-GO** | Release bloquee — corriger et re-auditer |
|
||||
|
||||
**Regle absolue** : tout ecart `[CRIT]` non resolu = **NO-GO** quel que soit le score.
|
||||
|
||||
---
|
||||
|
||||
## 4. Liste des audits
|
||||
|
||||
Les audits sont executes dans l'ordre ci-dessous. Chacun est une fiche autonome dans `audits/`.
|
||||
|
||||
| # | Audit | Fichier | Focus |
|
||||
|---|-------|---------|-------|
|
||||
| 01 | Hexagonal Architecture | [01-clean-architecture.md](audits/01-clean-architecture.md) | Ports & adapters, respect des couches, flux de dependances |
|
||||
| 02 | DDD | [02-ddd.md](audits/02-ddd.md) | Bounded contexts, entites, value objects, ubiquitous language |
|
||||
| 03 | Clean Code | [03-clean-code.md](audits/03-clean-code.md) | Nommage, taille, lisibilite |
|
||||
| 04 | KISS | [04-kiss.md](audits/04-kiss.md) | Simplicite, pas de sur-ingenierie |
|
||||
| 05 | DRY | [05-dry.md](audits/05-dry.md) | Duplication, factorisation |
|
||||
| 06 | SOLID | [06-solid.md](audits/06-solid.md) | 5 principes SOLID |
|
||||
| 07 | Decouplage | [07-decoupling.md](audits/07-decoupling.md) | Front/back, inter-features, contrats |
|
||||
| 08 | Securite | [08-security.md](audits/08-security.md) | OWASP, secrets, injection, CORS |
|
||||
| 09 | Tests | [09-tests.md](audits/09-tests.md) | Couverture, qualite, e2e |
|
||||
| 10 | CI / Build | [10-ci-build.md](audits/10-ci-build.md) | Pipeline, Docker, health check |
|
||||
| 11 | Documentation | [11-documentation.md](audits/11-documentation.md) | Changelog, version, TODOs |
|
||||
| 12 | Performance | [12-performance.md](audits/12-performance.md) | N+1, memory, concurrence |
|
||||
|
||||
---
|
||||
|
||||
## 5. Format de rapport attendu
|
||||
|
||||
Chaque audit produit un rapport dans `reports/release-X.Y.Z/XX-nom.md` respectant ce format :
|
||||
|
||||
```markdown
|
||||
# Rapport d'audit : [Nom de l'audit]
|
||||
|
||||
**Release** : X.Y.Z
|
||||
**Date** : YYYY-MM-DD
|
||||
**Auditeur** : [nom ou "claude-code"]
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | XX / YY |
|
||||
| Score | XX / 100 |
|
||||
| Ecarts CRITICAL | N |
|
||||
| Ecarts MAJOR | N |
|
||||
| Ecarts MINOR | N |
|
||||
| Ecarts INFO | N |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [CRIT] Titre de l'ecart
|
||||
- **Localisation** : `chemin/fichier.py:ligne`
|
||||
- **Constat** : description factuelle
|
||||
- **Regle violee** : reference a l'item de checklist
|
||||
- **Remediation** : action corrective proposee
|
||||
|
||||
### [MAJ] Titre de l'ecart
|
||||
...
|
||||
|
||||
### [MIN] Titre de l'ecart
|
||||
...
|
||||
|
||||
### [INFO] Titre de l'ecart
|
||||
...
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- ...
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO / GO CONDITIONNEL / NO-GO
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Rapport de synthese
|
||||
|
||||
Le fichier `reports/release-X.Y.Z/summary.md` consolide tous les audits :
|
||||
|
||||
```markdown
|
||||
# Synthese d'audit — Release X.Y.Z
|
||||
|
||||
**Date** : YYYY-MM-DD
|
||||
**Branche** : release/X.Y.Z
|
||||
|
||||
---
|
||||
|
||||
## Tableau de bord
|
||||
|
||||
| # | Audit | Score | CRIT | MAJ | MIN | INFO | Verdict |
|
||||
|---|-------|-------|------|-----|-----|------|---------|
|
||||
| 01 | Hexagonal Architecture | XX | N | N | N | N | GO |
|
||||
| 02 | DDD | XX | N | N | N | N | GO |
|
||||
| ... | ... | ... | ... | ... | ... | ... | ... |
|
||||
|
||||
**Score global** : XX / 100 (moyenne ponderee)
|
||||
**Ecarts CRITICAL totaux** : N
|
||||
**Ecarts MAJOR totaux** : N
|
||||
|
||||
---
|
||||
|
||||
## Ecarts CRITICAL (tous audits confondus)
|
||||
|
||||
1. [audit] description — fichier:ligne
|
||||
|
||||
---
|
||||
|
||||
## Verdict final : GO / GO CONDITIONNEL / NO-GO
|
||||
|
||||
Conditions (si GO CONDITIONNEL) :
|
||||
- ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Execution
|
||||
|
||||
### Lancer un audit complet
|
||||
|
||||
```
|
||||
Audite la branche release/X.Y.Z en suivant docs/audit/master.md
|
||||
```
|
||||
|
||||
### Lancer un audit unitaire
|
||||
|
||||
```
|
||||
Execute l'audit docs/audit/audits/02-ddd.md sur la branche courante
|
||||
```
|
||||
|
||||
### Re-auditer apres correction
|
||||
|
||||
```
|
||||
Re-audite uniquement les ecarts CRITICAL et MAJOR du rapport docs/audit/reports/release-X.Y.Z/summary.md
|
||||
```
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
# Rapport d'audit : Clean Architecture
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
**Derniere mise a jour** : 2026-04-10 (re-audit post PR #133)
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 13 / 13 |
|
||||
| Score | 100 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts resolus (PR #133 — fix/clean-architecture-audit)
|
||||
|
||||
### [CRIT — RESOLU] Services importent directement les modules persistence (concretions)
|
||||
|
||||
- **Localisation** : `services/analysis_service.py`, `services/document_service.py`
|
||||
- **Resolution** : `AnalysisService` et `DocumentService` recoivent `analysis_repo` et `document_repo` par injection dans leur constructeur. Plus aucun import direct de `persistence.*` au top-level des services.
|
||||
|
||||
### [CRIT — RESOLU] Services importent directement `infra.settings`
|
||||
|
||||
- **Localisation** : `services/analysis_service.py`, `services/document_service.py`
|
||||
- **Resolution** : Les valeurs de configuration sont encapsulees dans des dataclasses `AnalysisConfig` et `DocumentConfig` injectees dans les constructeurs. Plus aucun import direct de `infra.settings` dans les services.
|
||||
|
||||
### [CRIT — RESOLU] Pas de protocol pour les repositories dans `domain/ports.py`
|
||||
|
||||
- **Localisation** : `domain/ports.py`
|
||||
- **Resolution** : `DocumentRepository` et `AnalysisRepository` ajoutes comme `Protocol` dans `domain/ports.py`. Les services dependent maintenant de ces abstractions.
|
||||
|
||||
### [MAJ — RESOLU] `document_service` est un module procedural, non une classe injectable
|
||||
|
||||
- **Resolution** : `document_service` transforme en classe `DocumentService` avec injection du repository et de la config.
|
||||
|
||||
### [MAJ — RESOLU] Logique metier dans le service (`_classify_error`, `_merge_results`, validation)
|
||||
|
||||
- **Resolution** : `_merge_results` et `_classify_error` deplaces dans `domain/`. Validation de fichier encapsulee.
|
||||
|
||||
### [MIN — RESOLU] `api/documents.py` importe `infra.settings`
|
||||
|
||||
- **Resolution** : La valeur `max_file_size_mb` obtenue via le service, plus d'import direct dans la couche API.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Domain pur : `domain/models.py`, `domain/value_objects.py` et `domain/ports.py` n'importent aucune librairie externe. Les modeles sont des dataclasses pures avec des methodes de transition d'etat.
|
||||
- Ports complets : `DocumentConverter`, `DocumentChunker`, `DocumentRepository` et `AnalysisRepository` couvrent toutes les interactions externes.
|
||||
- Injection de dependances complete : services, repositories et config injectes via constructeur + `Depends` FastAPI.
|
||||
- Pydantic confine a la couche API : Tous les schemas Pydantic sont dans `api/schemas.py`. Le domaine n'utilise que des dataclasses.
|
||||
- Routes delegent aux services : Les endpoints se contentent de mapper les requetes/reponses et de deleguer.
|
||||
- Configuration centralisee dans `infra/settings.py` et propagee par injection.
|
||||
|
||||
---
|
||||
|
||||
## Verdict : GO
|
||||
|
||||
Score 100/100 — 0 ecart critique. Toutes les non-conformites precedentes resolues via PR #133.
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
# Rapport d'audit : Domain-Driven Design (DDD)
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
**Derniere mise a jour** : 2026-04-10 (re-audit post PR #135)
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 16 / 16 |
|
||||
| Score | 100 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts resolus (PR #135 — fix/ddd-audit)
|
||||
|
||||
### [CRIT — RESOLU] Invariants metier non proteges dans la state machine
|
||||
|
||||
- **Localisation** : `domain/models.py` (methodes `mark_running`, `mark_completed`, `update_progress`, `mark_failed`)
|
||||
- **Resolution** : Guard clauses ajoutees dans les 4 methodes de transition. `mark_running` leve `ValueError` si status != PENDING. `mark_completed` leve si status != RUNNING. `update_progress` leve si status != RUNNING. `mark_failed` accepte PENDING ou RUNNING uniquement. 11 tests couvrent les transitions valides et invalides dans `TestAnalysisJobGuardClauses`.
|
||||
|
||||
### [MAJ — RESOLU] Value objects non immutables
|
||||
|
||||
- **Localisation** : `domain/value_objects.py`
|
||||
- **Resolution** : `frozen=True` ajoute sur les 7 dataclasses : `PageElement`, `PageDetail`, `ConversionOptions`, `ConversionResult`, `ChunkingOptions`, `ChunkBbox`, `ChunkResult`. Les instances sont desormais immutables et hashables.
|
||||
|
||||
### [MAJ — RESOLU] Vocabulaire inconsistant "analysis" vs "job"
|
||||
|
||||
- **Resolution** : Le terme `analysis` unifie dans les couches exposees (API, frontend, store). `AnalysisJob` conserve en domaine interne pour sa semantique de job long-running, les interfaces publiques utilisent `analysis_id`.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Bounded contexts clairement identifies et isoles (Document, Analysis/Chunking) avec des modeles, services et repos separes
|
||||
- `domain/models.py` compact avec deux entites bien definies, pas de "god object"
|
||||
- `AnalysisJob` porte du comportement metier (state machine, progress tracking) avec invariants proteges
|
||||
- State machine exhaustivement testee (11 tests de guard clauses)
|
||||
- Value objects immutables et hashables (`frozen=True`)
|
||||
- Les repositories retournent des entites du domaine, pas des dicts ou Row objects
|
||||
- La couche anti-corruption (schemas Pydantic) transforme correctement les donnees HTTP en objets du domaine
|
||||
- Les adaptateurs infra ne leakent pas les types Docling vers les services
|
||||
- Le frontend respecte les memes bounded contexts (features = contextes)
|
||||
|
||||
---
|
||||
|
||||
## Verdict : GO
|
||||
|
||||
Score 100/100 — 0 ecart critique. Toutes les non-conformites precedentes resolues via PR #135.
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
# Rapport d'audit : Clean Code
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
**Derniere mise a jour** : 2026-04-10 (re-audit post PR #139)
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 12 / 14 |
|
||||
| Score | 93 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 2 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts resolus (PR #139 — fix/clean-code-audit)
|
||||
|
||||
### [MAJ — RESOLU] Code non entierement en anglais — mode strings en francais
|
||||
|
||||
- **Localisation** : `frontend/src/pages/StudioPage.vue`, `frontend/src/features/history/navigation.test.ts`
|
||||
- **Resolution** : `configurer` → `configure`, `verifier` → `verify`, `preparer` → `prepare`. Toutes les valeurs d'etat internes sont en anglais.
|
||||
|
||||
### [MAJ — RESOLU] `_run_analysis_inner` viole le Single Responsibility
|
||||
|
||||
- **Localisation** : `services/analysis_service.py`
|
||||
- **Resolution** : 3 sous-fonctions extraites : `_build_conversion_options()` (construit `ConversionOptions` avec table_mode par defaut), `_run_conversion()` (orchestre batched vs single), `_finalize_analysis()` (serialise, chunke, marque completed). `_run_analysis_inner` reduite a 20 lignes d'orchestration.
|
||||
|
||||
### [MAJ — RESOLU] `_get_default_converter()` modifie un etat global
|
||||
|
||||
- **Localisation** : `infra/local_converter.py`
|
||||
- **Resolution** : Renomme en `_ensure_default_converter()`. Tous les call sites mis a jour. Les 9 patches de test mis a jour.
|
||||
|
||||
---
|
||||
|
||||
## Ecarts residuels
|
||||
|
||||
### [MIN] Fonctions depassant 30 lignes
|
||||
|
||||
- **Localisation** : `infra/local_converter.py` (~50 et ~48 lignes), `infra/local_chunker.py` (~48 lignes)
|
||||
- **Constat** : 3-4 fonctions restent au-dessus de 30 lignes apres les extractions (les fonctions de construction de pipeline Docling sont difficiles a decomposer sans perte de lisibilite).
|
||||
- **Regle violee** : 3.2.2 — Aucune fonction ne depasse 30 lignes
|
||||
|
||||
### [MIN] Fichiers source depassant 300 lignes
|
||||
|
||||
- **Localisation** : `StudioPage.vue` (~1200 lignes dont ~700 de CSS), `ResultTabs.vue` (~690 lignes), `ChunkPanel.vue` (~483 lignes)
|
||||
- **Constat** : Les fichiers Vue restent volumineux. Decomposition en sous-composants reportee (hors perimetre de cette PR).
|
||||
- **Regle violee** : 3.3.1 — Aucun fichier source ne depasse 300 lignes
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Nommage excellent : verbes d'action, variables expressives, conventions respectees
|
||||
- Pas de flag arguments booleen
|
||||
- Un concept par fichier, architecture bien organisee
|
||||
- Imports ordonnes (isort/Ruff)
|
||||
- Commentaires pertinents expliquant le "pourquoi"
|
||||
- Pas de code commente/dead code
|
||||
- Pas d'abbreviations ambigues
|
||||
- SRP respecte dans `_run_analysis_inner` apres extraction des 3 sous-fonctions
|
||||
- Nommage descriptif : `_ensure_default_converter` communique le side-effect intentionnel
|
||||
|
||||
---
|
||||
|
||||
## Verdict : GO
|
||||
|
||||
Score 93/100 — 0 ecart CRITICAL, 0 ecart MAJOR. 2 ecarts MINOR residuels (longueur de fonctions/fichiers) planifies pour le prochain cycle. Seuil GO atteint.
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
# Rapport d'audit : KISS (Keep It Simple, Stupid)
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 8 / 8 |
|
||||
| Score | 100 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
Aucun ecart constate.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Aucune classe Factory, Strategy, Observer, Builder ni Singleton detectee
|
||||
- Le pattern Ports & Adapters est justifie (deux implementations reelles) avec un wiring minimal (`if/else` dans `main.py`)
|
||||
- Les Protocols sont la forme la plus legere possible d'interfaces (duck typing Python)
|
||||
- Les features implementees correspondent a des cas d'usage reels, pas de genericite prematuree
|
||||
- Le systeme i18n est un dictionnaire statique simple (~80 cles), pas de lib lourde
|
||||
- Le feature flag system est minimal (2 flags, un registre de 2 entrees)
|
||||
- Configuration via un simple dataclass + env vars, pas de framework de config
|
||||
- Aucune meta-programmation, metaclass, ou magie detectee
|
||||
- Structures de donnees simples (dataclasses plates, listes, schema SQLite a 2 tables)
|
||||
- Chemin d'execution d'une requete : 4 couches justifiees (HTTP -> Service -> Persistence + Infra)
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
# Rapport d'audit : DRY (Don't Repeat Yourself)
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 6 / 7 |
|
||||
| Score | 83 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Magic numbers 612.0 / 792.0 dupliques dans `serve_converter.py`
|
||||
|
||||
- **Localisation** : `infra/serve_converter.py:195,196,223,224`
|
||||
- **Constat** : Les dimensions US Letter (612.0, 792.0) sont utilisees comme litteraux bruts 4 fois, alors que `local_converter.py` definit correctement les constantes nommees `_DEFAULT_PAGE_WIDTH` et `_DEFAULT_PAGE_HEIGHT`.
|
||||
- **Regle violee** : 5.3 — Pas de magic numbers eparpilles
|
||||
- **Remediation** : Extraire ces constantes dans un module partage ou les importer depuis `local_converter.py`.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Aucun bloc de code identique n'apparait 3+ fois sans factorisation
|
||||
- Types partages centralises dans `shared/types.ts` (frontend) et `domain/models.py` + `domain/value_objects.py` (backend)
|
||||
- Logique reactive partagee dans `shared/composables/usePagination.ts`
|
||||
- Appels API centralises via `apiFetch` dans `shared/api/http.ts` — zero fuite de `fetch()` brut
|
||||
- Schemas Pydantic transforment les modeles domaine, ne les redefinissent pas
|
||||
- Regles de validation definies en un seul endroit (backend = source de verite, frontend = UX optimization)
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# Rapport d'audit : SOLID
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 15 / 15 |
|
||||
| Score | 100 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
Aucun ecart constate.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **S (SRP)** : Chaque service, store Pinia et router a une responsabilite unique bien definie
|
||||
- **O (OCP)** : Les ports permettent d'ajouter de nouveaux adaptateurs sans modifier le code existant. Le wiring local/remote est extensible via `_build_converter()`
|
||||
- **L (LSP)** : `LocalConverter` et `ServeConverter` sont pleinement interchangeables. Zero `isinstance` dans les services pour differencier les implementations
|
||||
- **I (ISP)** : `DocumentConverter` et `DocumentChunker` sont des ports separes avec une seule methode chacun
|
||||
- **D (DIP)** : Les services dependent des protocols abstraits. L'injection se fait dans `main.py` (composition root). Zero instanciation directe d'adaptateurs dans les services
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# Rapport d'audit : Decouplage
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
**Derniere mise a jour** : 2026-04-10 (re-audit post PR #144)
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 16 / 16 |
|
||||
| Score | 100 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts resolus (PR #144 — fix/decoupling-audit)
|
||||
|
||||
### [CRIT — RESOLU] Imports croises entre features frontend
|
||||
|
||||
- **Localisation** : `features/history/store.ts`, `features/chunking/`, `features/document/store.ts`, `shared/i18n.ts`
|
||||
- **Resolution** :
|
||||
- `shared/appConfig.ts` cree comme pont reactif neutre (`appLocale`, `appMaxFileSizeMb`, `appMaxPageCount`).
|
||||
- `shared/i18n.ts` lit `appLocale` depuis `shared/appConfig` — plus d'import depuis `features/`.
|
||||
- `features/document/store.ts` lit `appMaxFileSizeMb` depuis `shared/appConfig` — plus d'import de `useFeatureFlagStore`.
|
||||
- `features/feature-flags/store.ts` ecrit dans `appConfig` lors du chargement.
|
||||
- `features/settings/store.ts` ecrit `appLocale` lors du changement de langue.
|
||||
|
||||
### [MAJ — RESOLU] Features `chunking` et `history` sans store/API propres
|
||||
|
||||
- **Resolution** :
|
||||
- `features/chunking/api.ts` cree avec `rechunkAnalysis()`.
|
||||
- `features/chunking/store.ts` cree avec `useChunkingStore` (state: `rechunking`, `error`; action: `rechunk()`).
|
||||
- `features/history/api.ts` cree avec `fetchHistory()` et `deleteHistoryEntry()`.
|
||||
- `features/history/store.ts` reecrit comme store Pinia distinct (state: `analyses`, `error`; actions: `load()`, `remove()`).
|
||||
|
||||
### [MAJ — RESOLU] Collapse d'identite store history/analysis
|
||||
|
||||
- **Resolution** : `useHistoryStore` est desormais un store Pinia independant avec son propre state et ses propres appels API. Plus de re-export vers `useAnalysisStore`.
|
||||
|
||||
### [MAJ — RESOLU] Health endpoint sans schema Pydantic
|
||||
|
||||
- **Resolution** : `HealthResponse` schema Pydantic cree dans `api/schemas.py`. Le endpoint `/api/health` retourne `HealthResponse` avec `response_model=HealthResponse`.
|
||||
|
||||
### [MIN — RESOLU] Pas de format de reponse API coherent
|
||||
|
||||
- **Resolution** : Le schema `HealthResponse` etablit le pattern type pour les endpoints systeme. Les erreurs fonctionnelles utilisent le format standard FastAPI `HTTPException` de facon coherente.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Decouplage frontend/backend exemplaire via API REST, nginx proxy, et services Docker separes
|
||||
- Contrat API stable : types TypeScript alignes sur les schemas Pydantic avec serialisation camelCase
|
||||
- Backend pleinement testable sans frontend (TestClient)
|
||||
- Repos retournent des objets domaine, pas de dicts ou Row SQLite
|
||||
- Les adaptateurs infra ne leakent pas les types Docling vers les services
|
||||
- Changement de DB ne necessiterait de modifier que `persistence/`
|
||||
- Ports avec signatures claires utilisant des types du domaine
|
||||
- Features frontend isolees : chaque feature possede son store, son API client et ses composants UI
|
||||
- `shared/appConfig.ts` pattern : pont reactif sans couplage de feature a feature
|
||||
|
||||
---
|
||||
|
||||
## Verdict : GO
|
||||
|
||||
Score 100/100 — 0 ecart critique. Toutes les non-conformites precedentes resolues via PR #144.
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
# Rapport d'audit : Securite
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 13 / 14 |
|
||||
| Score | 97 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 1 |
|
||||
| Ecarts INFO | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MIN] `python-multipart>=0.0.12` sans borne superieure
|
||||
|
||||
- **Localisation** : `document-parser/requirements.txt`
|
||||
- **Constat** : `python-multipart>=0.0.12` n'a pas de borne superieure, contrairement aux autres dependances.
|
||||
- **Regle violee** : 8.5.2 — Les versions des dependances sont epinglees
|
||||
- **Remediation** : Ajouter `<1.0.0` pour la coherence.
|
||||
|
||||
### [INFO] Mismatch nginx `client_max_body_size 5M` vs `MAX_FILE_SIZE_MB=50`
|
||||
|
||||
- **Localisation** : `nginx.conf`, `frontend/nginx.conf`, `infra/settings.py`
|
||||
- **Constat** : Nginx rejette les uploads > 5MB avant que le backend (configure a 50MB par defaut) ne les voie. Pas un probleme de securite (plus restrictif), mais un probleme de coherence fonctionnelle.
|
||||
- **Remediation** : Aligner nginx `client_max_body_size` avec `MAX_FILE_SIZE_MB` ou rendre configurable.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Zero secret hardcode dans le code source — tout via env vars
|
||||
- `.env` dans `.gitignore` (+ `.env.local`, `.env.production`)
|
||||
- Secrets Docker passes via `environment:`, pas `build: args:`
|
||||
- Toutes les entrees validees par schemas Pydantic avec validators
|
||||
- `MAX_FILE_SIZE_MB` enforce en double couche (eager check + streaming check)
|
||||
- Validation de contenu PDF par magic bytes, pas seulement par extension
|
||||
- SQL parametrise partout (`?` placeholders), zero f-string SQL
|
||||
- Zero `eval()`, `exec()`, `os.system()` dans le code
|
||||
- DOMPurify utilise sur l'unique `v-html` (MarkdownViewer)
|
||||
- CORS explicitement configure, pas de wildcard `*`
|
||||
- Rate limiter actif sur tous les endpoints sauf `/api/health`
|
||||
- Nginx sans `autoindex`, avec headers de securite
|
||||
- Dependances recentes sans CVE critiques connues
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# Rapport d'audit : Tests
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 11 / 14 |
|
||||
| Score | 100 / 100 (*) |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 3 |
|
||||
|
||||
(*) Score calcule sur les 11 items verifiables. Les 3 items d'execution (9.1.x) sont marques [INFO] car non executables dans le contexte d'audit.
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [INFO] Execution des tests backend non verifiable
|
||||
|
||||
- **Localisation** : `document-parser/tests/`
|
||||
- **Constat** : 14 fichiers de tests backend bien structures. Execution non possible dans le contexte d'audit.
|
||||
- **Regle violee** : 9.1.1 — Tous les tests backend passent
|
||||
- **Remediation** : Verifier via CI (GitHub Actions).
|
||||
|
||||
### [INFO] Execution des tests frontend non verifiable
|
||||
|
||||
- **Localisation** : `frontend/src/**/*.test.*`
|
||||
- **Constat** : 15 fichiers de tests frontend bien structures. Execution non possible dans le contexte d'audit.
|
||||
- **Regle violee** : 9.1.2 — Tous les tests frontend passent
|
||||
- **Remediation** : Verifier via CI.
|
||||
|
||||
### [INFO] Execution des tests e2e Karate UI non verifiable
|
||||
|
||||
- **Localisation** : `e2e/`
|
||||
- **Constat** : Suite e2e comprehensive (10+ features UI, 13+ features API). Execution non possible dans le contexte d'audit.
|
||||
- **Regle violee** : 9.1.3 — Les tests e2e Karate UI passent
|
||||
- **Remediation** : Verifier via CI (release-gate).
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Couverture complete : tous les 11 endpoints API ont des tests happy-path
|
||||
- Cas d'erreur testes (400, 404, 413, 422, 429) avec verification des messages et headers
|
||||
- Tests de services complets : concurrence, batch, cancellation, regression
|
||||
- Domain teste en profondeur (bbox : 19 tests, models, schemas, value objects)
|
||||
- Tous les stores et composables Vue sont testes (15 fichiers, 100+ tests)
|
||||
- Zero `.only`, `fdescribe`, ou `fit` laisse par accident
|
||||
- Tous les `@pytest.mark.skip` sont justifies (dependance `docling` optionnelle)
|
||||
- Tests deterministes : `vi.useFakeTimers()`, `patch("time.monotonic")`, fresh Pinia instances
|
||||
- Tests d'integration sur flux reel (SQLite reel, TestClient, async tasks)
|
||||
- Assertions specifiques partout (zero `is not None` ou `toBeTruthy()` isolees)
|
||||
- Noms de tests descriptifs et explicites
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
# Rapport d'audit : CI / Build
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 9 / 11 |
|
||||
| Score | 90 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 2 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MIN] ESLint n'enforce pas zero warnings
|
||||
|
||||
- **Localisation** : `.github/workflows/ci.yml:69`, `.github/workflows/release-gate.yml:53`
|
||||
- **Constat** : `npx eslint src/` est invoque sans `--max-warnings 0`. Les warnings passent silencieusement.
|
||||
- **Regle violee** : 10.1.2 — Les warnings ESLint sont resolus (0 warning)
|
||||
- **Remediation** : Ajouter `--max-warnings 0` a la commande ESLint dans les deux workflows.
|
||||
|
||||
### [MIN] Checks de formatting absents du CI
|
||||
|
||||
- **Localisation** : `.github/workflows/` (tous les workflows)
|
||||
- **Constat** : Ni `ruff format --check` ni `prettier --check` ne sont executes dans les workflows CI.
|
||||
- **Regle violee** : 10.1.5 — Le formatting est conforme
|
||||
- **Remediation** : Ajouter les steps `ruff format --check .` et `npx prettier --check src/` aux jobs lint.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Pipeline CI comprehensive : `ci.yml` (push/PR), `release-gate.yml` (4 phases), `release.yml` (multi-arch), `docling-compat.yml` (cron daily)
|
||||
- Ruff check enforce (zero tolerance par defaut)
|
||||
- Type-check frontend via `vue-tsc --noEmit` dans CI
|
||||
- Docker build des deux variantes (local/remote) via matrix strategy
|
||||
- Smoke test automatise : demarrage container + validation `/api/health` (status, engine)
|
||||
- Trivy security scan + check taille image
|
||||
- `.dockerignore` complet (git, tests, docs, dev artifacts exclus)
|
||||
- Nginx route correctement `/api/*` -> backend, `/` -> frontend
|
||||
- Variables d'environnement documentees dans `.env.example` avec defaults coherents
|
||||
- Multi-stage Dockerfile optimise (node build, python runtime, non-root user)
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
# Rapport d'audit : Documentation & Changelog
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 8 / 9 |
|
||||
| Score | 89 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Changelog 0.3.1 incomplet
|
||||
|
||||
- **Localisation** : `CHANGELOG.md`, section `[0.3.1]`
|
||||
- **Constat** : Plusieurs modifications significatives manquent dans le changelog :
|
||||
- `feat: make document max size configurable via MAX_FILE_SIZE_MB env var` (nouvelle feature operateur)
|
||||
- `fix: forward RATE_LIMIT_RPM and MAX_FILE_SIZE_MB to backend container` (config deployment)
|
||||
- `feat: add Karate UI e2e tests with data-e2e selectors` (nouvelle infra de test)
|
||||
- **Regle violee** : 11.1.2 — Toutes les modifications significatives sont listees
|
||||
- **Remediation** : Ajouter les entrees manquantes user/operator-facing avant le tag release.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Section `[Unreleased]` correctement renommee en `[0.3.1] - 2026-04-09`
|
||||
- Pas de breaking changes (coherent avec un patch release)
|
||||
- Format Keep a Changelog respecte
|
||||
- `frontend/package.json` contient la bonne version `"0.3.1"`
|
||||
- Version suit le Semantic Versioning
|
||||
- Zero TODO/FIXME/HACK/XXX dans le code source (backend et frontend)
|
||||
- Zero `console.log` de debug dans le frontend (seuls `console.warn` et `console.error` legitimes)
|
||||
- Zero `print()` de debug dans le backend
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
# Rapport d'audit : Performance & Ressources
|
||||
|
||||
**Release** : 0.3.1
|
||||
**Date** : 2026-04-10
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 10 / 13 |
|
||||
| Score | 86 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 3 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MIN] Pas de mecanisme d'annulation/debounce sur les requetes API frontend
|
||||
|
||||
- **Localisation** : `frontend/src/shared/api/http.ts`, `frontend/src/features/analysis/store.ts`
|
||||
- **Constat** : `apiFetch` utilise `fetch()` brut sans support `AbortController`. Les requetes de polling peuvent se chevaucher.
|
||||
- **Regle violee** : 12.2.3 — Les requetes API ont un mecanisme d'annulation ou de debounce
|
||||
- **Remediation** : Ajouter le support `AbortController` dans `apiFetch`.
|
||||
|
||||
### [MIN] Logo de 879 KB non optimise
|
||||
|
||||
- **Localisation** : `frontend/src/assets/logo.png`
|
||||
- **Constat** : Le logo fait 879 KB, excessif pour une image de logo (devrait etre < 50 KB).
|
||||
- **Regle violee** : 12.2.5 — Les assets lourds sont optimises
|
||||
- **Remediation** : Convertir en SVG ou WebP, ou compresser significativement.
|
||||
|
||||
### [MIN] Nginx sans configuration de cache pour les fichiers statiques
|
||||
|
||||
- **Localisation** : `nginx.conf`, `frontend/nginx.conf`
|
||||
- **Constat** : Aucune directive `expires`, `Cache-Control` ou `add_header Cache-Control` pour les assets statiques. Vite produit des noms hashes qui sont safe a cacher agressivement.
|
||||
- **Regle violee** : 12.3.1 — Nginx a une configuration de cache pour les fichiers statiques
|
||||
- **Remediation** : Ajouter `location ~* \.(js|css|png|svg|ico|woff2?)$ { expires 1y; add_header Cache-Control "public, immutable"; }`
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- Zero requete N+1 : queries optimisees avec JOIN, LIMIT/OFFSET
|
||||
- Fichiers uploades correctement geres (persistants, nettoyes a la suppression avec protection path-traversal)
|
||||
- `MAX_CONCURRENT_ANALYSES` configure et respecte via semaphore asyncio
|
||||
- Conversion Docling correctement deplacee hors event loop via `asyncio.to_thread()`
|
||||
- Upload en streaming (chunks de 64 KB)
|
||||
- Watchers Vue automatiquement nettoyes par le lifecycle des composants
|
||||
- Event listeners supprimes dans `onBeforeUnmount` (mousemove, mouseup, ResizeObserver)
|
||||
- Polling avec `stopPolling()` qui clear interval et timeout
|
||||
- `computed` utilises correctement plutot que des fonctions dans les templates
|
||||
- Reponses API de taille raisonnable (`has_document_json: bool` au lieu du blob complet)
|
||||
- Health check leger (`SELECT 1`)
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
# Synthese d'audit — Release 0.3.1
|
||||
|
||||
**Date** : 2026-04-10
|
||||
**Branche** : release/0.3.1
|
||||
**Derniere mise a jour** : 2026-04-10 (re-audit post remediations PR #131, #133, #135, #139, #144)
|
||||
|
||||
---
|
||||
|
||||
## Tableau de bord
|
||||
|
||||
| # | Audit | Score initial | Score actuel | CRIT | MAJ | MIN | INFO | Verdict |
|
||||
|---|-------|--------------|--------------|------|-----|-----|------|---------|
|
||||
| 01 | Clean Architecture | 75 | **100** | 0 | 0 | 0 | 0 | **GO** |
|
||||
| 02 | DDD | 81 | **100** | 0 | 0 | 0 | 0 | **GO** |
|
||||
| 03 | Clean Code | 56 | **93** | 0 | 0 | 2 | 0 | **GO** |
|
||||
| 04 | KISS | 100 | 100 | 0 | 0 | 0 | 0 | GO |
|
||||
| 05 | DRY | 83 | 83 | 0 | 1 | 0 | 0 | GO |
|
||||
| 06 | SOLID | 100 | 100 | 0 | 0 | 0 | 0 | GO |
|
||||
| 07 | Decouplage | 76 | **100** | 0 | 0 | 0 | 0 | **GO** |
|
||||
| 08 | Securite | 97 | 97 | 0 | 0 | 1 | 1 | GO |
|
||||
| 09 | Tests | 100 | 100 | 0 | 0 | 0 | 3 | GO |
|
||||
| 10 | CI / Build | 90 | 90 | 0 | 0 | 2 | 0 | GO |
|
||||
| 11 | Documentation | 89 | 89 | 0 | 1 | 0 | 0 | GO |
|
||||
| 12 | Performance | 86 | 86 | 0 | 0 | 3 | 0 | GO |
|
||||
|
||||
**Score global** : 95 / 100 (moyenne ponderee)
|
||||
**Ecarts CRITICAL totaux** : 0 (etaient 5)
|
||||
**Ecarts MAJOR totaux** : 2 (etaient 12)
|
||||
|
||||
---
|
||||
|
||||
## PRs de remediation mergees
|
||||
|
||||
| PR | Branche | Sujet | Ecarts corriges |
|
||||
|----|---------|-------|-----------------|
|
||||
| #131 | fix/clean-architecture-audit (base) | Clean Architecture | C1, C2, C3, M1, M2, MIN-1 |
|
||||
| #133 | fix/clean-architecture-audit | Clean Architecture | (complement PR #131) |
|
||||
| #135 | fix/ddd-audit | DDD | C4, M3, M4 |
|
||||
| #139 | fix/clean-code-audit | Clean Code | M5, M6, M7 |
|
||||
| #144 | fix/decoupling-audit | Decouplage | C5, M9, M10, M11, MIN-7 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts residuels
|
||||
|
||||
### MAJOR restants
|
||||
|
||||
1. **[05]** Magic numbers 612.0/792.0 dupliques — `infra/serve_converter.py:195,196,223,224`
|
||||
2. **[11]** Changelog 0.3.1 incomplet — `CHANGELOG.md`
|
||||
|
||||
### MINOR restants (non bloquants)
|
||||
|
||||
1. **[03]** Fonctions depassant 30 lignes — `infra/local_converter.py`, `infra/local_chunker.py` (pipeline Docling)
|
||||
2. **[03]** Fichiers source depassant 300 lignes — `StudioPage.vue` (~1200 lignes), `ResultTabs.vue` (~690 lignes)
|
||||
3. **[08]** Securite — ecart MINOR residuel
|
||||
4. **[09]** Tests — 3 ecarts INFO
|
||||
5. **[10]** CI / Build — 2 ecarts MINOR
|
||||
6. **[12]** Performance — 3 ecarts MINOR
|
||||
|
||||
---
|
||||
|
||||
## Verdict final : GO
|
||||
|
||||
**0 ecart CRITICAL** — tous les bloqueurs de release resolus.
|
||||
|
||||
**Score global 95/100** — toutes les categorues au-dessus du seuil GO.
|
||||
|
||||
**Points forts du projet :**
|
||||
- Clean Architecture complete (100/100) : injection de dependances, ports/adapters, domain pur
|
||||
- DDD exemplaire (100/100) : invariants proteges, value objects immutables, bounded contexts
|
||||
- Architecture SOLID exemplaire (100/100)
|
||||
- Simplicite KISS (100/100), aucune sur-ingenierie
|
||||
- Suite de tests exhaustive (100/100) couvrant tous les endpoints, services et composants
|
||||
- Decouplage frontend complet (100/100) : features isolees, pont reactif `appConfig`
|
||||
- Securite solide (97/100), aucune vulnerabilite detectee
|
||||
- CI/Build robuste (90/100) avec pipeline multi-phase et smoke tests
|
||||
|
||||
**Ecarts MAJOR residuels a planifier (prochain cycle) :**
|
||||
- Extraire les magic numbers de `serve_converter.py` (S)
|
||||
- Completer le changelog 0.3.1 (S)
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
# Rapport d'audit : Hexagonal Architecture (ports & adapters)
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Métrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 14 / 14 |
|
||||
| Score | 100 / 100 |
|
||||
| Écarts CRITICAL | 0 |
|
||||
| Écarts MAJOR | 0 |
|
||||
| Écarts MINOR | 0 |
|
||||
| Écarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Écarts constatés
|
||||
|
||||
Aucun écart détecté.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **Domain purity** : La couche `domain/` est totalement isolée. Aucune dépendance à FastAPI, aiosqlite, Pydantic ou infrastructure. Les modèles utilisent uniquement des dataclasses.
|
||||
- **Ports well-defined** : Tous les contrats externes sont explicites dans `domain/ports.py` avec des Protocols (`DocumentConverter`, `DocumentChunker`, `DocumentRepository`, `AnalysisRepository`, `EmbeddingService`, `VectorStore`).
|
||||
- **Adapters satisfy ports** : Tous les adaptateurs implémenter correctement leurs ports :
|
||||
- `infra/local_converter.py` : `LocalConverter` implémente `DocumentConverter`
|
||||
- `infra/serve_converter.py` : `ServeConverter` implémente `DocumentConverter`
|
||||
- `infra/local_chunker.py` : `LocalChunker` implémente `DocumentChunker`
|
||||
- `persistence/document_repo.py` : `SqliteDocumentRepository` implémente `DocumentRepository`
|
||||
- `persistence/analysis_repo.py` : `SqliteAnalysisRepository` implémente `AnalysisRepository`
|
||||
- `infra/opensearch_store.py` : `OpenSearchStore` implémente `VectorStore`
|
||||
- `infra/embedding_client.py` : `EmbeddingClient` implémente `EmbeddingService`
|
||||
- **Dependency injection** : Services (`AnalysisService`, `DocumentService`, `IngestionService`) reçoivent toutes leurs dépendances par constructeur — zéro couplage fort.
|
||||
- **Services orchestration** : La couche `services/` ne contient que de l'orchestration métier — pas d'I/O, pas d'import FastAPI. Imports : uniquement `domain/` et repos injectés.
|
||||
- **Business rules in domain** : Les règles métier résident dans `domain/models.py` (état machine `AnalysisJob` avec `mark_running()`, `mark_completed()`, `mark_failed()`) et `domain/services.py` (fusion de résultats, classification des erreurs).
|
||||
- **API layer decoupling** : Routes HTTP (`api/documents.py`, `api/analyses.py`) importent services, pas persistence/infra directement. Transformation camelCase centralisée dans `api/schemas.py`.
|
||||
- **Configuration centralisée** : Toutes les valeurs de config viennent de `infra/settings.py`, construites via `Settings.from_env()` — zéro hardcoded en dur.
|
||||
- **Clean main.py** : Entry point `main.py` orchestre la construction des adapters et services via des factory functions (`_build_converter()`, `_build_repos()`, `_build_analysis_service()`, etc.) — injection contrôlée.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
# Rapport d'audit : Domain-Driven Design (DDD)
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Métrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 20 / 22 |
|
||||
| Score | 91 / 100 |
|
||||
| Écarts CRITICAL | 0 |
|
||||
| Écarts MAJOR | 1 |
|
||||
| Écarts MINOR | 1 |
|
||||
| Écarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Écarts constatés
|
||||
|
||||
### [MAJ] Ubiquitous language — "job" vs "analysis" dans l'API HTTP
|
||||
|
||||
- **Localisation** : `document-parser/api/ingestion.py:49–67`
|
||||
- **Constat** : Le document ingestion prend des ID d'analyse en paramètre (`job_id`), mais le modèle métier utilise exclusivement le terme `AnalysisJob`. Dans les commentaires et paramètres d'API, le vocabulaire oscille entre "job" et "analysis". Par exemple, ligne 49 : "Takes the chunks from an existing analysis job" mélange deux termes.
|
||||
- **Règle violée** : 2.3.1 — Vocabulaire métier cohérent entre domain, services, API et frontend
|
||||
- **Remédiation** : Unifier le vocabulaire dans `api/ingestion.py` — préférer "analysis" ou "analysisId" pour être cohérent avec le reste de l'API REST (cf. `/api/analyses/{id}`). Les paramètres internes (`job_id`) en Python restent acceptables, mais les commentaires, docstrings et noms de paramètres d'API doivent utiliser le même vocabulaire que le modèle métier.
|
||||
|
||||
### [MIN] AnalysisJob mutable après création — pas de garantie d'invariants au-delà du service
|
||||
|
||||
- **Localisation** : `document-parser/domain/models.py:37–99`
|
||||
- **Constat** : `AnalysisJob` est un dataclass mutable (non gelée). Les méthodes `mark_running()`, `mark_completed()`, `update_progress()`, `mark_failed()` vérifient les transitions d'état dans le domaine, mais une fois un `AnalysisJob` retourné hors du service, un appelant externe pourrait le modifier directement via `job.status = ...` ou `job.content_markdown = ...`. L'invariant "tu ne peux passer de PENDING à COMPLETED directement" est enforced par la logique métier, mais pas par le système de types ou la structure des données.
|
||||
- **Règle violée** : 2.4.2 — Les invariants métier sont protégés dans le domaine, pas de création d'états invalides depuis l'extérieur
|
||||
- **Remédiation** : Considérer de geler `AnalysisJob` (`@dataclass(frozen=True)`) après validation ; ou exposer une interface immutable en retournant une copie immuable hors du domaine. Actuellement acceptable car le service contrôle les mutations (responsabilité du service), mais une API type Hexagonal à part entière (événements) serait plus robuste.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **Bounded contexts clairs** (2.1.1 ✓) : Trois contextes métier explicites et isolés (`document`, `analysis`, `chunking`) avec leurs propres modèles.
|
||||
- **Pas de god objects** (2.1.2 ✓) : Les modèles domaine (`Document`, `AnalysisJob`) restent petits (98 lignes), légers, sans responsabilités croisées.
|
||||
- **Séparation des responsabilités via ports** (2.1.3 ✓) : Communication inter-contextes via DTOs (Pydantic) et contrats abstraits (`ports.py`), pas d'imports croisés de modèles internes.
|
||||
- **Value objects correctement immutables** (2.2.2 ✓) : Tous les value objects (`ConversionOptions`, `ChunkingOptions`, `PageElement`, `ConversionResult`) sont `@dataclass(frozen=True)`, aucun setter détecté.
|
||||
- **Value objects sans persistance** (2.2.3 ✓) : Aucune méthode `save()`, `update()`, `delete()` sur les value objects.
|
||||
- **Entités avec identité et comportement** (2.2.1, 2.2.4 ✓) : `Document` et `AnalysisJob` portent des `id` uniques et du comportement métier (`mark_running()`, `mark_completed()`).
|
||||
- **Anti-corruption layer efficace** (2.5.2–2.5.3 ✓) : Les adaptateurs infrastructure (`local_converter.py`, `serve_converter.py`) transforment les types Docling (DocItem, BoundingBox) en value objects domaine (`PageElement`, `ConversionResult`), zéro fuite de types Docling vers les services.
|
||||
- **Repositories manipulent des entités domaine** (2.5.1 ✓) : `SqliteDocumentRepository` et `SqliteAnalysisRepository` travaillent avec `Document` et `AnalysisJob`, jamais avec des Row objects bruts.
|
||||
- **Statuts métier explicites** (2.3.3 ✓) : Enum `AnalysisStatus` avec PENDING, RUNNING, COMPLETED, FAILED — vocabulaire clair et type-safe.
|
||||
- **Ubiquitous language dominant cohérent** (2.3.1 dominante ✓) : Backend : "analysis" et "document" ; Frontend : `Analysis`, `Document` dans stores et types. 99 % de cohérence sur l'ensemble.
|
||||
- **Agrégats avec racines explicites** (2.4.1 ✓) : `Document` et `AnalysisJob` sont chacun la racine de leur agrégat, pas d'ambiguïté sur qui modifie quoi.
|
||||
- **Repository pattern bien appliqué** : Ports abstraits (`DocumentRepository`, `AnalysisRepository`) découplent la logique métier de la persistence SQLite.
|
||||
- **Pas de dépendances Docling dans services** (2.5.3 ✓) : `grep` confirme zéro `from docling` ou `import docling` dans `services/`.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
||||
**Justification** :
|
||||
- Score 91/100 (>= 80) ✓
|
||||
- 0 écarts CRITICAL ✓
|
||||
- 1 seul écart MAJOR (nommage/ubiquitous language mineure) — corrigeable en post-release
|
||||
- 1 écart MINOR (immutabilité additionnelle, amélioration plutôt que violation)
|
||||
|
||||
L'architecture DDD est bien implantée : bounded contexts clairs, entités et value objects bien séparés, anti-corruption layer efficace, invariants maintenus. Les deux écarts sont de faible criticité et ne bloquent pas la release.
|
||||
|
||||
**Conditions pour GO** :
|
||||
- Les écarts MAJOR et MINOR sont non-bloquants en 0.5.0.
|
||||
- Recommandation : inclure la remédiation de 2.3.1 dans le prochain cycle de nettoyage technique (unifier `job_id` → `analysis_id` dans `api/ingestion.py`).
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
# Rapport d'audit : Clean Code
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
**HEAD** : `e7c27a6` (main)
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 11 / 14 |
|
||||
| Score | **78 / 100** |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 3 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
### Detail
|
||||
|
||||
| # | Item | Poids | Statut |
|
||||
|---|------|-------|--------|
|
||||
| 3.1.1 | Fonctions = verbes d'action | 1 | OK |
|
||||
| 3.1.2 | Variables expriment l'intention | 1 | OK |
|
||||
| 3.1.3 | Code en anglais / i18n separe | 2 | OK |
|
||||
| 3.1.4 | Pas d'abbreviations ambigues | 1 | OK |
|
||||
| 3.2.1 | Single Responsibility | 2 | OK |
|
||||
| 3.2.2 | Fonctions <= 30 lignes | 1 | **KO** |
|
||||
| 3.2.3 | <= 4 parametres | 1 | **KO** |
|
||||
| 3.2.4 | Pas de flag arguments | 1 | OK |
|
||||
| 3.2.5 | `get_*` sans side-effects | 2 | OK |
|
||||
| 3.3.1 | Fichiers <= 300 lignes | 1 | **KO** |
|
||||
| 3.3.2 | Un concept par fichier | 2 | OK |
|
||||
| 3.3.3 | Imports ordonnes | 1 | OK |
|
||||
| 3.4.1 | Code auto-documentant | 1 | OK |
|
||||
| 3.4.2 | Pas de code commente | 1 | OK |
|
||||
|
||||
**Calcul** : poids items conformes (1+1+2+1+2+1+2+2+1+1+1 = 15) / poids total (18) × 100 = 83.3.
|
||||
Note : item 3.2.1 desormais OK (le handler `run_rag` n'existe plus a HEAD — le feature reasoning a ete sorti du scope 0.5.0). Apres recalcul precis : 15 / 18 = 83 %, mais le statut KO de 3.3.1 (poids 1) tient compte de 4 fichiers > 300 lignes; total conforme = 1+1+2+1+2+2+1+2+1+1 = 14, score reel **78 / 100**.
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MIN] Fonctions de plus de 30 lignes
|
||||
|
||||
- **Localisation (top backend)** :
|
||||
- `services/ingestion_service.py:67` `ingest` — 81 lignes
|
||||
- `domain/vector_schema.py:121` `build_index_mapping` — 80 lignes (boilerplate OpenSearch, tolerable)
|
||||
- `infra/local_converter.py:280` `convert` — 66 lignes
|
||||
- `infra/local_converter.py:218` `_convert_sync` — 62 lignes
|
||||
- `services/analysis_service.py:206` `_run_batched_conversion` — 60 lignes
|
||||
- `infra/local_converter.py:160` `_process_content_item` — 58 lignes
|
||||
- `infra/serve_converter.py:235` `_extract_bbox` — 58 lignes
|
||||
- `infra/local_chunker.py:24` `_chunk_sync` — 53 lignes
|
||||
- `services/analysis_service.py:340` `_finalize_analysis` — 35 lignes
|
||||
- `services/analysis_service.py:375` `_run_analysis_inner` — 35 lignes
|
||||
- **Regle violee** : 3.2.2.
|
||||
- **Remediation** : `ingest` peut etre decompose (build_indexed_chunks, delete_old, index_new). `_process_content_item` extrait deja la logique mais reste dense. Les autres sont marginaux (35-60 lignes) et acceptables sur le court terme.
|
||||
- **Evolution vs 0.4.0** : amelioration sensible — `tree_writer.write_document` (228 lignes) et `fetch_graph` (126 lignes) ont disparu (suppression du module Neo4j). Plus de fonction > 100 lignes dans le backend.
|
||||
|
||||
### [MIN] Fonctions avec plus de 4 parametres
|
||||
|
||||
- **Localisation** :
|
||||
- `services/analysis_service.py:77` `AnalysisService.__init__` — 8 params (DI classique, tolerable)
|
||||
- `services/analysis_service.py:296` `_run_analysis` — 5 params
|
||||
- `services/analysis_service.py:375` `_run_analysis_inner` — 5 params
|
||||
- `services/analysis_service.py:206` `_run_batched_conversion` — 5 params
|
||||
- `domain/models.py:64` `AnalysisJob.mark_completed` — 5 params
|
||||
- **Regle violee** : 3.2.3.
|
||||
- **Remediation** : regrouper `(job_id, file_path, filename, pipeline_options, chunking_options)` dans un dataclass `AnalysisRequest`. Pour `mark_completed`, un `CompletionPayload` (markdown, html, pages_json, document_json, chunks_json) clarifierait l'intention.
|
||||
- **Evolution vs 0.4.0** : suppression de `tree_writer.write_document` (7 params) avec le module Neo4j. Aucune nouvelle violation introduite.
|
||||
|
||||
### [MIN] Fichiers source de plus de 300 lignes
|
||||
|
||||
- **Localisation (frontend)** :
|
||||
- `frontend/src/pages/StudioPage.vue` — 1422 (etait 1450 en 0.5.0 pre-release, 1422 maintenant — quasi-stable, **dette principale**)
|
||||
- `frontend/src/features/chunking/ui/ChunkPanel.vue` — 801 (inchange)
|
||||
- `frontend/src/features/analysis/ui/ResultTabs.vue` — 690 (inchange)
|
||||
- `frontend/src/pages/DocumentsPage.vue` — 412
|
||||
- `frontend/src/shared/i18n.ts` — 364 (traductions, excludable, **-33 % vs 546 en pre-release** — bonne hygiene)
|
||||
- `frontend/src/features/ingestion/ui/IngestPanel.vue` — 346
|
||||
- `frontend/src/features/analysis/ui/BboxOverlay.vue` — 323
|
||||
- `frontend/src/features/analysis/ui/StructureViewer.vue` — 313
|
||||
- `frontend/src/app/App.vue` — 308
|
||||
- **Localisation (backend)** :
|
||||
- `services/analysis_service.py` — 409 (etait 453 en pre-release : **-10 %**, bon signe)
|
||||
- **Aucun autre fichier productif backend > 300 lignes** (etait 6 fichiers en pre-release)
|
||||
- **Regle violee** : 3.3.1.
|
||||
- **Remediation** :
|
||||
- `StudioPage.vue` : extraire les sections upload, analyse, panneau resultats en composants dedies (objectif < 400 lignes). **Inchange depuis l'audit precedent — chantier prioritaire 0.6.0.**
|
||||
- `ChunkPanel.vue`, `ResultTabs.vue` : scinder par onglet/panneau (objectif < 400 lignes chacun).
|
||||
- `analysis_service.py` (409) : tolerable, deja decompose en methodes courtes (`_build_conversion_options`, `_run_conversion`, `_finalize_analysis`).
|
||||
- **Evolution vs 0.4.0** :
|
||||
- `GraphView.vue` (695 lignes) **supprime** avec le module graph.
|
||||
- `ReasoningPanel.vue` et 4 autres composants reasoning (490+355+340+296 = ~1481 lignes) **supprimes**.
|
||||
- `infra/neo4j/*` (~1000 lignes) **supprime**.
|
||||
- Backend : 1 seul fichier > 300 (vs 6 avant). Reduction massive de la dette de fichiers volumineux.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **MAJ `run_rag` resolu** : le handler `run_rag` (92 lignes, 9 responsabilites) a disparu — le feature `reasoning-trace` a ete sorti du scope 0.5.0. Plus aucun handler ne viole le SRP a HEAD.
|
||||
- **Reduction massive de la dette** : suppression du module `infra/neo4j/`, du feature `reasoning/` (frontend) et de `GraphView.vue`. Le code restant est plus focalise.
|
||||
- **Backend tres propre** : un seul fichier productif > 300 lignes (`analysis_service.py:409`), et il est bien decompose en methodes courtes.
|
||||
- **Zero TODO/FIXME/HACK/XXX** : grep sur les 4 mots-cles ne renvoie rien, ni en `document-parser/` ni en `frontend/src/`.
|
||||
- **Zero code commente** : grep sur `^[[:space:]]*#[[:space:]]*(def |return |import |class |variable=)` — aucun hit.
|
||||
- **Imports propres** : `ruff check document-parser/` passe sans erreur — `I` rule (isort) enforce, first-party `api|domain|persistence|services`.
|
||||
- **Conventions i18n respectees** : `shared/i18n.ts` reduit de 546 a 364 lignes, mais reste l'unique source FR/EN. Code anglais partout.
|
||||
- **Nommage explicite** : `_run_batched_conversion`, `_finalize_analysis`, `_build_conversion_options`, `mark_completed`, `find_latest_completed_by_document` — porteurs de sens.
|
||||
- **`get_*` sans side-effect** verifie sur `get_document`, `get_analysis`, `_get_service` — lecture pure.
|
||||
- **Aucun `flag argument`** identifie — les options dataclasses (`ConversionOptions`, `ChunkingOptions`, `AnalysisConfig`, `IngestionConfig`) remplacent les booleens proliferants.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO CONDITIONNEL
|
||||
|
||||
Score 78/100, zero CRITICAL, zero MAJOR. 3 MINOR (taille fichiers/fonctions/params).
|
||||
|
||||
**Amelioration vs 0.5.0 pre-release (72/100)** : +6 points. Le MAJ `run_rag` a ete leve par suppression du feature reasoning. Le backend a perdu 5 fichiers > 300 lignes. La dette frontend reste concentree sur `StudioPage.vue` (1422), `ChunkPanel.vue` (801) et `ResultTabs.vue` (690).
|
||||
|
||||
**Condition pour passer GO (>=80)** : decomposer au moins un des trois fichiers Vue volumineux (objectif < 400 lignes) avant 0.6.0, ou acter un plan de remediation explicite dans le changelog 0.5.0.
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
# Rapport d'audit : KISS (Keep It Simple, Stupid)
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 7 / 8 |
|
||||
| Score | 87.5 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 0 |
|
||||
| Ecarts MINOR | 1 |
|
||||
| Ecarts INFO | 2 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MIN] Unnecessary wrapper function `_to_response`
|
||||
- **Localisation** : `document-parser/api/documents.py:27-35`
|
||||
- **Constat** : La fonction `_to_response` n'effectue qu'une conversion directe 1:1 d'un objet `Document` en `DocumentResponse`. Aucune logique, validation, ou transformation de valeur. Le mapping est trivial et répété 4 fois (upload, list, get, preview).
|
||||
- **Regle violee** : Item 4.3 — Pas de fonction wrapper qui ne fait qu'appeler une autre fonction sans valeur ajoutee
|
||||
- **Remediation** : Utiliser la conversion Pydantic `DocumentResponse.model_validate()` directement dans les routes, ou supprimer la fonction si l'objet métier expose déjà les champs attendus.
|
||||
|
||||
### [MIN] Redundant property accessors in DocumentService
|
||||
- **Localisation** : `document-parser/services/document_service.py:55-61`
|
||||
- **Constat** : Les propriétés `max_file_size` et `max_file_size_mb` sont des accesseurs directs sur `_config`, sans transformation. `max_file_size` recalcule la conversion MB→bytes dans `__init__` et la stocke dans `_max_file_size`, puis l'expose via une propriété — double indirection inutile.
|
||||
- **Regle violee** : Item 4.3 — Pas de fonction wrapper qui ne fait qu'appeler une autre fonction sans valeur ajoutee
|
||||
- **Remediation** : Stocker directement `max_file_size_mb` dans le service et effectuer la conversion inline au moment de l'utilisation, ou exposer un unique accesseur.
|
||||
|
||||
### [INFO] DocumentConfig and IngestionConfig dataclass overhead
|
||||
- **Localisation** : `document-parser/services/document_service.py:29-35` et `document-parser/services/ingestion_service.py:27-30`
|
||||
- **Constat** : Deux petites dataclasses (`DocumentConfig`, `IngestionConfig`) avec 3-4 champs chacune pour la configuration injectée. Simple, mais pourrait utiliser directement le `Settings` singleton ou des tuples nommés simples.
|
||||
- **Regle violee** : Item 4.8 — Les structures de donnees utilisees sont les plus simples possibles
|
||||
- **Remediation** : Considérer de passer directement les valeurs de `Settings` au lieu d'une dataclass intermédiaire, ou fusionner dans une seule config centralisée.
|
||||
|
||||
### [INFO] Analysis store polling with nested setInterval/setTimeout
|
||||
- **Localisation** : `frontend/src/features/analysis/store.ts:69-101`
|
||||
- **Constat** : La fonction `startPolling()` crée deux timers imbriqués (`setInterval` pour le polling, `setTimeout` pour le timeout). La logique est simple mais le code aurait pu utiliser une abstraction unifiée (ex: `AbortController` ou une promesse avec timeout).
|
||||
- **Regle violee** : Item 4.6 — Pas d'indirection inutile — le chemin d'execution d'une requete ne traverse pas plus de couches que necessaire
|
||||
- **Remediation** : Encapsuler dans un helper utilitaire `withPollingTimeout(interval, maxDuration)` ou utiliser `Promise.race()` pour unifier la logique.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- ✓ Aucun design pattern complexe (Factory, Strategy, Observer, Builder, Singleton) détecté.
|
||||
- ✓ Aucune meta-programmation (`__metaclass__`, `type()`, `__init_subclass__`, `__class_getitem__`).
|
||||
- ✓ La configuration centralisée en `infra/settings.py` est simple et lisible ; validation explicite en `__post_init__()` sans magie.
|
||||
- ✓ Les services (`DocumentService`, `IngestionService`, `AnalysisService`) orchestrent correctement sans abstraction superflue.
|
||||
- ✓ Frontend : stores Pinia sont concis et ne font que des actions simples (état, fetches API).
|
||||
- ✓ Pas d'indirection excessive sur les couches HTTP → service → domain.
|
||||
- ✓ Rate limiter implémenté en ligne sans dépendance externe (slowapi) — choix pragmatique pour single-process.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
||||
**Justification** : Score 87.5 ≥ 80 ; zéro écarts CRITICAL ou MAJOR. Les deux [MIN] identifiés sont des optimisations cosmétiques (refactoring de petites fonctions utilitaires) qui n'impactent pas la maintenabilité ou la sécurité. Les [INFO] sont des suggestions pour la prochaine itération. Le codebase suit globalement le principe KISS : pas de sur-ingénierie, pas de patterns prématurés, structures de données simples.
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
# Rapport d'audit : DRY (Don't Repeat Yourself)
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 5 / 7 |
|
||||
| Score | 71 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 2 |
|
||||
| Ecarts MINOR | 1 |
|
||||
| Ecarts INFO | 2 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Magic API URL hardcodé en frontend
|
||||
|
||||
- **Localisation** : `frontend/src/features/settings/store.ts:23`
|
||||
- **Constat** : La chaîne `'http://localhost:8000'` est écrite en dur en tant que valeur par défaut de `apiUrl`. Aucune centralization dans un fichier de configuration partagé.
|
||||
- **Regle violee** : Item 5.3 (poids 2) — Les constantes doivent être nommées et centralisées, pas éparpillées.
|
||||
- **Remediation** : Créer `frontend/src/shared/config.ts` (ou similaire) pour exporter les endpoints de base et les constantes communes.
|
||||
|
||||
### [MAJ] Magic strings localStorage non centralisées en frontend
|
||||
|
||||
- **Localisation** : `frontend/src/features/settings/store.ts:24-27`
|
||||
- **Constat** : Les clés localStorage `'docling-theme'` et `'docling-locale'` apparaissent 2 fois chacune (lignes 24, 27 pour theme ; 25, 31 pour locale). Ces chaînes ne sont pas extraites en constantes nommées.
|
||||
- **Regle violee** : Item 5.3 (poids 2) — Les magic strings doivent être nommées et centralisées.
|
||||
- **Remediation** : Créer des constantes nommées (e.g., `STORAGE_KEY_THEME = 'docling-theme'`) dans `frontend/src/shared/appConfig.ts` ou un nouveau `frontend/src/shared/constants.ts`.
|
||||
|
||||
### [MIN] Magic string "uploaded" non centralisé en backend
|
||||
|
||||
- **Localisation** : `document-parser/api/schemas.py:49`
|
||||
- **Constat** : Le statut de document est hardcodé à `"uploaded"` comme défaut. Bien que cohérent avec le modèle métier, ce n'est pas une constante nommée — c'est une string brute en commentaire.
|
||||
- **Regle violee** : Item 5.3 (poids 2 → MIN car le field commente que c'est temporaire) — Les constantes doivent être nommées.
|
||||
- **Remediation** : Créer un enum ou constante `DOCUMENT_STATUS_UPLOADED = "uploaded"` dans `document-parser/domain/models.py` (déjà utilisé pour AnalysisStatus).
|
||||
|
||||
### [INFO] Validation de `table_mode` et `chunker_type` répétée
|
||||
|
||||
- **Localisation** : `document-parser/api/schemas.py:114-116` (table_mode) et `145-146` (chunker_type)
|
||||
- **Constat** : Les mêmes listes de valeurs valides ("accurate", "fast") et ("hybrid", "hierarchical") sont définies dans les validateurs Pydantic. Ces valeurs figurent aussi en dur dans les docstrings de `domain/value_objects.py` (lignes 41, 67).
|
||||
- **Regle violee** : Item 5.3 (poids 2 → INFO car la validation est centralisée au niveau du schéma) — Les constantes pourraient être partagées entre domain et api.
|
||||
- **Remediation** : Créer un module `document-parser/domain/constants.py` avec `TABLE_MODES = ("accurate", "fast")` et `CHUNKER_TYPES = ("hybrid", "hierarchical")`, puis l'importer dans les deux.
|
||||
|
||||
### [INFO] Logique de polling dupliquée entre features
|
||||
|
||||
- **Localisation** : `frontend/src/features/analysis/store.ts` (ligne 67+) et `frontend/src/features/ingestion/store.ts` (ligne 31+)
|
||||
- **Constat** : Deux stores implémentent indépendamment le même pattern de polling avec `setInterval`, `clearInterval`, et gestion d'erreurs avec retry logic. Le code n'est pas identique ligne par ligne, mais la structure (démarrage/arrêt avec setInterval, gestion de erreurs, retry counter) est dupliquée.
|
||||
- **Regle violee** : Item 5.4 (poids 1) — La logique reactive partagée devrait être dans `shared/composables/`.
|
||||
- **Remediation** : Extraire un composable réutilisable (e.g., `usePollAsync` ou `useAsyncPoller`) dans `frontend/src/shared/composables/` pour encapsuler le pattern setInterval + erreur + retry.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- ✅ Centralisation réussie de la config HTTP : tous les appels `apiFetch` passent par `frontend/src/shared/api/http.ts` — aucun fetch() direct détecté.
|
||||
- ✅ Schemas Pydantic correctement séparés des modèles métier : `api/schemas.py` utilise `AliasChoices` pour la sérialisation camelCase, il ne duplique pas les modèles (`domain/models.py` reste pur).
|
||||
- ✅ Validation centralisée au niveau de l'API : les `@field_validator` de Pydantic sont le seul point d'entrée pour valider les options de pipeline et chunking.
|
||||
- ✅ Constantes d'état (`AnalysisStatus` enum) bien factorisées en `domain/models.py` — aucune duplication de "PENDING", "RUNNING", "COMPLETED", "FAILED".
|
||||
- ✅ Queries SQL factorisées : `_SELECT_WITH_DOC` réutilisée 3 fois dans `persistence/analysis_repo.py`.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO CONDITIONNEL
|
||||
|
||||
**Justification** :
|
||||
- Score 71 / 100 (seuil: 80) → au-dessous de GO, mais aucun CRITICAL.
|
||||
- 2 MAJ trouvés : magic API URL + magic localStorage keys → doivent être centralisés avant release.
|
||||
- 1 MIN + 2 INFO pour amélioration future (pas bloquant immédiatement).
|
||||
|
||||
**Conditions pour GO** :
|
||||
1. Centraliser `http://localhost:8000` et `'docling-theme'`, `'docling-locale'` dans `frontend/src/shared/` (e.g., `config.ts` ou `constants.ts`).
|
||||
2. Exécuter la re-vérification après correction pour atteindre ≥ 80.
|
||||
|
||||
**Actions futures (prochain cycle)** :
|
||||
- Créer `document-parser/domain/constants.py` pour les validations (table_mode, chunker_type).
|
||||
- Extraire `usePollAsync` composable en `frontend/src/shared/composables/` pour réduire la duplication de logique reactive.
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
# Rapport d'audit : SOLID
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 18 / 20 |
|
||||
| Score | 85 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 1 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Liskov Substitution Principle — isinstance check sur adaptateur
|
||||
|
||||
- **Localisation** : `document-parser/services/analysis_service.py:356`
|
||||
- **Constat** : Le service `AnalysisService` utilise `isinstance(self._converter, ServeConverter)` pour differencier les implementations du port `DocumentConverter`. Ceci viole LSP car :
|
||||
1. Suppose que le port a une implementation concrète (`ServeConverter`) accessible
|
||||
2. Couple le service à une implementation spécifique via un import infra
|
||||
3. Empeche la substitution transparente d'autres adaptateurs (e.g., mock, Docling Cloud)
|
||||
- **Regle violee** : 6.3.3 — "Pas de `isinstance()` ou `type()` check pour differencier les implementations"
|
||||
- **Remediation** : Introduire une methode `supports_batching()` dans le port `DocumentConverter` ou passer un flag de configuration pour indiquer si le batching est disponible. Supprimer l'import infra du service.
|
||||
|
||||
### [MIN] Interface Segregation Principle — aperture OpenSearch directe
|
||||
|
||||
- **Localisation** : `document-parser/services/ingestion_service.py:197`
|
||||
- **Constat** : `ping()` accede directement à `self._vector_store._client.info()` — expose un detail d'implementation (OpenSearch client) au lieu de passer par le contrat du port.
|
||||
- **Regle violee** : 6.4.2 — "Aucun port ne force une implementation a definir des methodes qu'elle n'utilise pas"
|
||||
- **Remediation** : Ajouter une methode `ping()` au port `VectorStore` et l'utiliser via le contrat public.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
1. **S — Single Responsibility** : Services bien segreges (`DocumentService`, `AnalysisService`, `IngestionService`). Chaque service a une responsabilité unique.
|
||||
2. **S — Frontend Stores** : Pinia stores parfaitement segreges par feature (ingestion, analysis, document, search, reasoning, chunking, feature-flags, settings). Aucun store god-object.
|
||||
3. **O — Open/Closed** : Patterns d'injection via `_build_converter()`, `_build_chunker()`, `_build_repos()` permettent l'ajout d'adaptateurs sans modifier les services. Architecture très extensible.
|
||||
4. **I — Interface Segregation** : Ports `DocumentConverter`, `DocumentChunker`, `DocumentRepository`, `AnalysisRepository`, `EmbeddingService`, `VectorStore` sont bien segreges. Aucune god interface.
|
||||
5. **D — Dependency Inversion** : Services dependent correctement de protocoles abstraits (ports). Injection se fait en composition root (`main.py:_build_*`). Routes API ne instancient pas les services, ils sont injectes via `Depends()`.
|
||||
6. **API Router Organization** : Routes groupees par ressource (`documents.py`, `analyses.py`, `ingestion.py`, `reasoning.py`, `graph.py`) — pure OCP.
|
||||
7. **Type annotations** : Utilisation systématique de `TYPE_CHECKING` pour eviter les imports circulaires. Protocols comme ports — excellent design.
|
||||
8. **Frontend composables/stores** : Separation claire entre logique et UI. Chaque store est independant et recomposable.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
||||
**Conditions:**
|
||||
- [MAJ] Refactorer `_is_remote_converter()` pour utiliser une methode de configuration ou un flag au lieu de `isinstance()` infra.
|
||||
- [MIN] Ajouter `ping()` au port `VectorStore` et supprimer l'acces direct à `_client`.
|
||||
|
||||
Avec ces deux corrections simples, le score passe à 95/100 et le verdict devient **GO** sans conditions.
|
||||
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
# Rapport d'audit : Decouplage
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 13 / 15 |
|
||||
| Score | 86 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 1 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Inter-store coupling in history feature test
|
||||
|
||||
- **Localisation** : `frontend/src/features/history/navigation.test.ts:30-31, 67, 82-83, 143-144, 169-170, 188`
|
||||
- **Constat** : Le test `navigation.test.ts` importe et utilise directement `useAnalysisStore()` et `useDocumentStore()` pour orchestrer la navigation. Cela crée un couplage direct entre la feature history et les stores analysis/document, violant 7.2.2.
|
||||
- **Regle violee** : 7.2.2 — Les features ne s'importent pas mutuellement — la communication passe par `shared/` ou par les props/events Vue.
|
||||
- **Remediation** : Extraire la logique de navigation dans un utilitaire partagé ou un composable dans `shared/`. Passer les résultats via props/events plutôt que d'accéder directement aux stores d'autres features dans la logique métier.
|
||||
|
||||
### [MIN] Frontend API client lacks explicit mock layer support
|
||||
|
||||
- **Localisation** : `frontend/src/features/{analysis,document,chunking,search}/api.ts` — aucun pattern d'injection de mock ou de stub
|
||||
- **Constat** : Les fichiers `api.ts` utilisent `apiFetch` directement sans interface abstraite. Bien qu'il soit possible de mockerles appels au niveau des tests vitest, il n'existe pas d'interface explicite permettant au frontend de tourner sans backend (7.1.3).
|
||||
- **Regle violee** : 7.1.3 — Le frontend peut tourner avec un mock du backend (les appels API sont isoles dans des fichiers `api.ts` par feature).
|
||||
- **Remediation** : Considérer une abstraction plus explicite (ex: interface `ApiClient` injectable) ou documenter le pattern de mock pour chaque feature. Poids 2 = écart MINOR (les tests mocks existent mais l'intention n'est pas architecturalement explicite).
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- ✓ **Decouplage Frontend/Backend solide** : Aucun import croise, contrat API basé sur REST, types TypeScript alignés avec Pydantic via `_to_camel` (schemas.py). Frontend utilise `apiFetch` (shared/api/http.ts) comme couche unique.
|
||||
- ✓ **Hexagonal Architecture Backend** : Ports dans `domain/ports.py` définis avec types du domaine (ConversionResult, ChunkResult, etc.). Services importent uniquement `domain.ports` et `domain.value_objects`, zéro fuite de types docling.*. Repositories retournent des modèles du domaine (Document, AnalysisJob), pas des dicts.
|
||||
- ✓ **Schemas API fortement typés** : Tous les DTOs dans `api/schemas.py` utilisent Pydantic BaseModel strictement (pas de `dict` ou `Any`). Contrat cohérent : camelCase aliasing, validation intégrée (table_mode, images_scale, chunker_type).
|
||||
- ✓ **Isolation des features Frontend** : Chaque feature (analysis, document, chunking, history, search, settings, ingestion) a son propre store Pinia, API client et UI. Zéro imports croisés `from features/` → `from features/` sauf dans navigation.test.ts.
|
||||
- ✓ **Infrastructure Adapter Boundary** : `infra/local_converter.py` importe Docling (DoclingConverter, CodeItem, etc.) mais retourne uniquement `ConversionResult` (domaine). Services ne voient jamais les types docling.
|
||||
- ✓ **Configuration d'infra propre** : `docker-compose.yml` et `nginx.conf` établissent la séparation frontend/backend via proxy HTTP (`/api/` → `http://127.0.0.1:8000`). CORS déclaré explicitement.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO CONDITIONNEL
|
||||
|
||||
**Conditions** :
|
||||
1. Corriger l'inter-store coupling dans `frontend/src/features/history/navigation.test.ts` (MAJ).
|
||||
2. Documenter ou implémenter un pattern explicite de mock API pour le frontend (MIN — peut être adressé en 0.5.1).
|
||||
|
||||
**Score 86/100** : Satisfait >= 80 (GO). Un seul écart MAJOR, zéro CRITICAL. L'architecture hexagonale backend est correcte, le contrat API est solide, et le decouplage frontend/backend fonctionne. Le couplage histoire/analyse/document est une violation ponctuelle du design, facilement remédiable sans refactoring majeur.
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
# Rapport d'audit : Securite
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 16 / 18 |
|
||||
| Score | 91 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 2 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Hardcoded default Neo4j password in settings
|
||||
- **Localisation** : `document-parser/infra/settings.py:30,133`
|
||||
- **Constat** : La valeur par défaut `neo4j_password="changeme"` est codée en dur. Bien qu'elle soit remplacée par la variable d'environnement `NEO4J_PASSWORD` en production, cette valeur par défaut est dangereuse si elle est utilisée accidentellement en production.
|
||||
- **Regle violee** : 8.1.1 — Secrets en dur dans le code source (poids 3 → MAJ car c'est une valeur par défaut, pas une clé API réelle)
|
||||
- **Remediation** : Remplacer le défaut par une valeur vide ou None et lever une exception au démarrage si Neo4j est activé sans mot de passe défini.
|
||||
|
||||
### [MAJ] OpenSearch security plugin disabled in docker-compose.yml
|
||||
- **Localisation** : `docker-compose.yml:26`
|
||||
- **Constat** : `DISABLE_SECURITY_PLUGIN: "true"` désactive la sécurité OpenSearch. Cela expose l'instance à des accès non authentifiés si elle est accessible en réseau.
|
||||
- **Regle violee** : 8.4.1 — Configuration de sécurité réseau (poids 2)
|
||||
- **Remediation** : Activer le plugin de sécurité OpenSearch avec des credentials explicites en production. La configuration docker-compose.yml doit être destinée au développement local uniquement.
|
||||
|
||||
### [INFO] Rate limiter disabled by default in development
|
||||
- **Localisation** : `document-parser/infra/settings.py:24`
|
||||
- **Constat** : `rate_limit_rpm: int = 100` mais en développement local, la limite peut être désactivée (0). Le middleware rate limiter s'ajoute conditionnellement (`if settings.rate_limit_rpm > 0`).
|
||||
- **Regle violee** : Observation (poids 0)
|
||||
- **Remediation** : Documenter que le rate limiting est configuré à 100 RPM par défaut et doit rester actif en production. C'est une bonne pratique, pas une faille.
|
||||
|
||||
---
|
||||
|
||||
## Resultats detailles par domaine
|
||||
|
||||
### 8.1 Secrets et credentials
|
||||
|
||||
| Item | Verdict | Localisation | Details |
|
||||
|------|---------|--------------|---------|
|
||||
| 8.1.1 — Pas de clés API/tokens en dur | ✅ PASS | `document-parser/infra/settings.py:49,114` | `docling_serve_api_key` lu de l'env, pas de valeur par défaut dangereuse |
|
||||
| 8.1.2 — .env dans .gitignore | ✅ PASS | `.gitignore` | `.env`, `.env.local`, `.env.production` déclarés |
|
||||
| 8.1.3 — Secrets Docker en env vars | ⚠️ FAIL | `docker-compose.yml:76` | `NEO4J_PASSWORD=changeme` est un défaut, voir détail [MAJ] |
|
||||
|
||||
### 8.2 Validation des entrees
|
||||
|
||||
| Item | Verdict | Localisation | Details |
|
||||
|------|---------|--------------|---------|
|
||||
| 8.2.1 — Validation Pydantic | ✅ PASS | `document-parser/api/schemas.py` | Tous les DTOs utilise Pydantic avec validateurs (@field_validator) |
|
||||
| 8.2.2 — MAX_FILE_SIZE_MB configurée | ✅ PASS | `document-parser/infra/settings.py:122` | Défaut 50 MB, appliquée dans `document_service.py:68,69` et `api/documents.py:45-56` |
|
||||
| 8.2.3 — Types fichiers acceptés | ✅ PASS | `document-parser/services/document_service.py:71` | Seuls les PDFs acceptés (magic bytes `%PDF` vérifiés) |
|
||||
|
||||
### 8.3 Injection
|
||||
|
||||
| Item | Verdict | Localisation | Details |
|
||||
|------|---------|--------------|---------|
|
||||
| 8.3.1 — Parametres liés SQL | ✅ PASS | `document-parser/persistence/*.py` | Toutes les requêtes utilisent `?` (placeholders aiosqlite), zéro f-string |
|
||||
| 8.3.2 — Pas eval/exec/os.system | ✅ PASS | Scan complet | Aucune utilisation détectée |
|
||||
| 8.3.3 — DOMPurify pour HTML | ✅ PASS | `frontend/src/features/analysis/ui/MarkdownViewer.vue:9,17` | `DOMPurify.sanitize(marked.parse(...))` utilisé systématiquement |
|
||||
|
||||
### 8.4 CORS et reseau
|
||||
|
||||
| Item | Verdict | Localisation | Details |
|
||||
|------|---------|--------------|---------|
|
||||
| 8.4.1 — CORS explicites (pas de \*) | ✅ PASS | `document-parser/main.py:204` | `allow_origins=settings.cors_origins` (défaut localhost:3000,5173) |
|
||||
| 8.4.2 — Rate limiter actif | ✅ PASS | `document-parser/main.py:209-214` | Middleware RateLimiterMiddleware montée si `rate_limit_rpm > 0` |
|
||||
| 8.4.3 — Nginx secure (pas directory listing) | ✅ PASS | `nginx.conf:14` | `try_files $uri $uri/ /index.html;` pour SPA, pas d'autoindex |
|
||||
|
||||
### 8.5 Dependances
|
||||
|
||||
| Item | Verdict | Localisation | Details |
|
||||
|------|---------|--------------|---------|
|
||||
| 8.5.1 — Pas de CVE critiques | ✅ PASS | `document-parser/requirements.txt` | Versions epinglées, pas de vulnérabilités connues detectées |
|
||||
| 8.5.2 — Versions epinglees | ✅ PASS | `document-parser/requirements.txt`, `frontend/package.json` | Toutes les dépendances utilisent des bornes supérieures (ex: `>=2.0.0,<3.0.0`) |
|
||||
|
||||
### Infrastructure
|
||||
|
||||
| Item | Verdict | Localisation | Details |
|
||||
|------|---------|--------------|---------|
|
||||
| Non-root user | ✅ PASS | `Dockerfile:48` | `useradd appuser`, `su appuser` pour uvicorn |
|
||||
| Security headers Nginx | ✅ PASS | `nginx.conf:7-11` | X-Frame-Options, X-Content-Type-Options, X-XSS-Protection, Referrer-Policy |
|
||||
| File upload validation | ✅ PASS | `document-parser/api/documents.py:39-69` | Chunked reading + Content-Length check + magic bytes |
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **DOMPurify + marked** : Toutes les pages markdown/HTML utilisateur sont sanitisées avant rendu (MarkdownViewer.vue, ReasoningPanel.vue).
|
||||
- **Paramètres liés SQLite** : Zéro injection SQL possible — tous les paramètres utilisent `?` placeholders via aiosqlite.
|
||||
- **PDF magic bytes** : Validation stricte des uploads (fichiers PDF uniquement, vérification signature `%PDF`).
|
||||
- **Rate limiting** : Middleware glissant-window implémenté en local (en-mémoire), prêt pour Redis en multiprocess.
|
||||
- **CORS explicites** : Configuration stricte par défaut (localhost:3000, 5173), pas de wildcard.
|
||||
- **Secrets en env vars** : Tous les secrets sensibles (API keys, mots de passe) lus de l'environnement, jamais en dur (sauf valeurs par défaut non-prod).
|
||||
- **Nginx headers** : Mitigation XSS, clickjacking, et content-sniffing configurées.
|
||||
- **Non-root Docker** : Conteneur exécuté en tant qu'utilisateur `appuser` (principle of least privilege).
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO CONDITIONNEL
|
||||
|
||||
**Score** : 91 / 100 (seuil GO >= 80)
|
||||
|
||||
**Conditions requises avant merge** :
|
||||
|
||||
1. **[MAJ-1]** Remplacer le défaut `neo4j_password="changeme"` par une validation strikte au démarrage — soit une variable d'environnement obligatoire, soit une exception si Neo4j est activé sans password défini. (**P0**)
|
||||
|
||||
2. **[MAJ-2]** Documenter clairement que `docker-compose.yml` est destinée au développement **local uniquement**. Ajouter une section README expliquant que la sécurité OpenSearch doit être activée pour les déploiements en réseau. Optionnellement, créer une variante production-ready de docker-compose.yml avec sécurité OpenSearch activée. (**P1**)
|
||||
|
||||
**Pas de blocage GO** : Aucun [CRIT] détecté. Les deux [MAJ] sont adressables avant merge (l'une en 5 min de code, l'autre en documentation).
|
||||
|
||||
---
|
||||
|
||||
## Audits associes / tickets
|
||||
|
||||
- 08-security.md checklist : ✅ conforme (16/18 items)
|
||||
- Voir aussi : Audit 10 — CI/Build pour la pipeline d'intégration
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
# Rapport d'audit : Tests
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 17 / 18 |
|
||||
| Score | 94 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] 18 assertions vagues "assert X is not None"
|
||||
|
||||
- **Localisation** : `document-parser/tests/test_local_converter.py:70`, `test_repos.py:42,95,124,190,192`, `test_ingestion_service.py:119`, `test_analysis_service.py:354`, `test_models.py:61,77,87`, `test_chunking.py:276`, `test_api_endpoints.py:199`, `test_vector_store_port.py:47`, `test_pipeline_options.py:56`, `neo4j/test_document_roundtrip.py:24`, `neo4j/test_tree_writer.py:204`, `neo4j/test_chunk_writer.py:97`, `test_opensearch_store.py:110`
|
||||
- **Constat** : Plusieurs assertions backend testent l'existence (`assert X is not None`) sans vérifier le contenu ou les propriétés de X. Ces assertions ne capturent que la présence, pas la correction.
|
||||
- **Regle violee** : 9.3.5 — Les assertions sont specifiques (pas juste `assert result is not None`)
|
||||
- **Remediation** : Remplacer par des assertions précises : `assert result.field == expected_value` ou `assert len(result) > 0` avec vérification du contenu.
|
||||
- **Poids** : 2 (MAJOR) — Impact modéré sur la qualité des tests d'intégration
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
1. **Couverture d'endpoints robuste** (9.2.1) : 29 fichiers de tests backend couvrant tous les endpoints critiques (`/api/health`, `/api/documents`, `/api/analyses`, `/api/chunking`, etc.). Chaque endpoint a au moins un happy path avec TestClient.
|
||||
|
||||
2. **Test d'erreurs complets** (9.2.2) : Les tests couvrent 400, 404, et gestion d'erreurs métier. Exemples : `test_create_analysis_empty_document_id` (400), `test_get_analysis_not_found` (404), `test_create_analysis_document_not_found` (404), `test_upload_document_preview_page_out_of_range` (400).
|
||||
|
||||
3. **Services bien testés** (9.2.3) : Tests unitaires d'orchestration pour `AnalysisService`, `IngestionService`, `DocumentService` avec mocking des repos. Couverture des cas d'erreur : exception handling, cancellation, timeout classification.
|
||||
|
||||
4. **Domain value objects testes** (9.2.4) : `test_bbox.py` (193 tests sur 150+ lignes) couvre normalisations de coordonnées, page sizes, edge cases (zero-width, inverted, point bboxes). `test_chunking.py` teste `ChunkingOptions` avec defaults et custom values. `test_models.py` teste les state transitions.
|
||||
|
||||
5. **Composants Vue et stores critiques testes** (9.2.5) : 23 tests frontend colocalises (`*.test.ts`) couvrant stores Pinia (`ingestion/store.test.ts`, `analysis/store.test.ts`, `document/store.test.ts`, `feature-flags/store.test.ts`), composables (`useFeatureFlag.test.ts`), et API layers (`ingestion/api.test.ts`, `analysis/api.test.ts`).
|
||||
|
||||
6. **Pas d'anti-patterns de test** (9.3.1, 9.3.2) : Aucun `.only`, `fdescribe`, `fit`, ou `.skip()` sans justification trouvé. Tous les tests actifs et exécutés.
|
||||
|
||||
7. **Determinisme garanti** (9.3.3) :
|
||||
- Backend : pytest avec `asyncio_mode = auto`. Pas de `sleep()` ou dépendances réseau brutes trouvées.
|
||||
- Frontend : Vitest avec `vi.useFakeTimers()` / `vi.useRealTimers()` dans tests d'orchestration (polling, intervals). Mocks d'API systématiques (`vi.mock('./api')`).
|
||||
|
||||
8. **Mocking vs integration equilibre** (9.3.4) :
|
||||
- Tests d'endpoints : TestClient + mocks de services (flow réel HTTP mais dépendances mockées).
|
||||
- Tests de services : AsyncMock des repos/infra mais orchestration réelle testée.
|
||||
- E2E Karate UI : vrais workflows UI ; Karate API tests : vrais appels HTTP.
|
||||
|
||||
9. **Nommage explicite** (9.3.6) : Tous les tests ont des noms clairs décrivant le comportement. Exemples : `test_exception_marks_job_failed`, `test_bottomleft_origin_converted`, `test_full_pipeline`, `test_skips_deleted_chunks`, `test_ingest_and_verify`.
|
||||
|
||||
10. **E2E Karate bien structuré** : 15+ Gherkin features couvrant UI (upload, navigation, analyses), API workflows (health, ingestion, concurrent analyses). Conventions respectées (Background, waitFor, cleanup). UIRunner et E2ERunner organisent tests par tags (@critical, @ui, @smoke).
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO
|
||||
|
||||
**Justification** :
|
||||
- Score 94/100 dépasse le seuil GO (>= 80).
|
||||
- Zero écarts CRITICAL — aucune violation bloquante.
|
||||
- Un écart MAJOR (assertions vagues) est non-bloquant selon master.md §3 ("GO CONDITIONNEL si <= 3 MAJOR et 0 CRITICAL"). Cet écart est facilement corrigible en post-release.
|
||||
- Couverture de chemin critique solide (endpoints, services, domain, composants critiques, e2e).
|
||||
- Determinisme et mocking/integration balance correctement mis en place.
|
||||
|
||||
**Condition** : Les assertions vagues devraient être corrigées dans le prochain cycle (elles ne bloquent pas mais réduisent la qualité des assertions).
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
# Rapport d'audit : CI / Build
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 10 / 11 |
|
||||
| Score | 91 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Ruff warning UP038 — isinstance() union syntax not applied
|
||||
|
||||
- **Localisation** : `document-parser/infra/docling_tree.py:101`
|
||||
- **Constat** : Ruff rule UP038 (pyupgrade) detects that `isinstance(bbox, (list, tuple))` should use union syntax `isinstance(bbox, list | tuple)` for Python 3.12+ consistency.
|
||||
- **Regle violee** : 10.1.3 — Tous les warnings Ruff sont resolus (0 warning)
|
||||
- **Remediation** : Appliquer le fix unsafe : `ruff check . --fix --unsafe-fixes`, ou refactoriser manuellement la ligne 101 pour utiliser la syntaxe union.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **Pipeline CI robuste** : Workflows bien structures (ci.yml, release-gate.yml) avec phases paralleles et E2E complets (API + UI).
|
||||
- **.dockerignore complet** : Exclut correctement node_modules, .venv, .git, __pycache__ et autres artefacts inutiles.
|
||||
- **Multi-stage Docker** : Targets remote/local permettent flexibilite deployment ; caching GHA active.
|
||||
- **Health check operationnel** : Endpoint `/api/health` fonctionnel et teste en smoke-test avec validations de champs (status, engine).
|
||||
- **Nginx bien configure** : Routing `/api/*` → backend 127.0.0.1:8000, frontend statique sur `/`, timeouts 900s acceptables.
|
||||
- **Frontend builds deterministiques** : Type-check (vue-tsc) passe, ESLint zero-warning, Prettier format OK.
|
||||
- **Env vars documentees** : CORS_ORIGINS, DOCLING_SERVE_URL, RATE_LIMIT_RPM, MAX_FILE_SIZE_MB, etc. avec defaults cohaerents.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO CONDITIONNEL
|
||||
|
||||
**Conditions** :
|
||||
- Resoudre l'ecart MAJOR 10.1.3 : appliquer le fix Ruff UP038 a docling_tree.py:101 et reverifier `ruff check .` avant le merge final.
|
||||
|
||||
**Justification** :
|
||||
Score 91/100 avec 1 seul ecart (poids 1) maintient la release au-dessus du seuil 80 (GO). Aucun ecart CRITICAL. L'UP038 est une violation mineure de style/upgrade mais non bloquante fonctionnellement. La remediation est triviale (un fix unsafe ou 2 lignes manuelles).
|
||||
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
# Rapport d'audit : Documentation & Changelog
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 5 / 9 |
|
||||
| Score | **44 / 100** |
|
||||
| Ecarts CRITICAL | 1 |
|
||||
| Ecarts MAJOR | 2 |
|
||||
| Ecarts MINOR | 0 |
|
||||
| Ecarts INFO | 0 |
|
||||
|
||||
### Detail
|
||||
|
||||
| # | Item | Poids | Statut |
|
||||
|---|------|-------|--------|
|
||||
| 11.1.1 | `[Unreleased]` renommee en `[X.Y.Z] - YYYY-MM-DD` | 3 | **KO** |
|
||||
| 11.1.2 | Modifications de la release listees | 2 | **KO** |
|
||||
| 11.1.3 | Breaking changes identifies | 3 | OK |
|
||||
| 11.1.4 | Format Keep a Changelog | 1 | OK |
|
||||
| 11.2.1 | `package.json` a la bonne version | 2 | **KO** |
|
||||
| 11.2.2 | Semantic Versioning | 2 | OK |
|
||||
| 11.3.1 | Pas de TODO orphelin | 1 | OK |
|
||||
| 11.3.2 | Pas de `console.log` de debug | 2 | OK |
|
||||
| 11.3.3 | Pas de `print()` de debug | 2 | OK |
|
||||
|
||||
**Calcul** : poids conformes 3+1+2+2 = 8 / poids total 18 = 44.4 → 44.
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [CRIT] CHANGELOG.md sans section `[0.5.0]`
|
||||
|
||||
- **Localisation** : `CHANGELOG.md:7`
|
||||
- **Constat** : la derniere section est `## [0.4.0] - 2026-04-13`. Aucune section `[Unreleased]` ni `[0.5.0]` n'est presente. Pour une release nommee 0.5.0, la regle 11.1.1 (poids 3) exige une section `[0.5.0] - YYYY-MM-DD`.
|
||||
- **Regle violee** : 11.1.1.
|
||||
- **Impact** : aucune trace des nouveautes 0.5.0 (reasoning viewer, Neo4j integration, RAG endpoints). Un tag 0.5.0 cree depuis ce HEAD livrerait un changelog mensonger.
|
||||
- **Remediation** : ajouter section `## [0.5.0] - 2026-04-28` avec sous-sections `Added`, `Changed`, `Fixed` listtant les changements depuis 0.4.0 (reasoning-trace viewer, Neo4j graph storage, RAG endpoints, feature flags, env vars).
|
||||
|
||||
### [MAJ] Modifications de la release 0.5.0 non documentees
|
||||
|
||||
- **Localisation** : `CHANGELOG.md`
|
||||
- **Constat** : corollaire direct de l'ecart 11.1.1. Aucun bullet n'enumere les nouveautes de cette release (11.1.2, poids 2). Features visibles sur la branche (Neo4j TreeWriter/ChunkWriter, reasoning/ui components, `/api/documents/:id/graph` endpoint) ne sont pas listees.
|
||||
- **Regle violee** : 11.1.2.
|
||||
- **Remediation** : voir CRIT 11.1.1.
|
||||
|
||||
### [MAJ] `frontend/package.json` toujours a `0.4.0`
|
||||
|
||||
- **Localisation** : `frontend/package.json:3`
|
||||
- **Constat** : `"version": "0.4.0"`. Pour une release 0.5.0, il faut bumper a `"version": "0.5.0"` (regle 11.2.1, poids 2).
|
||||
- **Impact** : la version du frontend reste `0.4.0`, confusion utilisateur, tracing impossible du bundle vers la release 0.5.0.
|
||||
- **Remediation** : bumper a `"version": "0.5.0"` avant le tag final.
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- **Zero `console.log`/`console.debug`** dans le code frontend. Les 2 occurrences de `console.warn` (store.ts:85, ReasoningPanel.vue:150) respectent la convention permise.
|
||||
- **Zero `print()` de debug** dans le backend (hors tests).
|
||||
- **Zero TODO/FIXME/HACK/XXX** dans le code productif.
|
||||
- **Format Keep a Changelog correctement respecte** : preambule conforme, sections Added/Changed/Fixed/Fixed (v0.4.0), chronologie inverse.
|
||||
- **Semantic Versioning suivi** : sequence 0.1.0 → 0.2.0 → 0.3.0 → 0.3.1 → 0.4.0 → (0.5.0 en attente) coherente.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : NO-GO
|
||||
|
||||
Score 44/100 < 60 (seuil minimum) et **1 ecart CRITICAL non resolu** (regle absolue du master : tout `[CRIT]` non resolu = NO-GO).
|
||||
|
||||
Le release 0.5.0 ne peut pas partir (ne peut pas etre taguee) tant que :
|
||||
|
||||
1. **CHANGELOG.md** n'enumere pas les changements sous `## [0.5.0] - YYYY-MM-DD`.
|
||||
2. **frontend/package.json** n'est pas bumpe a `0.5.0`.
|
||||
|
||||
Recommendation : apres avoir resolu ces deux points, relancer l'audit 11.
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
# Rapport d'audit : Performance & Ressources
|
||||
|
||||
**Release** : 0.5.0
|
||||
**Date** : 2026-04-28
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Score de compliance
|
||||
|
||||
| Metrique | Valeur |
|
||||
|----------|--------|
|
||||
| Items conformes | 13 / 15 |
|
||||
| Score | 86.67 / 100 |
|
||||
| Ecarts CRITICAL | 0 |
|
||||
| Ecarts MAJOR | 1 |
|
||||
| Ecarts MINOR | 1 |
|
||||
| Ecarts INFO | 1 |
|
||||
|
||||
---
|
||||
|
||||
## Ecarts constates
|
||||
|
||||
### [MAJ] Blocage I/O synchrone dans endpoint async de prévisualisation
|
||||
|
||||
- **Localisation** : `document-parser/api/documents.py:115-116`
|
||||
- **Constat** : L'endpoint `/preview` lit le fichier PDF entier en synchrone (`with open(...); f.read()`) dans une fonction async, bloquant l'event loop FastAPI. Cela pénalise la performance pour les gros PDFs et empêche les autres requêtes d'être traitées pendant la lecture.
|
||||
- **Regle violee** : 12.1.4 (poids 3 → revu comme MAJ car contexte limité : endpoint unique, rarement appelé en rafale, mais violation du principle async)
|
||||
- **Remediation** : Utiliser `asyncio.to_thread()` ou une lecture asynchrone pour charger le fichier sans bloquer l'event loop.
|
||||
|
||||
### [MIN] Watchers Vue sans cleanup explicite en Pinia store
|
||||
|
||||
- **Localisation** : `frontend/src/features/settings/store.ts:27-39`
|
||||
- **Constat** : Les watchers `watch()` et `watchEffect()` définis au niveau du store (lignes 27-39) ne disposent pas explicitement de leur cleanup. En théorie, Pinia décharge les stores mais les watchers restent actifs si le store persiste en mémoire.
|
||||
- **Regle violee** : 12.2.1 (poids 2 → revu comme MIN car watchers non critiques et scope limité au store)
|
||||
- **Remediation** : Retourner une fonction `stop()` depuis le composable ou encapsuler dans `onUnmounted()`.
|
||||
|
||||
### [INFO] Absence de pagination explicite sur `/api/documents` et `/api/graph`
|
||||
|
||||
- **Localisation** : `document-parser/api/documents.py:72-76` et `document-parser/api/graph.py` (find_all implicit)
|
||||
- **Constat** : Les endpoints `GET /api/documents` et `GET /api/graph` retournent la liste entière sans pagination explicite (frontend assume la limite implicite). Si le nombre de documents/nœuds du graphe croît (>10k), la payload API et le render frontend deviennent coûteux.
|
||||
- **Regle violee** : 12.3.2 (poids 1, information)
|
||||
- **Remediation** : Ajouter des paramètres `limit` et `offset` optionnels pour paginer les listes (déjà implémentés au niveau repository avec `limit=200, offset=0`).
|
||||
|
||||
---
|
||||
|
||||
## Points positifs
|
||||
|
||||
- ✅ **Semaphore bien configuré** (12.1.3) — `MAX_CONCURRENT_ANALYSES` défini à 3, implémenté avec `asyncio.Semaphore()`, appliqué dans la méthode `_run_analysis()`.
|
||||
- ✅ **Fichiers temporaires supprimés** (12.1.2) — L'upload et la suppression de documents gèrent le disque correctement. Les fichiers sont supprimés lors de `delete()` après vérification du chemin.
|
||||
- ✅ **Conversion asynchrone** (12.1.4) — Docling (conversion CPU-intensive) est délégué via `asyncio.to_thread()` dans `LocalConverter`, libérant l'event loop.
|
||||
- ✅ **Lecture en chunks** (12.1.5) — L'endpoint `/upload` lit les uploads par chunks de 64 KB au lieu de charger le fichier entier en mémoire en une seule allocation.
|
||||
- ✅ **Watchers avec cleanup** (12.2.1, 12.2.2) — Dans les composants `StudioPage.vue`, `GraphView.vue`, `BboxOverlay.vue`, les event listeners sont correctement supprimés dans `onBeforeUnmount()` ou `onMounted()`.
|
||||
- ✅ **Pas de re-render excessif** (12.2.4) — Utilisation cohérente de `computed()` pour les états dérivés, pas de fonctions au template.
|
||||
- ✅ **Pas de requête N+1 évidente** (12.1.1) — Les repositories utilisent des `JOIN` explicites (ex: `_SELECT_WITH_DOC`) et pas de boucles avec requêtes unitaires.
|
||||
|
||||
---
|
||||
|
||||
## Verdict partiel : GO CONDITIONNEL
|
||||
|
||||
**Condition** : Appliquer la remediation [MAJ] pour l'endpoint `/preview` avant la release ou planifier la correction dans le sprint suivant avec une priorité haute. L'absence de fix n'est pas bloquante pour cette release (endpoint non critique, charge potentielle limitée), mais la dette technique doit être remboursée rapidement.
|
||||
|
||||
**Justification** :
|
||||
- Score 86.67 >= 80 → **GO**
|
||||
- 0 écart CRITICAL → aucun blocage de sécurité/intégrité
|
||||
- 1 écart MAJOR (preview I/O) → acceptable en GO CONDITIONNEL avec plan de remediation
|
||||
- 1 écart MINOR + 1 INFO → non bloquants
|
||||
|
||||
|
|
@ -1,933 +0,0 @@
|
|||
# Plan de remediation — Release 0.5.0
|
||||
|
||||
**Date** : 2026-04-22
|
||||
**Branche** : `feature/reasoning-trace` -> `release/0.5.0`
|
||||
**Entree** : [summary.md](summary.md) (audit complet)
|
||||
**Objectif** : passer de NO-GO (2 CRIT, 5 MAJ) a GO.
|
||||
|
||||
---
|
||||
|
||||
## Sequencement global
|
||||
|
||||
```
|
||||
Phase 1 (jour 1) Phase 2 (jour 2-3) Phase 3 (jour 4-5) Phase 4 (jour 5)
|
||||
------------- ----------------- ----------------- ---------------
|
||||
[M4 + B1] [M1 + M2 + M3 + Q1] [B2] [M5 + Q2-Q6]
|
||||
Port Graph/Converter Service refactor backend Decouplage frontend Cleanup + docs
|
||||
~1 jour ~1.5 jour ~2 jours ~0.5 jour
|
||||
```
|
||||
|
||||
**Dependances** :
|
||||
- B1 et M4 touchent le meme port -> faire ensemble.
|
||||
- M1/M2 beneficient de B1 (les services n'importent plus `infra.neo4j` avant qu'on casse leur code).
|
||||
- M3 est trivial mais profite du refactor M1 (nouveau `GraphService` peut lire `settings.max_graph_pages`).
|
||||
- Q1 tombe naturellement quand on migre la camelCase vers `api/schemas.py` dans le refactor M1.
|
||||
- B2 est isole (frontend only) et peut etre fait en parallele si une 2e personne.
|
||||
- M5 + Q5 + Q6 sont des edits ponctuels, dernier jour.
|
||||
|
||||
**Tests a maintenir verts a chaque phase** : `ruff check`, `pytest tests/`, `npm run lint`, `npm run type-check`, `npm run test:run`.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Ports (M4 + B1) ≈ 1 jour
|
||||
|
||||
### Step 1.1 : Elargir `DocumentConverter` port (resout M4)
|
||||
|
||||
**Fichier** : `document-parser/domain/ports.py`
|
||||
|
||||
```python
|
||||
class DocumentConverter(Protocol):
|
||||
async def convert(
|
||||
self,
|
||||
file_path: str,
|
||||
options: ConversionOptions,
|
||||
*,
|
||||
page_range: tuple[int, int] | None = None,
|
||||
) -> ConversionResult: ...
|
||||
|
||||
# NEW — resolves M4 (isinstance check)
|
||||
@property
|
||||
def supports_batching(self) -> bool:
|
||||
"""True if the converter can process a document in page batches.
|
||||
|
||||
Remote converters (ServeConverter) don't support batching because
|
||||
merging DoclingDocument fragments across HTTP calls is unsafe.
|
||||
"""
|
||||
...
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/infra/local_converter.py`
|
||||
|
||||
```python
|
||||
class LocalConverter:
|
||||
supports_batching: bool = True
|
||||
# ... reste inchange
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/infra/serve_converter.py`
|
||||
|
||||
```python
|
||||
class ServeConverter:
|
||||
supports_batching: bool = False
|
||||
# ... reste inchange
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/services/analysis_service.py`
|
||||
|
||||
```python
|
||||
# Supprimer la methode _is_remote_converter et son import ServeConverter
|
||||
# Remplacer l'appel :
|
||||
# is_remote = self._is_remote_converter()
|
||||
# if batch_size > 0 and total_pages > batch_size and not is_remote:
|
||||
# par :
|
||||
# if batch_size > 0 and total_pages > batch_size and self._converter.supports_batching:
|
||||
```
|
||||
|
||||
**Tests impactes** : `tests/test_serve_converter.py`, `tests/test_analysis_service.py` (mocker `supports_batching` sur le mock converter).
|
||||
|
||||
---
|
||||
|
||||
### Step 1.2 : Creer `GraphWriter` port (resout B1)
|
||||
|
||||
**Fichier** : `document-parser/domain/ports.py` (ajout)
|
||||
|
||||
```python
|
||||
@runtime_checkable
|
||||
class GraphWriter(Protocol):
|
||||
"""Port for persisting the DoclingDocument structure + chunks to a graph store.
|
||||
|
||||
Implementations (Neo4j, Nebula, …) mirror the tree and chunk structure so
|
||||
downstream features (graph view, reasoning traces) can query it without
|
||||
going through the primary SQLite store.
|
||||
"""
|
||||
|
||||
async def write_document(
|
||||
self,
|
||||
*,
|
||||
doc_id: str,
|
||||
filename: str,
|
||||
document_json: str,
|
||||
) -> None:
|
||||
"""Persist the DoclingDocument tree. Idempotent (replaces existing)."""
|
||||
...
|
||||
|
||||
async def write_chunks(
|
||||
self,
|
||||
*,
|
||||
doc_id: str,
|
||||
chunks_json: str,
|
||||
) -> None:
|
||||
"""Persist chunks with DERIVED_FROM edges. Idempotent."""
|
||||
...
|
||||
```
|
||||
|
||||
### Step 1.3 : Creer l'adapter `Neo4jGraphWriter`
|
||||
|
||||
**Nouveau fichier** : `document-parser/infra/neo4j/graph_writer.py`
|
||||
|
||||
```python
|
||||
"""Neo4jGraphWriter — GraphWriter port implementation over Neo4j.
|
||||
|
||||
Thin facade around the existing write_document / write_chunks free functions
|
||||
so the services can depend on the domain port instead of importing infra
|
||||
directly (audit 06-SOLID B1).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from infra.neo4j.chunk_writer import write_chunks as _write_chunks
|
||||
from infra.neo4j.tree_writer import write_document as _write_document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from infra.neo4j.driver import Neo4jDriver
|
||||
|
||||
|
||||
class Neo4jGraphWriter:
|
||||
"""Implements domain.ports.GraphWriter over a Neo4j driver."""
|
||||
|
||||
def __init__(self, driver: Neo4jDriver) -> None:
|
||||
self._driver = driver
|
||||
|
||||
async def write_document(
|
||||
self,
|
||||
*,
|
||||
doc_id: str,
|
||||
filename: str,
|
||||
document_json: str,
|
||||
) -> None:
|
||||
await _write_document(
|
||||
self._driver,
|
||||
doc_id=doc_id,
|
||||
filename=filename,
|
||||
document_json=document_json,
|
||||
)
|
||||
|
||||
async def write_chunks(self, *, doc_id: str, chunks_json: str) -> None:
|
||||
await _write_chunks(self._driver, doc_id=doc_id, chunks_json=chunks_json)
|
||||
```
|
||||
|
||||
### Step 1.4 : Cabler dans `main.py` + services
|
||||
|
||||
**Fichier** : `document-parser/main.py`
|
||||
|
||||
```python
|
||||
# Remplacer les signatures et le wiring :
|
||||
def _build_analysis_service(
|
||||
document_repo, analysis_repo, graph_writer: GraphWriter | None = None,
|
||||
) -> AnalysisService:
|
||||
...
|
||||
return AnalysisService(
|
||||
converter=converter,
|
||||
analysis_repo=analysis_repo,
|
||||
document_repo=document_repo,
|
||||
chunker=chunker,
|
||||
conversion_timeout=settings.conversion_timeout,
|
||||
max_concurrent=settings.max_concurrent_analyses,
|
||||
config=config,
|
||||
graph_writer=graph_writer, # remplace neo4j_driver=
|
||||
)
|
||||
|
||||
|
||||
async def lifespan(app: FastAPI):
|
||||
await init_db()
|
||||
document_repo, analysis_repo = _build_repos()
|
||||
app.state.analysis_repo = analysis_repo
|
||||
app.state.document_repo = document_repo
|
||||
app.state.neo4j = await _init_neo4j()
|
||||
|
||||
# NEW — build graph writer once, inject via port
|
||||
graph_writer = None
|
||||
if app.state.neo4j is not None:
|
||||
from infra.neo4j.graph_writer import Neo4jGraphWriter
|
||||
graph_writer = Neo4jGraphWriter(app.state.neo4j)
|
||||
app.state.graph_writer = graph_writer
|
||||
|
||||
app.state.analysis_service = _build_analysis_service(
|
||||
document_repo, analysis_repo, graph_writer=graph_writer,
|
||||
)
|
||||
...
|
||||
ingestion_service = _build_ingestion_service(graph_writer=graph_writer)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/services/analysis_service.py`
|
||||
|
||||
```python
|
||||
# Remplacer :
|
||||
# neo4j_driver=None
|
||||
# from infra.neo4j import write_document
|
||||
# await write_document(self._neo4j, ...)
|
||||
# par :
|
||||
# graph_writer: GraphWriter | None = None
|
||||
# await self._graph_writer.write_document(doc_id=..., filename=..., document_json=...)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/services/ingestion_service.py`
|
||||
|
||||
```python
|
||||
# Remplacer :
|
||||
# neo4j_driver=None
|
||||
# from infra.neo4j import write_chunks
|
||||
# await write_chunks(self._neo4j, doc_id=doc_id, chunks_json=chunks_json)
|
||||
# par :
|
||||
# graph_writer: GraphWriter | None = None
|
||||
# if self._graph_writer is not None:
|
||||
# await self._graph_writer.write_chunks(doc_id=doc_id, chunks_json=chunks_json)
|
||||
```
|
||||
|
||||
**Tests impactes** :
|
||||
- `tests/test_analysis_service.py` : mocker `GraphWriter` au lieu du driver neo4j.
|
||||
- `tests/test_ingestion_service.py` : idem.
|
||||
- `tests/neo4j/test_document_roundtrip.py` : ajouter un test pour `Neo4jGraphWriter` (verifier qu'il delegue correctement).
|
||||
|
||||
**Risques** :
|
||||
- La branche existante exposait `app.state.neo4j` (driver brut) sur d'autres consommateurs ? -> grep dans `api/*.py` montre seulement `api/graph.py` qui utilise le driver pour READ (fetch_graph). OK, pas de casse cote read.
|
||||
|
||||
**Check de validation** :
|
||||
```bash
|
||||
grep -rn "from infra.neo4j import" document-parser/services/
|
||||
# Attendu : 0 ligne
|
||||
grep -rn "from infra.serve_converter import" document-parser/services/
|
||||
# Attendu : 0 ligne
|
||||
grep -rn "isinstance" document-parser/services/
|
||||
# Attendu : 0 ligne
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Services (M1 + M2 + M3 + Q1) ≈ 1.5 jour
|
||||
|
||||
### Step 2.1 : Creer `GraphService` (resout M1 partie graph + M3)
|
||||
|
||||
**Fichier** : `document-parser/infra/settings.py`
|
||||
|
||||
```python
|
||||
# Ajouter :
|
||||
max_graph_pages: int = 200 # cap pour /graph et /reasoning-graph (413 au-dela)
|
||||
|
||||
# Et dans from_env() :
|
||||
max_graph_pages=int(os.environ.get("MAX_GRAPH_PAGES", "200")),
|
||||
```
|
||||
|
||||
**Nouveau fichier** : `document-parser/services/graph_service.py`
|
||||
|
||||
```python
|
||||
"""Graph service — orchestrates graph retrieval from Neo4j or SQLite fallback."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from infra.docling_graph import build_graph_payload
|
||||
from infra.neo4j.queries import GraphPayload, fetch_graph
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from domain.ports import AnalysisRepository
|
||||
from infra.neo4j.driver import Neo4jDriver
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphTooLargeError(Exception):
|
||||
page_count: int
|
||||
max_pages: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphNotFoundError(Exception):
|
||||
doc_id: str
|
||||
|
||||
|
||||
class GraphService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
analysis_repo: AnalysisRepository,
|
||||
neo4j_driver: Neo4jDriver | None = None,
|
||||
max_pages: int = 200,
|
||||
) -> None:
|
||||
self._analysis_repo = analysis_repo
|
||||
self._neo4j = neo4j_driver
|
||||
self._max_pages = max_pages
|
||||
|
||||
async def get_neo4j_graph(self, doc_id: str) -> GraphPayload:
|
||||
if self._neo4j is None:
|
||||
raise RuntimeError("Neo4j not configured")
|
||||
payload = await fetch_graph(self._neo4j, doc_id, max_pages=self._max_pages)
|
||||
if payload is None:
|
||||
raise GraphNotFoundError(doc_id=doc_id)
|
||||
if payload.truncated:
|
||||
raise GraphTooLargeError(page_count=payload.page_count, max_pages=self._max_pages)
|
||||
return payload
|
||||
|
||||
async def get_reasoning_graph(self, doc_id: str) -> GraphPayload:
|
||||
latest = await self._analysis_repo.find_latest_completed_by_document(doc_id)
|
||||
if latest is None or not latest.document_json:
|
||||
raise GraphNotFoundError(doc_id=doc_id)
|
||||
payload = build_graph_payload(
|
||||
latest.document_json,
|
||||
doc_id=doc_id,
|
||||
title=latest.document_filename or doc_id,
|
||||
max_pages=self._max_pages,
|
||||
)
|
||||
if payload.truncated:
|
||||
raise GraphTooLargeError(page_count=payload.page_count, max_pages=self._max_pages)
|
||||
return payload
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/api/graph.py` (simplifier)
|
||||
|
||||
```python
|
||||
# Devient :
|
||||
@router.get("/{doc_id}/graph", response_model=GraphResponse)
|
||||
async def get_document_graph(doc_id: str, request: Request) -> GraphResponse:
|
||||
svc = request.app.state.graph_service
|
||||
try:
|
||||
payload = await svc.get_neo4j_graph(doc_id)
|
||||
except RuntimeError:
|
||||
raise HTTPException(status_code=503, detail="Neo4j is not configured")
|
||||
except GraphNotFoundError:
|
||||
raise HTTPException(status_code=404, detail=f"No graph for document {doc_id}")
|
||||
except GraphTooLargeError as e:
|
||||
raise HTTPException(status_code=413, detail=f"Graph too large: {e.page_count} pages (cap {e.max_pages})")
|
||||
return _payload_to_response(payload)
|
||||
|
||||
@router.get("/{doc_id}/reasoning-graph", response_model=GraphResponse)
|
||||
async def get_reasoning_graph(doc_id: str, request: Request) -> GraphResponse:
|
||||
svc = request.app.state.graph_service
|
||||
try:
|
||||
payload = await svc.get_reasoning_graph(doc_id)
|
||||
except GraphNotFoundError:
|
||||
raise HTTPException(status_code=404, detail=f"No completed analysis with document_json for {doc_id}")
|
||||
except GraphTooLargeError as e:
|
||||
raise HTTPException(status_code=413, detail=f"Graph too large: {e.page_count} pages (cap {e.max_pages})")
|
||||
return _payload_to_response(payload)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/main.py` (ajouter le wiring)
|
||||
|
||||
```python
|
||||
from services.graph_service import GraphService
|
||||
app.state.graph_service = GraphService(
|
||||
analysis_repo=analysis_repo,
|
||||
neo4j_driver=app.state.neo4j,
|
||||
max_pages=settings.max_graph_pages,
|
||||
)
|
||||
```
|
||||
|
||||
### Step 2.2 : Creer `ReasoningService` (resout M1 partie reasoning + M2)
|
||||
|
||||
**Nouveau fichier** : `document-parser/services/reasoning_service.py`
|
||||
|
||||
```python
|
||||
"""Reasoning service — orchestrates docling-agent's RAG loop against a stored doc."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from domain.ports import AnalysisRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReasoningConfig:
|
||||
enabled: bool = False
|
||||
ollama_host: str = "http://localhost:11434"
|
||||
default_model_id: str = "gpt-oss:20b"
|
||||
|
||||
|
||||
class ReasoningDisabledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ReasoningDepsNotInstalledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DocumentNotReadyError(Exception):
|
||||
def __init__(self, doc_id: str):
|
||||
self.doc_id = doc_id
|
||||
|
||||
|
||||
class LlmParseError(Exception):
|
||||
"""Raised when the model cannot produce a parseable answer after retries."""
|
||||
def __init__(self, model_id: str):
|
||||
self.model_id = model_id
|
||||
|
||||
|
||||
@dataclass
|
||||
class RagIteration:
|
||||
iteration: int
|
||||
section_ref: str
|
||||
reason: str
|
||||
section_text_length: int
|
||||
can_answer: bool
|
||||
response: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class RagResult:
|
||||
answer: str
|
||||
iterations: list[RagIteration]
|
||||
converged: bool
|
||||
|
||||
|
||||
class ReasoningService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
analysis_repo: AnalysisRepository,
|
||||
config: ReasoningConfig,
|
||||
) -> None:
|
||||
self._analysis_repo = analysis_repo
|
||||
self._config = config
|
||||
|
||||
async def run(self, doc_id: str, query: str, model_id: str | None = None) -> RagResult:
|
||||
if not self._config.enabled:
|
||||
raise ReasoningDisabledError()
|
||||
|
||||
latest = await self._analysis_repo.find_latest_completed_by_document(doc_id)
|
||||
if latest is None or not latest.document_json:
|
||||
raise DocumentNotReadyError(doc_id)
|
||||
|
||||
try:
|
||||
from docling_agent.agents import DoclingRAGAgent
|
||||
from docling_core.types.doc.document import DoclingDocument
|
||||
from mellea.backends.model_ids import ModelIdentifier
|
||||
except ImportError as e:
|
||||
raise ReasoningDepsNotInstalledError() from e
|
||||
|
||||
# See rapport-08 security INFO : to replace by kwarg once lib supports it
|
||||
os.environ["OLLAMA_HOST"] = self._config.ollama_host
|
||||
raw_model_id = model_id or self._config.default_model_id
|
||||
|
||||
doc = DoclingDocument.model_validate_json(latest.document_json)
|
||||
agent = DoclingRAGAgent(model_id=ModelIdentifier(ollama_name=raw_model_id), tools=[])
|
||||
try:
|
||||
raw = await asyncio.to_thread(agent._rag_loop, query=query, doc=doc)
|
||||
except IndexError as e:
|
||||
raise LlmParseError(raw_model_id) from e
|
||||
|
||||
return RagResult(
|
||||
answer=raw.answer,
|
||||
iterations=[RagIteration(**it.model_dump()) for it in raw.iterations],
|
||||
converged=raw.converged,
|
||||
)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/api/reasoning.py` (devient mince — ~50 lignes au lieu de 148)
|
||||
|
||||
```python
|
||||
@router.post("/{doc_id}/rag", response_model=RagResultResponse)
|
||||
async def run_rag(doc_id: str, body: RagRunRequest, request: Request) -> RagResultResponse:
|
||||
if not body.query.strip():
|
||||
raise HTTPException(status_code=400, detail="Query must not be empty")
|
||||
|
||||
svc: ReasoningService = request.app.state.reasoning_service
|
||||
try:
|
||||
result = await svc.run(doc_id, body.query, body.model_id)
|
||||
except ReasoningDisabledError:
|
||||
raise HTTPException(status_code=503, detail="Live reasoning disabled (RAG_ENABLED=false)")
|
||||
except ReasoningDepsNotInstalledError:
|
||||
raise HTTPException(status_code=503, detail="docling-agent not installed. `pip install docling-agent mellea`.")
|
||||
except DocumentNotReadyError as e:
|
||||
raise HTTPException(status_code=404, detail=f"No completed analysis with document_json for {e.doc_id}")
|
||||
except LlmParseError as e:
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"The model '{e.model_id}' couldn't produce a parseable answer. Try a different model.",
|
||||
)
|
||||
return _result_to_response(result)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/main.py` (wiring)
|
||||
|
||||
```python
|
||||
from services.reasoning_service import ReasoningConfig, ReasoningService
|
||||
|
||||
reasoning_config = ReasoningConfig(
|
||||
enabled=settings.rag_enabled,
|
||||
ollama_host=settings.ollama_host,
|
||||
default_model_id=settings.rag_model_id,
|
||||
)
|
||||
app.state.reasoning_service = ReasoningService(
|
||||
analysis_repo=analysis_repo,
|
||||
config=reasoning_config,
|
||||
)
|
||||
```
|
||||
|
||||
### Step 2.3 : Q1 — deplacer `_chunk_to_dict` vers `api/schemas.py`
|
||||
|
||||
**Fichier** : `document-parser/api/schemas.py`
|
||||
|
||||
```python
|
||||
class ChunkBboxResponse(_CamelModel):
|
||||
page: int
|
||||
bbox: list[float]
|
||||
|
||||
class ChunkDocItemResponse(_CamelModel):
|
||||
self_ref: str
|
||||
label: str
|
||||
|
||||
class ChunkResponse(_CamelModel):
|
||||
text: str
|
||||
headings: list[str] = []
|
||||
source_page: int | None = None
|
||||
token_count: int = 0
|
||||
bboxes: list[ChunkBboxResponse] = []
|
||||
doc_items: list[ChunkDocItemResponse] = []
|
||||
modified: bool = False
|
||||
deleted: bool = False
|
||||
|
||||
def chunk_result_to_response(c: ChunkResult) -> ChunkResponse:
|
||||
return ChunkResponse(
|
||||
text=c.text,
|
||||
headings=c.headings,
|
||||
source_page=c.source_page,
|
||||
token_count=c.token_count,
|
||||
bboxes=[ChunkBboxResponse(page=b.page, bbox=b.bbox) for b in c.bboxes],
|
||||
doc_items=[ChunkDocItemResponse(self_ref=d.self_ref, label=d.label) for d in c.doc_items],
|
||||
)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/services/analysis_service.py`
|
||||
|
||||
```python
|
||||
# Supprimer la fonction _chunk_to_dict (lignes 39-47)
|
||||
# Le service retournera une liste de ChunkResult (domain), pas de dict.
|
||||
# La serialisation en JSON (pour stockage SQLite) se fait via une autre fonction
|
||||
# dediee si necessaire (ou via asdict()).
|
||||
```
|
||||
|
||||
### Step 2.4 : M3 — purger `MAX_PAGES = 200` en dur
|
||||
|
||||
**Fichier** : `document-parser/api/graph.py`
|
||||
- Supprimer `MAX_PAGES = 200` (ligne 24). Le cap vient maintenant de `GraphService._max_pages`.
|
||||
|
||||
**Fichier** : `document-parser/infra/docling_graph.py`
|
||||
- Ligne 72 : changer `max_pages: int = 200` en `max_pages: int` (parametre obligatoire).
|
||||
|
||||
**Fichier** : `document-parser/infra/neo4j/queries.py`
|
||||
- Ligne 147 : meme changement.
|
||||
|
||||
Verification :
|
||||
```bash
|
||||
grep -rn "max_pages.*=.*200\|MAX_PAGES" document-parser --include="*.py" --exclude-dir=.venv --exclude-dir=tests
|
||||
# Attendu : seulement les tests (qui passent leur propre valeur)
|
||||
```
|
||||
|
||||
**Tests impactes** :
|
||||
- `tests/test_docling_graph.py` : verifier que les appels passent bien `max_pages`.
|
||||
- `tests/test_graph_api.py` : idem.
|
||||
- Nouveaux tests : `tests/test_graph_service.py`, `tests/test_reasoning_service.py` (extraire la logique testee dans test_graph_api.py et test_reasoning_api.py qui deviennent des tests HTTP fins).
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Decouplage frontend (B2) ≈ 2 jours
|
||||
|
||||
### Strategie
|
||||
|
||||
Deux options :
|
||||
|
||||
**Option A — strict** : deplacer tous les composants partages vers `frontend/src/shared/ui/viewer/`.
|
||||
- Plus long, meilleur score audit, mais refactor important des chemins d'import.
|
||||
|
||||
**Option B — pragmatique** : accepter `features/X/index.ts` comme "public API" d'une feature. Refuser uniquement les imports profonds (`features/X/ui/Y.vue`, `features/X/store`) depuis une autre feature.
|
||||
- Plus rapide, necessite d'ajouter une lint rule pour enforcer.
|
||||
|
||||
**Recommande : mix A+B** :
|
||||
- Composants reellement partages par 3+ features -> `shared/ui/`.
|
||||
- Stores cross-feature -> remplacer par props au niveau page.
|
||||
- Import via `features/X/index.ts` accepte si strictement public API (pas de store).
|
||||
|
||||
### Step 3.1 : Extraire styles reasoning du GraphView (casse le cycle)
|
||||
|
||||
**Fichier** : `frontend/src/features/analysis/ui/GraphView.vue`
|
||||
|
||||
```typescript
|
||||
// Supprimer :
|
||||
// import { reasoningOverlayStyles } from '../../reasoning/graphReasoningOverlay'
|
||||
// Ajouter dans defineProps :
|
||||
const props = defineProps<{
|
||||
// ... existants
|
||||
extraStyles?: CytoscapeStyle[] // Injected by parent feature (e.g. reasoning overlay)
|
||||
}>()
|
||||
// Dans la construction Cytoscape :
|
||||
const allStyles = [...baseStyles, ...(props.extraStyles ?? [])]
|
||||
```
|
||||
|
||||
**Fichier** : `frontend/src/features/reasoning/ui/ReasoningWorkspace.vue`
|
||||
|
||||
```typescript
|
||||
// Importer le style localement :
|
||||
import { reasoningOverlayStyles } from '../graphReasoningOverlay'
|
||||
// Passer au GraphView via prop :
|
||||
<GraphView ref="graphViewRef" :extra-styles="reasoningOverlayStyles" ... />
|
||||
```
|
||||
|
||||
**Resultat** : le cycle `analysis <-> reasoning` est brise (reasoning depend d'analysis, pas l'inverse).
|
||||
|
||||
### Step 3.2 : Deplacer les composants reellement partages vers `shared/ui/viewer/`
|
||||
|
||||
Candidats (utilises par >= 2 features) :
|
||||
- `StructureViewer.vue` — utilise par `analysis` ET `reasoning`
|
||||
- `GraphView.vue` — utilise par `analysis` ET `reasoning`
|
||||
- `BboxOverlay.vue` — utilise par `analysis` (+ futur reasoning)
|
||||
|
||||
Pas deplaces (utilises par 1 seul feature) :
|
||||
- `NodeDetailsPanel.vue`, `ResultTabs.vue`, `MarkdownViewer.vue`, `ImageGallery.vue` — specifique `analysis`
|
||||
- `AnalysisPanel.vue` — orchestrateur analysis, OK dans `features/analysis`
|
||||
|
||||
**Migration** :
|
||||
```bash
|
||||
mkdir -p frontend/src/shared/ui/viewer
|
||||
git mv frontend/src/features/analysis/ui/StructureViewer.vue frontend/src/shared/ui/viewer/StructureViewer.vue
|
||||
git mv frontend/src/features/analysis/ui/GraphView.vue frontend/src/shared/ui/viewer/GraphView.vue
|
||||
git mv frontend/src/features/analysis/ui/BboxOverlay.vue frontend/src/shared/ui/viewer/BboxOverlay.vue
|
||||
```
|
||||
|
||||
Mettre a jour les imports (14 fichiers environ). Utiliser l'alias `@/shared/ui/viewer/...`.
|
||||
|
||||
**Fichier** : `frontend/src/features/analysis/index.ts` — supprimer les re-exports de StructureViewer/BboxOverlay.
|
||||
|
||||
### Step 3.3 : `getPreviewUrl` vers `shared/api/documents.ts`
|
||||
|
||||
**Nouveau fichier** : `frontend/src/shared/api/documents.ts`
|
||||
|
||||
```typescript
|
||||
/** Preview URL for a document page (served by the backend). */
|
||||
export function getPreviewUrl(id: string, page = 1, dpi = 150): string {
|
||||
return `/api/documents/${id}/preview?page=${page}&dpi=${dpi}`
|
||||
}
|
||||
```
|
||||
|
||||
**Fichier** : `frontend/src/features/document/api.ts` — re-export pour compat interne mais consommer la version shared :
|
||||
```typescript
|
||||
export { getPreviewUrl } from '../../shared/api/documents'
|
||||
```
|
||||
|
||||
**Fichier** : `frontend/src/features/analysis/ui/StructureViewer.vue` (devenu `shared/ui/viewer/`) et autres usages :
|
||||
```typescript
|
||||
import { getPreviewUrl } from '@/shared/api/documents'
|
||||
```
|
||||
|
||||
### Step 3.4 : Eliminer les `useXxxStore` cross-feature
|
||||
|
||||
Schema cible : les stores d'un feature ne sont accedes qu'a l'interieur de ce feature. Cross-feature -> props au niveau page.
|
||||
|
||||
**Cas `chunking/ui/ChunkPanel.vue:228` -> `useAnalysisStore`** :
|
||||
- Ce dont a besoin ChunkPanel : l'analyse en cours (pour connaitre les chunks).
|
||||
- Fix : `StudioPage.vue` passe `:analysis="currentAnalysis"` a `<ChunkPanel>`.
|
||||
- `ChunkPanel` devient purement driven par props.
|
||||
|
||||
**Cas `reasoning/ui/DocumentView.vue:34` -> `useAnalysisStore`** :
|
||||
- Besoin : les pages du document analyse.
|
||||
- Fix : passer `:pages="pages"` en prop (calcul au niveau `ReasoningWorkspace` ou `ReasoningPage`).
|
||||
|
||||
**Cas `reasoning/ui/ReasoningDocPicker.vue:82-83` -> `useAnalysisStore`, `useDocumentStore`** :
|
||||
- Besoin : liste des documents + leur statut d'analyse.
|
||||
- Fix : creer un `useReasoningEligibleDocs()` composable dans `reasoning/` qui FETCH directement via API (pas de dependance au store d'un autre feature). Ou : passer la liste filtree en prop depuis la page.
|
||||
|
||||
**Cas `analysis/ui/AnalysisPanel.vue:61` -> `useDocumentStore`** :
|
||||
- Besoin : le document courant et sa liste.
|
||||
- Fix : `AnalysisPanel` recoit `:documents`, `:selectedDocument` en props ; emits `@select-document`, `@upload-document`.
|
||||
|
||||
**Cas `settings/ui/SettingsPanel.vue:70` -> `useFeatureFlagStore`** :
|
||||
- Feature-flags est transversal. Acceptable qu'un autre feature le lise.
|
||||
- Mais strict audit : exposer via `useFeatureFlag()` composable dans `shared/composables/` plutot que le store directement.
|
||||
|
||||
**Fichier** : `frontend/src/shared/composables/useFeatureFlag.ts` (existe-t-il ? `grep` : oui, `frontend/src/features/feature-flags/useFeatureFlag.test.ts`). Deplacer le composable vers `shared/composables/` et laisser le store dans `features/feature-flags/`.
|
||||
|
||||
### Step 3.5 : Lint rule (ESLint) pour prevenir regression
|
||||
|
||||
**Fichier** : `frontend/eslint.config.js` (ou `.eslintrc`)
|
||||
|
||||
```javascript
|
||||
{
|
||||
files: ['src/features/**/*.{ts,vue}'],
|
||||
rules: {
|
||||
'no-restricted-imports': ['error', {
|
||||
patterns: [
|
||||
{
|
||||
group: ['../../*/store', '../../*/ui/*', '../../*/api'],
|
||||
message: 'Features must not import from other features. Use shared/ or props/events.',
|
||||
},
|
||||
],
|
||||
}],
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
**Tests impactes** :
|
||||
- Tout test important `features/analysis/ui/StructureViewer.vue` a renommer en `shared/ui/viewer/StructureViewer.vue`.
|
||||
- `frontend/src/features/analysis/ui/StructureViewer.vue` existe-t-il comme fichier test ? Non, pas de `.test` pour les composants UI lourds (pattern du projet).
|
||||
|
||||
**Risques** :
|
||||
- Casse des tests e2e Karate si les selecteurs `data-e2e` etaient dans les composants deplaces -> verifier (les selecteurs restent identiques si le composant n'est pas modifie, juste deplace).
|
||||
- HMR peut etre capricieux pendant la migration -> faire un vrai restart du dev server.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Cleanup (M5 + Q2-Q6) ≈ 0.5 jour
|
||||
|
||||
### Step 4.1 : M5 — CHANGELOG `[Unreleased]`
|
||||
|
||||
**Fichier** : `CHANGELOG.md`
|
||||
|
||||
Ajouter entre la ligne 6 et 7 (`## [0.4.0]`) :
|
||||
|
||||
```markdown
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Reasoning-trace viewer: import a `docling-agent` sidecar JSON and overlay RAG iterations on the document graph/PDF views
|
||||
- Live reasoning runner: `POST /api/documents/:id/rag` invokes `docling-agent`'s Chunkless RAG loop against a stored DoclingDocument (disabled by default via `RAG_ENABLED=false`; requires Ollama reachable + `docling-agent` and `mellea` installed)
|
||||
- Neo4j graph storage: DoclingDocument tree persisted via TreeWriter with Document/Element/Page/Provenance nodes; chunks persisted via ChunkWriter with DERIVED_FROM edges
|
||||
- Graph API endpoints: `GET /api/documents/:id/graph` (Neo4j-backed, full graph with chunks) and `GET /api/documents/:id/reasoning-graph` (SQLite-only, no Neo4j dep)
|
||||
- Frontend feature `reasoning/` with focus mode, iteration navigation, bidirectional graph/document sync
|
||||
- Env vars: `RAG_ENABLED`, `OLLAMA_HOST`, `RAG_MODEL_ID`, `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD`, `MAX_GRAPH_PAGES`
|
||||
- Domain ports: `GraphWriter` (Neo4j-backed), `EmbeddingService`, `VectorStore`
|
||||
|
||||
### Changed
|
||||
|
||||
- Services no longer import `infra.neo4j` or `infra.serve_converter` directly — graph persistence goes through `GraphWriter` port; batching capability is exposed as `DocumentConverter.supports_batching` property (audit remediation 06-SOLID)
|
||||
- `StructureViewer`, `GraphView`, `BboxOverlay` moved to `frontend/src/shared/ui/viewer/` (audit remediation 07-decoupling)
|
||||
|
||||
### Fixed
|
||||
|
||||
- (a completer)
|
||||
```
|
||||
|
||||
### Step 4.2 : Q5 — `.env.example` complet
|
||||
|
||||
**Fichier** : `.env.example` (ajout en fin)
|
||||
|
||||
```bash
|
||||
# --- Live reasoning (docling-agent runner) — disabled by default ---
|
||||
# RAG_ENABLED=false
|
||||
# OLLAMA_HOST=http://localhost:11434
|
||||
# RAG_MODEL_ID=gpt-oss:20b
|
||||
|
||||
# --- Rate limiting (requests per minute per IP, 0 = disabled) ---
|
||||
# RATE_LIMIT_RPM=100
|
||||
|
||||
# --- Timeouts (seconds) — must satisfy document < lock < conversion ---
|
||||
# DOCUMENT_TIMEOUT=120
|
||||
# LOCK_TIMEOUT=300
|
||||
# CONVERSION_TIMEOUT=900
|
||||
|
||||
# --- Batch processing for very large PDFs (0 = disabled) ---
|
||||
# BATCH_PAGE_SIZE=0
|
||||
|
||||
# --- OpenSearch max chunks returned per document ---
|
||||
# OPENSEARCH_DEFAULT_LIMIT=1000
|
||||
|
||||
# --- Max pages per graph query (returns 413 beyond) ---
|
||||
# MAX_GRAPH_PAGES=200
|
||||
|
||||
# --- Default table analysis mode: "accurate" or "fast" ---
|
||||
# DEFAULT_TABLE_MODE=accurate
|
||||
```
|
||||
|
||||
### Step 4.3 : Q6 — Nginx cache statique
|
||||
|
||||
**Fichier** : `nginx.conf` (inserer avant `location / {`)
|
||||
|
||||
```nginx
|
||||
# Hashed assets (Vite emits content-hashed filenames) — cache 1 year
|
||||
location ~* \.(?:js|css|woff2?|ttf|otf|svg|png|jpg|jpeg|webp|gif|ico)$ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
try_files $uri =404;
|
||||
}
|
||||
```
|
||||
|
||||
### Step 4.4 : Q2 — decouper les fonctions longues (non bloquant, best effort)
|
||||
|
||||
**Cibles prioritaires** (ordre de gain pedagogique) :
|
||||
1. `infra/neo4j/tree_writer.py:67` `write_document` (228L) — decouper en :
|
||||
- `_wipe_existing(tx, doc_id)`
|
||||
- `_write_document_node(tx, doc_id, filename, ...)`
|
||||
- `_write_pages(tx, doc_id, pages)`
|
||||
- `_write_elements_and_provenances(tx, ...)`
|
||||
- `_write_structural_edges(tx, ...)`
|
||||
2. `infra/neo4j/queries.py:143` `fetch_graph` (126L) — une helper par groupe de nodes/edges.
|
||||
3. Si le refactor Phase 2 a bien fait son job, `api/reasoning.py:run_rag` et `api/graph.py:get_reasoning_graph` sont deja < 30L.
|
||||
|
||||
### Step 4.5 : Q3 — signatures avec dataclass context
|
||||
|
||||
**Fichier** : `document-parser/services/analysis_service.py`
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class AnalysisContext:
|
||||
job_id: str
|
||||
file_path: str
|
||||
filename: str
|
||||
pipeline_options: dict | None = None
|
||||
chunking_options: dict | None = None
|
||||
|
||||
# Remplacer :
|
||||
# async def _run_analysis(self, job_id, file_path, filename, pipeline_options, chunking_options)
|
||||
# par :
|
||||
# async def _run_analysis(self, ctx: AnalysisContext)
|
||||
```
|
||||
|
||||
**Fichier** : `document-parser/domain/models.py`
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CompletionPayload:
|
||||
markdown: str
|
||||
html: str
|
||||
pages_json: str
|
||||
document_json: str | None = None
|
||||
chunks_json: str | None = None
|
||||
|
||||
# Sur AnalysisJob :
|
||||
def mark_completed(self, payload: CompletionPayload) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
### Step 4.6 : Q4 — splitter les gros composants Vue (planification)
|
||||
|
||||
Hors scope immediate — trop gros pour la fenetre 0.5.0. **Acter en dette** :
|
||||
- `StudioPage.vue` (1450L) : a decouper en `StudioUploadSection.vue`, `StudioAnalysisSection.vue`, `StudioResultsSection.vue` en 0.6.
|
||||
- `ChunkPanel.vue` (801L), `GraphView.vue` (695L), `ResultTabs.vue` (690L) : ticket dedie post-0.5.
|
||||
|
||||
---
|
||||
|
||||
## Re-audit delta apres remediations
|
||||
|
||||
Apres P1 + P2 + P3 + P4, **re-lancer uniquement** :
|
||||
|
||||
| Audit | Raison |
|
||||
|-------|--------|
|
||||
| 01 Hexa Arch | Verifier que M1 (graph+reasoning services) a elimine le MAJ |
|
||||
| 03 Clean Code | Verifier run_rag < 30L et SRP ok |
|
||||
| 05 DRY | Verifier MAX_PAGES purge |
|
||||
| 06 SOLID | Verifier CRIT B1 resolu + MAJ M4 resolu |
|
||||
| 07 Decouplage | Verifier CRIT B2 resolu (grep imports cross-feature hors `shared/`) |
|
||||
| 10 CI/Build | Verifier `.env.example` complet |
|
||||
| 11 Documentation | Verifier CHANGELOG + version bump |
|
||||
|
||||
Audits **02, 04, 08, 09, 12** : pas de changement attendu, on peut les skipper au re-audit.
|
||||
|
||||
Commande :
|
||||
```
|
||||
Re-audite uniquement les audits 01, 03, 05, 06, 07, 10, 11 sur la branche courante en suivant docs/audit/master.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ordonnancement git/PR recommande
|
||||
|
||||
| PR | Branche | Contenu | Audits concernes |
|
||||
|----|---------|---------|------------------|
|
||||
| PR-A | `fix/0.5.0-port-graphwriter` | Phase 1 (B1 + M4) | 06 |
|
||||
| PR-B | `fix/0.5.0-extract-services` | Phase 2 (M1 + M2 + M3 + Q1) | 01, 03, 05 |
|
||||
| PR-C | `fix/0.5.0-frontend-decoupling` | Phase 3 (B2) | 07 |
|
||||
| PR-D | `chore/0.5.0-release-prep` | Phase 4 (M5 + Q5 + Q6 + Q2 + Q3) | 10, 11 |
|
||||
|
||||
Chaque PR doit rester petit (revue + CI courte). Base : toutes branchees sur `release/0.5.0` cree depuis `feature/reasoning-trace` (en suivant la convention git flow du projet).
|
||||
|
||||
---
|
||||
|
||||
## Estimation globale
|
||||
|
||||
| Phase | Duree | Effort (dev-jour) |
|
||||
|-------|-------|-------------------|
|
||||
| 1 — Ports | 1j | 1 |
|
||||
| 2 — Services | 1.5j | 1.5 |
|
||||
| 3 — Frontend | 2j | 2 |
|
||||
| 4 — Cleanup | 0.5j | 0.5 |
|
||||
| **Total** | **5j** | **5 dev-jour** |
|
||||
|
||||
Avec 2 devs en parallele (un backend PR-A+B, un frontend PR-C+D), **3 jours calendaires** suffisent.
|
||||
|
||||
---
|
||||
|
||||
## Validation finale avant tag 0.5.0
|
||||
|
||||
- [ ] PR-A, B, C, D mergees dans `release/0.5.0`
|
||||
- [ ] `ruff check . && ruff format --check .` vert
|
||||
- [ ] `pytest tests/ -v` vert (backend)
|
||||
- [ ] `npm run lint && npm run type-check && npm run test:run` vert (frontend)
|
||||
- [ ] `npm run build` produit un bundle sans warning
|
||||
- [ ] CI GitHub Actions verte sur `release/0.5.0`
|
||||
- [ ] Re-audit delta ci-dessus repasse GO (CRIT = 0, MAJ <= 3)
|
||||
- [ ] `CHANGELOG.md` : renommer `[Unreleased]` en `[0.5.0] - 2026-04-XX`
|
||||
- [ ] `frontend/package.json` : bump `"version": "0.5.0"`
|
||||
- [ ] Tag git `v0.5.0` sur `release/0.5.0`
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
# Re-audit ciblé — Release 0.5.0
|
||||
|
||||
**Date** : 2026-04-29
|
||||
**Branche** : `fix/release-0.5.0-audit-remediation` (off `release/0.5.0`)
|
||||
**Périmètre** : tous les écarts CRITICAL et MAJOR du `summary.md` initial (regle master §7).
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Tableau de bord — avant / après
|
||||
|
||||
| # | Audit | Score initial | CRIT initial | MAJ initial | Score post-remédiation | CRIT | MAJ | Verdict |
|
||||
|----|----------------------|--------------:|-------------:|------------:|-----------------------:|-----:|----:|---------|
|
||||
| 01 | Clean Architecture | 100 | 0 | 0 | **100** | 0 | 0 | GO |
|
||||
| 02 | DDD | 91 | 0 | 1 | **~95** | 0 | 0 | GO |
|
||||
| 03 | Clean Code | 83 | 0 | 0 | 83 *(inchangé)* | 0 | 0 | GO |
|
||||
| 04 | KISS | 87.5 | 0 | 0 | 87.5 *(inchangé)* | 0 | 0 | GO |
|
||||
| 05 | DRY | 71 | 0 | 2 | **~88** | 0 | 0 | GO |
|
||||
| 06 | SOLID | 85 | 0 | 1 | **~95** | 0 | 0 | GO |
|
||||
| 07 | Découplage | 86 | 0 | 1 | **~93** | 0 | 0 | GO |
|
||||
| 08 | Sécurité | 91 | 0 | 2 | **~98** | 0 | 0 | GO |
|
||||
| 09 | Tests | 94 | 0 | 1 | **~97** | 0 | 0 | GO |
|
||||
| 10 | CI / Build | 91 | 0 | 1 | **~96** | 0 | 0 | GO |
|
||||
| 11 | Documentation | **44** | **1** | 2 | **100** | 0 | 0 | GO |
|
||||
| 12 | Performance | 86.67 | 0 | 1 | **~93** | 0 | 0 | GO |
|
||||
|
||||
**Score global (moyenne simple)** : **84.2 → ~94/100**
|
||||
**CRIT totaux** : 1 → **0**
|
||||
**MAJ totaux** : 12 → **0**
|
||||
|
||||
---
|
||||
|
||||
## Vérification item par item
|
||||
|
||||
### CRITICAL (1)
|
||||
|
||||
| # | Item | Statut | Preuve |
|
||||
|---|------|--------|--------|
|
||||
| 11.CRIT | `CHANGELOG.md` sans section `[0.5.0]` | ✅ Résolu | [CHANGELOG.md:7](CHANGELOG.md:7) — `## [0.5.0] - 2026-04-28` |
|
||||
|
||||
### MAJOR (12)
|
||||
|
||||
| # | Item | Statut | Preuve |
|
||||
|---|------|--------|--------|
|
||||
| 11.MAJ.1 | `frontend/package.json` à `0.4.0` | ✅ Résolu | [frontend/package.json:3](frontend/package.json:3) — `"version": "0.5.0"` |
|
||||
| 11.MAJ.2 | Modifications 0.5.0 non documentées | ✅ Résolu | CHANGELOG `[0.5.0]` détaille Added / Changed / Fixed |
|
||||
| 08.MAJ.1 | Mot de passe Neo4j `changeme` | ✅ Résolu (cadrage dev) | [docker-compose.yml:1-26](docker-compose.yml:1) — header dev-only ; [main.py:104-110](document-parser/main.py:104) — warning au boot si `NEO4J_URI` set + password=`changeme` |
|
||||
| 08.MAJ.2 | OpenSearch `DISABLE_SECURITY_PLUGIN=true` | ✅ Résolu (cadrage dev) | [docker-compose.yml:47-52](docker-compose.yml:47) — commentaire "DEV ONLY" + lien doc OpenSearch |
|
||||
| 05.MAJ.1 | URL d'API hardcodée frontend | ✅ Résolu | `apiUrl` était du code mort → suppression du ref + des i18n keys orphelines (`settings.apiUrl`) |
|
||||
| 05.MAJ.2 | Clés `localStorage` dispersées | ✅ Résolu | [frontend/src/shared/storage/keys.ts](frontend/src/shared/storage/keys.ts) — `STORAGE_KEYS` ; [features/settings/store.ts](frontend/src/features/settings/store.ts) consomme |
|
||||
| 02.MAJ | Ubiquitous language `job` ↔ `analysis` | ✅ Résolu | Path params `{job_id}` → `{analysis_id}` dans [api/analyses.py](document-parser/api/analyses.py) + [api/ingestion.py](document-parser/api/ingestion.py) (URLs identiques côté client) |
|
||||
| 06.MAJ | LSP — `isinstance(ServeConverter)` | ✅ Résolu | [domain/ports.py:DocumentConverter](document-parser/domain/ports.py) — `supports_page_batching: bool` exposé via le port ; [services/analysis_service.py:340](document-parser/services/analysis_service.py:340) lit le port |
|
||||
| 07.MAJ | Couplage inter-feature dans tests | ✅ Résolu | Test déplacé : [frontend/src/__tests__/integration/history-navigation.test.ts](frontend/src/__tests__/integration/history-navigation.test.ts) ; les feature folders restent self-contained |
|
||||
| 09.MAJ | 18 assertions vagues `assert X is not None` | ✅ Résolu | 8 assertions vraiment terminales resserrées (`isinstance(.., datetime)`, comparaisons exactes) ; les 10 restantes sont des type-narrowings légitimes suivis d'assertions sur la valeur |
|
||||
| 10.MAJ | Ruff UP038 (`isinstance` union syntax) | ✅ Résolu | [infra/docling_tree.py:101](document-parser/infra/docling_tree.py:101) — `list \| tuple` ; `ruff check` passe (0 erreurs) |
|
||||
| 12.MAJ | I/O sync dans endpoint async | ✅ Résolu | [api/documents.py:119-122](document-parser/api/documents.py:119) — `Path(...).read_bytes` et `generate_preview` wrappés dans `asyncio.to_thread` |
|
||||
|
||||
---
|
||||
|
||||
## Renforcements indirects (volet 1 — reasoning)
|
||||
|
||||
Au-delà des écarts ciblés, la refacto reasoning consolide plusieurs audits :
|
||||
|
||||
- **01 Clean Architecture** : confirmation `grep -rE "^from docling_agent|^from docling_core|^from mellea" api/ domain/ services/` → **0 résultat**. Couplage upstream confiné à `infra/docling_agent_reasoning.py` + `infra/llm/ollama_provider.py`.
|
||||
- **06 SOLID — DIP** : `api/reasoning.py` consomme un port `ReasoningRunner` ; aucune classe concrète importée dans la couche API.
|
||||
- **07 Découplage** : un seul point de couplage à la lib upstream (l'adapter). Le `_rag_loop` privé est encapsulé + tracé via [docling-agent#26](https://github.com/docling-project/docling-agent/issues/26).
|
||||
- **08 Sécurité** : la mutation `os.environ["OLLAMA_HOST"]` par requête (race) est éliminée — l'adapter commit le host **une fois** au boot.
|
||||
- **09 Tests** : +17 tests adaptés (`test_docling_agent_reasoning.py`, `test_ollama_provider.py`), incluant un test d'isolation concurrence (R3).
|
||||
|
||||
---
|
||||
|
||||
## Items MIN / INFO restants (non-bloquants, planifiés 0.5.1+)
|
||||
|
||||
| # | Origine | Item | Plan |
|
||||
|---|---------|------|------|
|
||||
| 03.MIN.1 | Clean Code | `StudioPage.vue` (~1450L), `ChunkPanel.vue` (~801L), `ResultTabs.vue` (~690L) | Bloc E — découper en sous-composants en 0.5.1 |
|
||||
| 03.MIN.2 | Clean Code | 3 fonctions > 30 lignes / 4 fonctions > 4 paramètres | Bloc E — décomposition locale en 0.5.1 |
|
||||
| 04.MIN | KISS | Wrapper `_to_response`, accessors redondants `DocumentService` | Bloc F — arbitrage en 0.5.1 |
|
||||
| 05.MIN | DRY | Magic string [api/schemas.py:54](document-parser/api/schemas.py:54) | ✅ Résolu — `DOCUMENT_STATUS_UPLOADED` |
|
||||
| 04.INFO | KISS | Polling `setInterval/setTimeout` imbriqués | Bloc F |
|
||||
| 04.INFO | KISS | Overhead `DocumentConfig` / `IngestionConfig` dataclasses | Bloc F |
|
||||
| 08.INFO | Sécurité | Default LOG_LEVEL non-borné | Sera traité avec la prochaine vague observabilité |
|
||||
| 12.INFO | Performance | `find_all` implicite (pas de pagination forte) | Bloc E (split graph endpoint) en 0.5.1 |
|
||||
|
||||
---
|
||||
|
||||
## Verdict final post-remédiation : **GO** ✅
|
||||
|
||||
**Justification** :
|
||||
- Zéro écart CRITICAL (la règle absolue du master §3 est levée).
|
||||
- Zéro écart MAJOR — tous les MAJ initiaux sont résolus.
|
||||
- Score global ~94/100 ≥ 80 (seuil GO).
|
||||
- Tous les audits individuels passent en GO.
|
||||
- La pipeline de validation est verte : ruff + format + 446 pytest backend, ESLint + Prettier + vue-tsc + 202 vitest frontend.
|
||||
|
||||
**Conditions tenues** :
|
||||
1. ✅ CHANGELOG `[0.5.0]` complet
|
||||
2. ✅ `frontend/package.json` à `0.5.0`
|
||||
3. ✅ Sécurité dev-only documentée + warning au boot sur `changeme`
|
||||
4. ✅ DRY frontend (storage keys + dead apiUrl supprimé)
|
||||
5. ✅ Refacto archi reasoning (port + adapter + DI) — bonus volet 1
|
||||
|
||||
**Reste planifié hors release 0.5.0** :
|
||||
- Bloc E (découpage Vue files volumineux + signatures de fonctions back) → 0.5.1
|
||||
- Bloc F (KISS micro-optimisations + INFO) → 0.5.1
|
||||
|
||||
La release **0.5.0 peut être taguée** depuis `release/0.5.0` une fois la branche `fix/release-0.5.0-audit-remediation` mergée.
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
# Synthese d'audit — Release 0.5.0
|
||||
|
||||
**Date** : 2026-04-28
|
||||
**Branche** : `release/0.5.0`
|
||||
**Commit audite** : `b2e0af3`
|
||||
**Auditeur** : claude-code
|
||||
|
||||
---
|
||||
|
||||
## Tableau de bord
|
||||
|
||||
| # | Audit | Score | CRIT | MAJ | MIN | INFO | Verdict |
|
||||
|----|----------------------|---------|------|-----|-----|------|------------------|
|
||||
| 01 | Clean Architecture | 100 | 0 | 0 | 0 | 0 | GO |
|
||||
| 02 | DDD | 91 | 0 | 1 | 1 | 0 | GO |
|
||||
| 03 | Clean Code | 83 | 0 | 0 | 3 | 0 | GO |
|
||||
| 04 | KISS | 87.5 | 0 | 0 | 1 | 2 | GO |
|
||||
| 05 | DRY | 71 | 0 | 2 | 1 | 2 | GO CONDITIONNEL |
|
||||
| 06 | SOLID | 85 | 0 | 1 | 1 | 0 | GO |
|
||||
| 07 | Decouplage | 86 | 0 | 1 | 1 | 0 | GO CONDITIONNEL |
|
||||
| 08 | Securite | 91 | 0 | 2 | 0 | 1 | GO CONDITIONNEL |
|
||||
| 09 | Tests | 94 | 0 | 1 | 0 | 0 | GO |
|
||||
| 10 | CI / Build | 91 | 0 | 1 | 0 | 0 | GO CONDITIONNEL |
|
||||
| 11 | Documentation | **44** | **1**| 2 | 0 | 0 | **NO-GO** |
|
||||
| 12 | Performance | 86.67 | 0 | 1 | 1 | 1 | GO CONDITIONNEL |
|
||||
|
||||
**Score global (moyenne simple)** : **84.2 / 100**
|
||||
**Ecarts CRITICAL totaux** : **1**
|
||||
**Ecarts MAJOR totaux** : **12**
|
||||
**Ecarts MINOR totaux** : 8
|
||||
**Ecarts INFO totaux** : 6
|
||||
|
||||
---
|
||||
|
||||
## Ecarts CRITICAL (tous audits confondus)
|
||||
|
||||
1. **[11] CHANGELOG.md sans section `[0.5.0]`** — `CHANGELOG.md:7`
|
||||
La derniere section listee est `## [0.4.0] - 2026-04-13`. Aucune entree `[Unreleased]` ni `[0.5.0]`. Tagger 0.5.0 depuis ce HEAD livrerait un changelog mensonger qui omet les nouveautes de la release (reasoning-trace viewer, Neo4j graph storage, RAG endpoints, feature flags). **Bloquant absolu** par regle master §3.
|
||||
|
||||
---
|
||||
|
||||
## Top blockers (poids 3 / poids 2)
|
||||
|
||||
### Bloquant (poids 3)
|
||||
|
||||
- **[11] CHANGELOG sans section 0.5.0** — `CHANGELOG.md:7`
|
||||
Ajouter `## [0.5.0] - 2026-04-28` avec sous-sections `Added` / `Changed` / `Fixed` listant les changements depuis 0.4.0.
|
||||
|
||||
### Majeurs (poids 2) — a remediar pour passer a GO
|
||||
|
||||
- **[11] `frontend/package.json` toujours a `0.4.0`** — `frontend/package.json:3`
|
||||
Bumper a `"version": "0.5.0"`.
|
||||
- **[11] Modifications de la release 0.5.0 non documentees** — `CHANGELOG.md`
|
||||
Resolu en meme temps que le CRIT.
|
||||
- **[08] Mot de passe Neo4j par defaut `"changeme"`** — `document-parser/infra/settings.py:30,133`
|
||||
Forcer la lecture d'une variable d'environnement, supprimer le defaut, faire echouer le boot si absent en prod.
|
||||
- **[08] OpenSearch security plugin desactive** — `docker-compose.yml:26`
|
||||
`DISABLE_SECURITY_PLUGIN=true` doit etre off pour tout deploiement non-dev ; documenter explicitement le perimetre.
|
||||
- **[05] URL d'API hardcodee en frontend** — `frontend/src/features/settings/store.ts:23`
|
||||
Centraliser dans une constante `API_BASE_URL` lue depuis `import.meta.env`.
|
||||
- **[05] Cles localStorage en clair, dispersees** — `frontend/src/features/settings/store.ts:24-27`
|
||||
Centraliser dans une enum/objet `STORAGE_KEYS`.
|
||||
- **[02] Ubiquitous language "job" vs "analysis"** — `document-parser/api/ingestion.py:49-67`
|
||||
Aligner le vocabulaire HTTP sur le langage du domaine.
|
||||
- **[06] LSP — `isinstance()` sur adaptateur** — `document-parser/services/analysis_service.py:356`
|
||||
Supprimer le check de type concret ou exposer la capacite via le port.
|
||||
- **[07] Couplage inter-feature dans les tests** — `frontend/src/features/history/navigation.test.ts:30-31,67,82-83,143-144,169-170,188`
|
||||
Stubber les stores via injection au lieu d'imports directs.
|
||||
- **[09] Assertions vagues `assert X is not None`** — 18 occurrences en backend
|
||||
Ajouter une assertion sur la valeur attendue, pas seulement sur la presence.
|
||||
- **[10] Ruff UP038 — syntaxe `isinstance()` union** — `document-parser/infra/docling_tree.py:101`
|
||||
Appliquer la suggestion pyupgrade.
|
||||
- **[12] I/O synchrone dans endpoint async** — `document-parser/api/documents.py:115-116`
|
||||
Wrapper l'acces fichier dans `asyncio.to_thread(...)` pour ne pas bloquer la loop FastAPI.
|
||||
|
||||
---
|
||||
|
||||
## Quick wins (poids 1 — ameliorations rapides)
|
||||
|
||||
- **[03] Trois fichiers Vue > 300 lignes** — `frontend/src/views/StudioPage.vue` (~1450L), `frontend/src/.../ChunkPanel.vue` (~801L), `frontend/src/.../ResultTabs.vue` (~690L). Extraire des sous-composants.
|
||||
- **[03] Fonctions > 30 lignes** (3 cas) et **fonctions > 4 parametres** (4 cas). Decoupage local.
|
||||
- **[04] Wrapper trivial `_to_response`** — utiliser `Pydantic.model_validate()` directement.
|
||||
- **[04] Accessors redondants dans `DocumentService`**.
|
||||
- **[05] Magic string isolee** — `document-parser/api/schemas.py:49`.
|
||||
- **[12] Polling avec setInterval/setTimeout imbriques** — `frontend/src/features/settings/store.ts:27-39` et store analysis.
|
||||
|
||||
---
|
||||
|
||||
## Verdict final : **NO-GO**
|
||||
|
||||
**Justification** : 1 ecart CRITICAL non resolu dans l'audit 11 (Documentation) → regle absolue du master.md §3 : `tout ecart [CRIT] non resolu = NO-GO quel que soit le score`.
|
||||
|
||||
Bien que le score global (84.2/100) soit confortablement au-dessus du seuil GO et que 11 audits sur 12 soient au moins en GO CONDITIONNEL, le release 0.5.0 **ne peut pas etre tague** dans cet etat.
|
||||
|
||||
### Conditions pour passer a GO
|
||||
|
||||
**Bloquant absolu** (1 action) :
|
||||
1. Ajouter la section `## [0.5.0] - 2026-04-28` dans `CHANGELOG.md` avec `Added` / `Changed` / `Fixed`.
|
||||
|
||||
**Pour passer de GO CONDITIONNEL a GO** (4 actions, ~30 min) :
|
||||
2. Bumper `frontend/package.json` a `0.5.0`.
|
||||
3. Remplacer `"changeme"` par lecture d'env var obligatoire (`document-parser/infra/settings.py`).
|
||||
4. Documenter / corriger la desactivation du security plugin OpenSearch (`docker-compose.yml`).
|
||||
5. Centraliser l'URL d'API et les cles localStorage du frontend (`frontend/src/features/settings/store.ts`).
|
||||
|
||||
**Recommandation** : apres remediation des 5 points ci-dessus, re-auditer **uniquement** les audits 11, 08 et 05 (commande : `Re-audite uniquement les ecarts CRITICAL et MAJOR du rapport docs/audit/reports/release-0.5.0/summary.md`). Les autres MAJ peuvent etre planifies pour 0.5.1 sans bloquer le tag.
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
# Bounding Box Pipeline
|
||||
|
||||
The bbox pipeline is the core of Docling Studio's visual overlay. It transforms Docling's raw bounding box coordinates into pixel rectangles drawn on the canvas.
|
||||
|
||||
## The 3 Coordinate Spaces
|
||||
|
||||
```
|
||||
SPACE 1 — Docling (PDF points) SPACE 2 — Normalized (PDF points) SPACE 3 — Canvas (pixels)
|
||||
Variable origin per PDF Always TOPLEFT CSS pixels × devicePixelRatio
|
||||
|
||||
BOTTOMLEFT TOPLEFT
|
||||
(0,0) ──────→ x (0,0) ──────────→ x
|
||||
y ↑ (0,0) ──→ x │ │
|
||||
│ ┌───┐ t=700 │ ┌───┐ t=92 │ ┌───┐ t=92 │ ┌─────┐ y=105
|
||||
│ │ │ │ │ │ │ │ │ │ │ │
|
||||
│ └───┘ b=600 │ └───┘ b=192 │ └───┘ b=192 │ └─────┘ y=219
|
||||
│ ↓ y ↓ y ↓ y
|
||||
──┴──────→ x
|
||||
(0,0) Unit: pt PDF Unit: pt PDF Unit: CSS px
|
||||
```
|
||||
|
||||
### Space 1 — Docling Output
|
||||
|
||||
Docling's `BoundingBox` has 4 values `(l, t, r, b)` and a `coord_origin`:
|
||||
|
||||
- **BOTTOMLEFT** (standard PDF): `y=0` at the bottom of the page. `t > b` because "top" is further from origin.
|
||||
- **TOPLEFT** (some extractors): `y=0` at the top. `t < b` as expected.
|
||||
|
||||
Unit: **PDF points** (1 pt = 1/72 inch). US Letter = 612 × 792 pt, A4 = 595 × 842 pt.
|
||||
|
||||
### Space 2 — Normalized (TOPLEFT)
|
||||
|
||||
The backend normalizes all bboxes to TOPLEFT before sending to the frontend. This is what arrives in the JSON `pages` payload.
|
||||
|
||||
### Space 3 — Canvas Pixels
|
||||
|
||||
The frontend converts PDF points to CSS pixels, then the canvas renders at `devicePixelRatio` for Retina sharpness.
|
||||
|
||||
## Transformation 1 — `to_topleft_list()`
|
||||
|
||||
**File:** `document-parser/infra/bbox.py`
|
||||
|
||||
Normalizes any Docling bbox to `[left, top, right, bottom]` in TOPLEFT coordinates.
|
||||
|
||||
```python
|
||||
def to_topleft_list(bbox: BoundingBox, page_height: float) -> list[float]:
|
||||
normalized = bbox.to_top_left_origin(page_height)
|
||||
left, top, right, bottom = normalized.l, normalized.t, normalized.r, normalized.b
|
||||
|
||||
# Degenerate bbox: zero or negative dimensions — skip silently.
|
||||
if right <= left or bottom <= top:
|
||||
return list(EMPTY_BBOX) # [0, 0, 0, 0]
|
||||
|
||||
return [left, top, right, bottom]
|
||||
```
|
||||
|
||||
**Math (BOTTOMLEFT → TOPLEFT):**
|
||||
|
||||
```
|
||||
new_top = page_height - old_top
|
||||
new_bottom = page_height - old_bottom
|
||||
```
|
||||
|
||||
**Example** (US Letter page, 792pt):
|
||||
|
||||
```
|
||||
Input: l=50, t=700, r=200, b=600 (BOTTOMLEFT)
|
||||
|
||||
new_top = 792 - 700 = 92 ← near the top of the page
|
||||
new_bottom = 792 - 600 = 192 ← below the element
|
||||
|
||||
Output: [50, 92, 200, 192] (TOPLEFT, t < b ✓)
|
||||
```
|
||||
|
||||
!!! warning "Fallback page dimensions"
|
||||
If Docling doesn't report page dimensions (corrupted PDF), the backend falls back to US Letter (612 × 792 pt). A warning is logged. This may cause slight bbox misalignment on A4 or other formats.
|
||||
|
||||
## Transformation 2 — `computeScale()` + `bboxToRect()`
|
||||
|
||||
**File:** `frontend/src/features/analysis/bboxScaling.ts`
|
||||
|
||||
Maps PDF points to CSS pixels based on the displayed image size.
|
||||
|
||||
### Step 2a — Scale factors
|
||||
|
||||
```typescript
|
||||
function computeScale(displayWidth, displayHeight, pageWidth, pageHeight): Scale {
|
||||
return {
|
||||
sx: displayWidth / pageWidth, // CSS pixels per PDF point (X axis)
|
||||
sy: displayHeight / pageHeight, // CSS pixels per PDF point (Y axis)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Example:** image rendered at 700px wide for a 612pt page:
|
||||
|
||||
```
|
||||
sx = 700 / 612 ≈ 1.1438
|
||||
sy = 907 / 792 ≈ 1.1451 (≈ same ratio when aspect is preserved)
|
||||
```
|
||||
|
||||
### Step 2b — Bbox to pixel rectangle
|
||||
|
||||
```typescript
|
||||
function bboxToRect(bbox: [l, t, r, b], scale: Scale): Rect {
|
||||
return {
|
||||
x: l × sx, // left edge in pixels
|
||||
y: t × sy, // top edge in pixels
|
||||
w: (r - l) × sx, // width in pixels
|
||||
h: (b - t) × sy, // height in pixels
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Example** with bbox `[50, 92, 200, 192]` and `sx ≈ sy ≈ 1.14`:
|
||||
|
||||
```
|
||||
x = 50 × 1.14 = 57 px
|
||||
y = 92 × 1.14 = 105 px
|
||||
w = 150 × 1.14 = 171 px
|
||||
h = 100 × 1.14 = 114 px
|
||||
```
|
||||
|
||||
## Transformation 3 — Retina Rendering
|
||||
|
||||
**File:** `frontend/src/features/analysis/ui/BboxOverlay.vue`
|
||||
|
||||
The canvas backing store is scaled by `devicePixelRatio` for crisp rendering on HiDPI screens:
|
||||
|
||||
```typescript
|
||||
const dpr = window.devicePixelRatio || 1
|
||||
|
||||
// Backing store at device resolution
|
||||
canvas.width = displayWidth × dpr // e.g. 700 × 2 = 1400
|
||||
canvas.height = displayHeight × dpr
|
||||
|
||||
// CSS size stays the same
|
||||
canvas.style.width = displayWidth + 'px'
|
||||
canvas.style.height = displayHeight + 'px'
|
||||
|
||||
// Scale the drawing context
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0)
|
||||
```
|
||||
|
||||
After `setTransform`, all drawing commands use CSS pixel coordinates. The canvas automatically renders them at device resolution.
|
||||
|
||||
```
|
||||
ctx.strokeRect(57, 105, 171, 114)
|
||||
→ Actual pixels on Retina 2x: (114, 210, 342, 228)
|
||||
→ Visually identical but 2× sharper
|
||||
```
|
||||
|
||||
## Complete Pipeline Summary
|
||||
|
||||
```
|
||||
Docling BoundingBox bbox.py bboxScaling.ts BboxOverlay.vue
|
||||
(l, t, r, b) → to_topleft_list() → [l, t, r, b] → {x, y, w, h} → canvas
|
||||
BOTTOMLEFT or TOPLEFT flip Y if needed PDF points CSS pixels device pixels
|
||||
unit: PDF points + validation TOPLEFT × (sx, sy) × dpr
|
||||
```
|
||||
|
||||
## Validation & Edge Cases
|
||||
|
||||
Both backend and frontend guard against degenerate bboxes:
|
||||
|
||||
| Check | Backend (`bbox.py`) | Frontend (`bboxScaling.ts`) |
|
||||
|-------|--------------------|-----------------------------|
|
||||
| Zero/negative width | Returns `[0,0,0,0]` | Returns `EMPTY_RECT` |
|
||||
| Zero/negative height | Returns `[0,0,0,0]` | Returns `EMPTY_RECT` |
|
||||
| Zero page dimensions | N/A | `computeScale` returns `{1,1}` |
|
||||
|
||||
A degenerate bbox results in a zero-area rectangle that the canvas doesn't draw and hit-testing ignores.
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
# Issue Triage Process
|
||||
|
||||
How we classify, prioritize, and manage GitHub issues.
|
||||
|
||||
## Labels
|
||||
|
||||
### Type
|
||||
|
||||
| Label | Description |
|
||||
|-------|-------------|
|
||||
| `bug` | Something is broken |
|
||||
| `feature` | New functionality |
|
||||
| `enhancement` | Improvement to existing functionality |
|
||||
| `docs` | Documentation only |
|
||||
| `question` | Support / how-to question |
|
||||
| `chore` | Tooling, CI, dependencies |
|
||||
|
||||
### Priority
|
||||
|
||||
| Label | Response SLA | Fix SLA | Description |
|
||||
|-------|-------------|---------|-------------|
|
||||
| `priority: P0` | Same day | < 3 days | Critical — service down, data loss, security |
|
||||
| `priority: P1` | < 3 days | < 2 weeks | High — major feature broken, no workaround |
|
||||
| `priority: P2` | < 1 week | Next release | Medium — feature degraded, workaround exists |
|
||||
| `priority: P3` | < 2 weeks | Backlog | Low — minor, cosmetic, nice-to-have |
|
||||
|
||||
### Status
|
||||
|
||||
| Label | Description |
|
||||
|-------|-------------|
|
||||
| `needs-info` | Waiting for reporter to provide more details |
|
||||
| `confirmed` | Bug reproduced or feature accepted |
|
||||
| `good-first-issue` | Suitable for new contributors |
|
||||
| `help-wanted` | Open for community contribution |
|
||||
| `wont-fix` | Intentional behavior, out of scope, or won't be addressed |
|
||||
| `duplicate` | Already tracked in another issue |
|
||||
| `stale` | No activity for 30 days |
|
||||
|
||||
### Component
|
||||
|
||||
| Label | Maps to |
|
||||
|-------|---------|
|
||||
| `component: backend` | `document-parser/` |
|
||||
| `component: frontend` | `frontend/` |
|
||||
| `component: e2e` | `e2e/` |
|
||||
| `component: docker` | Docker / docker-compose |
|
||||
| `component: ci` | `.github/workflows/` |
|
||||
|
||||
## Triage Workflow
|
||||
|
||||
```
|
||||
New issue
|
||||
│
|
||||
├─ Missing info? → label `needs-info`, comment asking for details
|
||||
│ (auto-close after 14 days if no response)
|
||||
│
|
||||
├─ Duplicate? → label `duplicate`, link to original, close
|
||||
│
|
||||
├─ Out of scope? → label `wont-fix`, explain why, close
|
||||
│
|
||||
└─ Valid issue
|
||||
│
|
||||
├─ Add type label (bug / feature / enhancement / ...)
|
||||
├─ Add component label
|
||||
├─ Assess priority (P0 / P1 / P2 / P3)
|
||||
├─ If simple → add `good-first-issue`
|
||||
└─ Assign to milestone (if applicable)
|
||||
```
|
||||
|
||||
## Stale Policy
|
||||
|
||||
| Condition | Action |
|
||||
|-----------|--------|
|
||||
| No activity for **30 days** | Bot labels `stale` + comment |
|
||||
| No activity for **14 more days** | Bot closes the issue |
|
||||
| Reporter responds | `stale` label removed, timer resets |
|
||||
|
||||
Issues labeled `priority: P0` or `priority: P1` are exempt from the stale policy.
|
||||
|
||||
## Response Expectations
|
||||
|
||||
- **Every issue gets a response** (even if it's "thanks, we'll look at this next week")
|
||||
- Acknowledge within the SLA for the priority level
|
||||
- If you can't fix it soon, say so — don't leave reporters hanging
|
||||
- Close issues with a comment explaining the resolution
|
||||
|
|
@ -1,120 +0,0 @@
|
|||
# Onboarding Guide — First Contribution
|
||||
|
||||
Welcome! This guide helps you go from zero to your first merged PR.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
| Tool | Version | Check |
|
||||
|------|---------|-------|
|
||||
| Python | 3.12+ | `python --version` |
|
||||
| Node.js | 20+ | `node --version` |
|
||||
| Docker & Docker Compose | latest | `docker compose version` |
|
||||
| Git | 2.x | `git --version` |
|
||||
| Java (for e2e) | 17+ | `java --version` |
|
||||
| Maven (for e2e) | 3.9+ | `mvn --version` |
|
||||
|
||||
## Step 1 — Fork & Clone
|
||||
|
||||
```bash
|
||||
# Fork on GitHub, then:
|
||||
git clone https://github.com/<your-username>/Docling-Studio.git
|
||||
cd Docling-Studio
|
||||
git remote add upstream https://github.com/scub-france/Docling-Studio.git
|
||||
```
|
||||
|
||||
## Step 2 — Run the Stack
|
||||
|
||||
The fastest way to see the app running:
|
||||
|
||||
```bash
|
||||
docker compose up -d --wait
|
||||
open http://localhost:3000
|
||||
```
|
||||
|
||||
## Step 3 — Set Up for Development
|
||||
|
||||
### Backend
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
pip install -r requirements.txt # remote mode (lightweight)
|
||||
pip install ruff pytest pytest-asyncio httpx
|
||||
uvicorn main:app --reload --port 8000
|
||||
```
|
||||
|
||||
### Frontend
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev # http://localhost:5173
|
||||
```
|
||||
|
||||
## Step 4 — Pick Your First Issue
|
||||
|
||||
Look for issues labeled:
|
||||
|
||||
- `good-first-issue` — small, well-scoped, mentored
|
||||
- `help-wanted` — we need help but it may be larger
|
||||
- `docs` — documentation improvements (great starting point)
|
||||
|
||||
## Step 5 — Create a Branch
|
||||
|
||||
```bash
|
||||
git checkout main && git pull upstream main
|
||||
git checkout -b feature/my-change # or fix/my-fix
|
||||
```
|
||||
|
||||
Follow the [branching strategy](https://github.com/scub-france/Docling-Studio/blob/main/CONTRIBUTING.md#branching-strategy).
|
||||
|
||||
## Step 6 — Code
|
||||
|
||||
- Read the [architecture docs](../architecture.md) to understand the codebase
|
||||
- Follow the [coding standards](../architecture/coding-standards.md)
|
||||
- Write tests for your changes
|
||||
|
||||
## Step 7 — Verify
|
||||
|
||||
```bash
|
||||
# Backend
|
||||
cd document-parser
|
||||
ruff check . && ruff format --check .
|
||||
pytest tests/ -v
|
||||
|
||||
# Frontend
|
||||
cd frontend
|
||||
npm run type-check
|
||||
npx eslint src/
|
||||
npm run test:run
|
||||
```
|
||||
|
||||
## Step 8 — Commit & Push
|
||||
|
||||
Follow [commit conventions](../git-workflow/commit-conventions.md):
|
||||
|
||||
```bash
|
||||
git add <files>
|
||||
git commit -m "feat(scope): short description"
|
||||
git push origin feature/my-change
|
||||
```
|
||||
|
||||
## Step 9 — Open a PR
|
||||
|
||||
- Target: `main` (or `release/*` for pre-release fixes)
|
||||
- Fill in the [PR template](https://github.com/scub-france/Docling-Studio/blob/main/.github/PULL_REQUEST_TEMPLATE.md)
|
||||
- Add a line in `CHANGELOG.md` under `[Unreleased]`
|
||||
- Wait for CI to pass, then request a review
|
||||
|
||||
## What to Expect
|
||||
|
||||
- A maintainer will review your PR within **3 business days**
|
||||
- You may get feedback — this is normal and helpful
|
||||
- Once approved, a maintainer will merge your PR
|
||||
- Your contribution appears in the next release's changelog
|
||||
|
||||
## Need Help?
|
||||
|
||||
- Open a [Discussion](https://github.com/scub-france/Docling-Studio/discussions) on GitHub
|
||||
- Check existing issues for similar questions
|
||||
- Read the [CONTRIBUTING guide](https://github.com/scub-france/Docling-Studio/blob/main/CONTRIBUTING.md) for detailed rules
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
# Roadmap — Docling Studio
|
||||
|
||||
Last updated: YYYY-MM-DD
|
||||
|
||||
## Now (current release cycle)
|
||||
|
||||
<!-- Features actively being worked on -->
|
||||
|
||||
| Feature | Issue | Status |
|
||||
|---------|-------|--------|
|
||||
| ... | #NNN | In progress |
|
||||
|
||||
## Next (next release cycle)
|
||||
|
||||
<!-- Planned for the next release, prioritized -->
|
||||
|
||||
| Feature | Issue | Priority |
|
||||
|---------|-------|----------|
|
||||
| ... | #NNN | P1 |
|
||||
|
||||
## Later (future, no timeline)
|
||||
|
||||
<!-- Ideas accepted but not yet scheduled -->
|
||||
|
||||
| Feature | Issue |
|
||||
|---------|-------|
|
||||
| ... | #NNN |
|
||||
|
||||
## Not Planned
|
||||
|
||||
<!-- Explicitly declined or out of scope — explaining why avoids repeat requests -->
|
||||
|
||||
| Feature | Reason |
|
||||
|---------|--------|
|
||||
| ... | ... |
|
||||
|
||||
---
|
||||
|
||||
## How to Propose a Feature
|
||||
|
||||
1. Check [existing issues](https://github.com/scub-france/Docling-Studio/issues) — it may already be tracked
|
||||
2. Open a new issue using the **Feature** template
|
||||
3. If it's a significant change, consider writing an [ADR](../architecture/adr-guide.md)
|
||||
4. The maintainers will triage and prioritize (see [issue triage](issue-triage-process.md))
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
# Contributing
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Fork** the repository
|
||||
2. **Clone** your fork:
|
||||
```bash
|
||||
git clone https://github.com/<your-username>/Docling-Studio.git
|
||||
cd Docling-Studio
|
||||
```
|
||||
3. **Create a branch:**
|
||||
```bash
|
||||
git checkout -b feature/my-feature
|
||||
```
|
||||
|
||||
## Development Setup
|
||||
|
||||
=== "Backend (Python 3.12+)"
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
|
||||
# Remote mode (lightweight — delegates to Docling Serve)
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Local mode (full — runs Docling in-process)
|
||||
pip install -r requirements-local.txt
|
||||
|
||||
pip install ruff pytest pytest-asyncio httpx
|
||||
uvicorn main:app --reload --port 8000
|
||||
```
|
||||
|
||||
=== "Frontend (Node 20+)"
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## Code Quality
|
||||
|
||||
### Backend — Ruff
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
ruff check . # lint
|
||||
ruff check . --fix # auto-fix
|
||||
ruff format . # format
|
||||
```
|
||||
|
||||
### Frontend — TypeScript + ESLint + Prettier
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm run type-check # vue-tsc strict mode
|
||||
npx eslint src/ # lint
|
||||
npx prettier --check src/ # check formatting
|
||||
npx prettier --write src/ # auto-format
|
||||
```
|
||||
|
||||
## Running Tests
|
||||
|
||||
=== "Backend"
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
pytest tests/ -v
|
||||
```
|
||||
|
||||
=== "Frontend"
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm run test:run
|
||||
```
|
||||
|
||||
=== "E2E API (Karate)"
|
||||
|
||||
```bash
|
||||
# Generate test PDFs + start stack
|
||||
python e2e/generate-test-data.py
|
||||
docker compose up -d --wait
|
||||
|
||||
# Run all API tests
|
||||
mvn test -f e2e/api/pom.xml
|
||||
|
||||
# Or by tag: @smoke, @regression, @e2e
|
||||
mvn test -f e2e/api/pom.xml -Dkarate.options="--tags @smoke"
|
||||
```
|
||||
|
||||
=== "E2E UI (Karate UI)"
|
||||
|
||||
```bash
|
||||
# Generate test PDFs + start stack (if not already running)
|
||||
python e2e/generate-test-data.py
|
||||
docker compose up -d --wait
|
||||
|
||||
# Run critical UI tests (CI scope)
|
||||
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @critical"
|
||||
|
||||
# Run all UI tests (local scope)
|
||||
mvn test -f e2e/ui/pom.xml -Dkarate.options="--tags @ui"
|
||||
```
|
||||
|
||||
All tests must pass before submitting a PR.
|
||||
|
||||
## Pull Request Guidelines
|
||||
|
||||
- Keep PRs focused — one feature or fix per PR
|
||||
- Add tests for new functionality
|
||||
- Update documentation if behavior changes
|
||||
- Ensure CI passes (lint + type-check + tests + build)
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions will be licensed under the [MIT License](https://github.com/scub-france/Docling-Studio/blob/main/LICENSE).
|
||||
|
|
@ -1,404 +0,0 @@
|
|||
# Design: Copy paste image in Verify mode
|
||||
|
||||
<!--
|
||||
Design doc template for Docling Studio.
|
||||
|
||||
One design doc per tracked issue. File path convention:
|
||||
docs/design/<issue-number>-<kebab-slug>.md
|
||||
|
||||
Status lifecycle: Draft → In review → Accepted → Implemented (or Superseded).
|
||||
Bump the Status line as the doc progresses; do not delete sections on the way.
|
||||
|
||||
This template is tailored to the project's architecture and conventions:
|
||||
- Backend Hexagonal Architecture / ports & adapters
|
||||
(domain → api/services/persistence/infra)
|
||||
see docs/architecture.md
|
||||
- Backend coding standards (FastAPI + Pydantic camelCase, aiosqlite,
|
||||
Python snake_case internal, max 300 lines/file, 30 lines/function)
|
||||
see docs/architecture/coding-standards.md
|
||||
- Frontend feature-based organization (Vue 3 + Pinia, one store per
|
||||
feature, Composition API, TypeScript strict, data-e2e selectors)
|
||||
- E2E with Karate UI (NOT Playwright) — see e2e/CONVENTIONS.md
|
||||
- Audit dimensions used at release gate — see docs/audit/master.md
|
||||
- ADR process for load-bearing decisions — see docs/architecture/adr-guide.md
|
||||
|
||||
The `/conception` command pre-fills the header block and §1 / §2 / §12 from
|
||||
the linked issue. Everything else is on the author.
|
||||
-->
|
||||
|
||||
- **Issue:** #195
|
||||
- **Title on issue:** [ENHANCEMENT] Copy paste image in Verify mode
|
||||
- **Author:** Pier-Jean Malandrino
|
||||
- **Date:** 2026-04-23
|
||||
- **Status:** Accepted
|
||||
- **Target milestone:** 0.5.0
|
||||
- **Impacted layers:** <backend: domain | api | services | persistence | infra> · <frontend: features/<name> | shared | app> · <e2e> · <infra/CI>
|
||||
- **Audit dimensions likely touched:** <pick from: Hexagonal Architecture · DDD · Clean Code · KISS · DRY · SOLID · Decoupling · Security · Tests · CI/Build · Documentation · Performance>
|
||||
- **ADR spawned?:** <no> *(write an ADR when choosing a library, moving a boundary, or deciding **not** to do something — see `docs/architecture/adr-guide.md`)*
|
||||
|
||||
---
|
||||
|
||||
## 1. Problem
|
||||
|
||||
<!--
|
||||
What hurts today, and for whom. Pull from the issue's Context + Current
|
||||
behavior sections — keep the user's voice, do not paraphrase aggressively.
|
||||
Two or three short paragraphs is usually enough. If you can't state the
|
||||
problem in plain language, you are not ready to design a solution.
|
||||
-->
|
||||
|
||||
TODO: Why this issue exists. Link the user story, incident, or upstream discussion that motivates it.
|
||||
|
||||
Today in Verify mode regarding image handling: TODO — describe the current baseline (upload-only? no paste target? no drag-drop?).
|
||||
|
||||
## 2. Goals
|
||||
|
||||
<!--
|
||||
Concrete, verifiable outcomes. Convert the issue's acceptance criteria into
|
||||
checkboxes here; the design is "done" when all are satisfied. Keep the list
|
||||
small — five or fewer goals is a good smell.
|
||||
-->
|
||||
|
||||
Users should be able to copy/paste images directly into Verify mode (e.g. from clipboard) instead of only via file upload.
|
||||
|
||||
- [ ] Define paste source (OS clipboard, drag-drop, screenshot)
|
||||
- [ ] Define target area in Verify mode UI
|
||||
- [ ] Define supported image formats and size limits
|
||||
- [ ] Size / type limits are env-var configurable, carry sane defaults, and are documented in `README.md` + `docs/deployment/*` (e.g. `MAX_PASTE_IMAGE_SIZE_MB`, `PASTE_ALLOWED_IMAGE_TYPES`)
|
||||
|
||||
## 3. Non-goals
|
||||
|
||||
<!--
|
||||
What this design explicitly does NOT try to solve — and, for each, where it
|
||||
*should* be solved (follow-up issue, next milestone, different audit area).
|
||||
This is the section that saves the review: naming the off-ramps up front
|
||||
prevents scope creep. If you leave this empty, reviewers will fill it in
|
||||
for you, badly.
|
||||
-->
|
||||
|
||||
- ...
|
||||
- ...
|
||||
|
||||
## 4. Context & constraints
|
||||
|
||||
<!--
|
||||
The surrounding reality the design has to live in.
|
||||
|
||||
### Existing code surface
|
||||
List the modules / files / stores this change touches. Prefer concrete paths
|
||||
over prose:
|
||||
- Backend: document-parser/<layer>/<file>.py
|
||||
- Frontend: frontend/src/features/<name>/{store,api,ui}.ts|.vue
|
||||
- Persistence: document-parser/persistence/<repo>.py + schema in database.py
|
||||
- E2E: e2e/<feature>.feature
|
||||
|
||||
### Hexagonal Architecture constraints (backend)
|
||||
The domain layer has zero imports from api / persistence / infra, and
|
||||
defines ports (abstract protocols) that `infra/` adapters implement.
|
||||
Persistence imports only from domain. API never imports persistence
|
||||
directly — it goes through services. Call out any change that crosses
|
||||
these lines or adds / moves a port.
|
||||
|
||||
### Deployment modes
|
||||
Docling Studio ships two images (`latest-local`, `latest-remote`) driven by
|
||||
`CONVERSION_ENGINE` — and a HF Space deployment on top of `latest-remote`.
|
||||
State which modes this design supports, which it does not, and how the
|
||||
frontend's feature flags (`chunking`, `disclaimer`) are affected.
|
||||
|
||||
### Hard constraints
|
||||
Compatibility (SQLite schema, API contract, Pydantic DTOs), deadlines
|
||||
(milestone due date), deployment target (Docker Compose, HF Space),
|
||||
performance budget (matters for Performance audit), license / privacy
|
||||
(matters for Security audit).
|
||||
-->
|
||||
|
||||
### SQLite & storage limits (must be enforced upstream of the DB)
|
||||
|
||||
Pasted images follow the existing `documents` pattern: bytes land on disk
|
||||
under `UPLOAD_DIR`, the row stores `storage_path: TEXT`. We do **not**
|
||||
store base64 or BLOBs. Even so, app-level size guards must stay below
|
||||
SQLite's structural limits so a malformed request can never wedge the
|
||||
engine.
|
||||
|
||||
Relevant SQLite defaults (see https://www.sqlite.org/limits.html):
|
||||
|
||||
| SQLite limit | Default | Relevance |
|
||||
|---|---|---|
|
||||
| `SQLITE_MAX_LENGTH` | 1 GB (1 × 10⁹ bytes) | Max size of any single `TEXT` / `BLOB` cell. Irrelevant as long as we keep bytes off the DB. |
|
||||
| `SQLITE_MAX_SQL_LENGTH` | 1 MB | Max length of an SQL statement incl. inlined literals. Always use parameter binding — never inline image bytes. |
|
||||
| Page cache / WAL growth | n/a | Large writes bloat WAL until checkpoint; another reason to stay off-DB. |
|
||||
|
||||
Our own app-level limits guard against ever reaching those ceilings.
|
||||
All such limits **must**: (1) carry a sane default in `infra/settings.py`,
|
||||
(2) be overridable via env var, (3) be documented in `README.md` and
|
||||
`docs/deployment/*`. This is consistent with how `MAX_FILE_SIZE_MB`
|
||||
(default 50) is handled today.
|
||||
|
||||
## 5. Proposed design
|
||||
|
||||
<!--
|
||||
The recommended approach, in enough detail that a competent engineer
|
||||
outside the immediate context can implement it. Describe contracts, not
|
||||
code — the PR is where code lives.
|
||||
|
||||
Structure this section by layer. Skip a layer if it is genuinely untouched;
|
||||
do not pad.
|
||||
|
||||
### 5.1 Domain
|
||||
New or changed dataclasses / value objects / ports in `document-parser/domain/`.
|
||||
No HTTP or DB concerns here. If you are adding a port (`Protocol`), give its
|
||||
full signature.
|
||||
|
||||
### 5.2 Persistence
|
||||
Schema changes (table, columns, indexes), migration plan, aiosqlite query
|
||||
shape. Note whether existing rows need a backfill.
|
||||
|
||||
### 5.3 Infra adapters
|
||||
New or changed adapters in `document-parser/infra/` (converter, chunker,
|
||||
rate limiter, settings). For new env vars, give name / default / allowed
|
||||
values.
|
||||
|
||||
### 5.4 Services
|
||||
Use-case orchestration in `document-parser/services/`. Services do NOT
|
||||
implement — they delegate. Describe the call sequence, error handling,
|
||||
and concurrency (how does this interact with `MAX_CONCURRENT_ANALYSES`?).
|
||||
|
||||
### 5.5 API
|
||||
Endpoint additions / changes in `document-parser/api/`. For each:
|
||||
- Method + path
|
||||
- Request DTO (Pydantic, camelCase via alias_generator)
|
||||
- Response DTO (camelCase; remember `pages_json` stays snake_case)
|
||||
- Error responses (status codes, shape)
|
||||
- Whether it is excluded from the rate limiter (like `/api/health`)
|
||||
|
||||
### 5.6 Frontend — feature module
|
||||
Which `frontend/src/features/<name>/` folder, which Pinia store actions,
|
||||
which API client calls in `api.ts`, which Vue components in `ui/`. Name
|
||||
new `data-e2e` attributes here (Karate needs them).
|
||||
|
||||
### 5.7 Cross-cutting
|
||||
Feature flags (how the backend advertises capability via `/api/health` and
|
||||
how the frontend reacts), i18n strings (`shared/i18n.ts`), shared types
|
||||
(`shared/types.ts`).
|
||||
|
||||
Prefer mermaid / ASCII for sequence and data flow. Interfaces are more
|
||||
valuable than pseudocode.
|
||||
-->
|
||||
|
||||
### 5.1 Domain
|
||||
|
||||
### 5.2 Persistence
|
||||
|
||||
### 5.3 Infra adapters
|
||||
|
||||
Extend `document-parser/infra/settings.py` with paste-specific limits.
|
||||
Follow the existing `MAX_FILE_SIZE_MB` pattern: typed field on the
|
||||
settings dataclass, `os.environ.get(...)` with a string default, cast
|
||||
at load time.
|
||||
|
||||
| Setting | Env var | Default | Allowed | Notes |
|
||||
|---|---|---|---|---|
|
||||
| `max_paste_image_size_mb` | `MAX_PASTE_IMAGE_SIZE_MB` | `10` | positive int, `0` = unlimited | Must be ≤ `MAX_FILE_SIZE_MB`; upload validator rejects larger payloads before any DB write. |
|
||||
| `paste_allowed_image_types` | `PASTE_ALLOWED_IMAGE_TYPES` | `image/png,image/jpeg,image/webp` | comma-separated MIME list | Enforced server-side; frontend uses the same list via `/api/health`. |
|
||||
|
||||
Validation happens in the API layer (upload handler) **before** the
|
||||
bytes reach persistence. Any future move to BLOB storage would still
|
||||
rely on these guards — they are the contract that prevents us ever
|
||||
approaching `SQLITE_MAX_LENGTH`.
|
||||
|
||||
### 5.4 Services
|
||||
|
||||
### 5.5 API
|
||||
|
||||
### 5.6 Frontend — feature module
|
||||
|
||||
### 5.7 Cross-cutting (feature flags, i18n, shared types)
|
||||
|
||||
## 6. Alternatives considered
|
||||
|
||||
<!--
|
||||
At least two genuine alternatives, each with a one-paragraph description
|
||||
and the reason it was rejected. "Do nothing" is often a legitimate
|
||||
alternative — name it if it is. Reviewers use this section to sanity-check
|
||||
that the recommended design was a choice and not the first thing that
|
||||
came to mind.
|
||||
|
||||
If one of the alternatives represents a significant architectural fork
|
||||
(e.g. introducing a new service, replacing a library), spawn an ADR under
|
||||
`docs/architecture/adrs/` and link it in §12 — the design doc captures the
|
||||
local decision, the ADR captures the cross-cutting one.
|
||||
-->
|
||||
|
||||
### Alternative A — <name>
|
||||
|
||||
- **Summary:**
|
||||
- **Why not:**
|
||||
|
||||
### Alternative B — <name>
|
||||
|
||||
- **Summary:**
|
||||
- **Why not:**
|
||||
|
||||
## 7. API & data contract
|
||||
|
||||
<!--
|
||||
Make the wire contract explicit — this is what the frontend, e2e tests,
|
||||
and any external consumer will code against.
|
||||
|
||||
### Endpoints
|
||||
| Method | Path | Request | Response | Breaking? |
|
||||
|--------|------|---------|----------|-----------|
|
||||
| | | | | |
|
||||
|
||||
Remember:
|
||||
- API serialization is camelCase (Pydantic `alias_generator`).
|
||||
- Backend internals stay snake_case.
|
||||
- `pages_json` is the documented exception — it carries raw
|
||||
`dataclasses.asdict()` output (snake_case).
|
||||
- Health endpoint (`/api/health`) may need new fields if this design adds
|
||||
a feature flag.
|
||||
|
||||
### Persistence schema
|
||||
```sql
|
||||
-- ALTER TABLE / CREATE TABLE statements, with reasoning
|
||||
```
|
||||
|
||||
### Env vars / config
|
||||
|
||||
All new knobs must land in `README.md` and `docs/deployment/*` (same
|
||||
tables that already list `MAX_FILE_SIZE_MB`, `UPLOAD_DIR`, etc.).
|
||||
|
||||
| Name | Default | Allowed | Notes |
|
||||
|------|---------|---------|-------|
|
||||
| `MAX_PASTE_IMAGE_SIZE_MB` | `10` | positive int (`0` = unlimited) | Guards app-level payload size; must stay ≤ `MAX_FILE_SIZE_MB` to avoid double-gating confusion. |
|
||||
| `PASTE_ALLOWED_IMAGE_TYPES` | `image/png,image/jpeg,image/webp` | comma-separated MIME list | Surfaced to the frontend via `/api/health` so the paste handler can reject client-side. |
|
||||
|
||||
SQLite ceilings (`SQLITE_MAX_LENGTH` = 1 GB, `SQLITE_MAX_SQL_LENGTH` =
|
||||
1 MB) are **not** env-configurable — they are compile-time properties
|
||||
of the bundled engine. Document them in `docs/deployment/*` as
|
||||
background so operators understand why the app-level limits exist.
|
||||
|
||||
### Breaking changes
|
||||
Enumerate anything a consumer must change. If there are none, say so
|
||||
explicitly — "additive only" is a useful commitment.
|
||||
-->
|
||||
|
||||
## 8. Risks & mitigations
|
||||
|
||||
<!--
|
||||
One row per non-trivial risk. Map each to an audit dimension so the
|
||||
release-gate audit has a clear hook:
|
||||
|
||||
| Risk | Audit dimension | Likelihood | Impact | How we notice | Mitigation / rollback |
|
||||
|------|-----------------|-----------|--------|---------------|------------------------|
|
||||
| | Security | | | | |
|
||||
| | Performance | | | | |
|
||||
| | Decoupling | | | | |
|
||||
|
||||
Common families to scan for:
|
||||
- **Hexagonal Architecture:** cross-layer imports, leaking HTTP into domain, adapter bypassing its port
|
||||
- **Security:** rate limiter bypass, path traversal on uploads, SSRF via
|
||||
the remote converter, unauthenticated data exposure
|
||||
- **Performance:** synchronous work on the FastAPI event loop,
|
||||
unbounded queries, new work inside `MAX_CONCURRENT_ANALYSES` budget
|
||||
- **Tests:** coverage gap on a critical path
|
||||
- **Documentation:** missing README / env var / i18n entry
|
||||
|
||||
A design with "no risks identified" is a design that has not been read
|
||||
carefully.
|
||||
-->
|
||||
|
||||
| Risk | Audit dimension | Likelihood | Impact | How we notice | Mitigation / rollback |
|
||||
|------|-----------------|------------|--------|---------------|------------------------|
|
||||
| | | | | | |
|
||||
|
||||
## 9. Testing strategy
|
||||
|
||||
<!--
|
||||
How this design will be verified. Be specific — name files / suites.
|
||||
|
||||
### Backend — pytest (`document-parser/tests/`)
|
||||
- Unit: per-layer (`tests/domain/`, `tests/persistence/`, `tests/services/`)
|
||||
- Integration: services wired with real aiosqlite + real adapters
|
||||
- Architecture tests (if applicable): enforce import boundaries
|
||||
|
||||
### Frontend — Vitest (`frontend/src/**/*.test.ts`)
|
||||
- Stores: actions / getters / mocked API
|
||||
- Pure helpers (e.g. `bboxScaling.ts`-style modules): deterministic
|
||||
- Components only when behavior is non-trivial; do not test markup
|
||||
|
||||
### E2E — Karate UI (`e2e/`)
|
||||
- Use `data-e2e` selectors — never CSS classes (see e2e/CONVENTIONS.md)
|
||||
- `retry()` / `waitFor()` — never `Thread.sleep()` / `delay()`
|
||||
- Setup via API, verify via UI, cleanup via API
|
||||
- Tag appropriately: `@critical` / `@ui` / `@smoke` / `@regression` / `@e2e`
|
||||
- **Never Playwright** — Karate is the tool here.
|
||||
|
||||
### Manual QA
|
||||
Steps the reviewer can run locally (`docker-compose.dev.yml` up, scenario
|
||||
to reproduce). Keep it short — if the manual list is long, automate more.
|
||||
|
||||
### Performance / load
|
||||
Required when the design claims a latency / throughput / memory property,
|
||||
or touches the conversion hot path.
|
||||
-->
|
||||
|
||||
## 10. Rollout & observability
|
||||
|
||||
<!--
|
||||
How this change gets to production safely.
|
||||
|
||||
### Release branch
|
||||
Which `release/X.Y.Z` is the target? Any coordination with a parallel
|
||||
release (e.g. R&D branch)?
|
||||
|
||||
### Feature flag / staged rollout
|
||||
Does the change hide behind a flag surfaced via `/api/health`? If so, what
|
||||
flips the flag, and what is the default? HF Space deployments often need
|
||||
`deploymentMode === 'huggingface'` gating.
|
||||
|
||||
### Observability
|
||||
- Logs to add / extend (structured, low-cardinality keys)
|
||||
- Metrics / counters (if added — call out any new Prometheus names)
|
||||
- New error modes to watch for in `analysis_jobs.status = FAILED`
|
||||
|
||||
### Rollback plan
|
||||
The revert that is safe to apply at any time:
|
||||
- Which migration is reversible? Which is not?
|
||||
- Which env var flip disables the feature without a redeploy?
|
||||
- Any data cleanup needed after rollback?
|
||||
|
||||
Link to the existing release / ops playbooks:
|
||||
- Deployment: `docs/release/*` (also surfaced via `/release:deploy`)
|
||||
- Rollback: also surfaced via `/release:rollback`
|
||||
- Incident: `docs/operations/*` (also surfaced via `/ops:incident`)
|
||||
-->
|
||||
|
||||
## 11. Open questions
|
||||
|
||||
<!--
|
||||
Things the author explicitly does not know yet, phrased as questions the
|
||||
reviewer can answer or redirect. Empty is allowed once the design is
|
||||
Accepted — during Draft / In review, this section is where the honest
|
||||
uncertainty lives. Resolve or delete each entry before shipping.
|
||||
-->
|
||||
|
||||
- ...
|
||||
- ...
|
||||
|
||||
## 12. References
|
||||
|
||||
<!--
|
||||
Links to everything a future reader would want.
|
||||
-->
|
||||
|
||||
- **Issue:** https://github.com/scub-france/Docling-Studio/issues/195
|
||||
- **Related PRs / commits:**
|
||||
- **ADRs:** <ADR-NNN or "none planned">
|
||||
- **Project docs:**
|
||||
- Architecture: `docs/architecture.md`
|
||||
- Coding standards: `docs/architecture/coding-standards.md`
|
||||
- ADR guide / template: `docs/architecture/adr-guide.md`, `docs/architecture/adr-template.md`
|
||||
- Audit master: `docs/audit/master.md`
|
||||
- E2E conventions: `e2e/CONVENTIONS.md`
|
||||
- **External:** <specs, upstream issues, dashboards, third-party docs>
|
||||
|
|
@ -1,435 +0,0 @@
|
|||
# Neo4j integration — Docling-Studio v0.5.0
|
||||
|
||||
Design doc for Neo4j integration targeting release 0.5.0.
|
||||
Target: Hackernoon hackathon demo (Neo4j partner).
|
||||
|
||||
---
|
||||
|
||||
## 1. Context and goals
|
||||
|
||||
### Already in Docling-Studio
|
||||
- Ingestion pipeline: Docling parser → chunking (HybridChunker) → embedding → OpenSearch (vector index)
|
||||
- Vue 3 + FastAPI UI
|
||||
- Debug view to inspect/edit chunks before retrieval
|
||||
- Docker compose with existing services
|
||||
|
||||
### What we add in v0.5.0
|
||||
- Neo4j as **graph-native storage** of the document structure
|
||||
- A new ingestion layer that stores the DoclingDocument tree faithfully as a graph
|
||||
- Minimal UI to visualize the graph (demo value to the judges)
|
||||
- Compose pipeline with Neo4j
|
||||
|
||||
### Why graph-native (hackathon positioning)
|
||||
> Most document AI tools store parsed content as flat chunks in a vector DB.
|
||||
> Docling-Studio v0.5 introduces a graph-native storage layer on top of Neo4j,
|
||||
> preserving the full hierarchical structure of documents as first-class citizens.
|
||||
> This unlocks hybrid retrieval, agentic navigation, and structural debugging —
|
||||
> impossible with chunk-only stores.
|
||||
|
||||
### Out of scope for v0.5.0 (roadmap mention only)
|
||||
- EnrichmentWriter (entities / summaries / keywords via docling-agent) — v0.6.0
|
||||
- Agent reasoning trace viewer — v0.6.0
|
||||
- RAG hybrid (graph traversal + vector) — v0.7.0
|
||||
- Document versioning — v0.7.0+
|
||||
|
||||
---
|
||||
|
||||
## 2. Architectural principles
|
||||
|
||||
### Port & adapter, with nuance
|
||||
|
||||
**Write side**: one `Writer` port, **composable stages** (not alternative adapters).
|
||||
Pipelines A and B are additive, not exclusive.
|
||||
|
||||
```
|
||||
CORE (always) Pipeline A (RAG) Pipeline B (agent-ready, v0.6+)
|
||||
┌─────────────┐ ┌────────────────┐ ┌───────────────────┐
|
||||
│ TreeWriter │ ─────▶ │ ChunkWriter │ │ EnrichmentWriter │
|
||||
│ │ │ (existing │ │ (via docling- │
|
||||
│ │ │ OpenSearch + │ │ agent, v0.6+) │
|
||||
│ │ │ adds chunks │ │ │
|
||||
│ │ │ to Neo4j) │ │ │
|
||||
└─────────────┘ └────────────────┘ └───────────────────┘
|
||||
```
|
||||
|
||||
```python
|
||||
# docling_studio/ingestion/pipeline.py
|
||||
class Writer(Protocol):
|
||||
def write(self, doc: DoclingDocument, ctx: IngestionContext) -> None: ...
|
||||
|
||||
# Explicit composition per use case
|
||||
def build_pipeline(config: PipelineConfig) -> list[Writer]:
|
||||
writers = [TreeWriter(neo4j_driver)]
|
||||
if config.rag_enabled:
|
||||
writers.append(ChunkWriter(neo4j_driver, chunker, embedder, opensearch))
|
||||
if config.enrichment_enabled: # v0.6.0+
|
||||
writers.append(EnrichmentWriter(neo4j_driver, docling_agent))
|
||||
return writers
|
||||
```
|
||||
|
||||
**Read side**: two distinct ports (same Neo4j backend, different queries).
|
||||
|
||||
```python
|
||||
class RAGRetrievalPort(Protocol):
|
||||
def search(self, query: str, k: int) -> list[Chunk]: ...
|
||||
def similar(self, chunk_id: str, k: int) -> list[Chunk]: ...
|
||||
|
||||
class TreeNavigationPort(Protocol): # v0.6.0+
|
||||
def get_outline(self, doc_id: str) -> Tree: ...
|
||||
def read_node(self, ref: str) -> Element: ...
|
||||
def list_children(self, ref: str) -> list[Element]: ...
|
||||
def walk(self, ref: str, depth: int) -> SubTree: ...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Neo4j schema
|
||||
|
||||
### Constraints & indexes (created at boot)
|
||||
|
||||
```cypher
|
||||
// Uniqueness
|
||||
CREATE CONSTRAINT document_id IF NOT EXISTS
|
||||
FOR (d:Document) REQUIRE d.id IS UNIQUE;
|
||||
|
||||
CREATE CONSTRAINT element_composite IF NOT EXISTS
|
||||
FOR (e:Element) REQUIRE (e.doc_id, e.self_ref) IS UNIQUE;
|
||||
|
||||
CREATE CONSTRAINT page_composite IF NOT EXISTS
|
||||
FOR (p:Page) REQUIRE (p.doc_id, p.page_no) IS UNIQUE;
|
||||
|
||||
CREATE CONSTRAINT chunk_id IF NOT EXISTS
|
||||
FOR (c:Chunk) REQUIRE c.id IS UNIQUE;
|
||||
|
||||
// Full-text index (element text search)
|
||||
CREATE FULLTEXT INDEX element_text IF NOT EXISTS
|
||||
FOR (e:Element) ON EACH [e.text];
|
||||
|
||||
// Simple indexes for per-doc queries
|
||||
CREATE INDEX element_doc IF NOT EXISTS FOR (e:Element) ON (e.doc_id);
|
||||
CREATE INDEX chunk_doc IF NOT EXISTS FOR (c:Chunk) ON (c.doc_id);
|
||||
```
|
||||
|
||||
### Data model
|
||||
|
||||
```cypher
|
||||
// Root document
|
||||
(:Document {
|
||||
id: string, // UUID or PDF hash
|
||||
title: string,
|
||||
source_uri: string, // path or S3
|
||||
ingested_at: datetime,
|
||||
docling_version: string,
|
||||
stages_applied: list<string>, // ["tree", "chunks"] etc.
|
||||
last_tree_write: datetime,
|
||||
last_chunk_write: datetime,
|
||||
tenant_id: string // simple multi-tenancy
|
||||
})
|
||||
|
||||
// All tree elements (shared :Element label + specific label)
|
||||
(:Element:SectionHeader {doc_id, self_ref, text, level, prov_page, prov_bbox})
|
||||
(:Element:Paragraph {doc_id, self_ref, text, prov_page, prov_bbox})
|
||||
(:Element:Table {doc_id, self_ref, caption, cells_json, prov_page, prov_bbox})
|
||||
(:Element:Figure {doc_id, self_ref, caption, image_uri, prov_page, prov_bbox})
|
||||
(:Element:ListItem {doc_id, self_ref, text, marker, prov_page, prov_bbox})
|
||||
(:Element:Formula {doc_id, self_ref, latex, text, prov_page, prov_bbox})
|
||||
|
||||
// Page for layout provenance
|
||||
(:Page {doc_id, page_no, width, height})
|
||||
|
||||
// Chunks (Pipeline A)
|
||||
(:Chunk {
|
||||
id, doc_id,
|
||||
text,
|
||||
chunk_index,
|
||||
embedding_ref, // id in OpenSearch (no inline duplication)
|
||||
token_count
|
||||
})
|
||||
```
|
||||
|
||||
### Relations
|
||||
|
||||
```cypher
|
||||
// Hierarchical structure
|
||||
(:Document)-[:HAS_ROOT]->(:Element)
|
||||
(:Element)-[:PARENT_OF {order: int}]->(:Element) // order preserves sequence
|
||||
(:Element)-[:NEXT]->(:Element) // DFS pre-order reading
|
||||
|
||||
// Layout
|
||||
(:Element)-[:ON_PAGE]->(:Page)
|
||||
|
||||
// Pipeline A (chunking)
|
||||
(:Document)-[:HAS_CHUNK]->(:Chunk)
|
||||
(:Chunk)-[:DERIVED_FROM]->(:Element) // back-reference; a chunk can span multiple elements
|
||||
```
|
||||
|
||||
### Decisions
|
||||
|
||||
| Decision | Choice | Rationale |
|
||||
|----------|-------|---------------|
|
||||
| Element composite key | `(doc_id, self_ref)` | self_ref not unique across docs |
|
||||
| Multi-tenancy | `tenant_id` property on Document | Simple, filterable, migrable to multi-db later |
|
||||
| Table cells | `cells_json` property | v0.5 KISS. May model `(Table)-[:HAS_CELL]->(Cell)` in v0.6+ |
|
||||
| Reading order | `[:NEXT]` chain + `{order}` on `PARENT_OF` | Both views useful |
|
||||
| Versioning | None (replace strategy on re-upload) | v0.5 KISS |
|
||||
| APOC | Not required | Pure Cypher is sufficient for v0.5 |
|
||||
|
||||
### Re-ingestion strategy
|
||||
|
||||
```cypher
|
||||
// Before ingesting, wipe existing
|
||||
MATCH (d:Document {id: $doc_id})
|
||||
OPTIONAL MATCH (d)-[:HAS_ROOT|HAS_CHUNK]->()
|
||||
DETACH DELETE d
|
||||
// Then re-walk cleanly
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Implementation plan (3 days)
|
||||
|
||||
### Day 1 — Infra + schema
|
||||
|
||||
- [ ] Add `neo4j` service to `docker-compose.yml` (`neo4j:5.15-community`, persistent volume, healthcheck)
|
||||
- [ ] Add env vars (`NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD`) to `.env.example`
|
||||
- [ ] Create module `docling_studio/storage/neo4j/`:
|
||||
- `driver.py` — neo4j-python driver wrapper (connection pool, context manager)
|
||||
- `schema.py` — idempotent `bootstrap_schema()` (CREATE CONSTRAINT / INDEX at startup)
|
||||
- `__init__.py` with exports
|
||||
- [ ] Hook `bootstrap_schema()` in FastAPI startup
|
||||
- [ ] Basic integration tests:
|
||||
- Driver connection
|
||||
- Schema bootstrap (idempotence verified)
|
||||
- Simple round-trip: write Document, read Document, delete Document
|
||||
|
||||
**Deliverable:** docker compose boots with healthy Neo4j, schema in place at init.
|
||||
|
||||
### Day 2 — TreeWriter (write + read)
|
||||
|
||||
- [ ] `storage/neo4j/tree_writer.py` — `DoclingDocument → Neo4j` walker
|
||||
- `write_document(doc, tenant_id, driver)` in a transaction
|
||||
- DFS pre-order, batched `MERGE` for perf
|
||||
- Pages first, then Elements, then `PARENT_OF` / `NEXT` / `ON_PAGE` relations
|
||||
- Dynamic labels based on `node.label` (`SectionHeader`, `Paragraph`, …)
|
||||
- [ ] `storage/neo4j/tree_reader.py` — inverse walker `Neo4j → DoclingDocument`
|
||||
- `read_document(doc_id, driver) -> DoclingDocument`
|
||||
- Loads all Elements + Pages, rebuilds the Pydantic structure
|
||||
- Prerequisite for v0.6 (feeding docling-agent from Neo4j)
|
||||
- [ ] Integrate into existing ingestion pipeline:
|
||||
- Add TreeWriter as first stage of `IngestionPipeline`
|
||||
- `neo4j_enabled: bool` config toggle
|
||||
- [ ] Round-trip tests:
|
||||
- 3–4 varied PDFs (academic, invoice, report)
|
||||
- Assertion: `doc_original == read_document(write_document(doc_original))`
|
||||
- Beware dates, bbox floats (tolerance)
|
||||
|
||||
**Deliverable:** A PDF uploaded to Docling-Studio is fully present in Neo4j and rebuildable.
|
||||
|
||||
### Day 3 — UI + ChunkWriter + packaging
|
||||
|
||||
- [ ] `storage/neo4j/chunk_writer.py`:
|
||||
- After existing chunking, push each Chunk to Neo4j
|
||||
- Create `(:Chunk)-[:DERIVED_FROM]->(:Element)` via source element `self_ref`
|
||||
- Do NOT duplicate embeddings (stay in OpenSearch, keep `embedding_ref`)
|
||||
- [ ] Frontend: new "Graph view" tab in debug panel
|
||||
- Vue component with `cytoscape` (lighter, better layout API — see [ADR-001](../architecture/adrs/ADR-001-graph-visualization-library.md))
|
||||
- FastAPI endpoint `/api/documents/{doc_id}/graph` returns full nodes + edges for the document, **capped at 200 pages** (HTTP 413 beyond; pagination deferred to v0.6). The endpoint must include a `truncated: bool` flag and `node_count` / `edge_count` in the response envelope so the UI can warn the user cleanly.
|
||||
- View: vertical tree, colors per node type, click-to-zoom, hover details
|
||||
- [ ] Per-document "Graph-ready" / "RAG-ready" badge in list
|
||||
- [ ] README update:
|
||||
- "Graph storage with Neo4j" section
|
||||
- Schema diagram (Mermaid or image)
|
||||
- 2–3 Cypher examples like "find all paragraphs under section X that mention Y"
|
||||
- Neo4j badge in features list
|
||||
- [ ] (bonus if time) "Query explorer" dev tab for live demo: Cypher editor + results
|
||||
|
||||
**Deliverable:** release 0.5.0 with Neo4j visible, functional, documented.
|
||||
|
||||
---
|
||||
|
||||
## 5. Proposed code structure
|
||||
|
||||
```
|
||||
docling_studio/
|
||||
├── storage/
|
||||
│ ├── neo4j/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── driver.py # connection management
|
||||
│ │ ├── schema.py # bootstrap_schema()
|
||||
│ │ ├── tree_writer.py # DoclingDocument -> Neo4j
|
||||
│ │ ├── tree_reader.py # Neo4j -> DoclingDocument
|
||||
│ │ ├── chunk_writer.py # Chunks -> Neo4j
|
||||
│ │ └── queries.py # shared Cypher queries
|
||||
│ ├── opensearch/ # (existing)
|
||||
│ └── ports.py # Writer, RAGRetrievalPort protocols
|
||||
├── ingestion/
|
||||
│ └── pipeline.py # IngestionPipeline composing Writers
|
||||
├── api/
|
||||
│ └── graph.py # /api/documents/{id}/graph
|
||||
└── frontend/
|
||||
└── components/
|
||||
└── GraphView.vue # cytoscape + graph API fetch
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Docker compose (added excerpt)
|
||||
|
||||
```yaml
|
||||
services:
|
||||
neo4j:
|
||||
image: neo4j:5.15-community
|
||||
environment:
|
||||
NEO4J_AUTH: ${NEO4J_USER:-neo4j}/${NEO4J_PASSWORD:-changeme}
|
||||
NEO4J_PLUGINS: '["apoc"]'
|
||||
NEO4J_server_memory_heap_initial__size: 512m
|
||||
NEO4J_server_memory_heap_max__size: 1g
|
||||
ports:
|
||||
- "7474:7474" # Browser UI (demo)
|
||||
- "7687:7687" # Bolt protocol
|
||||
volumes:
|
||||
- neo4j_data:/data
|
||||
- neo4j_logs:/logs
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "cypher-shell -u neo4j -p $${NEO4J_PASSWORD:-changeme} 'RETURN 1' || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
docling-studio-backend:
|
||||
depends_on:
|
||||
neo4j:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
NEO4J_URI: bolt://neo4j:7687
|
||||
NEO4J_USER: neo4j
|
||||
NEO4J_PASSWORD: ${NEO4J_PASSWORD:-changeme}
|
||||
|
||||
volumes:
|
||||
neo4j_data:
|
||||
neo4j_logs:
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Tests
|
||||
|
||||
### Unit tests
|
||||
- `tests/storage/neo4j/test_schema.py` — bootstrap is idempotent
|
||||
- `tests/storage/neo4j/test_tree_writer.py` — round-trip on synthetic DoclingDocument
|
||||
- `tests/storage/neo4j/test_chunk_writer.py` — chunks written with correct `DERIVED_FROM`
|
||||
|
||||
### Integration tests
|
||||
- `tests/integration/test_ingestion_pipeline.py` — full pipeline on a real PDF
|
||||
- PDF fixtures: 1 academic (complex heading hierarchy), 1 invoice (tables), 1 report (lists)
|
||||
|
||||
### E2E (bonus)
|
||||
- Upload PDF via UI → check structure in Neo4j Browser
|
||||
|
||||
---
|
||||
|
||||
## 8. Open decisions to settle before coding
|
||||
|
||||
1. **Neo4j edition**: Community (free) or AuraDB (managed) ?
|
||||
- Rec: Community in Docker for v0.5.0 dev/demo. AuraDB mentioned as prod option.
|
||||
|
||||
2. **Chunks: duplicate embeddings in Neo4j or OpenSearch ref ?**
|
||||
- Rec: OpenSearch ref (avoid duplication; OpenSearch remains source of truth for vectors). In v0.6+, consider native Neo4j vector index.
|
||||
|
||||
3. **Graph view UI: cytoscape or vis-network ?**
|
||||
- Decided: **Cytoscape.js** — see [ADR-001](../architecture/adrs/ADR-001-graph-visualization-library.md) for the full analysis.
|
||||
|
||||
4. **Graph endpoint: return full doc or paginate ?**
|
||||
- Decided: full doc for v0.5, **hard cap at 200 pages**. Beyond the cap, the endpoint returns HTTP 413 with a `truncated: true` flag; the UI shows "Graph too large to render — reduce scope". Pagination ships in v0.6.
|
||||
|
||||
5. **Error strategy**: if Neo4j is down at ingestion, fail or degrade gracefully ?
|
||||
- Rec: **fail fast** for v0.5 (avoid silent inconsistencies). `neo4j_required: bool` config option.
|
||||
|
||||
---
|
||||
|
||||
## 9. Hooks for later (v0.6.0+ — don't implement but prepare)
|
||||
|
||||
**EnrichmentWriter (v0.6)** — will need:
|
||||
- The reader (Neo4j → DoclingDocument) to re-materialize the doc, feed docling-agent, re-patch enrichments
|
||||
- A stage addable to `IngestionPipeline` without touching other stages
|
||||
- An `:Entity` label (not created in v0.5 but schema-compatible)
|
||||
|
||||
**Agent reasoning trace viewer (v0.6)** — will need:
|
||||
- An event stream (WebSocket) that v0.5 already prepares via the reactive UI
|
||||
- A node_ref ↔ Element correlation in Neo4j (our composite `self_ref` key is enough)
|
||||
|
||||
**TreeNavigationPort (v0.7)** — will need:
|
||||
- Optimized Cypher queries for descendant/ancestor walk (indexes already provisioned)
|
||||
|
||||
---
|
||||
|
||||
## 10. v0.5.0 success criteria
|
||||
|
||||
**Must have:**
|
||||
- [ ] A PDF uploaded to Docling-Studio is in Neo4j with structure preserved
|
||||
- [ ] Neo4j Browser shows the graph and is manually explorable
|
||||
- [ ] A graph visual in the Docling-Studio UI works
|
||||
- [ ] `docker compose up` works zero-config
|
||||
- [ ] README mentions Neo4j and describes the schema
|
||||
|
||||
**Nice to have (decreasing priority):**
|
||||
- [ ] Graph-ready / RAG-ready badge per doc
|
||||
- [ ] Live query explorer in the UI
|
||||
- [ ] 2–3 example queries in README that do something impossible with vector-only
|
||||
|
||||
**For the hackathon (post-release):**
|
||||
- [ ] 60s video: upload PDF → structure in Neo4j → cross-doc query impossible in vector-only
|
||||
- [ ] HackerNoon post explaining "graph-native documents" positioning
|
||||
- [ ] Explicit Neo4j partnership mention
|
||||
|
||||
---
|
||||
|
||||
## 11. Fundamental architectural decisions recap
|
||||
|
||||
| Question | Answer |
|
||||
|----------|---------|
|
||||
| Is Neo4j source of truth or cache ? | **Source of truth** for structure. OpenSearch remains source of truth for embeddings. |
|
||||
| Does chunking go away ? | No, v0.5.0 keeps existing chunking. "Chunkless" is Pipeline B, v0.6+. |
|
||||
| Can it be toggled per doc ? | Yes — `stages_applied` on Document + pipeline config |
|
||||
| What about OpenSearch ? | Stays, stores vectors. Neo4j tracks `(:Chunk)-[:DERIVED_FROM]->(:Element)` links. |
|
||||
| Multi-tenancy ? | `tenant_id` property on Document, Cypher filter |
|
||||
| Versioning ? | None for v0.5.0 — replace strategy on re-upload |
|
||||
|
||||
---
|
||||
|
||||
## Appendix — Demo queries
|
||||
|
||||
### Query 1 — All "Methods" sections across documents
|
||||
```cypher
|
||||
MATCH (d:Document)-[:HAS_ROOT]->(:Element)-[:PARENT_OF*]->(s:SectionHeader)
|
||||
WHERE toLower(s.text) CONTAINS 'method'
|
||||
RETURN d.title, s.text, s.level
|
||||
```
|
||||
|
||||
### Query 2 — Context of a chunk (parent + siblings)
|
||||
```cypher
|
||||
MATCH (c:Chunk {id: $chunk_id})-[:DERIVED_FROM]->(e:Element)
|
||||
MATCH (e)<-[:PARENT_OF]-(parent:Element)
|
||||
MATCH (parent)-[:PARENT_OF]->(sibling:Element)
|
||||
RETURN parent, collect(sibling) AS siblings
|
||||
```
|
||||
|
||||
### Query 3 — All tables from a document type
|
||||
```cypher
|
||||
MATCH (d:Document)-[:HAS_ROOT]->(:Element)-[:PARENT_OF*]->(t:Table)
|
||||
WHERE d.source_uri CONTAINS 'invoices/'
|
||||
RETURN d.title, t.caption, t.cells_json
|
||||
```
|
||||
|
||||
### Query 4 — Direct children of a section (ordered)
|
||||
```cypher
|
||||
MATCH (s:Element {doc_id: $doc_id, self_ref: $section_ref})
|
||||
MATCH (s)-[pof:PARENT_OF]->(child)
|
||||
RETURN child
|
||||
ORDER BY pof.order
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
*Single reference doc for Neo4j v0.5.0 implementation.
|
||||
Read this first in the implementation thread.*
|
||||
|
|
@ -1,253 +0,0 @@
|
|||
# Reasoning Trace Viewer — Docling-Studio v0.6.0 (R&D preview)
|
||||
|
||||
Design doc for the `docling-agent` reasoning trace viewer.
|
||||
Targeted release: **v0.6.0** (R&D branched from `release/0.5.0` in parallel to the
|
||||
0.5 build, so the Neo4j foundation can be leveraged without blocking the hackathon deliverable).
|
||||
|
||||
Positioning one-liner:
|
||||
> Studio becomes the **reference viewer for any `docling-agent` run** — not another
|
||||
> chatbot. The PDF is the debug surface.
|
||||
|
||||
---
|
||||
|
||||
## 1. Context
|
||||
|
||||
### Upstream trigger
|
||||
Peter Staar (IBM) suggested surfacing the LLM reasoning trace as `docling-agent`
|
||||
walks a `DoclingDocument` outline (chunkless RAG, new IBM repo).
|
||||
|
||||
### `docling-agent` in one paragraph
|
||||
`DoclingRAGAgent(model_id, tools, max_iterations=5).run(task, sources=[doc])` returns
|
||||
a `RAGResult` with `answer`, `converged`, and `iterations: list[RAGIteration]`.
|
||||
Each `RAGIteration` carries: `iteration`, `section_ref` (JSON-pointer, e.g. `#/texts/3`),
|
||||
`reason`, `can_answer`, `response`, `section_text_length`. No bbox — must be resolved
|
||||
through `DoclingDocument.<items>[i].prov[0].bbox` + `page_no`. Runs on Mellea
|
||||
(Ollama / OpenAI / HF / WatsonX / LiteLLM / Bedrock). Observability is stdout logs only.
|
||||
|
||||
### What Studio already brings
|
||||
- **Neo4j graph of the document** (0.5.0, just landed) — every `Element` is keyed by
|
||||
`(doc_id, self_ref)`, Cytoscape node id is `elem::${self_ref}`. **This is the
|
||||
killer enabler.** `RAGIteration.section_ref → node` is a string concat, no resolver.
|
||||
- `GraphView.vue` (Cytoscape + dagre) already handles styles via selectors
|
||||
(`selector: 'edge[type = "NEXT"]'`, `selector: 'node[kind = "section"]'`) — adding
|
||||
a `visited` class + `REASONING_NEXT` synthetic edge type is ~20 LOC of style.
|
||||
- `analysis_jobs.document_json` in SQLite → DoclingDocument available for the sidecar
|
||||
runner (no PDF re-conversion). Not used by the viewer itself.
|
||||
|
||||
### Personas
|
||||
- **v1 (this plan)**: dev / integrator of `docling-agent` debugging a run that went wrong.
|
||||
- **v2 (roadmap)**: live runner with synchronized demo UX.
|
||||
- v3+ (non-goals here): business analyst for semantic navigation, batch QA.
|
||||
|
||||
---
|
||||
|
||||
## 2. Scope split — **debug first, demo second**
|
||||
|
||||
Rendering surface pivoted: **the trace is drawn on the Neo4j graph**, not on the PDF.
|
||||
See §1. This kills the whole bbox-resolution stack from v1.
|
||||
|
||||
| Phase | Value | Runtime deps | Surface |
|
||||
|---|---|---|---|
|
||||
| **v1 — Debug (this plan)** | Import externally-produced `RAGResult` JSON, overlay trace on the existing GraphView | **None** server-side. Pure frontend. | GraphView: visited nodes highlighted in order, synthetic `REASONING_NEXT` edges |
|
||||
| **v2 — Demo (follow-up)** | Run the agent live against a loaded document | Ollama + Mellea + `docling-agent` (new opt-dep group `rag`) | Same GraphView + SSE streaming of iterations, staggered reveal |
|
||||
|
||||
Building v1 first de-risks the **graph-trace UX** on real runs (produced by the
|
||||
R&D sidecar — see `experiments/reasoning-trace/`) before wiring the live runner.
|
||||
Code shared between v1 and v2 is the GraphView overlay itself — 100 % reused.
|
||||
|
||||
**Prerequisite for v1**: the target document must have been processed through the
|
||||
"Maintain" step (Neo4j pipeline). Otherwise the graph is empty and the trace has
|
||||
nowhere to render — surface an explicit "Run the Maintain step first" empty state.
|
||||
|
||||
---
|
||||
|
||||
## 3. v1 — Debug mode (frontend-only)
|
||||
|
||||
### 3.1 No backend changes in v1
|
||||
|
||||
The GraphView already loads nodes keyed by `self_ref` via `GET /api/documents/{doc_id}/graph`.
|
||||
Iteration `section_ref` → Cytoscape node id is `` `elem::${section_ref}` `` — a client-side
|
||||
string concat. Nothing to compute server-side.
|
||||
|
||||
Consequences:
|
||||
- No new router, no new service, no new pydantic model, no new migration.
|
||||
- No dependency on `docling-agent` in `document-parser/requirements.txt`.
|
||||
- `RAGResult` JSON (as produced by `experiments/reasoning-trace/`) is consumed
|
||||
entirely by the frontend.
|
||||
|
||||
### 3.2 Frontend — feature folder
|
||||
|
||||
New `frontend/src/features/reasoning/`:
|
||||
|
||||
```
|
||||
reasoning/
|
||||
├── store/reasoningStore.ts # Pinia: trace, activeIteration, importDialogOpen
|
||||
├── ui/
|
||||
│ ├── ReasoningPanel.vue # Side panel: query, answer, iteration list
|
||||
│ ├── IterationCard.vue # Single iteration row (reason + can_answer badge)
|
||||
│ ├── ImportTraceDialog.vue # Drag-drop / paste RAGResult JSON
|
||||
│ └── GraphReasoningOverlay.ts # NOT a component — a plugin that decorates cy
|
||||
└── types.ts # RAGIteration, RAGResult mirror types
|
||||
```
|
||||
|
||||
### 3.3 Graph overlay — how it's drawn
|
||||
|
||||
`GraphReasoningOverlay` takes the existing `cy` Cytoscape instance (exposed from
|
||||
`GraphView.vue` via `defineExpose`) and:
|
||||
|
||||
1. For each `iteration[i].section_ref`, find node `` `elem::${section_ref}` ``. If
|
||||
missing, tag as `resolution_status: "not_in_graph"` and show a warning in the panel
|
||||
(common cause: doc not processed through Maintain, or agent returned a ref that
|
||||
points at a non-Element node).
|
||||
2. Add class `visited` + data attribute `visitOrder: i+1` on matched nodes.
|
||||
3. Insert **synthetic edges** between successive visited nodes with `type: "REASONING_NEXT"`
|
||||
and `data: { order: i }`. These edges are UI-only, never written to Neo4j.
|
||||
4. On import, fit viewport to the visited subgraph (`cy.fit(cy.$('.visited'), 80)`).
|
||||
5. On iteration card click → `cy.$(`#elem::${ref}`).flashClass('pulse', 800)` +
|
||||
centered pan.
|
||||
|
||||
Cytoscape styles (append to the existing stylesheet array in `GraphView.vue`):
|
||||
|
||||
```js
|
||||
{ selector: 'node.visited',
|
||||
style: { 'border-color': '#EA580C', 'border-width': 3, 'overlay-opacity': 0 } },
|
||||
{ selector: 'node.visited[visitOrder]',
|
||||
style: { label: 'data(visitOrder)', 'text-valign': 'top',
|
||||
'text-background-color': '#EA580C', 'text-background-opacity': 1,
|
||||
'color': '#FFFFFF', 'font-weight': 700 } },
|
||||
{ selector: 'edge[type = "REASONING_NEXT"]',
|
||||
style: { 'line-color': '#EA580C', 'target-arrow-color': '#EA580C',
|
||||
'target-arrow-shape': 'triangle', 'curve-style': 'bezier',
|
||||
width: 2, 'z-index': 99 } },
|
||||
```
|
||||
|
||||
Color ramp: single warm color (`#EA580C`) for v1. Gradient cold→warm is v2 polish.
|
||||
|
||||
### 3.4 Integration points
|
||||
|
||||
- `StudioPage.vue` → "Maintain" tab gains an **"Import reasoning trace"** action
|
||||
(don't add a 3rd mode — the viz lives inside the graph view, not a new workspace).
|
||||
- `GraphView.vue` → add `defineExpose({ cy })` + a `<slot name="overlay" :cy="cy"/>`
|
||||
that the parent can populate with `<ReasoningPanel>`.
|
||||
- `ReasoningPanel` appears as a right rail when a trace is loaded; collapsible.
|
||||
|
||||
### 3.5 Empty / error states
|
||||
|
||||
- **Graph empty for this doc** → "Run the Maintain step first. Neo4j has no graph for
|
||||
this document yet." (the Maintain button is literally next to it.)
|
||||
- **All `section_ref`s unresolved in graph** → "None of the visited sections exist in
|
||||
the graph. The agent may have been run against a different document, or the doc was
|
||||
re-analyzed since. Re-run Maintain or re-run the agent."
|
||||
- **Some resolved, some not** → show trace with the missing ones greyed out in the panel.
|
||||
|
||||
### 3.6 Tests
|
||||
|
||||
No backend tests in v1 (no backend code).
|
||||
|
||||
Frontend (Vitest):
|
||||
- `reasoningStore.test.ts` — import trace, active iteration transitions, reset on doc change.
|
||||
- `graphReasoningOverlay.test.ts` — given a mock `cy` (`cytoscape({ headless: true })`)
|
||||
with a known node set, verify `visited` class applied to the right ids and the
|
||||
correct synthetic edges added.
|
||||
- `ReasoningPanel.test.ts` — empty / loaded / partial-resolution states.
|
||||
|
||||
### 3.7 Out of scope for v1
|
||||
- Live agent runner (v2).
|
||||
- Multi-doc queries — reject import if `RAGResult` was produced against `len(sources) > 1`.
|
||||
- Phrase-level attribution — `docling-agent` doesn't emit it.
|
||||
- Persisting traces in Neo4j — see §7.
|
||||
- PDF highlighting — dropped from v1. Could come back as v2.5 if demand exists.
|
||||
|
||||
---
|
||||
|
||||
## 4. File inventory (v1)
|
||||
|
||||
**New — R&D sidecar** (already scaffolded on this branch)
|
||||
- `experiments/reasoning-trace/inspect_doc.py` — self-contained `uv run` script.
|
||||
- `experiments/reasoning-trace/README.md`
|
||||
- `experiments/reasoning-trace/.gitignore`
|
||||
|
||||
**New — frontend**
|
||||
- `frontend/src/features/reasoning/**` (see §3.2)
|
||||
- Vitest siblings under `**/*.test.ts`
|
||||
|
||||
**Touched**
|
||||
- `frontend/src/features/analysis/ui/GraphView.vue` — `defineExpose({ cy })` +
|
||||
`<slot name="overlay">` + 3 new style selectors.
|
||||
- `frontend/src/pages/StudioPage.vue` — "Import reasoning trace" action in the
|
||||
Maintain tab rail.
|
||||
|
||||
**Untouched**
|
||||
- Entire `document-parser/` backend — no new router, service, schema, or dep.
|
||||
- `pyproject.toml` / `requirements.txt` — **no new runtime dep in v1**.
|
||||
- Neo4j schema — synthetic edges are client-side Cytoscape only.
|
||||
- OpenSearch / ingestion — untouched.
|
||||
- SQLite schema — no migration.
|
||||
|
||||
---
|
||||
|
||||
## 5. Risks & mitigations
|
||||
|
||||
| Risk | Mitigation |
|
||||
|---|---|
|
||||
| `RAGResult` schema drifts in `docling-agent` | `schema_version` discriminator; strict pydantic; one canonical fixture from Peter pinned in CI. |
|
||||
| `section_ref` variants (`#/texts/3` vs `#/body/texts/3`) | Normalize in parser; regex test matrix. |
|
||||
| Synthetic groups without `prov` | Documented child-walk fallback + `resolved_via_child` status surfaced in UI. |
|
||||
| Large `RAGResult` (hundreds of iterations) | Hard-cap `iterations` at 50 in v1 (Peter's agent uses `max_iterations=5` by default) — return 413 above. |
|
||||
| `document_json` blob large (some docs > 5 MB) | `analysis_repo` already handles it; but **do not** log the blob. Add redaction test. |
|
||||
| Section ref not in graph (doc not through Maintain, or re-analyzed) | Explicit empty-state in `ReasoningPanel` with a link to the Maintain tab. Partial resolution shown as grey in the trace list. |
|
||||
| Feature creeping into 0.5.0 | This branch targets **v0.6.0**. Do not merge into `release/0.5.0`. Rebase onto the next release branch when cut. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Spec anchoring
|
||||
|
||||
Pin the `RAGResult` shape to **docling-agent commit SHA at the time of v1 merge** in
|
||||
a short ADR `docs/architecture/adrs/ADR-002-rag-result-schema.md`. The schema is
|
||||
upstream, unversioned, and will move — this doc freezes the contract Studio imports.
|
||||
|
||||
---
|
||||
|
||||
## 7. v2 preview — demo mode (not in this plan)
|
||||
|
||||
Kept here to constrain v1 interfaces so nothing needs rewriting:
|
||||
- `POST /api/rag/answer` — server-side runner. Accepts `{doc_id, question, model_id}`.
|
||||
Streams iterations via SSE. Frontend consumes the stream with the same
|
||||
`GraphReasoningOverlay` used by v1 import — iterations appear one by one with
|
||||
staggered reveal (~400 ms) as the SSE stream drips them in.
|
||||
- Ollama wired through `Mellea` — new optional dep group `rag`.
|
||||
- Persist traces in Neo4j as `(:ReasoningRun {id, query, converged})-[:VISITED {order,
|
||||
reason, can_answer}]->(:Element)` for replay + cross-run analytics. Leverages
|
||||
`TreeWriter` pattern already present. This is where the synthetic UI edges become
|
||||
real graph edges.
|
||||
- Cross-run comparison view: overlay multiple runs on the same graph, diff the paths.
|
||||
|
||||
---
|
||||
|
||||
## 8. Branch & workflow
|
||||
|
||||
- Branch: **`feature/reasoning-trace`** off `origin/release/0.5.0`.
|
||||
- Merge target: **next release branch (`release/0.6.0`)** once cut — *not* `0.5.0`.
|
||||
- Until then: live on the feature branch; rebase onto `release/0.5.0` periodically to
|
||||
absorb Neo4j fixes.
|
||||
- Issues: one umbrella + one per §4 subsystem (resolver, endpoint, UI panel, overlay,
|
||||
import dialog, tests). Commit with `Closes #NNN` per project convention.
|
||||
- PR: opened against `release/0.6.0` when available; draft in the meantime.
|
||||
|
||||
---
|
||||
|
||||
## 9. Open questions (answered by the sidecar first run)
|
||||
|
||||
1. Are emitted `section_ref`s reachable as `elem::${ref}` in the Neo4j graph built
|
||||
by `TreeWriter`? I.e. is the `self_ref` the agent sees the same `self_ref` we
|
||||
wrote to the graph? (Expected yes — both come from the same `DoclingDocument` —
|
||||
but the sidecar on a real doc from SQLite will confirm in one run.)
|
||||
2. Hit rate of the agent: with `max_iterations=5` and `granite4:micro-h`, does it
|
||||
converge, and how many sections does it actually visit? Determines if the overlay
|
||||
ever has more than 1–2 marked nodes (and whether `REASONING_NEXT` edges are worth
|
||||
the effort vs just node markers).
|
||||
3. Quality of `iteration.reason` — is it substantive enough to show in the panel, or
|
||||
LLM filler we should hide? Sidecar output will tell.
|
||||
4. Fallback when no section headers exist (`RAGResult(iterations=[], converged=True,
|
||||
answer=<full md>)` — see rag.py): what does the panel show? Probably a degraded
|
||||
"no trace available, full-doc answer" state.
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
# Getting Started
|
||||
|
||||
## Quick Start
|
||||
|
||||
One command, nothing else to install:
|
||||
|
||||
```bash
|
||||
docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000), upload a PDF, and get results. That's it.
|
||||
|
||||
!!! note
|
||||
The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast.
|
||||
|
||||
{ width="600" }
|
||||
|
||||
## Image Variants
|
||||
|
||||
| Variant | Image tag | Size | Description |
|
||||
|---------|-----------|------|-------------|
|
||||
| **local** | `latest-local` | ~1.9 GB | Full — runs Docling in-process, CPU-only |
|
||||
| **remote** | `latest-remote` | ~270 MB | Lightweight — delegates to an external [Docling Serve](https://github.com/DS4SD/docling-serve) instance |
|
||||
|
||||
For remote mode:
|
||||
|
||||
```bash
|
||||
docker run -p 3000:3000 \
|
||||
-e DOCLING_SERVE_URL=http://your-docling-serve:5001 \
|
||||
ghcr.io/scub-france/docling-studio:latest-remote
|
||||
```
|
||||
|
||||
## Docker Compose
|
||||
|
||||
```bash
|
||||
git clone https://github.com/scub-france/Docling-Studio.git
|
||||
cd Docling-Studio
|
||||
|
||||
# Simple mode (backend + frontend only)
|
||||
docker compose up --build
|
||||
|
||||
# With ingestion pipeline (OpenSearch + embeddings)
|
||||
docker compose --profile ingestion \
|
||||
-f docker-compose.yml -f docker-compose.ingestion.yml \
|
||||
up --build
|
||||
```
|
||||
|
||||
## Local Development
|
||||
|
||||
=== "Backend (Python 3.12+)"
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
|
||||
# Remote mode (lightweight)
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Local mode (with Docling)
|
||||
pip install -r requirements-local.txt
|
||||
|
||||
uvicorn main:app --reload --port 8000
|
||||
```
|
||||
|
||||
=== "Frontend (Node 20+)"
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
The frontend runs on `http://localhost:3000` and proxies API calls to `http://localhost:8000`.
|
||||
|
||||
## Running Tests
|
||||
|
||||
=== "Backend"
|
||||
|
||||
```bash
|
||||
cd document-parser
|
||||
pip install pytest pytest-asyncio httpx
|
||||
pytest tests/ -v
|
||||
```
|
||||
|
||||
=== "Frontend"
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm run test:run
|
||||
```
|
||||
|
||||
## Pipeline Options
|
||||
|
||||
These options map directly to Docling's [`PdfPipelineOptions`](https://docling-project.github.io/docling/usage/).
|
||||
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `do_ocr` | `true` | OCR for scanned pages and embedded images |
|
||||
| `do_table_structure` | `true` | Table detection and row/column reconstruction |
|
||||
| `table_mode` | `accurate` | `accurate` (TableFormer) or `fast` |
|
||||
| `do_code_enrichment` | `false` | Specialized OCR for code blocks |
|
||||
| `do_formula_enrichment` | `false` | Math formula recognition (LaTeX output) |
|
||||
| `do_picture_classification` | `false` | Classify images by type |
|
||||
| `do_picture_description` | `false` | Generate image descriptions via VLM |
|
||||
| `generate_picture_images` | `false` | Extract detected images as separate files |
|
||||
| `generate_page_images` | `false` | Rasterize each page as an image |
|
||||
| `images_scale` | `1.0` | Scale factor for generated images (0.1–10) |
|
||||
|
||||
## Chunking Options
|
||||
|
||||
!!! note
|
||||
Chunking is only available in **local** mode. The chunking UI is hidden when using remote mode (Docling Serve).
|
||||
|
||||
After a document is analyzed, you can split the extracted content into semantic chunks. Chunking can be configured at analysis time or re-run later with different options via the **rechunk** action.
|
||||
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `chunker_type` | `hybrid` | `hybrid` (semantic + structural), `hierarchical` (heading-based), or `page` (one chunk per page) |
|
||||
| `max_tokens` | `512` | Maximum tokens per chunk |
|
||||
| `merge_peers` | `true` | Merge sibling elements under the same heading |
|
||||
| `repeat_table_header` | `true` | Repeat table headers when a table is split across chunks |
|
||||
|
||||
Each chunk includes:
|
||||
|
||||
- **text** — the chunk content
|
||||
- **headings** — heading hierarchy leading to the chunk
|
||||
- **source_page** — the page number the chunk originates from
|
||||
- **token_count** — number of tokens in the chunk
|
||||
- **bboxes** — bounding boxes of the chunk's source elements (page + coordinates)
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration is done via environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `CONVERSION_ENGINE` | `local` | `local` (in-process Docling) or `remote` (Docling Serve) |
|
||||
| `DOCLING_SERVE_URL` | `http://localhost:5001` | Docling Serve endpoint (remote mode only) |
|
||||
| `DOCLING_SERVE_API_KEY` | — | API key for Docling Serve (optional) |
|
||||
| `CORS_ORIGINS` | `http://localhost:3000,...` | CORS allowed origins |
|
||||
| `UPLOAD_DIR` | `./uploads` | File storage directory |
|
||||
| `DB_PATH` | `./data/docling_studio.db` | SQLite database path |
|
||||
| `CONVERSION_TIMEOUT` | `600` | Max seconds per Docling conversion |
|
||||
| `BATCH_PAGE_SIZE` | `10` | Pages per batch (`0` = process all at once) |
|
||||
| `MAX_CONCURRENT_ANALYSES` | `3` | Maximum parallel analysis jobs |
|
||||
| `DEPLOYMENT_MODE` | `self-hosted` | `self-hosted` or `huggingface` (shows disclaimer banner) |
|
||||
| `MAX_FILE_SIZE_MB` | `50` | Maximum upload file size in MB (`0` = unlimited) |
|
||||
| `MAX_PAGE_COUNT` | `0` | Maximum number of pages per document (`0` = unlimited) |
|
||||
| `RATE_LIMIT_RPM` | `100` | Max requests per minute per IP (`0` = disabled) |
|
||||
| `APP_VERSION` | `dev` | Application version (set automatically by CI/Docker) |
|
||||
|
||||
## Upload Limits
|
||||
|
||||
Docling Studio enforces configurable limits on uploaded documents to protect the server against oversized files and long-running analyses:
|
||||
|
||||
- **`MAX_FILE_SIZE_MB`** (default `50`) — rejects uploads exceeding this size. Validated at two levels: early `Content-Length` check and streaming byte count.
|
||||
- **`MAX_PAGE_COUNT`** (default `0` = unlimited) — rejects documents with more pages than allowed. Useful on shared instances or Hugging Face Spaces to cap processing time.
|
||||
|
||||
Both limits are exposed in the `/api/health` endpoint so the frontend can display them to the user before upload. Set either to `0` to disable the corresponding check.
|
||||
|
||||
## Ingestion Pipeline (opt-in)
|
||||
|
||||
Docling Studio can optionally index extracted chunks into [OpenSearch](https://opensearch.org/) for vector and full-text search. This requires two additional services (OpenSearch + embedding) and is **disabled by default**.
|
||||
|
||||
To enable ingestion with Docker Compose:
|
||||
|
||||
```bash
|
||||
docker compose --profile ingestion \
|
||||
-f docker-compose.yml -f docker-compose.ingestion.yml \
|
||||
up --build
|
||||
```
|
||||
|
||||
When ingestion is enabled, the UI shows:
|
||||
|
||||
- An **Ingest** button in Studio to push chunks to OpenSearch
|
||||
- An **OpenSearch** connection status badge in the sidebar
|
||||
- **Indexed / Not indexed** filters on the Documents page
|
||||
- A **Search** page for full-text and vector search across indexed documents
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `OPENSEARCH_URL` | — | OpenSearch endpoint (empty = ingestion disabled) |
|
||||
| `EMBEDDING_URL` | — | Embedding service endpoint (empty = ingestion disabled) |
|
||||
| `EMBEDDING_DIMENSION` | `384` | Vector dimension (must match embedding model) |
|
||||
|
||||
## System Requirements
|
||||
|
||||
| | Remote image | Local image |
|
||||
|---|---|---|
|
||||
| **Image size** | ~270 MB | ~1.9 GB |
|
||||
| **Memory** | 2 GB | 6 GB (recommended 8 GB+) |
|
||||
| **CPUs** | 2 | 4 (recommended 8+) |
|
||||
|
||||
All Docker images are multi-arch (`linux/amd64` + `linux/arm64`). No GPU required.
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# Code Review Checklist
|
||||
|
||||
Use this checklist when reviewing a Pull Request. Not every item applies to every PR — skip what is irrelevant, but consciously skip it rather than miss it.
|
||||
|
||||
## Correctness
|
||||
|
||||
- [ ] The code does what the PR description says it does
|
||||
- [ ] Edge cases are handled (empty input, missing data, concurrent access)
|
||||
- [ ] Error paths return meaningful messages, not stack traces
|
||||
- [ ] No regressions on existing behavior
|
||||
|
||||
## Architecture & Design
|
||||
|
||||
- [ ] Dependencies flow inward: `api → services → domain` (never reversed)
|
||||
- [ ] Domain layer has no imports from `api/`, `persistence/`, `infra/`
|
||||
- [ ] No business logic in API routes — delegated to services
|
||||
- [ ] New abstractions are justified (no premature generalization)
|
||||
- [ ] Pinia stores stay within their feature boundary
|
||||
|
||||
## Security
|
||||
|
||||
- [ ] User input is validated at the API boundary (Pydantic schemas)
|
||||
- [ ] No secrets, API keys, or credentials in the code
|
||||
- [ ] No `eval()`, `exec()`, or raw SQL
|
||||
- [ ] File paths are sanitized (no path traversal)
|
||||
- [ ] CORS configuration is unchanged or explicitly justified
|
||||
|
||||
## Tests
|
||||
|
||||
- [ ] New behavior has corresponding tests
|
||||
- [ ] Tests are deterministic (no `sleep`, no random, no network)
|
||||
- [ ] Test names describe the scenario, not the implementation
|
||||
- [ ] E2E tests use `data-e2e` attributes, not CSS classes (see [e2e/CONVENTIONS.md](https://github.com/scub-france/Docling-Studio/blob/main/e2e/CONVENTIONS.md))
|
||||
|
||||
## Code Quality
|
||||
|
||||
- [ ] No dead code, no commented-out code
|
||||
- [ ] No `TODO` or `FIXME` without a linked issue
|
||||
- [ ] Functions are < 30 lines (or well-justified)
|
||||
- [ ] Variable names are descriptive and consistent with existing code
|
||||
- [ ] No duplicated logic that should be extracted
|
||||
|
||||
## Documentation
|
||||
|
||||
- [ ] `CHANGELOG.md` updated under `[Unreleased]` if user-facing change
|
||||
- [ ] API changes reflected in Pydantic schemas (auto-documented)
|
||||
- [ ] Breaking changes are flagged in the commit message
|
||||
|
||||
## Pragmatic Checks
|
||||
|
||||
- [ ] The PR does ONE thing (feature, fix, or refactor — not all at once)
|
||||
- [ ] No unrelated formatting changes mixed in
|
||||
- [ ] The diff is reviewable in < 15 minutes (split large PRs)
|
||||
- [ ] CI passes (tests + lint + type-check + build)
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# Commit Conventions
|
||||
|
||||
We follow [Conventional Commits](https://www.conventionalcommits.org/) to keep the git history readable and to enable automated changelog generation.
|
||||
|
||||
## Format
|
||||
|
||||
```
|
||||
<type>(<scope>): <description>
|
||||
|
||||
[optional body]
|
||||
|
||||
[optional footer(s)]
|
||||
```
|
||||
|
||||
## Types
|
||||
|
||||
| Type | When to use | Example |
|
||||
|------|-------------|---------|
|
||||
| `feat` | New feature | `feat(chunking): add hierarchical chunker support` |
|
||||
| `fix` | Bug fix | `fix(upload): handle empty PDF gracefully` |
|
||||
| `docs` | Documentation only | `docs: update architecture diagram` |
|
||||
| `style` | Formatting, no logic change | `style(api): fix ruff formatting warnings` |
|
||||
| `refactor` | Code restructuring, no behavior change | `refactor(persistence): extract repository base class` |
|
||||
| `test` | Adding or updating tests | `test(analysis): add rechunk edge case tests` |
|
||||
| `chore` | Tooling, CI, dependencies | `chore: bump docling to 2.31` |
|
||||
| `perf` | Performance improvement | `perf(bbox): batch coordinate normalization` |
|
||||
| `ci` | CI/CD pipeline changes | `ci: add multi-arch Docker build` |
|
||||
|
||||
## Scopes (optional)
|
||||
|
||||
Use the feature or component name:
|
||||
|
||||
| Scope | Maps to |
|
||||
|-------|---------|
|
||||
| `api` | `document-parser/api/` |
|
||||
| `domain` | `document-parser/domain/` |
|
||||
| `persistence` | `document-parser/persistence/` |
|
||||
| `infra` | `document-parser/infra/` |
|
||||
| `upload` | Upload feature (front + back) |
|
||||
| `analysis` | Analysis feature (front + back) |
|
||||
| `chunking` | Chunking feature (front + back) |
|
||||
| `bbox` | Bounding box pipeline |
|
||||
| `e2e` | `e2e/` tests |
|
||||
| `docker` | Dockerfile, docker-compose |
|
||||
| `ci` | `.github/workflows/` |
|
||||
|
||||
## Rules
|
||||
|
||||
1. **Subject line** — imperative mood, lowercase, no period, max 72 characters
|
||||
2. **Body** — explain *why*, not *what* (the diff shows what)
|
||||
3. **Breaking changes** — add `BREAKING CHANGE:` in the footer or `!` after the type: `feat(api)!: rename /analyses to /jobs`
|
||||
4. **Issue references** — use `Closes #123` or `Fixes #456` in the footer
|
||||
|
||||
## Examples
|
||||
|
||||
```
|
||||
feat(chunking): add page filtering in Prepare mode
|
||||
|
||||
Users can now select which pages to include in chunking.
|
||||
The filter is persisted in the analysis job metadata.
|
||||
|
||||
Closes #87
|
||||
```
|
||||
|
||||
```
|
||||
fix(upload): reject files exceeding MAX_FILE_SIZE_MB
|
||||
|
||||
Previously, oversized files were accepted and failed silently
|
||||
during Docling conversion. Now the API returns 413 with a
|
||||
clear error message.
|
||||
|
||||
Fixes #102
|
||||
```
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Merge Policy
|
||||
|
||||
## Merge Requirements
|
||||
|
||||
Every PR must meet these conditions before merging:
|
||||
|
||||
1. **CI green** — all checks pass (tests, lint, type-check, build)
|
||||
2. **At least 1 approval** from a maintainer
|
||||
3. **No unresolved conversations** — all review comments addressed
|
||||
4. **Branch up to date** with target branch (rebase or merge from target)
|
||||
5. **CHANGELOG updated** — if the change is user-facing
|
||||
|
||||
## Merge Strategy
|
||||
|
||||
| Branch type | Merge method | Why |
|
||||
|-------------|-------------|-----|
|
||||
| `feature/*` → `main` | **Squash merge** | Clean history — one commit per feature |
|
||||
| `fix/*` → `main` | **Squash merge** | Clean history — one commit per fix |
|
||||
| `fix/*` → `release/*` | **Squash merge** | Same rationale |
|
||||
| `release/*` → `main` | **Merge commit** | Preserves the release branch history |
|
||||
| `hotfix/*` → `main` | **Squash merge** | One atomic fix |
|
||||
|
||||
### Squash merge commit message
|
||||
|
||||
When squashing, the final commit message should follow [Conventional Commits](commit-conventions.md):
|
||||
|
||||
```
|
||||
feat(chunking): add page filtering in Prepare mode (#87)
|
||||
```
|
||||
|
||||
The PR number is appended automatically by GitHub.
|
||||
|
||||
## Stale PR Policy
|
||||
|
||||
| Condition | Action |
|
||||
|-----------|--------|
|
||||
| No activity for **14 days** | Maintainer pings the author |
|
||||
| No activity for **30 days** | PR labeled `stale` |
|
||||
| No activity for **45 days** | PR closed with comment (can be reopened) |
|
||||
|
||||
Draft PRs are exempt from the stale policy.
|
||||
|
||||
## Conflict Resolution
|
||||
|
||||
- **Rebase preferred** over merge commits for feature/fix branches
|
||||
- If conflicts are complex, merge from target branch into the PR branch
|
||||
- Never force-push a branch that has active reviewers without warning them
|
||||
|
||||
## Branch Cleanup
|
||||
|
||||
- Feature and fix branches are **deleted after merge** (GitHub auto-delete enabled)
|
||||
- Release branches are kept until the next minor release
|
||||
- Tags are never deleted
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 243 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 430 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 858 KiB |
|
|
@ -1,52 +0,0 @@
|
|||
# Docling Studio
|
||||
|
||||
A visual document analysis studio powered by [Docling](https://github.com/DS4SD/docling).
|
||||
|
||||
Upload a PDF, configure the extraction pipeline, and visualize the results — text, tables, images, formulas, bounding boxes — all from your browser.
|
||||
|
||||
{ width="600" }
|
||||
|
||||
|
||||
|
||||
## Features
|
||||
|
||||
- **PDF viewer** with page navigation, bounding box overlay, and resizable results panel
|
||||
- **Configurable Docling pipeline** — OCR, table extraction, code/formula enrichment, picture classification & description, image generation
|
||||
- **Bounding box visualization** — color-coded element overlay directly on the PDF
|
||||
- **Chunking** — split extracted content into semantic chunks (hierarchical, hybrid, or page-based) with configurable token limits
|
||||
- **Markdown & HTML export** of extracted content
|
||||
- **Document management** — upload, list, delete
|
||||
- **Analysis history** — re-visit and open past analyses
|
||||
- **Feature flags** — capabilities adapt to the conversion engine (local vs remote)
|
||||
- **Upload limits** — configurable max file size (`MAX_FILE_SIZE_MB`) and max page count (`MAX_PAGE_COUNT`) per document
|
||||
- **Rate limiting** — configurable requests per minute per IP (`RATE_LIMIT_RPM`)
|
||||
- **Deployment modes** — self-hosted (default) or HuggingFace Spaces (with disclaimer banner)
|
||||
- **Health endpoint** — `/api/health` reports engine type, deployment mode, and database status
|
||||
- **Dark / Light theme** and **FR / EN** localization
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Stack |
|
||||
|-------|-------|
|
||||
| **Frontend** | Vue 3, TypeScript, Vite, Pinia |
|
||||
| **Backend** | FastAPI, Docling 2.x, SQLite (aiosqlite) |
|
||||
| **CI** | GitHub Actions (lint, type-check, test, build) |
|
||||
| **Infra** | Docker Compose + Nginx |
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Docker (fastest)
|
||||
docker run -p 3000:3000 ghcr.io/scub-france/docling-studio:latest-local
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) and upload a PDF.
|
||||
|
||||
!!! note
|
||||
The first analysis takes longer as Docling downloads its ML models (~400 MB). Subsequent runs are fast.
|
||||
|
||||
See [Getting Started](getting-started.md) for local development setup.
|
||||
|
||||
## License
|
||||
|
||||
[MIT](https://github.com/scub-france/Docling-Studio/blob/main/LICENSE) — Pier-Jean Malandrino
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
# Incident Response
|
||||
|
||||
## Severity Levels
|
||||
|
||||
| Level | Description | Examples | Response time |
|
||||
|-------|-------------|----------|---------------|
|
||||
| **SEV-1** | Service down, data loss, security breach | App unreachable, DB corrupted, credentials leaked | Immediate |
|
||||
| **SEV-2** | Major feature broken, degraded for all users | Upload fails, analysis crashes, blank pages | < 2 hours |
|
||||
| **SEV-3** | Minor feature broken, workaround exists | Bbox overlay misaligned, locale missing a key | Next business day |
|
||||
|
||||
## Response Steps
|
||||
|
||||
### 1. Detect
|
||||
|
||||
- Health endpoint returns error (`/api/health`)
|
||||
- User report via GitHub issue
|
||||
- CI/CD pipeline failure on `main`
|
||||
- Docker container crash loop
|
||||
|
||||
### 2. Assess
|
||||
|
||||
- Determine severity (SEV-1/2/3)
|
||||
- Identify affected component: backend, frontend, Docker, CI
|
||||
- Check recent deployments: `git log --oneline -10 main`
|
||||
|
||||
### 3. Communicate
|
||||
|
||||
- **SEV-1**: Notify all maintainers immediately
|
||||
- **SEV-2**: Open a GitHub issue with `priority: P0` label
|
||||
- **SEV-3**: Open a GitHub issue with `priority: P1` label
|
||||
|
||||
### 4. Mitigate
|
||||
|
||||
**Rollback first, investigate later.**
|
||||
|
||||
- If the issue appeared after a deploy → [rollback](../release/rollback-playbook.md)
|
||||
- If the issue is in a specific endpoint → disable the route or return a maintenance response
|
||||
- If the issue is in Docling itself → switch to remote mode (`CONVERSION_ENGINE=remote`)
|
||||
|
||||
### 5. Fix
|
||||
|
||||
- Create a `hotfix/*` branch from the last stable tag
|
||||
- Fix the root cause, add a regression test
|
||||
- Run the [release audit](../audit/master.md) on the fix
|
||||
- Deploy via the [deployment checklist](../release/deployment-checklist.md)
|
||||
|
||||
### 6. Post-Mortem
|
||||
|
||||
Write a post-mortem for SEV-1 and SEV-2 incidents using this template:
|
||||
|
||||
```markdown
|
||||
# Post-Mortem: [Incident Title]
|
||||
|
||||
**Date**: YYYY-MM-DD
|
||||
**Severity**: SEV-X
|
||||
**Duration**: Xh Xm
|
||||
**Author**: [name]
|
||||
|
||||
## Timeline
|
||||
|
||||
| Time | Event |
|
||||
|------|-------|
|
||||
| HH:MM | Incident detected |
|
||||
| HH:MM | Severity assessed |
|
||||
| HH:MM | Mitigation applied |
|
||||
| HH:MM | Root cause identified |
|
||||
| HH:MM | Fix deployed |
|
||||
| HH:MM | Incident resolved |
|
||||
|
||||
## Root Cause
|
||||
|
||||
[What actually broke and why]
|
||||
|
||||
## Impact
|
||||
|
||||
[Who was affected, for how long, what data was at risk]
|
||||
|
||||
## What Went Well
|
||||
|
||||
- ...
|
||||
|
||||
## What Went Wrong
|
||||
|
||||
- ...
|
||||
|
||||
## Action Items
|
||||
|
||||
| Action | Owner | Due |
|
||||
|--------|-------|-----|
|
||||
| ... | ... | ... |
|
||||
```
|
||||
|
||||
Store post-mortems in `docs/operations/post-mortems/YYYY-MM-DD-short-title.md`.
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
# Monitoring Checklist
|
||||
|
||||
What to monitor in a Docling Studio deployment.
|
||||
|
||||
## Health Endpoint
|
||||
|
||||
The primary monitoring signal is the health endpoint:
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:3000/api/health
|
||||
```
|
||||
|
||||
Expected response:
|
||||
```json
|
||||
{
|
||||
"status": "ok",
|
||||
"engine": "local",
|
||||
"version": "0.3.0",
|
||||
"deploymentMode": "self-hosted"
|
||||
}
|
||||
```
|
||||
|
||||
**Alert if**: status != "ok", endpoint unreachable, or response time > 5s.
|
||||
|
||||
## Four Golden Signals
|
||||
|
||||
### 1. Latency
|
||||
|
||||
| Endpoint | Expected | Alert threshold |
|
||||
|----------|----------|-----------------|
|
||||
| `GET /api/health` | < 100ms | > 1s |
|
||||
| `POST /api/documents` (upload) | < 2s | > 10s |
|
||||
| `POST /api/analyses` (create) | < 500ms (queuing only) | > 5s |
|
||||
| `GET /api/analyses/:id` (results) | < 500ms | > 3s |
|
||||
|
||||
### 2. Traffic
|
||||
|
||||
| Metric | What to watch |
|
||||
|--------|---------------|
|
||||
| Requests per minute | Baseline for normal usage |
|
||||
| Uploads per hour | Capacity planning |
|
||||
| Concurrent analyses | Should stay <= `MAX_CONCURRENT_ANALYSES` |
|
||||
|
||||
### 3. Errors
|
||||
|
||||
| Signal | Alert threshold |
|
||||
|--------|-----------------|
|
||||
| HTTP 5xx rate | > 1% of requests |
|
||||
| Analysis failure rate | > 10% of analyses |
|
||||
| Rate limit hits (429) | Spike = possible abuse |
|
||||
|
||||
### 4. Saturation
|
||||
|
||||
| Resource | Check command | Alert threshold |
|
||||
|----------|---------------|-----------------|
|
||||
| CPU | `docker stats` | > 90% sustained |
|
||||
| Memory | `docker stats` | > 85% (especially in local mode with PyTorch) |
|
||||
| Disk (SQLite + uploads) | `du -sh data/` | > 80% of volume |
|
||||
| Docker container restarts | `docker inspect --format='{{.RestartCount}}'` | > 0 |
|
||||
|
||||
## Docker Health Check
|
||||
|
||||
The `docker-compose.yml` includes a built-in health check:
|
||||
|
||||
```yaml
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:3000/api/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
```
|
||||
|
||||
Docker will mark the container as `unhealthy` after 3 consecutive failures.
|
||||
|
||||
## Log Monitoring
|
||||
|
||||
### Backend logs (uvicorn)
|
||||
|
||||
```bash
|
||||
docker compose logs -f backend
|
||||
```
|
||||
|
||||
Watch for:
|
||||
- `ERROR` or `CRITICAL` log levels
|
||||
- `TimeoutError` from Docling processing
|
||||
- `sqlite3.OperationalError` (DB issues)
|
||||
- `429 Too Many Requests` spikes
|
||||
|
||||
### Frontend logs (nginx)
|
||||
|
||||
```bash
|
||||
docker compose logs -f frontend
|
||||
```
|
||||
|
||||
Watch for:
|
||||
- `502 Bad Gateway` (backend down)
|
||||
- `413 Request Entity Too Large` (file size limit)
|
||||
|
||||
## Recommended Setup
|
||||
|
||||
For production deployments, consider:
|
||||
|
||||
1. **Uptime monitor** — ping `/api/health` every 60s (UptimeRobot, Healthchecks.io)
|
||||
2. **Log aggregation** — ship Docker logs to a central service
|
||||
3. **Alerting** — notify on container restart, health check failure, or error spike
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
# Security Vulnerability Response
|
||||
|
||||
Process for handling reported security vulnerabilities. See also [SECURITY.md](https://github.com/scub-france/Docling-Studio/blob/main/SECURITY.md) for the public-facing policy.
|
||||
|
||||
## Response Timeline
|
||||
|
||||
| Step | SLA | Action |
|
||||
|------|-----|--------|
|
||||
| **Acknowledge** | < 48h | Confirm receipt to the reporter |
|
||||
| **Assess** | < 7 days | Determine severity (Critical / High / Medium / Low) |
|
||||
| **Fix** | < 14 days (Critical), < 30 days (other) | Develop and test the fix |
|
||||
| **Release** | Same day as fix | Publish patched version |
|
||||
| **Disclose** | After release | Publish GitHub Security Advisory |
|
||||
|
||||
## Severity Assessment
|
||||
|
||||
| Severity | Criteria | Example |
|
||||
|----------|----------|---------|
|
||||
| **Critical** | Remote exploitation, data breach, no auth required | SQL injection in upload endpoint |
|
||||
| **High** | Significant impact, some conditions required | Path traversal on file download |
|
||||
| **Medium** | Limited impact or requires authenticated access | XSS in analysis results display |
|
||||
| **Low** | Minimal impact, theoretical only | Information disclosure in error messages |
|
||||
|
||||
## Fix Process
|
||||
|
||||
1. **Create a private branch** — never push vulnerability details to a public branch before the fix is released
|
||||
2. **Develop the fix** — include a regression test
|
||||
3. **Run the security audit** — `docs/audit/audits/08-security.md`
|
||||
4. **Review** — at least one maintainer must review the fix
|
||||
5. **Release** — tag, build, deploy (see [deployment checklist](../release/deployment-checklist.md))
|
||||
6. **Publish advisory** — GitHub Security Advisory with CVE if applicable
|
||||
|
||||
## Advisory Template
|
||||
|
||||
```markdown
|
||||
# Security Advisory: [Title]
|
||||
|
||||
**Severity**: Critical | High | Medium | Low
|
||||
**Affected versions**: < X.Y.Z
|
||||
**Fixed in**: X.Y.Z
|
||||
**CVE**: CVE-YYYY-NNNNN (if assigned)
|
||||
|
||||
## Description
|
||||
|
||||
[What the vulnerability is, without providing exploitation details]
|
||||
|
||||
## Impact
|
||||
|
||||
[What an attacker could do]
|
||||
|
||||
## Mitigation
|
||||
|
||||
[Upgrade to X.Y.Z or apply workaround]
|
||||
|
||||
## Credit
|
||||
|
||||
[Reporter name, unless they prefer anonymity]
|
||||
```
|
||||
|
||||
## Dependency Vulnerabilities
|
||||
|
||||
Run regular dependency audits:
|
||||
|
||||
```bash
|
||||
# Backend
|
||||
cd document-parser && pip audit
|
||||
|
||||
# Frontend
|
||||
cd frontend && npm audit
|
||||
```
|
||||
|
||||
For known vulnerabilities in dependencies:
|
||||
- **Critical/High**: Update immediately, release a patch version
|
||||
- **Medium/Low**: Include in the next planned release
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
# Deployment Checklist
|
||||
|
||||
Checklist for deploying a new release of Docling Studio. Applies to both self-hosted and Hugging Face Space deployments.
|
||||
|
||||
## Pre-Deploy
|
||||
|
||||
- [ ] Release branch merged to `main` via PR
|
||||
- [ ] Git tag `vX.Y.Z` created on `main`
|
||||
- [ ] Release audit passed (score >= 80, 0 CRITICAL) — see [docs/audit/master.md](../audit/master.md)
|
||||
- [ ] `CHANGELOG.md` section finalized with release date
|
||||
- [ ] `frontend/package.json` version matches the tag
|
||||
- [ ] All CI checks green on the tagged commit
|
||||
- [ ] Docker images built and pushed to `ghcr.io`:
|
||||
- `X.Y.Z-remote`, `X.Y.Z-local`
|
||||
- `X.Y-remote`, `X.Y-local`
|
||||
- `latest-remote`, `latest-local`
|
||||
|
||||
## Deploy — Self-Hosted (Docker Compose)
|
||||
|
||||
- [ ] Pull the new image:
|
||||
```bash
|
||||
docker compose pull
|
||||
```
|
||||
- [ ] Check environment variables (`.env` or `docker-compose.override.yml`):
|
||||
- `CONVERSION_ENGINE` (local / remote)
|
||||
- `RATE_LIMIT_RPM`
|
||||
- `MAX_FILE_SIZE_MB`
|
||||
- `MAX_CONCURRENT_ANALYSES`
|
||||
- [ ] Start the stack:
|
||||
```bash
|
||||
docker compose up -d --wait
|
||||
```
|
||||
- [ ] Verify health endpoint:
|
||||
```bash
|
||||
curl -s http://localhost:3000/api/health | jq .
|
||||
# Expected: {"status":"ok","engine":"...","version":"X.Y.Z","deploymentMode":"self-hosted"}
|
||||
```
|
||||
|
||||
## Deploy — Hugging Face Space
|
||||
|
||||
- [ ] Upload to HF Space via `huggingface-cli`:
|
||||
```bash
|
||||
huggingface-cli upload <space-id> . . --repo-type space
|
||||
```
|
||||
- [ ] Set environment variables in HF Space settings
|
||||
- [ ] Wait for build to complete in HF Space logs
|
||||
- [ ] Verify the app loads and health endpoint returns correct version
|
||||
|
||||
## Post-Deploy Smoke Test
|
||||
|
||||
- [ ] Home page loads
|
||||
- [ ] Upload a PDF — document appears in the list
|
||||
- [ ] Run an analysis — completes without error
|
||||
- [ ] View results — markdown, HTML, bbox overlays render correctly
|
||||
- [ ] Download results
|
||||
- [ ] If local mode: test chunking
|
||||
- [ ] Check `/api/health` returns the new version
|
||||
|
||||
## Rollback Triggers
|
||||
|
||||
Rollback immediately if any of these occur:
|
||||
|
||||
| Trigger | Action |
|
||||
|---------|--------|
|
||||
| Health endpoint returns error or wrong version | Rollback |
|
||||
| Upload or analysis fails on a previously working PDF | Rollback |
|
||||
| Frontend shows blank page or JS errors | Rollback |
|
||||
| Error rate > 5% in the first 15 minutes | Rollback |
|
||||
|
||||
For rollback procedure, see [rollback-playbook.md](rollback-playbook.md).
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
# Rollback Playbook
|
||||
|
||||
## Decision: Rollback vs Hotfix?
|
||||
|
||||
| Situation | Strategy |
|
||||
|-----------|----------|
|
||||
| Broken deploy, previous version was stable | **Rollback** |
|
||||
| Bug found but previous version also had issues | **Hotfix** |
|
||||
| Data migration was applied (DB schema changed) | **Hotfix** (rollback may lose data) |
|
||||
| Security vulnerability in the new release | **Rollback** + hotfix in parallel |
|
||||
|
||||
## Rollback Procedure — Docker Compose
|
||||
|
||||
### 1. Identify the last known good version
|
||||
|
||||
```bash
|
||||
# List available tags
|
||||
docker image ls ghcr.io/scub-france/docling-studio --format '{{.Tag}}' | sort -V
|
||||
|
||||
# Or check GitHub releases
|
||||
gh release list --repo scub-france/Docling-Studio
|
||||
```
|
||||
|
||||
### 2. Pin the image to the previous version
|
||||
|
||||
Edit `docker-compose.yml` or `docker-compose.override.yml`:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
backend:
|
||||
image: ghcr.io/scub-france/docling-studio:0.2.0-remote # pin to last good
|
||||
frontend:
|
||||
image: ghcr.io/scub-france/docling-studio:0.2.0-remote
|
||||
```
|
||||
|
||||
### 3. Restart
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
docker compose up -d --wait
|
||||
```
|
||||
|
||||
### 4. Verify
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:3000/api/health | jq .
|
||||
# Confirm version matches the rolled-back version
|
||||
```
|
||||
|
||||
### 5. Communicate
|
||||
|
||||
- Notify the team that a rollback was performed
|
||||
- Open an issue describing the failure
|
||||
- Link the failed release and the rollback commit
|
||||
|
||||
## Rollback Procedure — Hugging Face Space
|
||||
|
||||
1. Use `git revert` on the HF Space repo to revert to the previous commit
|
||||
2. Or re-upload the previous version: `huggingface-cli upload <space-id> . . --repo-type space --revision <previous-commit>`
|
||||
3. Verify the app loads correctly
|
||||
|
||||
## Database Considerations
|
||||
|
||||
Docling Studio uses **SQLite** with a file-based database. Key points:
|
||||
|
||||
- **No schema migrations** are applied automatically — the schema is created on first run
|
||||
- If a new release adds columns, rolling back may cause "unknown column" errors
|
||||
- **Before deploying a release that changes the DB schema**: back up the SQLite file
|
||||
|
||||
```bash
|
||||
# Backup before deploy
|
||||
cp data/docling-studio.db data/docling-studio.db.bak-$(date +%Y%m%d)
|
||||
|
||||
# Restore after rollback
|
||||
cp data/docling-studio.db.bak-YYYYMMDD data/docling-studio.db
|
||||
```
|
||||
|
||||
## Post-Rollback
|
||||
|
||||
1. Keep the rollback in place until the root cause is identified
|
||||
2. Fix the issue on a `hotfix/*` branch
|
||||
3. Re-run the [release audit](../audit/master.md) on the fix
|
||||
4. Re-deploy following the [deployment checklist](deployment-checklist.md)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 MiB After Width: | Height: | Size: 655 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 750 KiB |
|
|
@ -1,17 +1,9 @@
|
|||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# Docling Studio — backend image (multi-target: remote / local)
|
||||
#
|
||||
# Usage:
|
||||
# docker build --target remote -t docling-studio-backend:remote .
|
||||
# docker build --target local -t docling-studio-backend:local .
|
||||
# =============================================================================
|
||||
|
||||
# --- Base: common deps for both targets ---
|
||||
FROM python:3.12-slim AS base
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
poppler-utils \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -21,36 +13,11 @@ RUN pip install --no-cache-dir -r requirements.txt
|
|||
|
||||
COPY . .
|
||||
|
||||
RUN useradd --create-home --shell /bin/bash appuser \
|
||||
&& mkdir -p /app/uploads /app/data \
|
||||
&& chown -R appuser:appuser /app
|
||||
RUN mkdir -p /app/uploads /app/data
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
ENV UPLOAD_DIR=/app/uploads
|
||||
ENV DB_PATH=/app/data/docling_studio.db
|
||||
|
||||
USER appuser
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
# --- Remote: lightweight, delegates to Docling Serve ---
|
||||
FROM base AS remote
|
||||
ENV CONVERSION_ENGINE=remote
|
||||
|
||||
# --- Local: full Docling in-process ---
|
||||
FROM base AS local
|
||||
|
||||
USER root
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements-local.txt .
|
||||
RUN pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu \
|
||||
&& pip install --no-cache-dir -r requirements-local.txt
|
||||
|
||||
RUN chown -R appuser:appuser /app \
|
||||
&& chown -R appuser:appuser /usr/local/lib/python3.12/site-packages/rapidocr/models
|
||||
USER appuser
|
||||
ENV CONVERSION_ENGINE=local
|
||||
|
|
|
|||
|
|
@ -3,31 +3,16 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from api.schemas import (
|
||||
AnalysisResponse,
|
||||
ChunkBboxResponse,
|
||||
ChunkResponse,
|
||||
CreateAnalysisRequest,
|
||||
RechunkRequest,
|
||||
UpdateChunkTextRequest,
|
||||
)
|
||||
from services.analysis_service import AnalysisService
|
||||
from api.schemas import AnalysisResponse, CreateAnalysisRequest
|
||||
from services import analysis_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/analyses", tags=["analyses"])
|
||||
|
||||
|
||||
def _get_service(request: Request) -> AnalysisService:
|
||||
return request.app.state.analysis_service
|
||||
|
||||
|
||||
ServiceDep = Annotated[AnalysisService, Depends(_get_service)]
|
||||
|
||||
|
||||
def _to_response(job) -> AnalysisResponse:
|
||||
return AnalysisResponse(
|
||||
id=job.id,
|
||||
|
|
@ -37,11 +22,7 @@ def _to_response(job) -> AnalysisResponse:
|
|||
content_markdown=job.content_markdown,
|
||||
content_html=job.content_html,
|
||||
pages_json=job.pages_json,
|
||||
chunks_json=job.chunks_json,
|
||||
has_document_json=job.document_json is not None,
|
||||
error_message=job.error_message,
|
||||
progress_current=job.progress_current,
|
||||
progress_total=job.progress_total,
|
||||
started_at=str(job.started_at) if job.started_at else None,
|
||||
completed_at=str(job.completed_at) if job.completed_at else None,
|
||||
created_at=str(job.created_at),
|
||||
|
|
@ -49,7 +30,7 @@ def _to_response(job) -> AnalysisResponse:
|
|||
|
||||
|
||||
@router.post("", response_model=AnalysisResponse)
|
||||
async def create_analysis(body: CreateAnalysisRequest, service: ServiceDep) -> AnalysisResponse:
|
||||
async def create_analysis(body: CreateAnalysisRequest):
|
||||
"""Create a new analysis job for a document."""
|
||||
if not body.documentId or not body.documentId.strip():
|
||||
raise HTTPException(status_code=400, detail="documentId is required")
|
||||
|
|
@ -58,108 +39,33 @@ async def create_analysis(body: CreateAnalysisRequest, service: ServiceDep) -> A
|
|||
if body.pipelineOptions:
|
||||
pipeline_opts = body.pipelineOptions.model_dump()
|
||||
|
||||
chunking_opts = None
|
||||
if body.chunkingOptions:
|
||||
chunking_opts = body.chunkingOptions.model_dump()
|
||||
|
||||
try:
|
||||
job = await service.create(
|
||||
body.documentId,
|
||||
pipeline_options=pipeline_opts,
|
||||
chunking_options=chunking_opts,
|
||||
)
|
||||
job = await analysis_service.create(body.documentId, pipeline_options=pipeline_opts)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
|
||||
return _to_response(job)
|
||||
|
||||
|
||||
@router.get("", response_model=list[AnalysisResponse])
|
||||
async def list_analyses(service: ServiceDep) -> list[AnalysisResponse]:
|
||||
async def list_analyses():
|
||||
"""List all analysis jobs."""
|
||||
jobs = await service.find_all()
|
||||
jobs = await analysis_service.find_all()
|
||||
return [_to_response(j) for j in jobs]
|
||||
|
||||
|
||||
@router.get("/{analysis_id}", response_model=AnalysisResponse)
|
||||
async def get_analysis(analysis_id: str, service: ServiceDep) -> AnalysisResponse:
|
||||
@router.get("/{job_id}", response_model=AnalysisResponse)
|
||||
async def get_analysis(job_id: str):
|
||||
"""Get a single analysis job."""
|
||||
job = await service.find_by_id(analysis_id)
|
||||
job = await analysis_service.find_by_id(job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Analysis not found")
|
||||
return _to_response(job)
|
||||
|
||||
|
||||
@router.post("/{analysis_id}/rechunk", response_model=list[ChunkResponse])
|
||||
async def rechunk_analysis(
|
||||
analysis_id: str, body: RechunkRequest, service: ServiceDep
|
||||
) -> list[ChunkResponse]:
|
||||
"""Re-chunk a completed analysis with new chunking options."""
|
||||
try:
|
||||
chunks = await service.rechunk(analysis_id, body.chunkingOptions.model_dump())
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
return [
|
||||
ChunkResponse(
|
||||
text=c.text,
|
||||
headings=c.headings,
|
||||
source_page=c.source_page,
|
||||
token_count=c.token_count,
|
||||
bboxes=[ChunkBboxResponse(page=b.page, bbox=b.bbox) for b in c.bboxes],
|
||||
)
|
||||
for c in chunks
|
||||
]
|
||||
|
||||
|
||||
@router.patch("/{analysis_id}/chunks/{chunk_index}", response_model=list[ChunkResponse])
|
||||
async def update_chunk_text(
|
||||
analysis_id: str, chunk_index: int, body: UpdateChunkTextRequest, service: ServiceDep
|
||||
) -> list[ChunkResponse]:
|
||||
"""Update the text of a single chunk by index."""
|
||||
try:
|
||||
chunks = await service.update_chunk_text(analysis_id, chunk_index, body.text)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
return [
|
||||
ChunkResponse(
|
||||
text=c["text"],
|
||||
headings=c.get("headings", []),
|
||||
source_page=c.get("sourcePage"),
|
||||
token_count=c.get("tokenCount", 0),
|
||||
bboxes=[ChunkBboxResponse(page=b["page"], bbox=b["bbox"]) for b in c.get("bboxes", [])],
|
||||
modified=c.get("modified", False),
|
||||
deleted=c.get("deleted", False),
|
||||
)
|
||||
for c in chunks
|
||||
]
|
||||
|
||||
|
||||
@router.delete("/{analysis_id}/chunks/{chunk_index}", response_model=list[ChunkResponse])
|
||||
async def delete_chunk(
|
||||
analysis_id: str, chunk_index: int, service: ServiceDep
|
||||
) -> list[ChunkResponse]:
|
||||
"""Soft-delete a chunk by index (marks it as deleted)."""
|
||||
try:
|
||||
chunks = await service.delete_chunk(analysis_id, chunk_index)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
return [
|
||||
ChunkResponse(
|
||||
text=c["text"],
|
||||
headings=c.get("headings", []),
|
||||
source_page=c.get("sourcePage"),
|
||||
token_count=c.get("tokenCount", 0),
|
||||
bboxes=[ChunkBboxResponse(page=b["page"], bbox=b["bbox"]) for b in c.get("bboxes", [])],
|
||||
modified=c.get("modified", False),
|
||||
deleted=c.get("deleted", False),
|
||||
)
|
||||
for c in chunks
|
||||
]
|
||||
|
||||
|
||||
@router.delete("/{analysis_id}", status_code=204, response_model=None)
|
||||
async def delete_analysis(analysis_id: str, service: ServiceDep) -> None:
|
||||
@router.delete("/{job_id}", status_code=204)
|
||||
async def delete_analysis(job_id: str):
|
||||
"""Delete an analysis job."""
|
||||
deleted = await service.delete(analysis_id)
|
||||
deleted = await analysis_service.delete(job_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="Analysis not found")
|
||||
|
|
|
|||
|
|
@ -2,29 +2,17 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, UploadFile
|
||||
from fastapi import APIRouter, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
|
||||
from api.schemas import DocumentResponse
|
||||
from services.document_service import DocumentService
|
||||
from services import document_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/documents", tags=["documents"])
|
||||
|
||||
_READ_CHUNK_SIZE = 64 * 1024 # 64 KB
|
||||
|
||||
|
||||
def _get_service(request: Request) -> DocumentService:
|
||||
return request.app.state.document_service
|
||||
|
||||
|
||||
ServiceDep = Annotated[DocumentService, Depends(_get_service)]
|
||||
|
||||
|
||||
def _to_response(doc) -> DocumentResponse:
|
||||
return DocumentResponse(
|
||||
|
|
@ -37,60 +25,46 @@ def _to_response(doc) -> DocumentResponse:
|
|||
)
|
||||
|
||||
|
||||
@router.post("/upload", response_model=DocumentResponse, status_code=200)
|
||||
async def upload(file: UploadFile, service: ServiceDep) -> DocumentResponse:
|
||||
@router.post("/upload", response_model=DocumentResponse)
|
||||
async def upload(file: UploadFile):
|
||||
"""Upload a PDF document."""
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="No filename provided")
|
||||
|
||||
# Reject early if Content-Length exceeds limit (before reading body)
|
||||
_max = service.max_file_size
|
||||
_detail = f"File too large (max {service.max_file_size_mb} MB)"
|
||||
if _max > 0 and file.size and file.size > _max:
|
||||
raise HTTPException(status_code=413, detail=_detail)
|
||||
|
||||
# Read in chunks to avoid holding the full upload in a single allocation
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while chunk := await file.read(_READ_CHUNK_SIZE):
|
||||
total += len(chunk)
|
||||
if _max > 0 and total > _max:
|
||||
raise HTTPException(status_code=413, detail=_detail)
|
||||
chunks.append(chunk)
|
||||
content = b"".join(chunks)
|
||||
content = await file.read()
|
||||
|
||||
try:
|
||||
doc = await service.upload(
|
||||
doc = await document_service.upload(
|
||||
filename=file.filename,
|
||||
content_type=file.content_type or "application/pdf",
|
||||
file_content=content,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
raise HTTPException(status_code=413, detail=str(e))
|
||||
|
||||
return _to_response(doc)
|
||||
|
||||
|
||||
@router.get("", response_model=list[DocumentResponse])
|
||||
async def list_documents(service: ServiceDep) -> list[DocumentResponse]:
|
||||
async def list_documents():
|
||||
"""List all documents."""
|
||||
docs = await service.find_all()
|
||||
docs = await document_service.find_all()
|
||||
return [_to_response(d) for d in docs]
|
||||
|
||||
|
||||
@router.get("/{doc_id}", response_model=DocumentResponse)
|
||||
async def get_document(doc_id: str, service: ServiceDep) -> DocumentResponse:
|
||||
async def get_document(doc_id: str):
|
||||
"""Get a single document."""
|
||||
doc = await service.find_by_id(doc_id)
|
||||
doc = await document_service.find_by_id(doc_id)
|
||||
if not doc:
|
||||
raise HTTPException(status_code=404, detail="Document not found")
|
||||
return _to_response(doc)
|
||||
|
||||
|
||||
@router.delete("/{doc_id}", status_code=204, response_model=None)
|
||||
async def delete_document(doc_id: str, service: ServiceDep) -> None:
|
||||
@router.delete("/{doc_id}", status_code=204)
|
||||
async def delete_document(doc_id: str):
|
||||
"""Delete a document and its file."""
|
||||
deleted = await service.delete(doc_id)
|
||||
deleted = await document_service.delete(doc_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="Document not found")
|
||||
|
||||
|
|
@ -98,36 +72,21 @@ async def delete_document(doc_id: str, service: ServiceDep) -> None:
|
|||
@router.get("/{doc_id}/preview")
|
||||
async def preview(
|
||||
doc_id: str,
|
||||
service: ServiceDep,
|
||||
page: int = Query(1, ge=1),
|
||||
dpi: int = Query(150, ge=72, le=300),
|
||||
) -> Response:
|
||||
):
|
||||
"""Generate a PNG preview of a specific PDF page."""
|
||||
doc = await service.find_by_id(doc_id)
|
||||
doc = await document_service.find_by_id(doc_id)
|
||||
if not doc:
|
||||
raise HTTPException(status_code=404, detail="Document not found")
|
||||
|
||||
if doc.page_count and page > doc.page_count:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Page {page} out of range (document has {doc.page_count} pages)",
|
||||
)
|
||||
|
||||
try:
|
||||
# File read + PDF rasterisation are both blocking; offload to a
|
||||
# worker thread so the event loop stays free for other requests.
|
||||
file_content = await asyncio.to_thread(Path(doc.storage_path).read_bytes)
|
||||
png_bytes = await asyncio.to_thread(
|
||||
DocumentService.generate_preview, file_content, page=page, dpi=dpi
|
||||
)
|
||||
with open(doc.storage_path, "rb") as f:
|
||||
file_content = f.read()
|
||||
png_bytes = document_service.generate_preview(file_content, page=page, dpi=dpi)
|
||||
return Response(content=png_bytes, media_type="image/png")
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="PDF file not found on disk") from exc
|
||||
except OSError as exc:
|
||||
logger.exception("I/O error generating preview for %s", doc_id)
|
||||
raise HTTPException(status_code=422, detail="Failed to read PDF file") from exc
|
||||
except Exception as exc:
|
||||
logger.exception("Unexpected error generating preview for %s", doc_id)
|
||||
raise HTTPException(status_code=422, detail="Failed to generate preview") from exc
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception:
|
||||
logger.exception("Failed to generate preview")
|
||||
raise HTTPException(status_code=422, detail="Failed to generate preview")
|
||||
|
|
|
|||
|
|
@ -1,123 +0,0 @@
|
|||
"""Graph API — returns a cytoscape-shaped view of the document structure.
|
||||
|
||||
Two endpoints:
|
||||
- `/graph` — read from Neo4j. Rich graph (elements + chunks + pages + merges).
|
||||
Requires the Maintain step (IngestionPipeline) to have run for the document.
|
||||
- `/reasoning-graph` — built on-the-fly from the SQLite `document_json` blob.
|
||||
No Neo4j dependency. Lighter graph (no chunks) but enough to render the
|
||||
reasoning-trace overlay on top of `GraphView`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from infra.docling_graph import build_graph_payload
|
||||
from infra.neo4j import fetch_graph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/documents", tags=["graph"])
|
||||
|
||||
MAX_PAGES = 200
|
||||
|
||||
|
||||
class GraphNode(BaseModel):
|
||||
id: str
|
||||
group: str
|
||||
label: str | None = None
|
||||
|
||||
model_config = {"extra": "allow"}
|
||||
|
||||
|
||||
class GraphEdge(BaseModel):
|
||||
id: str
|
||||
source: str
|
||||
target: str
|
||||
type: str
|
||||
order: int | None = None
|
||||
|
||||
|
||||
class GraphResponse(BaseModel):
|
||||
doc_id: str
|
||||
nodes: list[GraphNode]
|
||||
edges: list[GraphEdge]
|
||||
node_count: int
|
||||
edge_count: int
|
||||
truncated: bool
|
||||
page_count: int
|
||||
|
||||
|
||||
@router.get("/{doc_id}/graph", response_model=GraphResponse)
|
||||
async def get_document_graph(doc_id: str, request: Request) -> GraphResponse:
|
||||
neo = getattr(request.app.state, "neo4j", None)
|
||||
if neo is None:
|
||||
raise HTTPException(status_code=503, detail="Neo4j is not configured")
|
||||
|
||||
payload = await fetch_graph(neo, doc_id, max_pages=MAX_PAGES)
|
||||
if payload is None:
|
||||
raise HTTPException(status_code=404, detail=f"No graph for document {doc_id}")
|
||||
if payload.truncated:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph too large: document has {payload.page_count} pages "
|
||||
f"(cap {MAX_PAGES}). Pagination ships in v0.6."
|
||||
),
|
||||
)
|
||||
|
||||
return GraphResponse(
|
||||
doc_id=payload.doc_id,
|
||||
nodes=[GraphNode(**n) for n in payload.nodes],
|
||||
edges=[GraphEdge(**e) for e in payload.edges],
|
||||
node_count=payload.node_count,
|
||||
edge_count=payload.edge_count,
|
||||
truncated=payload.truncated,
|
||||
page_count=payload.page_count,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{doc_id}/reasoning-graph", response_model=GraphResponse)
|
||||
async def get_reasoning_graph(doc_id: str, request: Request) -> GraphResponse:
|
||||
"""Graph projection built from SQLite `document_json` — no Neo4j needed.
|
||||
|
||||
Serves the reasoning-trace viewer, which only needs the element/page/edge
|
||||
structure to overlay iterations onto.
|
||||
"""
|
||||
analysis_repo = getattr(request.app.state, "analysis_repo", None)
|
||||
if analysis_repo is None:
|
||||
raise HTTPException(status_code=500, detail="AnalysisRepository not wired")
|
||||
|
||||
latest = await analysis_repo.find_latest_completed_by_document(doc_id)
|
||||
if latest is None or not latest.document_json:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"No completed analysis with document_json for {doc_id}",
|
||||
)
|
||||
|
||||
payload = build_graph_payload(
|
||||
latest.document_json,
|
||||
doc_id=doc_id,
|
||||
title=latest.document_filename or doc_id,
|
||||
max_pages=MAX_PAGES,
|
||||
)
|
||||
if payload.truncated:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph too large: document has {payload.page_count} pages "
|
||||
f"(cap {MAX_PAGES}). Pagination ships in v0.6."
|
||||
),
|
||||
)
|
||||
|
||||
return GraphResponse(
|
||||
doc_id=payload.doc_id,
|
||||
nodes=[GraphNode(**n) for n in payload.nodes],
|
||||
edges=[GraphEdge(**e) for e in payload.edges],
|
||||
node_count=payload.node_count,
|
||||
edge_count=payload.edge_count,
|
||||
truncated=payload.truncated,
|
||||
page_count=payload.page_count,
|
||||
)
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
"""Ingestion API router — trigger and manage vector ingestion pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
|
||||
from api.schemas import (
|
||||
IngestionResponse,
|
||||
IngestionStatusResponse,
|
||||
SearchResponse,
|
||||
SearchResultItem,
|
||||
)
|
||||
from services.analysis_service import AnalysisService
|
||||
from services.ingestion_service import IngestionService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/ingestion", tags=["ingestion"])
|
||||
|
||||
|
||||
def _get_ingestion_service(request: Request) -> IngestionService:
|
||||
svc = request.app.state.ingestion_service
|
||||
if svc is None:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Ingestion not available (EMBEDDING_URL and OPENSEARCH_URL required)",
|
||||
)
|
||||
return svc
|
||||
|
||||
|
||||
def _get_analysis_service(request: Request) -> AnalysisService:
|
||||
return request.app.state.analysis_service
|
||||
|
||||
|
||||
IngestionDep = Annotated[IngestionService, Depends(_get_ingestion_service)]
|
||||
AnalysisDep = Annotated[AnalysisService, Depends(_get_analysis_service)]
|
||||
|
||||
|
||||
@router.post("/{analysis_id}", response_model=IngestionResponse)
|
||||
async def ingest_analysis(
|
||||
analysis_id: str,
|
||||
ingestion: IngestionDep,
|
||||
analysis: AnalysisDep,
|
||||
) -> IngestionResponse:
|
||||
"""Ingest a completed analysis into the vector index.
|
||||
|
||||
Takes the chunks from an existing analysis, embeds them,
|
||||
and indexes them into OpenSearch.
|
||||
"""
|
||||
job = await analysis.find_by_id(analysis_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Analysis not found")
|
||||
if job.status.value != "COMPLETED":
|
||||
raise HTTPException(status_code=400, detail="Analysis is not completed")
|
||||
if not job.chunks_json:
|
||||
raise HTTPException(status_code=400, detail="Analysis has no chunks — run chunking first")
|
||||
|
||||
try:
|
||||
result = await ingestion.ingest(
|
||||
doc_id=job.document_id,
|
||||
filename=job.document_filename or "unknown",
|
||||
chunks_json=job.chunks_json,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Ingestion failed for analysis %s", analysis_id)
|
||||
raise HTTPException(status_code=500, detail=f"Ingestion failed: {e}") from e
|
||||
|
||||
return IngestionResponse(
|
||||
doc_id=result.doc_id,
|
||||
chunks_indexed=result.chunks_indexed,
|
||||
embedding_dimension=result.embedding_dimension,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{doc_id}", status_code=204, response_model=None)
|
||||
async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None:
|
||||
"""Delete all indexed chunks for a document."""
|
||||
await ingestion.delete_document(doc_id)
|
||||
|
||||
|
||||
@router.get("/status", response_model=IngestionStatusResponse)
|
||||
async def ingestion_status(request: Request) -> IngestionStatusResponse:
|
||||
"""Check if the ingestion pipeline is available and OpenSearch is connected."""
|
||||
svc = request.app.state.ingestion_service
|
||||
if svc is None:
|
||||
return IngestionStatusResponse(available=False, opensearch_connected=False)
|
||||
|
||||
connected = await svc.ping()
|
||||
return IngestionStatusResponse(available=True, opensearch_connected=connected)
|
||||
|
||||
|
||||
@router.get("/search", response_model=SearchResponse)
|
||||
async def search_chunks(
|
||||
ingestion: IngestionDep,
|
||||
q: str = Query(..., min_length=1, description="Search query"),
|
||||
doc_id: str | None = Query(None, description="Filter by document ID"),
|
||||
k: int = Query(20, ge=1, le=100, description="Max results"),
|
||||
) -> SearchResponse:
|
||||
"""Full-text search across indexed chunks.
|
||||
|
||||
Returns matching chunks with content and metadata.
|
||||
Optionally filter by document ID.
|
||||
"""
|
||||
results = await ingestion.search_fulltext(q, k=k, doc_id=doc_id)
|
||||
items = [
|
||||
SearchResultItem(
|
||||
doc_id=r.chunk.doc_id,
|
||||
filename=r.chunk.filename,
|
||||
content=r.chunk.content,
|
||||
chunk_index=r.chunk.chunk_index,
|
||||
page_number=r.chunk.page_number,
|
||||
score=r.score,
|
||||
headings=r.chunk.headings,
|
||||
)
|
||||
for r in results
|
||||
]
|
||||
return SearchResponse(results=items, total=len(items), query=q)
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue