efc27932dd
Closes the 12 MAJ raised by the release/0.5.0 audit pipeline (cf.
docs/audit/reports/release-0.5.0/summary.md → summary-reaudit.md).
Volet 1 — Reasoning architecture (audits 01/02/06/07 strengthening)
* Domain ports: LLMProvider, ReasoningRunner, ReasoningParseError
* Domain DTOs: LLMProviderType, ReasoningResult, ReasoningIteration
* infra/llm/ollama_provider.py — OllamaProvider with health_check
* infra/docling_agent_reasoning.py — runner adapter, encapsulates the
private _rag_loop call (tracked at docling-project/docling-agent#26),
commits OLLAMA_HOST once at boot (eliminates the per-request env race),
translates upstream IndexError into ReasoningParseError
* api/reasoning.py — zero coupling to docling-agent / mellea / docling-core,
consumes app.state.reasoning_runner via the port
* main.py — DI wires OllamaProvider + DoclingAgentReasoningRunner at boot
when REASONING_ENABLED=true and deps are importable
* Rename RAG_* env vars → REASONING_*, endpoint /rag → /reasoning,
type RAGResult → ReasoningResult, frontend feature flag wiring,
i18n strings, tests, docs (BREAKING — pre-1.0 surface, no external
consumers in production)
* 17 new tests: adapter unit tests with sys.modules stubs, OllamaProvider
httpx tests, R3 concurrent-host isolation, R6 multi-iteration trace
serialization, R13 Protocol conformance via isinstance
* E2E Karate scenario: nav-reasoning hidden when REASONING_ENABLED=false
* README — Live Reasoning section (env vars, archi, link to issue #26)
Bloc B — Security (audit 08, dev-only context)
* docker-compose.yml — DEV DEFAULTS header, OpenSearch DISABLE_SECURITY_PLUGIN
flagged as dev-only with link to OpenSearch security docs
* main.py — boot warning if NEO4J_URI is set with the default 'changeme'
password, so prod operators can't silently inherit it
Bloc C — DRY frontend (audit 05)
* shared/storage/keys.ts — STORAGE_KEYS centralised (theme, locale)
* features/settings/store.ts — dead apiUrl ref + orphan i18n keys removed
* api/schemas.py — DOCUMENT_STATUS_UPLOADED constant
Bloc D — Quality (audits 02/06/07/09/10/12)
* domain/ports.py — DocumentConverter.supports_page_batching property
(LSP fix, replaces isinstance(ServeConverter) check)
* domain/ports.py — VectorStore.ping() (encapsulation, replaces
_vector_store._client.info() reach-around)
* api/analyses.py + api/ingestion.py — path params {job_id} → {analysis_id}
aligned with the user-facing terminology (URLs unchanged)
* api/documents.py — Path.read_bytes() + generate_preview() wrapped in
asyncio.to_thread, unblocks the FastAPI event loop on /preview
* infra/docling_tree.py — PEP 604 union for isinstance (Ruff UP038)
* src/__tests__/integration/ — cross-feature integration test relocated
out of features/history/ so feature folders stay self-contained
* Tightened terminal `assert X is not None` checks (isinstance(.., datetime),
exact value comparisons)
Validation
* 446 backend pytest, 202 frontend vitest — all green
* ruff + ruff format + ESLint + Prettier + vue-tsc clean
* Re-audit verdict: 0 CRIT / 0 MAJ, score ~94/100, GO
Closes #200
119 lines
4 KiB
Python
119 lines
4 KiB
Python
"""Ingestion API router — trigger and manage vector ingestion pipeline."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Annotated
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
|
|
|
from api.schemas import (
|
|
IngestionResponse,
|
|
IngestionStatusResponse,
|
|
SearchResponse,
|
|
SearchResultItem,
|
|
)
|
|
from services.analysis_service import AnalysisService
|
|
from services.ingestion_service import IngestionService
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api/ingestion", tags=["ingestion"])
|
|
|
|
|
|
def _get_ingestion_service(request: Request) -> IngestionService:
|
|
svc = request.app.state.ingestion_service
|
|
if svc is None:
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail="Ingestion not available (EMBEDDING_URL and OPENSEARCH_URL required)",
|
|
)
|
|
return svc
|
|
|
|
|
|
def _get_analysis_service(request: Request) -> AnalysisService:
|
|
return request.app.state.analysis_service
|
|
|
|
|
|
IngestionDep = Annotated[IngestionService, Depends(_get_ingestion_service)]
|
|
AnalysisDep = Annotated[AnalysisService, Depends(_get_analysis_service)]
|
|
|
|
|
|
@router.post("/{analysis_id}", response_model=IngestionResponse)
|
|
async def ingest_analysis(
|
|
analysis_id: str,
|
|
ingestion: IngestionDep,
|
|
analysis: AnalysisDep,
|
|
) -> IngestionResponse:
|
|
"""Ingest a completed analysis into the vector index.
|
|
|
|
Takes the chunks from an existing analysis, embeds them,
|
|
and indexes them into OpenSearch.
|
|
"""
|
|
job = await analysis.find_by_id(analysis_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Analysis not found")
|
|
if job.status.value != "COMPLETED":
|
|
raise HTTPException(status_code=400, detail="Analysis is not completed")
|
|
if not job.chunks_json:
|
|
raise HTTPException(status_code=400, detail="Analysis has no chunks — run chunking first")
|
|
|
|
try:
|
|
result = await ingestion.ingest(
|
|
doc_id=job.document_id,
|
|
filename=job.document_filename or "unknown",
|
|
chunks_json=job.chunks_json,
|
|
)
|
|
except Exception as e:
|
|
logger.exception("Ingestion failed for analysis %s", analysis_id)
|
|
raise HTTPException(status_code=500, detail=f"Ingestion failed: {e}") from e
|
|
|
|
return IngestionResponse(
|
|
doc_id=result.doc_id,
|
|
chunks_indexed=result.chunks_indexed,
|
|
embedding_dimension=result.embedding_dimension,
|
|
)
|
|
|
|
|
|
@router.delete("/{doc_id}", status_code=204, response_model=None)
|
|
async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None:
|
|
"""Delete all indexed chunks for a document."""
|
|
await ingestion.delete_document(doc_id)
|
|
|
|
|
|
@router.get("/status", response_model=IngestionStatusResponse)
|
|
async def ingestion_status(request: Request) -> IngestionStatusResponse:
|
|
"""Check if the ingestion pipeline is available and OpenSearch is connected."""
|
|
svc = request.app.state.ingestion_service
|
|
if svc is None:
|
|
return IngestionStatusResponse(available=False, opensearch_connected=False)
|
|
|
|
connected = await svc.ping()
|
|
return IngestionStatusResponse(available=True, opensearch_connected=connected)
|
|
|
|
|
|
@router.get("/search", response_model=SearchResponse)
|
|
async def search_chunks(
|
|
ingestion: IngestionDep,
|
|
q: str = Query(..., min_length=1, description="Search query"),
|
|
doc_id: str | None = Query(None, description="Filter by document ID"),
|
|
k: int = Query(20, ge=1, le=100, description="Max results"),
|
|
) -> SearchResponse:
|
|
"""Full-text search across indexed chunks.
|
|
|
|
Returns matching chunks with content and metadata.
|
|
Optionally filter by document ID.
|
|
"""
|
|
results = await ingestion.search_fulltext(q, k=k, doc_id=doc_id)
|
|
items = [
|
|
SearchResultItem(
|
|
doc_id=r.chunk.doc_id,
|
|
filename=r.chunk.filename,
|
|
content=r.chunk.content,
|
|
chunk_index=r.chunk.chunk_index,
|
|
page_number=r.chunk.page_number,
|
|
score=r.score,
|
|
headings=r.chunk.headings,
|
|
)
|
|
for r in results
|
|
]
|
|
return SearchResponse(results=items, total=len(items), query=q)
|