Merge pull request #156 from scub-france/feature/ingestion-pipeline

feat: ingestion pipeline, My Documents, and ingest button (#72-#76)
This commit is contained in:
Pier-Jean Malandrino 2026-04-10 22:39:52 +02:00 committed by GitHub
commit b32781a055
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 1154 additions and 17 deletions

View file

@ -16,6 +16,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this
- OpenSearch adapter (`OpenSearchStore`): kNN vector search, full-text search, bulk indexing, document CRUD - OpenSearch adapter (`OpenSearchStore`): kNN vector search, full-text search, bulk indexing, document CRUD
- Embedding microservice (`embedding-service/`): sentence-transformers REST API with batch processing and Dockerfile - Embedding microservice (`embedding-service/`): sentence-transformers REST API with batch processing and Dockerfile
- `EmbeddingService` port and `EmbeddingClient` HTTP adapter for remote embedding generation - `EmbeddingService` port and `EmbeddingClient` HTTP adapter for remote embedding generation
- Orchestrated ingestion pipeline: Docling → chunking → embedding → OpenSearch indexing (idempotent)
- Ingestion REST API: `POST /api/ingestion/{jobId}`, `DELETE /api/ingestion/{docId}`, `GET /api/ingestion/status`
- Production docker-compose with OpenSearch and embedding service
- E2E Karate test for full ingestion workflow (PDF → chunks in OpenSearch)
- My Documents screen: search, filter (all/indexed/not indexed), sort (name/date), ingestion status badges
- Ingest button in Studio: one-click ingestion from completed analysis with progress feedback
### Fixed ### Fixed

View file

@ -1,4 +1,38 @@
services: services:
# --- OpenSearch (single-node, security disabled) ---
opensearch:
image: opensearchproject/opensearch:2
environment:
discovery.type: single-node
DISABLE_SECURITY_PLUGIN: "true"
OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
volumes:
- opensearch_data:/usr/share/opensearch/data
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
interval: 10s
timeout: 5s
retries: 10
# --- Embedding service (sentence-transformers) ---
embedding:
build:
context: ./embedding-service
environment:
EMBEDDING_MODEL: ${EMBEDDING_MODEL:-all-MiniLM-L6-v2}
EMBEDDING_BATCH_SIZE: ${EMBEDDING_BATCH_SIZE:-64}
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:8001/health || exit 1"]
interval: 15s
timeout: 10s
retries: 20
start_period: 120s
deploy:
resources:
limits:
memory: 2g
# --- Backend (FastAPI) ---
document-parser: document-parser:
build: build:
context: ./document-parser context: ./document-parser
@ -15,11 +49,19 @@ services:
RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100} RATE_LIMIT_RPM: ${RATE_LIMIT_RPM:-100}
MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50} MAX_FILE_SIZE_MB: ${MAX_FILE_SIZE_MB:-50}
BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-0} BATCH_PAGE_SIZE: ${BATCH_PAGE_SIZE:-0}
OPENSEARCH_URL: http://opensearch:9200
EMBEDDING_URL: http://embedding:8001
depends_on:
opensearch:
condition: service_healthy
embedding:
condition: service_healthy
deploy: deploy:
resources: resources:
limits: limits:
memory: 4g memory: 4g
# --- Frontend (nginx) ---
frontend: frontend:
build: build:
context: ./frontend context: ./frontend
@ -29,5 +71,6 @@ services:
- document-parser - document-parser
volumes: volumes:
opensearch_data:
uploads_data: uploads_data:
db_data: db_data:

View file

@ -0,0 +1,82 @@
"""Ingestion API router — trigger and manage vector ingestion pipeline."""
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request
from api.schemas import IngestionResponse, IngestionStatusResponse
from services.analysis_service import AnalysisService
from services.ingestion_service import IngestionService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/ingestion", tags=["ingestion"])
def _get_ingestion_service(request: Request) -> IngestionService:
svc = request.app.state.ingestion_service
if svc is None:
raise HTTPException(
status_code=503,
detail="Ingestion not available (EMBEDDING_URL and OPENSEARCH_URL required)",
)
return svc
def _get_analysis_service(request: Request) -> AnalysisService:
return request.app.state.analysis_service
IngestionDep = Annotated[IngestionService, Depends(_get_ingestion_service)]
AnalysisDep = Annotated[AnalysisService, Depends(_get_analysis_service)]
@router.post("/{job_id}", response_model=IngestionResponse)
async def ingest_analysis(
job_id: str,
ingestion: IngestionDep,
analysis: AnalysisDep,
) -> IngestionResponse:
"""Ingest a completed analysis into the vector index.
Takes the chunks from an existing analysis job, embeds them,
and indexes them into OpenSearch.
"""
job = await analysis.find_by_id(job_id)
if not job:
raise HTTPException(status_code=404, detail="Analysis not found")
if job.status.value != "COMPLETED":
raise HTTPException(status_code=400, detail="Analysis is not completed")
if not job.chunks_json:
raise HTTPException(status_code=400, detail="Analysis has no chunks — run chunking first")
try:
result = await ingestion.ingest(
doc_id=job.document_id,
filename=job.document_filename or "unknown",
chunks_json=job.chunks_json,
)
except Exception as e:
logger.exception("Ingestion failed for job %s", job_id)
raise HTTPException(status_code=500, detail=f"Ingestion failed: {e}") from e
return IngestionResponse(
doc_id=result.doc_id,
chunks_indexed=result.chunks_indexed,
embedding_dimension=result.embedding_dimension,
)
@router.delete("/{doc_id}", status_code=204)
async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None:
"""Delete all indexed chunks for a document."""
await ingestion.delete_document(doc_id)
@router.get("/status", response_model=IngestionStatusResponse)
async def ingestion_status(request: Request) -> IngestionStatusResponse:
"""Check if the ingestion pipeline is available."""
available = request.app.state.ingestion_service is not None
return IngestionStatusResponse(available=available)

View file

@ -180,3 +180,13 @@ class RechunkRequest(BaseModel):
chunkingOptions: ChunkingOptionsRequest = Field( chunkingOptions: ChunkingOptionsRequest = Field(
validation_alias=AliasChoices("chunkingOptions", "chunking_options") validation_alias=AliasChoices("chunkingOptions", "chunking_options")
) )
class IngestionResponse(_CamelModel):
doc_id: str
chunks_indexed: int
embedding_dimension: int
class IngestionStatusResponse(_CamelModel):
available: bool

View file

@ -20,6 +20,7 @@ from fastapi.middleware.cors import CORSMiddleware
from api.analyses import router as analyses_router from api.analyses import router as analyses_router
from api.documents import router as documents_router from api.documents import router as documents_router
from api.ingestion import router as ingestion_router
from api.schemas import HealthResponse from api.schemas import HealthResponse
from infra.rate_limiter import RateLimiterMiddleware from infra.rate_limiter import RateLimiterMiddleware
from infra.settings import settings from infra.settings import settings
@ -28,6 +29,7 @@ from persistence.database import get_connection, init_db
from persistence.document_repo import SqliteDocumentRepository from persistence.document_repo import SqliteDocumentRepository
from services.analysis_service import AnalysisConfig, AnalysisService from services.analysis_service import AnalysisConfig, AnalysisService
from services.document_service import DocumentConfig, DocumentService from services.document_service import DocumentConfig, DocumentService
from services.ingestion_service import IngestionConfig, IngestionService
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
@ -87,6 +89,28 @@ def _build_analysis_service(
) )
def _build_ingestion_service() -> IngestionService | None:
"""Build the ingestion service — only if embedding + opensearch are configured."""
if not settings.embedding_url or not settings.opensearch_url:
logger.info("Ingestion disabled (EMBEDDING_URL or OPENSEARCH_URL not set)")
return None
from infra.embedding_client import EmbeddingClient
from infra.opensearch_store import OpenSearchStore
embedding = EmbeddingClient(settings.embedding_url)
vector_store = OpenSearchStore(settings.opensearch_url)
config = IngestionConfig(
embedding_dimension=settings.embedding_dimension,
)
logger.info(
"Ingestion enabled (embedding=%s, opensearch=%s)",
settings.embedding_url,
settings.opensearch_url,
)
return IngestionService(embedding, vector_store, config)
def _build_document_service( def _build_document_service(
document_repo: SqliteDocumentRepository, document_repo: SqliteDocumentRepository,
analysis_repo: SqliteAnalysisRepository, analysis_repo: SqliteAnalysisRepository,
@ -114,6 +138,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
document_repo, analysis_repo = _build_repos() document_repo, analysis_repo = _build_repos()
app.state.analysis_service = _build_analysis_service(document_repo, analysis_repo) app.state.analysis_service = _build_analysis_service(document_repo, analysis_repo)
app.state.document_service = _build_document_service(document_repo, analysis_repo) app.state.document_service = _build_document_service(document_repo, analysis_repo)
app.state.ingestion_service = _build_ingestion_service()
logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine) logger.info("Docling Studio backend ready (engine=%s)", settings.conversion_engine)
yield yield
@ -128,7 +153,7 @@ app.add_middleware(
CORSMiddleware, CORSMiddleware,
allow_origins=settings.cors_origins, allow_origins=settings.cors_origins,
allow_credentials=True, allow_credentials=True,
allow_methods=["GET", "POST", "DELETE", "OPTIONS"], allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"],
allow_headers=["Content-Type", "Authorization"], allow_headers=["Content-Type", "Authorization"],
) )
if settings.rate_limit_rpm > 0: if settings.rate_limit_rpm > 0:
@ -140,6 +165,7 @@ if settings.rate_limit_rpm > 0:
app.include_router(documents_router) app.include_router(documents_router)
app.include_router(analyses_router) app.include_router(analyses_router)
app.include_router(ingestion_router)
@app.get("/api/health", response_model=HealthResponse) @app.get("/api/health", response_model=HealthResponse)

View file

@ -0,0 +1,166 @@
"""Ingestion service — orchestrates Docling → embedding → OpenSearch.
Chains the full ingestion pipeline:
1. Convert document via Docling (reuse existing analysis)
2. Chunk with selected strategy
3. Embed all chunk texts via EmbeddingService
4. Index into OpenSearch via VectorStore
Idempotent: re-ingesting a document deletes old chunks before re-indexing.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass
from typing import TYPE_CHECKING
from domain.vector_schema import (
ChunkBboxEntry,
ChunkOrigin,
IndexedChunk,
build_index_mapping,
)
if TYPE_CHECKING:
from domain.ports import EmbeddingService, VectorStore
logger = logging.getLogger(__name__)
@dataclass
class IngestionConfig:
"""Configuration for the ingestion pipeline."""
index_name: str = "docling-studio-chunks"
embedding_dimension: int = 384
@dataclass
class IngestionResult:
"""Result of an ingestion pipeline run."""
doc_id: str
chunks_indexed: int
embedding_dimension: int
class IngestionService:
"""Orchestrates the embedding + indexing pipeline."""
def __init__(
self,
embedding_service: EmbeddingService,
vector_store: VectorStore,
config: IngestionConfig | None = None,
) -> None:
self._embedding = embedding_service
self._vector_store = vector_store
self._config = config or IngestionConfig()
async def ensure_index(self) -> None:
"""Ensure the vector index exists with the correct mapping."""
mapping = build_index_mapping(self._config.embedding_dimension)
await self._vector_store.ensure_index(self._config.index_name, mapping)
async def ingest(
self,
doc_id: str,
filename: str,
chunks_json: str,
*,
binary_hash: str | None = None,
) -> IngestionResult:
"""Run the embedding + indexing pipeline on pre-chunked data.
This method is idempotent: it deletes any existing chunks for the
document before re-indexing.
Args:
doc_id: Unique document identifier.
filename: Original filename.
chunks_json: JSON-serialized list of chunk dicts (from analysis).
binary_hash: Optional hash of the source file for provenance.
Returns:
IngestionResult with the number of chunks indexed.
"""
await self.ensure_index()
chunks_data: list[dict] = json.loads(chunks_json)
active_chunks = [c for c in chunks_data if not c.get("deleted")]
if not active_chunks:
logger.info("No active chunks for doc %s — skipping ingestion", doc_id)
return IngestionResult(doc_id=doc_id, chunks_indexed=0, embedding_dimension=0)
# 1. Embed all chunk texts
texts = [c["text"] for c in active_chunks]
logger.info("Embedding %d chunks for doc %s", len(texts), doc_id)
embeddings = await self._embedding.embed(texts)
# 2. Build IndexedChunk domain objects
origin = (
ChunkOrigin(binary_hash=binary_hash or "", filename=filename) if binary_hash else None
)
indexed_chunks: list[IndexedChunk] = []
for i, (chunk_data, embedding) in enumerate(zip(active_chunks, embeddings, strict=True)):
bboxes = [
ChunkBboxEntry(
page=b["page"],
x=b["bbox"][0] if b.get("bbox") else 0,
y=b["bbox"][1] if b.get("bbox") else 0,
w=(b["bbox"][2] - b["bbox"][0]) if b.get("bbox") and len(b["bbox"]) >= 4 else 0,
h=(b["bbox"][3] - b["bbox"][1]) if b.get("bbox") and len(b["bbox"]) >= 4 else 0,
)
for b in chunk_data.get("bboxes", [])
]
indexed_chunks.append(
IndexedChunk(
doc_id=doc_id,
filename=filename,
content=chunk_data["text"],
embedding=embedding,
chunk_index=i,
chunk_type=chunk_data.get("chunkType", "text"),
page_number=chunk_data.get("sourcePage", 0) or 0,
bboxes=bboxes,
headings=chunk_data.get("headings", []),
origin=origin,
)
)
# 3. Delete old chunks (idempotent re-indexing)
deleted = await self._vector_store.delete_document(self._config.index_name, doc_id)
if deleted:
logger.info("Deleted %d old chunks for doc %s", deleted, doc_id)
# 4. Index new chunks
indexed = await self._vector_store.index_chunks(self._config.index_name, indexed_chunks)
logger.info("Indexed %d/%d chunks for doc %s", indexed, len(indexed_chunks), doc_id)
return IngestionResult(
doc_id=doc_id,
chunks_indexed=indexed,
embedding_dimension=len(embeddings[0]) if embeddings else 0,
)
async def delete_document(self, doc_id: str) -> int:
"""Remove all indexed chunks for a document."""
return await self._vector_store.delete_document(self._config.index_name, doc_id)
async def search(
self,
query: str,
*,
k: int = 10,
doc_id: str | None = None,
) -> list:
"""Semantic search: embed the query then find nearest chunks."""
embeddings = await self._embedding.embed([query])
return await self._vector_store.search_similar(
self._config.index_name,
embeddings[0],
k=k,
doc_id=doc_id,
)

View file

@ -0,0 +1,114 @@
"""Tests for the ingestion API endpoints (api.ingestion)."""
from __future__ import annotations
from unittest.mock import AsyncMock
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from api.ingestion import router
from domain.models import AnalysisJob
from services.ingestion_service import IngestionResult
@pytest.fixture
def mock_ingestion_service() -> AsyncMock:
svc = AsyncMock()
svc.ingest.return_value = IngestionResult(
doc_id="doc-1", chunks_indexed=5, embedding_dimension=384
)
svc.delete_document.return_value = 3
return svc
@pytest.fixture
def mock_analysis_service() -> AsyncMock:
svc = AsyncMock()
job = AnalysisJob(document_id="doc-1")
job.document_filename = "test.pdf"
job.mark_running()
job.mark_completed(
markdown="# Test",
html="<h1>Test</h1>",
pages_json="[]",
document_json='{"doc": true}',
chunks_json='[{"text": "hello"}]',
)
svc.find_by_id.return_value = job
return svc
@pytest.fixture
def client(mock_ingestion_service: AsyncMock, mock_analysis_service: AsyncMock) -> TestClient:
app = FastAPI()
app.include_router(router)
app.state.ingestion_service = mock_ingestion_service
app.state.analysis_service = mock_analysis_service
return TestClient(app)
class TestIngestAnalysis:
def test_ingest_success(self, client: TestClient) -> None:
resp = client.post("/api/ingestion/job-1")
assert resp.status_code == 200
data = resp.json()
assert data["docId"] == "doc-1"
assert data["chunksIndexed"] == 5
assert data["embeddingDimension"] == 384
def test_ingest_not_found(self, client: TestClient, mock_analysis_service: AsyncMock) -> None:
mock_analysis_service.find_by_id.return_value = None
resp = client.post("/api/ingestion/missing")
assert resp.status_code == 404
def test_ingest_not_completed(
self, client: TestClient, mock_analysis_service: AsyncMock
) -> None:
job = AnalysisJob(document_id="doc-1")
mock_analysis_service.find_by_id.return_value = job
resp = client.post("/api/ingestion/job-1")
assert resp.status_code == 400
def test_ingest_no_chunks(self, client: TestClient, mock_analysis_service: AsyncMock) -> None:
job = AnalysisJob(document_id="doc-1")
job.mark_running()
job.mark_completed(markdown="x", html="x", pages_json="[]")
mock_analysis_service.find_by_id.return_value = job
resp = client.post("/api/ingestion/job-1")
assert resp.status_code == 400
class TestDeleteIngested:
def test_delete_success(self, client: TestClient) -> None:
resp = client.delete("/api/ingestion/doc-1")
assert resp.status_code == 204
class TestIngestionStatus:
def test_available(self, client: TestClient) -> None:
resp = client.get("/api/ingestion/status")
assert resp.status_code == 200
assert resp.json()["available"] is True
def test_not_available(self) -> None:
app = FastAPI()
app.include_router(router)
app.state.ingestion_service = None
app.state.analysis_service = AsyncMock()
tc = TestClient(app)
resp = tc.get("/api/ingestion/status")
assert resp.status_code == 200
assert resp.json()["available"] is False
class TestIngestionDisabled:
def test_returns_503_when_disabled(self) -> None:
app = FastAPI()
app.include_router(router)
app.state.ingestion_service = None
app.state.analysis_service = AsyncMock()
tc = TestClient(app)
resp = tc.post("/api/ingestion/job-1")
assert resp.status_code == 503

View file

@ -0,0 +1,150 @@
"""Tests for the ingestion service (services.ingestion_service)."""
from __future__ import annotations
import json
from unittest.mock import AsyncMock
import pytest
from services.ingestion_service import IngestionConfig, IngestionService
def _make_chunks_json(count: int = 3, *, with_deleted: bool = False) -> str:
chunks = []
for i in range(count):
chunk = {
"text": f"chunk text {i}",
"headings": [f"Heading {i}"],
"sourcePage": i + 1,
"tokenCount": 10,
"bboxes": [{"page": i + 1, "bbox": [0.0, 0.0, 100.0, 50.0]}],
}
if with_deleted and i == count - 1:
chunk["deleted"] = True
chunks.append(chunk)
return json.dumps(chunks)
@pytest.fixture
def mock_embedding() -> AsyncMock:
svc = AsyncMock()
svc.embed.return_value = [[0.1, 0.2, 0.3]] * 3
return svc
@pytest.fixture
def mock_vector_store() -> AsyncMock:
store = AsyncMock()
store.ensure_index.return_value = None
store.delete_document.return_value = 0
store.index_chunks.return_value = 3
return store
@pytest.fixture
def service(mock_embedding: AsyncMock, mock_vector_store: AsyncMock) -> IngestionService:
return IngestionService(
embedding_service=mock_embedding,
vector_store=mock_vector_store,
config=IngestionConfig(index_name="test-idx", embedding_dimension=3),
)
class TestIngest:
async def test_full_pipeline(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
result = await service.ingest("doc-1", "test.pdf", _make_chunks_json(3))
assert result.doc_id == "doc-1"
assert result.chunks_indexed == 3
mock_embedding.embed.assert_awaited_once()
texts = mock_embedding.embed.call_args[0][0]
assert len(texts) == 3
mock_vector_store.ensure_index.assert_awaited_once()
mock_vector_store.delete_document.assert_awaited_once_with("test-idx", "doc-1")
mock_vector_store.index_chunks.assert_awaited_once()
indexed = mock_vector_store.index_chunks.call_args[0][1]
assert len(indexed) == 3
assert indexed[0].doc_id == "doc-1"
assert indexed[0].filename == "test.pdf"
assert indexed[0].embedding == [0.1, 0.2, 0.3]
async def test_skips_deleted_chunks(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
mock_embedding.embed.return_value = [[0.1, 0.2, 0.3]] * 2
mock_vector_store.index_chunks.return_value = 2
result = await service.ingest("doc-1", "test.pdf", _make_chunks_json(3, with_deleted=True))
assert result.chunks_indexed == 2
texts = mock_embedding.embed.call_args[0][0]
assert len(texts) == 2
async def test_empty_chunks(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
result = await service.ingest("doc-1", "test.pdf", json.dumps([]))
assert result.chunks_indexed == 0
mock_embedding.embed.assert_not_awaited()
async def test_idempotent_deletes_old(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
mock_vector_store.delete_document.return_value = 5
await service.ingest("doc-1", "test.pdf", _make_chunks_json(3))
mock_vector_store.delete_document.assert_awaited_once_with("test-idx", "doc-1")
async def test_bbox_conversion(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
mock_embedding.embed.return_value = [[0.1, 0.2, 0.3]]
mock_vector_store.index_chunks.return_value = 1
await service.ingest("doc-1", "test.pdf", _make_chunks_json(1))
indexed = mock_vector_store.index_chunks.call_args[0][1]
bbox = indexed[0].bboxes[0]
assert bbox.x == 0.0
assert bbox.y == 0.0
assert bbox.w == 100.0
assert bbox.h == 50.0
async def test_with_binary_hash(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
mock_embedding = service._embedding
mock_embedding.embed.return_value = [[0.1]] * 1
await service.ingest("doc-1", "test.pdf", _make_chunks_json(1), binary_hash="abc123")
indexed = mock_vector_store.index_chunks.call_args[0][1]
assert indexed[0].origin is not None
assert indexed[0].origin.binary_hash == "abc123"
class TestDeleteDocument:
async def test_delegates_to_vector_store(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
mock_vector_store.delete_document.return_value = 3
result = await service.delete_document("doc-1")
assert result == 3
class TestSearch:
async def test_embeds_and_searches(
self, service: IngestionService, mock_embedding: AsyncMock, mock_vector_store: AsyncMock
) -> None:
mock_embedding.embed.return_value = [[0.5, 0.6, 0.7]]
mock_vector_store.search_similar.return_value = []
await service.search("test query", k=5)
mock_embedding.embed.assert_awaited_once_with(["test query"])
mock_vector_store.search_similar.assert_awaited_once()
class TestEnsureIndex:
async def test_calls_vector_store(
self, service: IngestionService, mock_vector_store: AsyncMock
) -> None:
await service.ensure_index()
mock_vector_store.ensure_index.assert_awaited_once()
call_args = mock_vector_store.ensure_index.call_args
assert call_args[0][0] == "test-idx"

View file

@ -0,0 +1,59 @@
@e2e @ingestion
Feature: Ingestion pipeline — PDF → chunks → embeddings → OpenSearch
Background:
* url baseUrl
Scenario: Upload PDF, analyze with chunking, ingest into OpenSearch, verify
# Step 1: Check ingestion is available
Given path '/api/ingestion/status'
When method GET
Then status 200
And match response.available == true
# Step 2: Upload a PDF
Given path '/api/documents/upload'
And multipart file file = { read: 'classpath:common/data/generated/medium.pdf', filename: 'medium.pdf', contentType: 'application/pdf' }
When method POST
Then status 200
* def docId = response.id
# Step 3: Create analysis with chunking
Given path '/api/analyses'
And request { documentId: '#(docId)', pipelineOptions: { doOcr: true, tableMode: 'fast' }, chunkingOptions: { chunkerType: 'hybrid', maxTokens: 256 } }
When method POST
Then status 200
* def jobId = response.id
# Step 4: Poll until completed
Given path '/api/analyses', jobId
And retry until response.status == 'COMPLETED' || response.status == 'FAILED'
When method GET
Then status 200
And match response.status == 'COMPLETED'
And match response.chunksJson == '#string'
# Step 5: Trigger ingestion
Given path '/api/ingestion', jobId
When method POST
Then status 200
And match response.docId == docId
And match response.chunksIndexed == '#number'
And assert response.chunksIndexed > 0
And match response.embeddingDimension == '#number'
And assert response.embeddingDimension > 0
# Step 6: Cleanup — delete ingested data
Given path '/api/ingestion', docId
When method DELETE
Then status 204
# Step 7: Cleanup — delete analysis and document
Given path '/api/analyses', jobId
When method DELETE
Then status 204
Given path '/api/documents', docId
When method DELETE
Then status 204

View file

@ -2,6 +2,8 @@ FROM python:3.12-slim
WORKDIR /app WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
# Install dependencies first (cache layer) # Install dependencies first (cache layer)
COPY requirements.txt . COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt

View file

@ -0,0 +1,48 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { ingestAnalysis, deleteIngested, fetchIngestionStatus } from './api'
const mockFetch = vi.fn()
vi.stubGlobal('fetch', mockFetch)
beforeEach(() => {
mockFetch.mockReset()
})
describe('ingestAnalysis', () => {
it('posts to /api/ingestion/:jobId', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ docId: 'doc-1', chunksIndexed: 5, embeddingDimension: 384 }),
})
const result = await ingestAnalysis('job-1')
expect(mockFetch).toHaveBeenCalledWith(
'/api/ingestion/job-1',
expect.objectContaining({ method: 'POST' }),
)
expect(result.chunksIndexed).toBe(5)
})
})
describe('deleteIngested', () => {
it('deletes /api/ingestion/:docId', async () => {
mockFetch.mockResolvedValue({ ok: true, status: 204, json: () => Promise.resolve(null) })
await deleteIngested('doc-1')
expect(mockFetch).toHaveBeenCalledWith(
'/api/ingestion/doc-1',
expect.objectContaining({ method: 'DELETE' }),
)
})
})
describe('fetchIngestionStatus', () => {
it('gets /api/ingestion/status', async () => {
mockFetch.mockResolvedValue({
ok: true,
status: 200,
json: () => Promise.resolve({ available: true }),
})
const result = await fetchIngestionStatus()
expect(result.available).toBe(true)
})
})

View file

@ -0,0 +1,25 @@
import { apiFetch } from '../../shared/api/http'
export interface IngestionResult {
docId: string
chunksIndexed: number
embeddingDimension: number
}
export interface IngestionStatus {
available: boolean
}
export function ingestAnalysis(jobId: string): Promise<IngestionResult> {
return apiFetch<IngestionResult>(`/api/ingestion/${jobId}`, {
method: 'POST',
})
}
export function deleteIngested(docId: string): Promise<unknown> {
return apiFetch(`/api/ingestion/${docId}`, { method: 'DELETE' })
}
export function fetchIngestionStatus(): Promise<IngestionStatus> {
return apiFetch<IngestionStatus>('/api/ingestion/status')
}

View file

@ -0,0 +1 @@
export { useIngestionStore } from './store'

View file

@ -0,0 +1,66 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { setActivePinia, createPinia } from 'pinia'
import { useIngestionStore } from './store'
import * as api from './api'
vi.mock('./api', () => ({
fetchIngestionStatus: vi.fn(),
ingestAnalysis: vi.fn(),
deleteIngested: vi.fn(),
}))
beforeEach(() => {
setActivePinia(createPinia())
vi.clearAllMocks()
})
describe('useIngestionStore', () => {
describe('checkAvailability', () => {
it('sets available to true when API responds', async () => {
vi.mocked(api.fetchIngestionStatus).mockResolvedValue({ available: true })
const store = useIngestionStore()
await store.checkAvailability()
expect(store.available).toBe(true)
})
it('sets available to false on error', async () => {
vi.mocked(api.fetchIngestionStatus).mockRejectedValue(new Error('fail'))
const store = useIngestionStore()
await store.checkAvailability()
expect(store.available).toBe(false)
})
})
describe('ingest', () => {
it('calls API and tracks ingested doc', async () => {
vi.mocked(api.ingestAnalysis).mockResolvedValue({
docId: 'doc-1',
chunksIndexed: 5,
embeddingDimension: 384,
})
const store = useIngestionStore()
const result = await store.ingest('job-1')
expect(result?.chunksIndexed).toBe(5)
expect(store.ingestedDocs['doc-1']).toBe(5)
expect(store.ingesting).toBe(false)
})
it('sets error on failure', async () => {
vi.mocked(api.ingestAnalysis).mockRejectedValue(new Error('fail'))
const store = useIngestionStore()
const result = await store.ingest('job-1')
expect(result).toBeNull()
expect(store.error).toBe('fail')
})
})
describe('deleteIngested', () => {
it('removes doc from tracked map', async () => {
vi.mocked(api.deleteIngested).mockResolvedValue(null)
const store = useIngestionStore()
store.ingestedDocs['doc-1'] = 5
await store.deleteIngested('doc-1')
expect(store.ingestedDocs['doc-1']).toBeUndefined()
})
})
})

View file

@ -0,0 +1,56 @@
import { defineStore } from 'pinia'
import { ref } from 'vue'
import * as api from './api'
export const useIngestionStore = defineStore('ingestion', () => {
const available = ref(false)
const ingesting = ref(false)
const error = ref<string | null>(null)
/** Map of docId → chunks indexed count (tracks which docs are ingested) */
const ingestedDocs = ref<Record<string, number>>({})
async function checkAvailability(): Promise<void> {
try {
const status = await api.fetchIngestionStatus()
available.value = status.available
} catch {
available.value = false
}
}
async function ingest(jobId: string): Promise<api.IngestionResult | null> {
ingesting.value = true
error.value = null
try {
const result = await api.ingestAnalysis(jobId)
ingestedDocs.value[result.docId] = result.chunksIndexed
return result
} catch (e) {
error.value = (e as Error).message || 'Ingestion failed'
console.error('Ingestion failed', e)
return null
} finally {
ingesting.value = false
}
}
async function deleteIngested(docId: string): Promise<void> {
try {
await api.deleteIngested(docId)
delete ingestedDocs.value[docId]
} catch (e) {
error.value = (e as Error).message || 'Failed to delete ingested data'
console.error('Failed to delete ingested data', e)
}
}
return {
available,
ingesting,
error,
ingestedDocs,
checkAvailability,
ingest,
deleteIngested,
}
})

View file

@ -2,13 +2,40 @@
<div class="documents-page"> <div class="documents-page">
<div class="page-header"> <div class="page-header">
<h1 class="page-title">{{ t('nav.documents') }}</h1> <h1 class="page-title">{{ t('nav.documents') }}</h1>
<div class="header-actions">
<input
v-model="searchQuery"
type="text"
class="search-input"
:placeholder="t('ingestion.search')"
/>
<div class="filter-group">
<button
v-for="f in filters"
:key="f.value"
class="filter-btn"
:class="{ active: activeFilter === f.value }"
@click="activeFilter = f.value"
>
{{ f.label }}
</button>
</div>
<div class="sort-group">
<button class="sort-btn" :class="{ active: sortBy === 'name' }" @click="sortBy = 'name'">
{{ t('ingestion.sortName') }}
</button>
<button class="sort-btn" :class="{ active: sortBy === 'date' }" @click="sortBy = 'date'">
{{ t('ingestion.sortDate') }}
</button>
</div>
</div>
</div> </div>
<div class="page-content"> <div class="page-content">
<div v-if="docStore.documents.length === 0" class="tab-empty"> <div v-if="filteredDocs.length === 0" class="tab-empty">
{{ t('history.emptyDocs') }} {{ t('history.emptyDocs') }}
</div> </div>
<div v-else class="doc-items"> <div v-else class="doc-items">
<div v-for="doc in docStore.documents" :key="doc.id" class="doc-row"> <div v-for="doc in filteredDocs" :key="doc.id" class="doc-row">
<div class="doc-row-info"> <div class="doc-row-info">
<svg class="doc-row-icon" viewBox="0 0 20 20" fill="currentColor"> <svg class="doc-row-icon" viewBox="0 0 20 20" fill="currentColor">
<path <path
@ -26,15 +53,39 @@
</span> </span>
</div> </div>
</div> </div>
<button class="doc-row-delete" @click="docStore.remove(doc.id)"> <div class="doc-row-actions">
<svg viewBox="0 0 20 20" fill="currentColor"> <span
<path v-if="ingestionStore.ingestedDocs[doc.id]"
fill-rule="evenodd" class="status-badge indexed"
d="M9 2a1 1 0 00-.894.553L7.382 4H4a1 1 0 000 2v10a2 2 0 002 2h8a2 2 0 002-2V6a1 1 0 100-2h-3.382l-.724-1.447A1 1 0 0011 2H9zM7 8a1 1 0 012 0v6a1 1 0 11-2 0V8zm5-1a1 1 0 00-1 1v6a1 1 0 102 0V8a1 1 0 00-1-1z" :title="t('ingestion.chunksIndexed', { n: ingestionStore.ingestedDocs[doc.id] })"
clip-rule="evenodd" >
/> {{ t('ingestion.indexed') }}
</svg> <span class="badge-count">{{ ingestionStore.ingestedDocs[doc.id] }}</span>
</button> </span>
<span v-else class="status-badge not-indexed">
{{ t('ingestion.notIndexed') }}
</span>
<button
class="action-btn"
:title="t('ingestion.openInStudio')"
@click="openInStudio(doc)"
>
<svg viewBox="0 0 20 20" fill="currentColor">
<path
d="M10.394 2.08a1 1 0 00-.788 0l-7 3a1 1 0 000 1.84L5.25 8.051a.999.999 0 01.356-.257l4-1.714a1 1 0 11.788 1.838l-2.727 1.17 1.94.831a1 1 0 00.787 0l7-3a1 1 0 000-1.838l-7-3z"
/>
</svg>
</button>
<button class="action-btn delete" @click="handleDelete(doc.id)">
<svg viewBox="0 0 20 20" fill="currentColor">
<path
fill-rule="evenodd"
d="M9 2a1 1 0 00-.894.553L7.382 4H4a1 1 0 000 2v10a2 2 0 002 2h8a2 2 0 002-2V6a1 1 0 100-2h-3.382l-.724-1.447A1 1 0 0011 2H9zM7 8a1 1 0 012 0v6a1 1 0 11-2 0V8zm5-1a1 1 0 00-1 1v6a1 1 0 102 0V8a1 1 0 00-1-1z"
clip-rule="evenodd"
/>
</svg>
</button>
</div>
</div> </div>
</div> </div>
</div> </div>
@ -42,21 +93,75 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import { onMounted } from 'vue' import { computed, onMounted, ref } from 'vue'
import { useRouter } from 'vue-router'
import { useDocumentStore } from '../features/document/store' import { useDocumentStore } from '../features/document/store'
import { useIngestionStore } from '../features/ingestion/store'
import { useI18n } from '../shared/i18n' import { useI18n } from '../shared/i18n'
import { formatSize } from '../shared/format' import { formatSize } from '../shared/format'
import type { Document } from '../shared/types'
const docStore = useDocumentStore() const docStore = useDocumentStore()
const ingestionStore = useIngestionStore()
const router = useRouter()
const { t } = useI18n() const { t } = useI18n()
const searchQuery = ref('')
const activeFilter = ref<'all' | 'indexed' | 'not-indexed'>('all')
const sortBy = ref<'name' | 'date'>('date')
const filters = computed(() => [
{ value: 'all' as const, label: t('ingestion.filterAll') },
{ value: 'indexed' as const, label: t('ingestion.filterIndexed') },
{ value: 'not-indexed' as const, label: t('ingestion.filterNotIndexed') },
])
const filteredDocs = computed(() => {
let docs = [...docStore.documents]
// Search filter
if (searchQuery.value.trim()) {
const q = searchQuery.value.toLowerCase()
docs = docs.filter((d) => d.filename.toLowerCase().includes(q))
}
// Status filter
if (activeFilter.value === 'indexed') {
docs = docs.filter((d) => ingestionStore.ingestedDocs[d.id])
} else if (activeFilter.value === 'not-indexed') {
docs = docs.filter((d) => !ingestionStore.ingestedDocs[d.id])
}
// Sort
if (sortBy.value === 'name') {
docs.sort((a, b) => a.filename.localeCompare(b.filename))
} else {
docs.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
}
return docs
})
function formatDate(iso: string) { function formatDate(iso: string) {
if (!iso) return '' if (!iso) return ''
return new Date(iso).toLocaleString() return new Date(iso).toLocaleString()
} }
function openInStudio(doc: Document) {
docStore.select(doc.id)
router.push('/studio')
}
async function handleDelete(docId: string) {
if (ingestionStore.ingestedDocs[docId]) {
await ingestionStore.deleteIngested(docId)
}
await docStore.remove(docId)
}
onMounted(() => { onMounted(() => {
docStore.load() docStore.load()
ingestionStore.checkAvailability()
}) })
</script> </script>
@ -72,6 +177,11 @@ onMounted(() => {
padding: 16px 24px; padding: 16px 24px;
border-bottom: 1px solid var(--border); border-bottom: 1px solid var(--border);
flex-shrink: 0; flex-shrink: 0;
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
flex-wrap: wrap;
} }
.page-title { .page-title {
@ -80,6 +190,57 @@ onMounted(() => {
color: var(--text); color: var(--text);
} }
.header-actions {
display: flex;
align-items: center;
gap: 12px;
}
.search-input {
padding: 6px 12px;
border: 1px solid var(--border);
border-radius: var(--radius-sm);
background: var(--bg);
color: var(--text);
font-size: 13px;
width: 180px;
outline: none;
transition: border-color var(--transition);
}
.search-input:focus {
border-color: var(--accent);
}
.filter-group,
.sort-group {
display: flex;
gap: 2px;
background: var(--bg-surface);
border-radius: var(--radius-sm);
padding: 2px;
border: 1px solid var(--border);
}
.filter-btn,
.sort-btn {
padding: 4px 10px;
border: none;
background: none;
color: var(--text-secondary);
font-size: 12px;
font-weight: 500;
border-radius: 4px;
cursor: pointer;
transition: all var(--transition);
}
.filter-btn.active,
.sort-btn.active {
background: var(--accent);
color: white;
}
.page-content { .page-content {
flex: 1; flex: 1;
overflow-y: auto; overflow-y: auto;
@ -152,7 +313,41 @@ onMounted(() => {
font-family: 'IBM Plex Mono', monospace; font-family: 'IBM Plex Mono', monospace;
} }
.doc-row-delete { .doc-row-actions {
display: flex;
align-items: center;
gap: 8px;
flex-shrink: 0;
}
.status-badge {
display: inline-flex;
align-items: center;
gap: 4px;
padding: 3px 8px;
border-radius: 10px;
font-size: 11px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.03em;
}
.status-badge.indexed {
background: rgba(34, 197, 94, 0.15);
color: var(--success);
}
.status-badge.not-indexed {
background: rgba(156, 163, 175, 0.15);
color: var(--text-muted);
}
.badge-count {
font-family: 'IBM Plex Mono', monospace;
font-size: 10px;
}
.action-btn {
background: none; background: none;
border: none; border: none;
padding: 6px; padding: 6px;
@ -164,14 +359,21 @@ onMounted(() => {
transition: all var(--transition); transition: all var(--transition);
} }
.doc-row:hover .doc-row-delete { .doc-row:hover .action-btn {
opacity: 1; opacity: 1;
} }
.doc-row-delete:hover {
.action-btn:hover {
color: var(--accent);
background: rgba(249, 115, 22, 0.1);
}
.action-btn.delete:hover {
color: var(--error); color: var(--error);
background: rgba(239, 68, 68, 0.1); background: rgba(239, 68, 68, 0.1);
} }
.doc-row-delete svg {
.action-btn svg {
width: 16px; width: 16px;
height: 16px; height: 16px;
} }

View file

@ -94,6 +94,23 @@
</svg> </svg>
{{ analysisStore.running ? t('studio.analyzing') : t('studio.run') }} {{ analysisStore.running ? t('studio.analyzing') : t('studio.run') }}
</button> </button>
<button
v-if="canIngest"
class="topbar-btn ingest"
data-e2e="ingest-btn"
:disabled="ingestionStore.ingesting"
@click="runIngestion"
>
<div v-if="ingestionStore.ingesting" class="spinner-sm" />
<svg v-else viewBox="0 0 20 20" fill="currentColor" class="btn-icon">
<path
fill-rule="evenodd"
d="M3 17a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1zM6.293 6.707a1 1 0 010-1.414l3-3a1 1 0 011.414 0l3 3a1 1 0 01-1.414 1.414L11 5.414V13a1 1 0 11-2 0V5.414L7.707 6.707a1 1 0 01-1.414 0z"
clip-rule="evenodd"
/>
</svg>
{{ ingestionStore.ingesting ? t('ingestion.ingesting') : t('ingestion.ingest') }}
</button>
</div> </div>
</div> </div>
@ -459,6 +476,7 @@ import { ref, computed, watch, nextTick, onMounted, onBeforeUnmount, reactive }
import { useRoute, useRouter } from 'vue-router' import { useRoute, useRouter } from 'vue-router'
import { useDocumentStore } from '../features/document/store' import { useDocumentStore } from '../features/document/store'
import { useAnalysisStore } from '../features/analysis/store' import { useAnalysisStore } from '../features/analysis/store'
import { useIngestionStore } from '../features/ingestion/store'
import { DocumentUpload, DocumentList } from '../features/document/index' import { DocumentUpload, DocumentList } from '../features/document/index'
import { ResultTabs } from '../features/analysis/index' import { ResultTabs } from '../features/analysis/index'
import BboxOverlay from '../features/analysis/ui/BboxOverlay.vue' import BboxOverlay from '../features/analysis/ui/BboxOverlay.vue'
@ -472,6 +490,7 @@ const route = useRoute()
const router = useRouter() const router = useRouter()
const documentStore = useDocumentStore() const documentStore = useDocumentStore()
const analysisStore = useAnalysisStore() const analysisStore = useAnalysisStore()
const ingestionStore = useIngestionStore()
const { t } = useI18n() const { t } = useI18n()
const chunkingEnabled = useFeatureFlag('chunking') const chunkingEnabled = useFeatureFlag('chunking')
@ -528,6 +547,14 @@ const pipelineOptions = reactive<PipelineOptions>({
images_scale: 1.0, images_scale: 1.0,
}) })
const canIngest = computed(() => {
return (
ingestionStore.available &&
analysisStore.currentAnalysis?.status === 'COMPLETED' &&
analysisStore.currentAnalysis?.chunksJson != null
)
})
const hasAnalysisResults = computed(() => { const hasAnalysisResults = computed(() => {
return ( return (
analysisStore.currentAnalysis?.status === 'COMPLETED' && analysisStore.currentPages?.length > 0 analysisStore.currentAnalysis?.status === 'COMPLETED' && analysisStore.currentPages?.length > 0
@ -564,6 +591,11 @@ async function runAnalysis() {
await analysisStore.run(documentStore.selectedId, { ...pipelineOptions }) await analysisStore.run(documentStore.selectedId, { ...pipelineOptions })
} }
async function runIngestion() {
if (!analysisStore.currentAnalysis?.id) return
await ingestionStore.ingest(analysisStore.currentAnalysis.id)
}
function addMore() { function addMore() {
documentStore.selectedId = null documentStore.selectedId = null
} }
@ -598,6 +630,7 @@ watch(
onMounted(async () => { onMounted(async () => {
await documentStore.load() await documentStore.load()
analysisStore.load() analysisStore.load()
ingestionStore.checkAvailability()
// Restore analysis from history via query param // Restore analysis from history via query param
const analysisId = route.query.analysisId const analysisId = route.query.analysisId
@ -811,6 +844,21 @@ onBeforeUnmount(() => {
cursor: not-allowed; cursor: not-allowed;
} }
.topbar-btn.ingest {
background: var(--success);
border-color: var(--success);
color: white;
}
.topbar-btn.ingest:hover:not(:disabled) {
filter: brightness(1.1);
}
.topbar-btn.ingest:disabled {
opacity: 0.6;
cursor: not-allowed;
}
.topbar-btn .btn-icon { .topbar-btn .btn-icon {
width: 16px; width: 16px;
height: 16px; height: 16px;

View file

@ -131,6 +131,23 @@ const messages: Messages = {
'chunking.batchNotice': 'chunking.batchNotice':
'Le chunking n\u2019est pas disponible pour cette analyse. Les documents volumineux trait\u00e9s par batch ne g\u00e9n\u00e8rent pas la structure interne n\u00e9cessaire au d\u00e9coupage.', 'Le chunking n\u2019est pas disponible pour cette analyse. Les documents volumineux trait\u00e9s par batch ne g\u00e9n\u00e8rent pas la structure interne n\u00e9cessaire au d\u00e9coupage.',
// Ingestion / My Documents
'ingestion.ingest': 'Ingérer',
'ingestion.ingesting': 'Ingestion...',
'ingestion.reindex': 'Ré-indexer',
'ingestion.indexed': 'Indexé',
'ingestion.notIndexed': 'Non indexé',
'ingestion.chunksIndexed': '{n} chunks indexés',
'ingestion.openInStudio': 'Ouvrir dans le Studio',
'ingestion.deleteIndex': "Supprimer de l'index",
'ingestion.unavailable': 'Ingestion non disponible',
'ingestion.filterAll': 'Tous',
'ingestion.filterIndexed': 'Indexés',
'ingestion.filterNotIndexed': 'Non indexés',
'ingestion.sortName': 'Nom',
'ingestion.sortDate': 'Date',
'ingestion.search': 'Rechercher...',
// Pagination // Pagination
'pagination.pageOf': 'Page {current} sur {total}', 'pagination.pageOf': 'Page {current} sur {total}',
'pagination.perPage': '/ page', 'pagination.perPage': '/ page',
@ -266,6 +283,22 @@ const messages: Messages = {
'chunking.batchNotice': 'chunking.batchNotice':
'Chunking is not available for this analysis. Large documents processed in batch mode do not generate the internal structure required for chunking.', 'Chunking is not available for this analysis. Large documents processed in batch mode do not generate the internal structure required for chunking.',
'ingestion.ingest': 'Ingest',
'ingestion.ingesting': 'Ingesting...',
'ingestion.reindex': 'Re-index',
'ingestion.indexed': 'Indexed',
'ingestion.notIndexed': 'Not indexed',
'ingestion.chunksIndexed': '{n} chunks indexed',
'ingestion.openInStudio': 'Open in Studio',
'ingestion.deleteIndex': 'Remove from index',
'ingestion.unavailable': 'Ingestion unavailable',
'ingestion.filterAll': 'All',
'ingestion.filterIndexed': 'Indexed',
'ingestion.filterNotIndexed': 'Not indexed',
'ingestion.sortName': 'Name',
'ingestion.sortDate': 'Date',
'ingestion.search': 'Search...',
'pagination.pageOf': 'Page {current} of {total}', 'pagination.pageOf': 'Page {current} of {total}',
'pagination.perPage': '/ page', 'pagination.perPage': '/ page',