diff --git a/document-parser/api/schemas.py b/document-parser/api/schemas.py index 8744411..ad034d9 100644 --- a/document-parser/api/schemas.py +++ b/document-parser/api/schemas.py @@ -233,3 +233,63 @@ class SearchResponse(_CamelModel): results: list[SearchResultItem] total: int query: str + + +# --------------------------------------------------------------------------- +# Stores (#251) +# --------------------------------------------------------------------------- + + +class StoreInfoResponse(_CamelModel): + """Read model for `GET /api/stores`.""" + + name: str + slug: str + type: str + embedder: str + is_default: bool + document_count: int + chunk_count: int + connected: bool + error_message: str | None = None + + +class StoreResponse(_CamelModel): + """Detailed read model for `GET /api/stores/{slug}`.""" + + id: str + name: str + slug: str + kind: str + embedder: str + is_default: bool + config: dict + created_at: str | datetime + + +class StoreCreateRequest(_CamelModel): + name: str + slug: str + kind: str + embedder: str + config: dict = Field(default_factory=dict) + is_default: bool = False + + +class StoreUpdateRequest(_CamelModel): + """Partial update — every field is optional. Use `slug` to rename.""" + + name: str | None = None + slug: str | None = None + kind: str | None = None + embedder: str | None = None + config: dict | None = None + is_default: bool | None = None + + +class StoreDocEntryResponse(_CamelModel): + doc_id: str + filename: str + state: str + chunk_count: int + pushed_at: str | None = None diff --git a/document-parser/api/stores.py b/document-parser/api/stores.py new file mode 100644 index 0000000..3f11dc5 --- /dev/null +++ b/document-parser/api/stores.py @@ -0,0 +1,170 @@ +"""Stores API router — CRUD on ingestion targets (#251).""" + +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, HTTPException, Request, Response + +from api.schemas import ( + StoreCreateRequest, + StoreDocEntryResponse, + StoreInfoResponse, + StoreResponse, + StoreUpdateRequest, +) +from domain.value_objects import StoreKind +from services.store_service import ( + StoreService, + StoreServiceError, +) + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/stores", tags=["stores"]) + + +def _get_service(request: Request) -> StoreService: + return request.app.state.store_service + + +ServiceDep = Annotated[StoreService, Depends(_get_service)] + + +def _parse_kind(value: str) -> StoreKind: + try: + return StoreKind(value) + except ValueError as exc: + valid = ", ".join(k.value for k in StoreKind) + raise HTTPException( + status_code=422, + detail=f"Unknown store kind '{value}'. Valid kinds: {valid}.", + ) from exc + + +def _store_to_response(store) -> StoreResponse: + return StoreResponse( + id=store.id, + name=store.name, + slug=store.slug, + kind=store.kind.value, + embedder=store.embedder, + is_default=store.is_default, + config=store.config, + created_at=str(store.created_at), + ) + + +def _info_to_response(view) -> StoreInfoResponse: + return StoreInfoResponse( + name=view.name, + slug=view.slug, + type=view.kind, + embedder=view.embedder, + is_default=view.is_default, + document_count=view.document_count, + chunk_count=view.chunk_count, + connected=view.connected, + error_message=view.error_message, + ) + + +def _doc_entry_to_response(entry) -> StoreDocEntryResponse: + return StoreDocEntryResponse( + doc_id=entry.doc_id, + filename=entry.filename, + state=entry.state, + chunk_count=entry.chunk_count, + pushed_at=entry.pushed_at, + ) + + +@router.get("", response_model=list[StoreInfoResponse]) +async def list_stores(service: ServiceDep) -> list[StoreInfoResponse]: + views = await service.list_stores() + return [_info_to_response(v) for v in views] + + +@router.post("", response_model=StoreResponse, status_code=201) +async def create_store( + payload: StoreCreateRequest, + service: ServiceDep, +) -> StoreResponse: + kind = _parse_kind(payload.kind) + try: + store = await service.create_store( + name=payload.name, + slug=payload.slug, + kind=kind, + embedder=payload.embedder, + config=payload.config, + is_default=payload.is_default, + ) + except StoreServiceError as exc: + raise HTTPException(status_code=exc.http_status, detail=str(exc)) from exc + return _store_to_response(store) + + +@router.get("/{slug}", response_model=StoreResponse) +async def get_store(slug: str, service: ServiceDep) -> StoreResponse: + try: + store = await service.get_by_slug(slug) + except StoreServiceError as exc: + raise HTTPException(status_code=exc.http_status, detail=str(exc)) from exc + return _store_to_response(store) + + +@router.patch("/{slug}", response_model=StoreResponse) +async def update_store( + slug: str, + payload: StoreUpdateRequest, + service: ServiceDep, +) -> StoreResponse: + kind = _parse_kind(payload.kind) if payload.kind is not None else None + try: + store = await service.update_store( + slug, + name=payload.name, + new_slug=payload.slug, + kind=kind, + embedder=payload.embedder, + config=payload.config, + is_default=payload.is_default, + ) + except StoreServiceError as exc: + raise HTTPException(status_code=exc.http_status, detail=str(exc)) from exc + return _store_to_response(store) + + +@router.delete("/{slug}", status_code=204) +async def delete_store(slug: str, service: ServiceDep) -> Response: + try: + await service.delete_store(slug) + except StoreServiceError as exc: + raise HTTPException(status_code=exc.http_status, detail=str(exc)) from exc + return Response(status_code=204) + + +@router.get("/{slug}/documents", response_model=list[StoreDocEntryResponse]) +async def list_store_documents( + slug: str, + service: ServiceDep, +) -> list[StoreDocEntryResponse]: + try: + entries = await service.list_documents(slug) + except StoreServiceError as exc: + raise HTTPException(status_code=exc.http_status, detail=str(exc)) from exc + return [_doc_entry_to_response(e) for e in entries] + + +@router.delete("/{slug}/documents/{doc_id}", status_code=204) +async def remove_store_document( + slug: str, + doc_id: str, + service: ServiceDep, +) -> Response: + try: + await service.remove_document(slug, doc_id) + except StoreServiceError as exc: + raise HTTPException(status_code=exc.http_status, detail=str(exc)) from exc + return Response(status_code=204) diff --git a/document-parser/main.py b/document-parser/main.py index e344f2f..58d2778 100644 --- a/document-parser/main.py +++ b/document-parser/main.py @@ -22,14 +22,18 @@ from api.analyses import router as analyses_router from api.documents import router as documents_router from api.ingestion import router as ingestion_router from api.schemas import HealthResponse +from api.stores import router as stores_router from infra.rate_limiter import RateLimiterMiddleware from infra.settings import settings from persistence.analysis_repo import SqliteAnalysisRepository from persistence.database import get_connection, init_db from persistence.document_repo import SqliteDocumentRepository +from persistence.document_store_link_repo import SqliteDocumentStoreLinkRepository +from persistence.store_repo import SqliteStoreRepository from services.analysis_service import AnalysisConfig, AnalysisService from services.document_service import DocumentConfig, DocumentService from services.ingestion_service import IngestionConfig, IngestionService +from services.store_service import StoreService logging.basicConfig( level=logging.INFO, @@ -187,6 +191,15 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: document_repo, analysis_repo, neo4j_driver=app.state.neo4j ) app.state.document_service = _build_document_service(document_repo, analysis_repo) + store_repo = SqliteStoreRepository() + link_repo = SqliteDocumentStoreLinkRepository() + app.state.store_repo = store_repo + app.state.document_store_link_repo = link_repo + app.state.store_service = StoreService( + store_repo=store_repo, + link_repo=link_repo, + document_repo=document_repo, + ) ingestion_service = _build_ingestion_service(neo4j_driver=app.state.neo4j) app.state.ingestion_service = ingestion_service if ingestion_service is not None: @@ -224,6 +237,7 @@ if settings.rate_limit_rpm > 0: app.include_router(documents_router) app.include_router(analyses_router) +app.include_router(stores_router) # Graph view — mounted regardless; individual requests 503 if Neo4j is absent. from api.graph import router as graph_router # noqa: E402 diff --git a/document-parser/services/store_service.py b/document-parser/services/store_service.py index e94f77f..bee40a5 100644 --- a/document-parser/services/store_service.py +++ b/document-parser/services/store_service.py @@ -20,6 +20,7 @@ from domain.models import Store from domain.value_objects import StoreKind if TYPE_CHECKING: + from persistence.document_repo import SqliteDocumentRepository from persistence.document_store_link_repo import SqliteDocumentStoreLinkRepository from persistence.store_repo import SqliteStoreRepository @@ -67,6 +68,17 @@ class StoreInfoView: error_message: str | None = None +@dataclass +class StoreDocEntryView: + """Read model for `GET /api/stores/{slug}/documents`.""" + + doc_id: str + filename: str + state: str + chunk_count: int + pushed_at: str | None + + def _validate_slug(slug: str) -> None: if not slug or not _SLUG_PATTERN.match(slug): raise StoreValidationError( @@ -91,9 +103,11 @@ class StoreService: self, store_repo: SqliteStoreRepository, link_repo: SqliteDocumentStoreLinkRepository, + document_repo: SqliteDocumentRepository | None = None, ): self._stores = store_repo self._links = link_repo + self._documents = document_repo async def list_stores(self) -> list[StoreInfoView]: stores = await self._stores.find_all() @@ -217,6 +231,40 @@ class StoreService: await self._stores.clear_default_except(store.id) return store + async def list_documents(self, slug: str) -> list[StoreDocEntryView]: + store = await self.get_by_slug(slug) + links = await self._links.find_for_store(store.id) + if self._documents is None: + return [ + StoreDocEntryView( + doc_id=link.document_id, + filename=link.document_id, + state=link.state.value, + chunk_count=0, + pushed_at=str(link.last_push_at) if link.last_push_at else None, + ) + for link in links + ] + entries: list[StoreDocEntryView] = [] + for link in links: + doc = await self._documents.find_by_id(link.document_id) + entries.append( + StoreDocEntryView( + doc_id=link.document_id, + filename=doc.filename if doc else link.document_id, + state=link.state.value, + chunk_count=0, + pushed_at=str(link.last_push_at) if link.last_push_at else None, + ) + ) + return entries + + async def remove_document(self, slug: str, doc_id: str) -> None: + store = await self.get_by_slug(slug) + removed = await self._links.delete(doc_id, store.id) + if not removed: + raise StoreNotFoundError(f"document '{doc_id}' is not linked to store '{slug}'") + async def delete_store(self, slug: str) -> None: store = await self.get_by_slug(slug) if store.slug == "default": diff --git a/document-parser/tests/test_api_stores.py b/document-parser/tests/test_api_stores.py new file mode 100644 index 0000000..df619a9 --- /dev/null +++ b/document-parser/tests/test_api_stores.py @@ -0,0 +1,200 @@ +"""Tests for `/api/stores/*` HTTP endpoints (#251).""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from domain.models import Document, DocumentStoreLink +from domain.value_objects import DocumentStoreLinkState +from main import app +from persistence.database import init_db +from persistence.document_repo import SqliteDocumentRepository +from persistence.document_store_link_repo import SqliteDocumentStoreLinkRepository +from persistence.store_repo import SqliteStoreRepository +from services.store_service import StoreService + + +@pytest.fixture(autouse=True) +async def setup_db_and_state(monkeypatch, tmp_path): + db_path = str(tmp_path / "test.db") + monkeypatch.setattr("persistence.database.DB_PATH", db_path) + await init_db() + + store_repo = SqliteStoreRepository() + link_repo = SqliteDocumentStoreLinkRepository() + document_repo = SqliteDocumentRepository() + + original = getattr(app.state, "store_service", None) + app.state.store_service = StoreService(store_repo, link_repo, document_repo) + yield + app.state.store_service = original + + +@pytest.fixture +def client(): + return TestClient(app, raise_server_exceptions=False) + + +VALID_OS = { + "name": "rh", + "slug": "rh", + "kind": "opensearch", + "embedder": "bge-m3", + "config": {"indexName": "rh"}, +} + + +class TestList: + def test_list_returns_seeded_default(self, client): + resp = client.get("/api/stores") + assert resp.status_code == 200 + slugs = [s["slug"] for s in resp.json()] + assert "default" in slugs + + def test_list_uses_camel_case(self, client): + resp = client.get("/api/stores") + body = resp.json()[0] + assert "documentCount" in body + assert "isDefault" in body + + +class TestCreate: + def test_create_201(self, client): + resp = client.post("/api/stores", json=VALID_OS) + assert resp.status_code == 201, resp.text + body = resp.json() + assert body["slug"] == "rh" + assert body["isDefault"] is False + + def test_create_duplicate_slug_409(self, client): + client.post("/api/stores", json=VALID_OS) + resp = client.post("/api/stores", json={**VALID_OS, "name": "rh-2"}) + assert resp.status_code == 409 + + def test_create_invalid_kind_422(self, client): + resp = client.post("/api/stores", json={**VALID_OS, "kind": "pinecone"}) + assert resp.status_code == 422 + + def test_create_missing_index_422(self, client): + resp = client.post("/api/stores", json={**VALID_OS, "config": {}}) + assert resp.status_code == 422 + + def test_create_bad_slug_422(self, client): + resp = client.post("/api/stores", json={**VALID_OS, "slug": "RH Corp"}) + assert resp.status_code == 422 + + +class TestRead: + def test_get_by_slug(self, client): + client.post("/api/stores", json=VALID_OS) + resp = client.get("/api/stores/rh") + assert resp.status_code == 200 + body = resp.json() + assert body["slug"] == "rh" + assert body["embedder"] == "bge-m3" + + def test_get_unknown_404(self, client): + resp = client.get("/api/stores/missing") + assert resp.status_code == 404 + + +class TestUpdate: + def test_patch_embedder(self, client): + client.post("/api/stores", json=VALID_OS) + resp = client.patch("/api/stores/rh", json={"embedder": "bge-large"}) + assert resp.status_code == 200 + assert resp.json()["embedder"] == "bge-large" + + def test_patch_rename_slug(self, client): + client.post("/api/stores", json=VALID_OS) + resp = client.patch("/api/stores/rh", json={"slug": "rh-v2"}) + assert resp.status_code == 200 + assert resp.json()["slug"] == "rh-v2" + assert client.get("/api/stores/rh").status_code == 404 + assert client.get("/api/stores/rh-v2").status_code == 200 + + def test_patch_unknown_404(self, client): + resp = client.patch("/api/stores/missing", json={"embedder": "x"}) + assert resp.status_code == 404 + + +class TestDelete: + def test_delete_204(self, client): + client.post("/api/stores", json=VALID_OS) + resp = client.delete("/api/stores/rh") + assert resp.status_code == 204 + assert client.get("/api/stores/rh").status_code == 404 + + def test_delete_default_409(self, client): + resp = client.delete("/api/stores/default") + assert resp.status_code == 409 + + async def test_delete_with_links_409(self, client): + client.post("/api/stores", json=VALID_OS) + # Seed a link from outside (service-level access). + await SqliteDocumentRepository().insert( + Document(id="d-1", filename="t.pdf", storage_path="/tmp/t.pdf") + ) + store = await SqliteStoreRepository().find_by_slug("rh") + assert store is not None + await SqliteDocumentStoreLinkRepository().upsert( + DocumentStoreLink( + id="l-1", + document_id="d-1", + store_id=store.id, + state=DocumentStoreLinkState.INGESTED, + ) + ) + resp = client.delete("/api/stores/rh") + assert resp.status_code == 409 + + +class TestStoreDocuments: + async def test_list_documents_for_store(self, client): + client.post("/api/stores", json=VALID_OS) + await SqliteDocumentRepository().insert( + Document(id="d-1", filename="hr.pdf", storage_path="/tmp/hr.pdf") + ) + store = await SqliteStoreRepository().find_by_slug("rh") + assert store is not None + await SqliteDocumentStoreLinkRepository().upsert( + DocumentStoreLink( + id="l-1", + document_id="d-1", + store_id=store.id, + state=DocumentStoreLinkState.INGESTED, + ) + ) + resp = client.get("/api/stores/rh/documents") + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 1 + assert body[0]["docId"] == "d-1" + assert body[0]["filename"] == "hr.pdf" + + async def test_remove_document_204(self, client): + client.post("/api/stores", json=VALID_OS) + await SqliteDocumentRepository().insert( + Document(id="d-1", filename="hr.pdf", storage_path="/tmp/hr.pdf") + ) + store = await SqliteStoreRepository().find_by_slug("rh") + assert store is not None + await SqliteDocumentStoreLinkRepository().upsert( + DocumentStoreLink( + id="l-1", + document_id="d-1", + store_id=store.id, + state=DocumentStoreLinkState.INGESTED, + ) + ) + resp = client.delete("/api/stores/rh/documents/d-1") + assert resp.status_code == 204 + # Now empty. + listing = client.get("/api/stores/rh/documents") + assert listing.json() == [] + + def test_remove_unknown_doc_404(self, client): + client.post("/api/stores", json=VALID_OS) + resp = client.delete("/api/stores/rh/documents/missing") + assert resp.status_code == 404