From b968ea230e368dc65971fc316c2917e1deeeac68 Mon Sep 17 00:00:00 2001 From: Pier-Jean Malandrino Date: Fri, 10 Apr 2026 20:35:03 +0200 Subject: [PATCH] feat(architecture): define vector index metadata schema Closes #68 --- .env.example | 3 + CHANGELOG.md | 1 + document-parser/domain/vector_schema.py | 166 +++++++++++++++++ document-parser/infra/settings.py | 6 + document-parser/tests/test_vector_schema.py | 197 ++++++++++++++++++++ 5 files changed, 373 insertions(+) create mode 100644 document-parser/domain/vector_schema.py create mode 100644 document-parser/tests/test_vector_schema.py diff --git a/.env.example b/.env.example index 0f23b23..bb9ec43 100644 --- a/.env.example +++ b/.env.example @@ -39,3 +39,6 @@ # OpenSearch URL (used by docker-compose.dev.yml, auto-set to service name) # OPENSEARCH_URL=http://opensearch:9200 + +# Embedding vector dimension (default: 384 for Granite Embedding 30M / all-MiniLM-L6-v2) +# EMBEDDING_DIMENSION=384 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aab9c8..1bff0be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this - Inline chunk text editing: double-click or edit button to modify chunk text, with save/cancel and "modified" badge - Docker Compose dev stack (`docker-compose.dev.yml`) with OpenSearch, Dashboards, hot-reload backend and Vite frontend - Soft-delete chunks: delete button with confirmation dialog, chunks hidden from UI but preserved in data +- Vector index metadata schema: `IndexedChunk` domain model, OpenSearch mapping builder, configurable embedding dimension ### Fixed diff --git a/document-parser/domain/vector_schema.py b/document-parser/domain/vector_schema.py new file mode 100644 index 0000000..7e52ba1 --- /dev/null +++ b/document-parser/domain/vector_schema.py @@ -0,0 +1,166 @@ +"""Vector index schema — data contract for OpenSearch ingestion and inspection. + +This module defines the standard metadata schema for the vector index used by +the ingestion pipeline (0.4.0) and the inspection UI (0.5.0). + +Field usage by milestone: + ┌────────────┬────────────────────────┬───────────────────────────────┬──────────────┐ + │ Field │ 0.4.0 (write) │ 0.5.0 (read) │ Source │ + ├────────────┼────────────────────────┼───────────────────────────────┼──────────────┤ + │ content │ Full-text search │ Chunk panel display │ Docling std │ + │ embedding │ Indexed │ kNN semantic search │ Docling std │ + │ doc_items │ Indexed │ Element type filtering │ Docling std │ + │ headings │ Indexed │ Section hierarchy display │ Docling std │ + │ origin │ Indexed │ Document provenance │ Docling std │ + │ bboxes │ Written at ingestion │ Chunk↔bbox highlight │ Studio │ + │ page_number│ Written at ingestion │ Split view navigation │ Studio │ + │ chunk_index│ Written at ingestion │ Chunk ordering in panel │ Studio │ + │ chunk_type │ Written at ingestion │ Metadata panel │ Studio │ + │ doc_id │ Document linking │ Document list navigation │ Studio │ + │ filename │ "My Documents" list │ Display │ Studio │ + └────────────┴────────────────────────┴───────────────────────────────┴──────────────┘ +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +# -- Value objects for a single indexed chunk ---------------------------------- + +DEFAULT_EMBEDDING_DIMENSION = 384 # Granite Embedding 30M (sentence-transformers) +DEFAULT_INDEX_NAME = "docling-studio-chunks" + + +@dataclass(frozen=True) +class ChunkBboxEntry: + """Bounding box for a chunk region on a specific page.""" + + page: int + x: float + y: float + w: float + h: float + + +@dataclass(frozen=True) +class DocItemRef: + """Reference to a Docling DocItem (element in the document structure).""" + + self_ref: str + label: str # text, table, picture, list, etc. + + +@dataclass(frozen=True) +class ChunkOrigin: + """Provenance metadata — links a chunk back to its source document binary.""" + + binary_hash: str + filename: str + + +@dataclass(frozen=True) +class IndexedChunk: + """A single chunk ready to be indexed in the vector store. + + This is the domain-level representation of a document in the OpenSearch index. + It combines Docling-standard fields (content, embedding, doc_items, headings, + origin) with Docling Studio enriched fields (bboxes, page_number, chunk_index, + chunk_type, doc_id, filename). + """ + + doc_id: str + filename: str + content: str + embedding: list[float] + chunk_index: int + chunk_type: str # text, table, picture, list, etc. + page_number: int + bboxes: list[ChunkBboxEntry] = field(default_factory=list) + headings: list[str] = field(default_factory=list) + doc_items: list[DocItemRef] = field(default_factory=list) + origin: ChunkOrigin | None = None + + def to_dict(self) -> dict: + """Serialize to a dict matching the OpenSearch index mapping.""" + result: dict = { + "doc_id": self.doc_id, + "filename": self.filename, + "content": self.content, + "embedding": self.embedding, + "chunk_index": self.chunk_index, + "chunk_type": self.chunk_type, + "page_number": self.page_number, + "bboxes": [ + {"page": b.page, "x": b.x, "y": b.y, "w": b.w, "h": b.h} for b in self.bboxes + ], + "headings": self.headings, + "doc_items": [{"self_ref": d.self_ref, "label": d.label} for d in self.doc_items], + } + if self.origin: + result["origin"] = { + "binary_hash": self.origin.binary_hash, + "filename": self.origin.filename, + } + return result + + +# -- Index mapping template ---------------------------------------------------- + + +def build_index_mapping(embedding_dimension: int = DEFAULT_EMBEDDING_DIMENSION) -> dict: + """Build the OpenSearch index mapping for the chunk index. + + Args: + embedding_dimension: Vector dimension for the knn_vector field. + Defaults to 384 (Granite Embedding 30M / all-MiniLM-L6-v2). + """ + return { + "settings": { + "index": { + "knn": True, + }, + }, + "mappings": { + "properties": { + "doc_id": {"type": "keyword"}, + "filename": {"type": "keyword"}, + "content": {"type": "text", "analyzer": "standard"}, + "embedding": { + "type": "knn_vector", + "dimension": embedding_dimension, + "method": { + "engine": "faiss", + "name": "hnsw", + }, + }, + "chunk_index": {"type": "integer"}, + "chunk_type": {"type": "keyword"}, + "page_number": {"type": "integer"}, + "bboxes": { + "type": "nested", + "properties": { + "page": {"type": "integer"}, + "x": {"type": "float"}, + "y": {"type": "float"}, + "w": {"type": "float"}, + "h": {"type": "float"}, + }, + }, + "headings": {"type": "text"}, + "doc_items": { + "type": "nested", + "properties": { + "self_ref": {"type": "keyword"}, + "label": {"type": "keyword"}, + }, + }, + "origin": { + "type": "object", + "properties": { + "binary_hash": {"type": "keyword"}, + "filename": {"type": "keyword"}, + }, + }, + }, + }, + } diff --git a/document-parser/infra/settings.py b/document-parser/infra/settings.py index 843ea59..bc7dd49 100644 --- a/document-parser/infra/settings.py +++ b/document-parser/infra/settings.py @@ -23,6 +23,8 @@ class Settings: max_file_size_mb: int = 50 # upload limit in MB (0 = unlimited) rate_limit_rpm: int = 100 # requests per minute per IP (0 = disabled) batch_page_size: int = 0 # 0 = disabled, > 0 = pages per batch + opensearch_url: str = "" # empty = disabled + embedding_dimension: int = 384 # Granite Embedding 30M / all-MiniLM-L6-v2 upload_dir: str = "./uploads" db_path: str = "./data/docling_studio.db" cors_origins: list[str] = field( @@ -51,6 +53,8 @@ class Settings: errors.append(f"rate_limit_rpm must be >= 0 (got {self.rate_limit_rpm})") if self.batch_page_size < 0: errors.append(f"batch_page_size must be >= 0 (got {self.batch_page_size})") + if self.embedding_dimension < 1: + errors.append(f"embedding_dimension must be >= 1 (got {self.embedding_dimension})") if self.default_table_mode not in ("accurate", "fast"): errors.append( f"default_table_mode must be 'accurate' or 'fast' (got '{self.default_table_mode}')" @@ -90,6 +94,8 @@ class Settings: max_file_size_mb=int(os.environ.get("MAX_FILE_SIZE_MB", "50")), rate_limit_rpm=int(os.environ.get("RATE_LIMIT_RPM", "100")), batch_page_size=int(os.environ.get("BATCH_PAGE_SIZE", "0")), + opensearch_url=os.environ.get("OPENSEARCH_URL", ""), + embedding_dimension=int(os.environ.get("EMBEDDING_DIMENSION", "384")), upload_dir=os.environ.get("UPLOAD_DIR", "./uploads"), db_path=os.environ.get("DB_PATH", "./data/docling_studio.db"), cors_origins=[o.strip() for o in cors_raw.split(",")], diff --git a/document-parser/tests/test_vector_schema.py b/document-parser/tests/test_vector_schema.py new file mode 100644 index 0000000..d201280 --- /dev/null +++ b/document-parser/tests/test_vector_schema.py @@ -0,0 +1,197 @@ +"""Tests for vector index schema — value objects and OpenSearch mapping.""" + +from __future__ import annotations + +import pytest + +from domain.vector_schema import ( + DEFAULT_EMBEDDING_DIMENSION, + DEFAULT_INDEX_NAME, + ChunkBboxEntry, + ChunkOrigin, + DocItemRef, + IndexedChunk, + build_index_mapping, +) + + +class TestChunkBboxEntry: + def test_construction(self): + bbox = ChunkBboxEntry(page=1, x=10.0, y=20.0, w=100.0, h=50.0) + assert bbox.page == 1 + assert bbox.x == 10.0 + assert bbox.w == 100.0 + + def test_frozen(self): + bbox = ChunkBboxEntry(page=1, x=0, y=0, w=10, h=10) + with pytest.raises(AttributeError): + bbox.page = 2 # type: ignore[misc] + + +class TestDocItemRef: + def test_construction(self): + ref = DocItemRef(self_ref="#/texts/0", label="text") + assert ref.self_ref == "#/texts/0" + assert ref.label == "text" + + +class TestChunkOrigin: + def test_construction(self): + origin = ChunkOrigin(binary_hash="abc123", filename="doc.pdf") + assert origin.binary_hash == "abc123" + assert origin.filename == "doc.pdf" + + +class TestIndexedChunk: + def _make_chunk(self, **overrides) -> IndexedChunk: + defaults = { + "doc_id": "doc-1", + "filename": "test.pdf", + "content": "Hello world", + "embedding": [0.1] * 384, + "chunk_index": 0, + "chunk_type": "text", + "page_number": 1, + } + defaults.update(overrides) + return IndexedChunk(**defaults) + + def test_minimal_chunk(self): + chunk = self._make_chunk() + assert chunk.doc_id == "doc-1" + assert chunk.content == "Hello world" + assert chunk.bboxes == [] + assert chunk.headings == [] + assert chunk.doc_items == [] + assert chunk.origin is None + + def test_full_chunk(self): + chunk = self._make_chunk( + bboxes=[ChunkBboxEntry(page=1, x=10, y=20, w=100, h=50)], + headings=["Chapter 1", "Section A"], + doc_items=[DocItemRef(self_ref="#/texts/0", label="text")], + origin=ChunkOrigin(binary_hash="abc", filename="test.pdf"), + ) + assert len(chunk.bboxes) == 1 + assert chunk.headings == ["Chapter 1", "Section A"] + assert chunk.doc_items[0].label == "text" + assert chunk.origin.binary_hash == "abc" + + def test_to_dict_minimal(self): + chunk = self._make_chunk() + d = chunk.to_dict() + assert d["doc_id"] == "doc-1" + assert d["filename"] == "test.pdf" + assert d["content"] == "Hello world" + assert d["embedding"] == [0.1] * 384 + assert d["chunk_index"] == 0 + assert d["chunk_type"] == "text" + assert d["page_number"] == 1 + assert d["bboxes"] == [] + assert d["headings"] == [] + assert d["doc_items"] == [] + assert "origin" not in d + + def test_to_dict_full(self): + chunk = self._make_chunk( + bboxes=[ChunkBboxEntry(page=1, x=10.5, y=20.0, w=100.0, h=50.0)], + headings=["H1"], + doc_items=[DocItemRef(self_ref="#/texts/0", label="text")], + origin=ChunkOrigin(binary_hash="abc", filename="test.pdf"), + ) + d = chunk.to_dict() + assert d["bboxes"] == [{"page": 1, "x": 10.5, "y": 20.0, "w": 100.0, "h": 50.0}] + assert d["headings"] == ["H1"] + assert d["doc_items"] == [{"self_ref": "#/texts/0", "label": "text"}] + assert d["origin"] == {"binary_hash": "abc", "filename": "test.pdf"} + + def test_frozen(self): + chunk = self._make_chunk() + with pytest.raises(AttributeError): + chunk.content = "modified" # type: ignore[misc] + + +class TestBuildIndexMapping: + def test_default_dimension(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + assert props["embedding"]["dimension"] == 384 + assert props["embedding"]["type"] == "knn_vector" + assert props["embedding"]["method"]["engine"] == "faiss" + assert props["embedding"]["method"]["name"] == "hnsw" + + def test_custom_dimension(self): + mapping = build_index_mapping(embedding_dimension=768) + assert mapping["mappings"]["properties"]["embedding"]["dimension"] == 768 + + def test_knn_enabled(self): + mapping = build_index_mapping() + assert mapping["settings"]["index"]["knn"] is True + + def test_all_fields_present(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + expected_fields = { + "doc_id", + "filename", + "content", + "embedding", + "chunk_index", + "chunk_type", + "page_number", + "bboxes", + "headings", + "doc_items", + "origin", + } + assert set(props.keys()) == expected_fields + + def test_bboxes_nested_type(self): + mapping = build_index_mapping() + bboxes = mapping["mappings"]["properties"]["bboxes"] + assert bboxes["type"] == "nested" + assert "page" in bboxes["properties"] + assert "x" in bboxes["properties"] + assert "y" in bboxes["properties"] + assert "w" in bboxes["properties"] + assert "h" in bboxes["properties"] + + def test_doc_items_nested_type(self): + mapping = build_index_mapping() + doc_items = mapping["mappings"]["properties"]["doc_items"] + assert doc_items["type"] == "nested" + assert "self_ref" in doc_items["properties"] + assert "label" in doc_items["properties"] + + def test_origin_object_type(self): + mapping = build_index_mapping() + origin = mapping["mappings"]["properties"]["origin"] + assert origin["type"] == "object" + assert "binary_hash" in origin["properties"] + assert "filename" in origin["properties"] + + def test_content_uses_standard_analyzer(self): + mapping = build_index_mapping() + content = mapping["mappings"]["properties"]["content"] + assert content["type"] == "text" + assert content["analyzer"] == "standard" + + def test_keyword_fields(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + for field_name in ("doc_id", "filename", "chunk_type"): + assert props[field_name]["type"] == "keyword", f"{field_name} should be keyword" + + def test_integer_fields(self): + mapping = build_index_mapping() + props = mapping["mappings"]["properties"] + for field_name in ("chunk_index", "page_number"): + assert props[field_name]["type"] == "integer", f"{field_name} should be integer" + + +class TestConstants: + def test_default_embedding_dimension(self): + assert DEFAULT_EMBEDDING_DIMENSION == 384 + + def test_default_index_name(self): + assert DEFAULT_INDEX_NAME == "docling-studio-chunks"