canonical metadata keys: source_revision + content_type, bump to 0.50.0

CHANGELOG.md

@@ -12,6 +12,7 @@
 ### Changed
 
 - `haiku-rag serve` renamed to `haiku-rag mcp` (only MCP is left). `--mcp-port` renamed to `--port`. Update any `claude_desktop_config.json` from `["serve", "--mcp", "--stdio"]` to `["mcp", "--stdio"]`.
+- `document.metadata` now uses source-agnostic keys: `source_revision` (was `etag` — S3-only and never populated for FS, so periodic sweeps re-ingested every file) and `content_type` (was `contentType`, snake_case for consistency). The v0.50.0 startup migration rewrites existing documents. All four source adapters (FS, HTTP, S3, future WebDAV) now write their native revision (mtime_ns, ETag, etc.) under the same key, fixing the regression where FS sources never short-circuited on unchanged files.
 - Drop `list_documents` and `get_document` from the default RAG skill's tool set; the skill now exposes only `search` and `cite`. Both tools dumped unbounded content into the agent's context (full document lists, full document bodies) and `get_document` returned no chunk_ids so its output was structurally uncitable. The analysis skill already covers these uses programmatically — `await list_documents()` and `Path('/documents/{id}/content.txt').read_text()` inside `execute_code`. The tool branches remain in `create_skill_tools` and the `skill_generator` `AVAILABLE_TOOLS` set so users can still opt in when building custom skills.
 
 ## [0.48.1] - 2026-05-21

docs/ingester.md

@@ -80,10 +80,10 @@ ingester:
 ```
 
 ETags are the cheap-skip key. Each sweep lists the prefix, compares
-the listed ETag against the document's stored `metadata["etag"]`, and
+the listed ETag against the document's stored `metadata["source_revision"]`,
-only fetches keys whose ETag has changed. If the bytes turn out to
+and only fetches keys whose ETag has changed. If the bytes turn out to
 match the stored MD5 (multipart re-upload landing a new ETag on the
-same content), only the etag is refreshed — no re-chunk.
+same content), only the revision is refreshed — no re-chunk.
 
 `storage_options` follows the same convention as `lancedb.storage_options` —
 the dict is passed straight to obstore (the Rust `object_store` library

haiku_rag_slim/haiku/rag/client/documents.py

@@ -231,10 +231,12 @@ async def _ingest_fetch_result(
         )
 
     source_metadata: dict = {
-        "contentType": result.content_type,
+        "content_type": result.content_type,
         "md5": result.content_hash,
         **result.extra_metadata,
     }
+    if result.revision is not None:
+        source_metadata["source_revision"] = result.revision
 
     if result.disk_path is not None:
         target_path = result.disk_path
@@ -384,10 +386,11 @@ async def create_document_from_source(
     existing_doc = await client.get_document_by_uri(stored_uri)
 
     # Cheap revision-based short-circuit: only worth a HEAD when we have a
-    # stored revision to compare against. S3 doc metadata persists "etag";
+    # stored revision to compare against. All sources persist their native
-    # FS/HTTP currently don't, so this branch is effectively S3-only today.
+    # revision (mtime_ns for FS, ETag for S3, ETag/Last-Modified for HTTP)
+    # under the canonical "source_revision" metadata key.
     stored_revision = (
-        (existing_doc.metadata or {}).get("etag") if existing_doc else None
+        (existing_doc.metadata or {}).get("source_revision") if existing_doc else None
     )
     if existing_doc and stored_revision:
         current_revision = await fetcher.head(source_str)
@@ -406,14 +409,16 @@ async def create_document_from_source(
         fetch_span.set_attribute("content_hash", result.content_hash)
 
     # MD5 short-circuit: the bytes are unchanged even if the revision wasn't.
-    # Refresh the source-derived metadata (etag may have rolled) but skip
+    # Refresh the source-derived metadata (revision may have rolled) but skip
     # convert/embed/store entirely.
     if existing_doc and existing_doc.metadata.get("md5") == result.content_hash:
         source_meta: dict = {
-            "contentType": result.content_type,
+            "content_type": result.content_type,
             "md5": result.content_hash,
             **result.extra_metadata,
         }
+        if result.revision is not None:
+            source_meta["source_revision"] = result.revision
         return await _refresh_doc_metadata(
             client,
             existing_doc,

haiku_rag_slim/haiku/rag/ingester/sources/http.py

@@ -14,14 +14,15 @@ from haiku.rag.ingester.sources.base import (
 
 
 def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]:
+    """Return (canonical_revision, extras). ETag is the stronger validator
+    and is the canonical revision when present; Last-Modified backs it up.
+    Last-Modified always goes into extras as separate per-source provenance
+    (some pipelines want both signals)."""
     extra: dict[str, str] = {}
     etag = (headers.get("etag") or "").strip('"').strip()
     last_modified = (headers.get("last-modified") or "").strip()
-    if etag:
-        extra["etag"] = etag
     if last_modified:
         extra["last_modified"] = last_modified
-    # Prefer ETag — it's a stronger validator. Fall back to Last-Modified.
     revision = etag or last_modified or None
     return revision, extra
 

haiku_rag_slim/haiku/rag/ingester/sources/s3.py

@@ -93,17 +93,12 @@ class S3Source:
         if not content_type:
             content_type = "application/octet-stream"
 
-        extra: dict[str, str] = {}
-        if etag is not None:
-            extra["etag"] = etag
-
         return FetchResult(
             uri=uri,
             body=body,
             content_type=content_type,
             content_hash=hashlib.md5(body, usedforsecurity=False).hexdigest(),
             revision=etag,
-            extra_metadata=extra,
         )
 
     async def discover(

haiku_rag_slim/haiku/rag/ingester/workers/pipeline.py

@@ -114,7 +114,7 @@ async def run_job(client: "HaikuRAG", job: Job) -> JobResult:
             metadata = result.metadata or {}
             return JobResult(
                 document_id=result.id,
-                revision=metadata.get("etag"),
+                revision=metadata.get("source_revision"),
                 content_hash=metadata.get("md5"),
             )
         except BaseException as exc:

haiku_rag_slim/haiku/rag/store/upgrades/__init__.py

@@ -90,6 +90,9 @@ from haiku.rag.store.upgrades.v0_45_0 import (
 from haiku.rag.store.upgrades.v0_48_0 import (
     upgrade_backfill_heading_hierarchy as upgrade_0_48_0_heading_hierarchy,
 )
+from haiku.rag.store.upgrades.v0_50_0 import (
+    upgrade_canonical_metadata_keys as upgrade_0_50_0_canonical_metadata_keys,
+)
 
 upgrades.append(upgrade_0_20_0_docling)
 upgrades.append(upgrade_0_23_1_contextualize)
@@ -98,3 +101,4 @@ upgrades.append(upgrade_0_38_0_split_pages)
 upgrades.append(upgrade_0_40_0_document_items)
 upgrades.append(upgrade_0_45_0_extract_picture_bytes)
 upgrades.append(upgrade_0_48_0_heading_hierarchy)
+upgrades.append(upgrade_0_50_0_canonical_metadata_keys)

haiku_rag_slim/haiku/rag/store/upgrades/v0_50_0.py

@@ -0,0 +1,85 @@
+import json
+import logging
+
+from haiku.rag.store.engine import Store
+from haiku.rag.store.upgrades import Upgrade
+from haiku.rag.utils import escape_sql_string
+
+logger = logging.getLogger(__name__)
+
+PROGRESS_INTERVAL = 50
+
+
+def _normalize_metadata(meta: dict) -> tuple[dict, bool]:
+    """Return (normalized_metadata, changed). Renames `etag` → `source_revision`
+    and `contentType` → `content_type`. Pre-existing canonical keys win on
+    conflict so repeat migrations are no-ops."""
+    changed = False
+    out = dict(meta)
+
+    if "etag" in out:
+        if "source_revision" not in out:
+            out["source_revision"] = out["etag"]
+        del out["etag"]
+        changed = True
+
+    if "contentType" in out:
+        if "content_type" not in out:
+            out["content_type"] = out["contentType"]
+        del out["contentType"]
+        changed = True
+
+    return out, changed
+
+
+async def _apply_canonical_metadata_keys(store: Store) -> None:
+    """Rewrite document.metadata so revision lives under the source-agnostic
+    `source_revision` key and `contentType` becomes `content_type`. Documents
+    whose metadata already uses the canonical keys are untouched."""
+    rows = (
+        await store.documents_table.query().select(["id", "metadata"]).to_arrow()
+    ).to_pylist()
+    total = len(rows)
+    logger.info("Normalising document metadata keys across %d documents", total)
+    rewritten = 0
+    skipped = 0
+
+    for idx, row in enumerate(rows, 1):
+        doc_id = row["id"]
+        raw = row.get("metadata") or "{}"
+        try:
+            meta = json.loads(raw)
+        except Exception:  # pragma: no cover
+            logger.warning(
+                "Could not parse metadata JSON for document %s; skipping", doc_id
+            )
+            skipped += 1
+            continue
+
+        normalized, changed = _normalize_metadata(meta)
+        if not changed:
+            continue
+
+        safe_id = escape_sql_string(doc_id)
+        await store.documents_table.update(
+            {"metadata": json.dumps(normalized)},
+            where=f"id = '{safe_id}'",
+        )
+        rewritten += 1
+
+        if idx % PROGRESS_INTERVAL == 0 or idx == total:
+            logger.info("Progress: %d/%d (%d rewritten)", idx, total, rewritten)
+
+    logger.info(
+        "Metadata key normalisation complete: %d rewritten, %d skipped of %d",
+        rewritten,
+        skipped,
+        total,
+    )
+
+
+upgrade_canonical_metadata_keys = Upgrade(
+    version="0.50.0",
+    apply=_apply_canonical_metadata_keys,
+    description="Rename document.metadata keys: etag→source_revision, contentType→content_type",
+)

haiku_rag_slim/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag-slim"
 description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies"
-version = "0.48.1"
+version = "0.50.0"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }

pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag"
 description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling"
-version = "0.48.1"
+version = "0.50.0"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -30,7 +30,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.48.1",
+    "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.50.0",
 ]
 
 [project.scripts]
@@ -38,9 +38,9 @@ haiku-rag = "haiku.rag.cli:cli"
 
 [project.optional-dependencies]
 tui = ["textual>=8.2.4"]
-s3 = ["haiku.rag-slim[s3]==0.48.1"]
+s3 = ["haiku.rag-slim[s3]==0.50.0"]
-cross-encoder = ["haiku.rag-slim[cross-encoder]==0.48.1"]
+cross-encoder = ["haiku.rag-slim[cross-encoder]==0.50.0"]
-ingester = ["haiku.rag-slim[ingester]==0.48.1"]
+ingester = ["haiku.rag-slim[ingester]==0.50.0"]
 
 [build-system]
 requires = ["hatchling"]

tests/ingester/test_http_source.py

@@ -60,7 +60,7 @@ async def test_fetch_returns_bytes_and_md5_and_etag():
     assert result.content_type == "text/markdown"
     # etag preferred over last-modified, surrounding quotes stripped
     assert result.revision == "abc123"
-    assert result.extra_metadata["etag"] == "abc123"
+    assert "etag" not in result.extra_metadata
     assert result.extra_metadata["last_modified"] == "Wed, 21 Oct 2025 07:28:00 GMT"
 
 

tests/ingester/test_pipeline.py

@@ -44,7 +44,11 @@ async def test_upsert_calls_create_document_from_source_and_returns_metadata():
         id="doc-42",
         content="x",
         uri="https://example.com/a.pdf",
-        metadata={"md5": "abcd", "etag": "xyz", "contentType": "application/pdf"},
+        metadata={
+            "md5": "abcd",
+            "source_revision": "xyz",
+            "content_type": "application/pdf",
+        },
     )
 
     result = await run_job(

tests/ingester/test_revision_round_trip.py

@@ -0,0 +1,98 @@
+"""Regression: every source's revision must round-trip from FetchResult →
+document.metadata → sync_state, so the next sweep can recognise the file
+as unchanged. Catches the FS-specific bug where revision was lost in the
+pipeline and every periodic sweep re-enqueued every file forever.
+"""
+
+import pytest
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.ingester.sources.base import SourceEventKind
+from haiku.rag.ingester.sources.fs import FSSource
+
+
+@pytest.mark.asyncio
+async def test_fs_ingest_writes_source_revision_to_metadata(temp_db_path, tmp_path):
+    file_path = tmp_path / "doc.md"
+    file_path.write_text("hello")
+    expected_revision = str(file_path.stat().st_mtime_ns)
+
+    async with HaikuRAG(temp_db_path, create=True) as client:
+        doc = await client.create_document_from_source(file_path)
+
+    assert doc.metadata["source_revision"] == expected_revision
+
+
+@pytest.mark.asyncio
+async def test_fs_second_sweep_emits_unchanged_after_ingest(temp_db_path, tmp_path):
+    """The full round-trip: ingest a file, build a sync_state-shaped snapshot
+    from document.metadata, hand it to FSSource.discover() — must see
+    UNCHANGED, not UPSERT. This is exactly what the periodic poller does."""
+    file_path = tmp_path / "doc.md"
+    file_path.write_text("hello")
+
+    async with HaikuRAG(temp_db_path, create=True) as client:
+        doc = await client.create_document_from_source(file_path)
+
+    assert doc.uri is not None
+    snapshot = {doc.uri: doc.metadata["source_revision"]}
+
+    src = FSSource(root=tmp_path)
+    kinds: list[SourceEventKind] = []
+    async for event in src.discover(since=snapshot):
+        kinds.append(event.kind)
+
+    assert kinds == [SourceEventKind.UNCHANGED]
+
+
+@pytest.mark.asyncio
+async def test_fs_second_sweep_emits_upsert_when_file_changes(temp_db_path, tmp_path):
+    """Counterpart to the unchanged test: a file modified after ingest still
+    triggers UPSERT. Ensures the round-trip doesn't accidentally over-skip."""
+    file_path = tmp_path / "doc.md"
+    file_path.write_text("hello")
+
+    async with HaikuRAG(temp_db_path, create=True) as client:
+        doc = await client.create_document_from_source(file_path)
+
+    # Bump mtime by writing different content; snapshot still has the old revision.
+    assert doc.uri is not None
+    snapshot = {doc.uri: doc.metadata["source_revision"]}
+    file_path.write_text("hello world")
+    # st_mtime_ns has nanosecond resolution; a write_text always advances it
+    # on any sane filesystem, but assert anyway to make the intent explicit.
+    assert str(file_path.stat().st_mtime_ns) != doc.metadata["source_revision"]
+
+    src = FSSource(root=tmp_path)
+    kinds: list[SourceEventKind] = []
+    async for event in src.discover(since=snapshot):
+        kinds.append(event.kind)
+
+    assert kinds == [SourceEventKind.UPSERT]
+
+
+@pytest.mark.asyncio
+async def test_fs_head_short_circuit_skips_fetch_for_unchanged_revision(
+    temp_db_path, tmp_path, monkeypatch
+):
+    """The one-shot (add-src) path: if the existing doc has source_revision
+    and HEAD reports the same value, fetch must not run."""
+    file_path = tmp_path / "doc.md"
+    file_path.write_text("hello")
+
+    async with HaikuRAG(temp_db_path, create=True) as client:
+        first = await client.create_document_from_source(file_path)
+
+        fetch_calls: list[str] = []
+        original_fetch = FSSource.fetch
+
+        async def _track_fetch(self: FSSource, uri: str):  # type: ignore[no-untyped-def]
+            fetch_calls.append(uri)
+            return await original_fetch(self, uri)
+
+        monkeypatch.setattr(FSSource, "fetch", _track_fetch)
+
+        second = await client.create_document_from_source(file_path)
+
+    assert second.id == first.id
+    assert fetch_calls == []

tests/ingester/test_s3_source.py

@@ -107,7 +107,7 @@ async def test_fetch_returns_bytes_md5_etag(fake_obstore_io):
     assert result.content_hash == hashlib.md5(body, usedforsecurity=False).hexdigest()
     assert result.content_type == "text/plain"
     assert result.revision == "abc123"
-    assert result.extra_metadata["etag"] == "abc123"
+    assert result.extra_metadata == {}
     head_async.assert_awaited_once()
     get_async.assert_awaited_once()
 
@@ -121,7 +121,7 @@ async def test_fetch_handles_missing_etag(fake_obstore_io):
     src = S3Source(uri="s3://bucket/")
     result = await src.fetch("s3://bucket/file.txt")
     assert result.revision is None
-    assert "etag" not in result.extra_metadata
+    assert result.extra_metadata == {}
 
 
 @pytest.mark.asyncio

tests/ingester/test_serve_integration.py

@@ -52,7 +52,7 @@ async def _wait_for(predicate, *, timeout: float = 5.0, interval: float = 0.05):
 
 def _mock_client(docs_root) -> AsyncMock:
     """A HaikuRAG mock that returns a fresh Document for each URI it's asked
-    to ingest, mirroring real metadata shape (contentType + md5)."""
+    to ingest, mirroring real metadata shape (content_type + md5)."""
     client = AsyncMock(spec=HaikuRAG)
 
     counter = {"n": 0}
@@ -63,7 +63,7 @@ def _mock_client(docs_root) -> AsyncMock:
             id=f"doc-{counter['n']}",
             content="x",
             uri=uri,
-            metadata={"contentType": "text/markdown", "md5": f"md5-{counter['n']}"},
+            metadata={"content_type": "text/markdown", "md5": f"md5-{counter['n']}"},
         )
 
     client.create_document_from_source.side_effect = _fake_create

tests/ingester/test_workers.py

@@ -59,7 +59,10 @@ def _pool(client, jobs, sync, **kwargs) -> WorkerPool:
 @pytest.mark.asyncio
 async def test_drain_marks_job_succeeded_and_writes_sync_state(client, jobs, sync):
     client.create_document_from_source.return_value = Document(
-        id="doc-1", content="x", uri="s3://b/k.md", metadata={"md5": "m1", "etag": "e1"}
+        id="doc-1",
+        content="x",
+        uri="s3://b/k.md",
+        metadata={"md5": "m1", "source_revision": "e1"},
     )
     job = await jobs.enqueue("src", "s3://b/k.md", JobOp.UPSERT, revision="e0")
     assert job is not None
@@ -179,7 +182,7 @@ async def test_unknown_exception_caught_and_marked_dead(client, jobs, sync):
 @pytest.mark.asyncio
 async def test_workers_drain_queue_after_start(client, jobs, sync):
     client.create_document_from_source.return_value = Document(
-        id="doc", content="x", uri="u", metadata={"md5": "m", "etag": "e"}
+        id="doc", content="x", uri="u", metadata={"md5": "m", "source_revision": "e"}
     )
     for i in range(5):
         await jobs.enqueue("src", f"u{i}", JobOp.UPSERT)

tests/store/test_v0_50_0_migration.py

@@ -0,0 +1,119 @@
+import json
+
+import pytest
+
+from haiku.rag.store.engine import DocumentRecord, Store
+
+
+@pytest.mark.asyncio
+class TestV0_50_0Migration:
+    """v0.50.0 normalises document.metadata to source-agnostic keys."""
+
+    async def test_renames_etag_and_content_type(self, temp_db_path):
+        async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
+            await store.set_haiku_version("0.48.1")
+            await store.documents_table.add(
+                [
+                    DocumentRecord(
+                        id="doc-s3",
+                        content="x",
+                        uri="s3://b/k",
+                        metadata=json.dumps(
+                            {
+                                "etag": "abc123",
+                                "contentType": "application/pdf",
+                                "md5": "deadbeef",
+                            }
+                        ),
+                    ),
+                    DocumentRecord(
+                        id="doc-fs",
+                        content="y",
+                        uri="file:///tmp/x.md",
+                        metadata=json.dumps(
+                            {
+                                "contentType": "text/markdown",
+                                "md5": "feedface",
+                            }
+                        ),
+                    ),
+                ]
+            )
+
+        async with Store(temp_db_path, skip_migration_check=True) as store:
+            applied = await store.migrate()
+            assert any("0.50.0" in d for d in applied)
+
+            rows = await store.documents_table.query().to_list()
+            by_id = {r["id"]: json.loads(r["metadata"]) for r in rows}
+
+            assert by_id["doc-s3"] == {
+                "source_revision": "abc123",
+                "content_type": "application/pdf",
+                "md5": "deadbeef",
+            }
+            assert by_id["doc-fs"] == {
+                "content_type": "text/markdown",
+                "md5": "feedface",
+            }
+
+    async def test_idempotent_on_already_migrated(self, temp_db_path):
+        async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
+            await store.set_haiku_version("0.48.1")
+            await store.documents_table.add(
+                [
+                    DocumentRecord(
+                        id="d",
+                        content="x",
+                        uri="s3://b/k",
+                        metadata=json.dumps(
+                            {
+                                "source_revision": "abc",
+                                "content_type": "text/plain",
+                                "md5": "m",
+                            }
+                        ),
+                    )
+                ]
+            )
+
+        async with Store(temp_db_path, skip_migration_check=True) as store:
+            await store.migrate()
+            rows = await store.documents_table.query().to_list()
+            assert json.loads(rows[0]["metadata"]) == {
+                "source_revision": "abc",
+                "content_type": "text/plain",
+                "md5": "m",
+            }
+
+    async def test_preserves_existing_canonical_keys_on_collision(self, temp_db_path):
+        """If both legacy and canonical keys are present, the canonical wins
+        and the legacy is dropped — defends against partial-migration states."""
+        async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
+            await store.set_haiku_version("0.48.1")
+            await store.documents_table.add(
+                [
+                    DocumentRecord(
+                        id="d",
+                        content="x",
+                        uri="s3://b/k",
+                        metadata=json.dumps(
+                            {
+                                "etag": "legacy",
+                                "source_revision": "canonical",
+                                "contentType": "text/old",
+                                "content_type": "text/new",
+                            }
+                        ),
+                    )
+                ]
+            )
+
+        async with Store(temp_db_path, skip_migration_check=True) as store:
+            await store.migrate()
+            rows = await store.documents_table.query().to_list()
+            meta = json.loads(rows[0]["metadata"])
+            assert meta == {
+                "source_revision": "canonical",
+                "content_type": "text/new",
+            }

tests/test_client.py

@@ -209,9 +209,9 @@ async def test_client_create_document_from_source(temp_db_path):
             assert doc.id is not None
             assert doc.content == test_content
             assert doc.uri == temp_path.as_uri()
-            assert "contentType" in doc.metadata
+            assert "content_type" in doc.metadata
             assert "md5" in doc.metadata
-            assert doc.metadata["contentType"] == "text/plain"
+            assert doc.metadata["content_type"] == "text/plain"
 
             # Test create_document_from_source with string path
             doc2 = await client.create_document_from_source(source=str(temp_path))
@@ -220,7 +220,7 @@ async def test_client_create_document_from_source(temp_db_path):
             assert doc2.id is not None
             assert doc2.content == test_content
             assert doc2.uri == temp_path.as_uri()
-            assert "contentType" in doc2.metadata
+            assert "content_type" in doc2.metadata
             assert "md5" in doc2.metadata
 
 
@@ -374,7 +374,7 @@ async def test_client_create_document_from_directory(temp_db_path):
                 assert doc.id is not None
                 assert doc.uri is not None
                 assert "md5" in doc.metadata
-                assert "contentType" in doc.metadata
+                assert "content_type" in doc.metadata
 
             uris = [doc.uri for doc in result if doc.uri]
             assert any("doc1.txt" in uri for uri in uris)
@@ -404,9 +404,9 @@ async def test_client_create_document_from_url(temp_db_path):
             assert "test content" in doc.content
             assert doc.uri == "https://example.com/test.html"
             assert doc.metadata["source_type"] == "web"
-            assert "contentType" in doc.metadata
+            assert "content_type" in doc.metadata
             assert "md5" in doc.metadata
-            assert doc.metadata["contentType"] == "text/html"
+            assert doc.metadata["content_type"] == "text/html"
 
 
 @pytest.mark.vcr()
@@ -432,9 +432,9 @@ async def test_client_create_document_from_url_with_different_content_types(
             assert doc.id is not None
             assert "Test JSON" in doc.content
             assert doc.uri == "https://api.example.com/data.json"
-            assert "contentType" in doc.metadata
+            assert "content_type" in doc.metadata
             assert "md5" in doc.metadata
-            assert doc.metadata["contentType"] == "application/json"
+            assert doc.metadata["content_type"] == "application/json"
 
         # Test plain text content
         mock_text_response = AsyncMock()
@@ -451,9 +451,9 @@ async def test_client_create_document_from_url_with_different_content_types(
             assert doc.id is not None
             assert doc.content == "This is plain text content from a URL."
             assert doc.uri == "https://example.com/readme.txt"
-            assert "contentType" in doc.metadata
+            assert "content_type" in doc.metadata
             assert "md5" in doc.metadata
-            assert doc.metadata["contentType"] == "text/plain"
+            assert doc.metadata["content_type"] == "text/plain"
 
 
 @pytest.mark.vcr()
@@ -523,7 +523,7 @@ def test_get_extension_from_content_type_or_url():
 
 @pytest.mark.vcr()
 async def test_client_metadata_content_type_and_md5(temp_db_path):
-    """Test that contentType and md5 metadata are correctly set."""
+    """Test that content_type and md5 metadata are correctly set."""
     import hashlib
 
     async with HaikuRAG(temp_db_path, create=True) as client:
@@ -538,7 +538,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path):
             doc = await client.create_document_from_source(temp_path)
             assert isinstance(doc, Document)
 
-            assert doc.metadata["contentType"] == "text/plain"
+            assert doc.metadata["content_type"] == "text/plain"
             assert doc.metadata["md5"] == expected_md5
 
             mock_response = AsyncMock()
@@ -552,7 +552,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path):
                 )
                 assert isinstance(url_doc, Document)
 
-                assert url_doc.metadata["contentType"] == "text/plain"
+                assert url_doc.metadata["content_type"] == "text/plain"
                 assert url_doc.metadata["md5"] == expected_md5
 
 

tests/test_s3_source.py

@@ -77,7 +77,7 @@ async def test_create_document_from_s3_new(fake_obstore_io, temp_db_path):
         doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt")
 
     assert doc.uri == "s3://my-bucket/folder/file.txt"
-    assert doc.metadata["etag"] == "abc123"  # quotes stripped
+    assert doc.metadata["source_revision"] == "abc123"  # quotes stripped
     assert doc.metadata["md5"]  # real content MD5
     assert doc.metadata["md5"] != "abc123"
     head_async.assert_awaited_once()
@@ -114,7 +114,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk(
 ):
     """Multipart re-upload of same content: etag changes, MD5 doesn't.
 
-    Expected: GET runs to verify, but no re-chunk; only metadata.etag updates.
+    Expected: GET runs to verify, but no re-chunk; only metadata.source_revision updates.
     """
     head_async, get_async = fake_obstore_io
     text = b"S3 hosted content"
@@ -134,7 +134,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk(
 
     assert second.id == first.id
     assert second.metadata["md5"] == original_md5
-    assert second.metadata["etag"] == "def456-2"
+    assert second.metadata["source_revision"] == "def456-2"
     assert second.updated_at >= original_updated_at
     assert get_async.await_count == 2  # initial create + etag-changed compare
 
@@ -158,7 +158,7 @@ async def test_create_document_from_s3_etag_changed_md5_changed_rechunks(
 
     assert second.id == first.id
     assert second.metadata["md5"] != first.metadata["md5"]
-    assert second.metadata["etag"] == "new999"
+    assert second.metadata["source_revision"] == "new999"
     assert "different text now" in second.content
 
 

uv.lock

@@ -1466,7 +1466,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.48.1"
+version = "0.50.0"
 source = { editable = "." }
 dependencies = [
     { name = "haiku-rag-slim", extra = ["cohere", "cross-encoder", "docling", "mxbai", "tui", "voyageai", "zeroentropy"] },
@@ -1558,7 +1558,7 @@ requires-dist = [
 
 [[package]]
 name = "haiku-rag-slim"
-version = "0.48.1"
+version = "0.50.0"
 source = { editable = "haiku_rag_slim" }
 dependencies = [
     { name = "docling-core" },