From 13cbadeb6fa862f7f9744899d717c45389c5fa02 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 22 May 2026 15:21:38 +0300 Subject: [PATCH] canonical metadata keys: source_revision + content_type, bump to 0.50.0 --- CHANGELOG.md | 1 + docs/ingester.md | 6 +- haiku_rag_slim/haiku/rag/client/documents.py | 17 ++- .../haiku/rag/ingester/sources/http.py | 7 +- .../haiku/rag/ingester/sources/s3.py | 5 - .../haiku/rag/ingester/workers/pipeline.py | 2 +- .../haiku/rag/store/upgrades/__init__.py | 4 + .../haiku/rag/store/upgrades/v0_50_0.py | 85 +++++++++++++ haiku_rag_slim/pyproject.toml | 2 +- pyproject.toml | 10 +- tests/ingester/test_http_source.py | 2 +- tests/ingester/test_pipeline.py | 6 +- tests/ingester/test_revision_round_trip.py | 98 +++++++++++++++ tests/ingester/test_s3_source.py | 4 +- tests/ingester/test_serve_integration.py | 4 +- tests/ingester/test_workers.py | 7 +- tests/store/test_v0_50_0_migration.py | 119 ++++++++++++++++++ tests/test_client.py | 26 ++-- tests/test_s3_source.py | 8 +- uv.lock | 4 +- 20 files changed, 366 insertions(+), 51 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/store/upgrades/v0_50_0.py create mode 100644 tests/ingester/test_revision_round_trip.py create mode 100644 tests/store/test_v0_50_0_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c6db2cc..f29d9ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Changed - `haiku-rag serve` renamed to `haiku-rag mcp` (only MCP is left). `--mcp-port` renamed to `--port`. Update any `claude_desktop_config.json` from `["serve", "--mcp", "--stdio"]` to `["mcp", "--stdio"]`. +- `document.metadata` now uses source-agnostic keys: `source_revision` (was `etag` — S3-only and never populated for FS, so periodic sweeps re-ingested every file) and `content_type` (was `contentType`, snake_case for consistency). The v0.50.0 startup migration rewrites existing documents. All four source adapters (FS, HTTP, S3, future WebDAV) now write their native revision (mtime_ns, ETag, etc.) under the same key, fixing the regression where FS sources never short-circuited on unchanged files. - Drop `list_documents` and `get_document` from the default RAG skill's tool set; the skill now exposes only `search` and `cite`. Both tools dumped unbounded content into the agent's context (full document lists, full document bodies) and `get_document` returned no chunk_ids so its output was structurally uncitable. The analysis skill already covers these uses programmatically — `await list_documents()` and `Path('/documents/{id}/content.txt').read_text()` inside `execute_code`. The tool branches remain in `create_skill_tools` and the `skill_generator` `AVAILABLE_TOOLS` set so users can still opt in when building custom skills. ## [0.48.1] - 2026-05-21 diff --git a/docs/ingester.md b/docs/ingester.md index cc1378ad..e119a074 100644 --- a/docs/ingester.md +++ b/docs/ingester.md @@ -80,10 +80,10 @@ ingester: ``` ETags are the cheap-skip key. Each sweep lists the prefix, compares -the listed ETag against the document's stored `metadata["etag"]`, and -only fetches keys whose ETag has changed. If the bytes turn out to +the listed ETag against the document's stored `metadata["source_revision"]`, +and only fetches keys whose ETag has changed. If the bytes turn out to match the stored MD5 (multipart re-upload landing a new ETag on the -same content), only the etag is refreshed — no re-chunk. +same content), only the revision is refreshed — no re-chunk. `storage_options` follows the same convention as `lancedb.storage_options` — the dict is passed straight to obstore (the Rust `object_store` library diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index f66325bc..27cd1adf 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -231,10 +231,12 @@ async def _ingest_fetch_result( ) source_metadata: dict = { - "contentType": result.content_type, + "content_type": result.content_type, "md5": result.content_hash, **result.extra_metadata, } + if result.revision is not None: + source_metadata["source_revision"] = result.revision if result.disk_path is not None: target_path = result.disk_path @@ -384,10 +386,11 @@ async def create_document_from_source( existing_doc = await client.get_document_by_uri(stored_uri) # Cheap revision-based short-circuit: only worth a HEAD when we have a - # stored revision to compare against. S3 doc metadata persists "etag"; - # FS/HTTP currently don't, so this branch is effectively S3-only today. + # stored revision to compare against. All sources persist their native + # revision (mtime_ns for FS, ETag for S3, ETag/Last-Modified for HTTP) + # under the canonical "source_revision" metadata key. stored_revision = ( - (existing_doc.metadata or {}).get("etag") if existing_doc else None + (existing_doc.metadata or {}).get("source_revision") if existing_doc else None ) if existing_doc and stored_revision: current_revision = await fetcher.head(source_str) @@ -406,14 +409,16 @@ async def create_document_from_source( fetch_span.set_attribute("content_hash", result.content_hash) # MD5 short-circuit: the bytes are unchanged even if the revision wasn't. - # Refresh the source-derived metadata (etag may have rolled) but skip + # Refresh the source-derived metadata (revision may have rolled) but skip # convert/embed/store entirely. if existing_doc and existing_doc.metadata.get("md5") == result.content_hash: source_meta: dict = { - "contentType": result.content_type, + "content_type": result.content_type, "md5": result.content_hash, **result.extra_metadata, } + if result.revision is not None: + source_meta["source_revision"] = result.revision return await _refresh_doc_metadata( client, existing_doc, diff --git a/haiku_rag_slim/haiku/rag/ingester/sources/http.py b/haiku_rag_slim/haiku/rag/ingester/sources/http.py index d37052d2..df23182b 100644 --- a/haiku_rag_slim/haiku/rag/ingester/sources/http.py +++ b/haiku_rag_slim/haiku/rag/ingester/sources/http.py @@ -14,14 +14,15 @@ from haiku.rag.ingester.sources.base import ( def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]: + """Return (canonical_revision, extras). ETag is the stronger validator + and is the canonical revision when present; Last-Modified backs it up. + Last-Modified always goes into extras as separate per-source provenance + (some pipelines want both signals).""" extra: dict[str, str] = {} etag = (headers.get("etag") or "").strip('"').strip() last_modified = (headers.get("last-modified") or "").strip() - if etag: - extra["etag"] = etag if last_modified: extra["last_modified"] = last_modified - # Prefer ETag — it's a stronger validator. Fall back to Last-Modified. revision = etag or last_modified or None return revision, extra diff --git a/haiku_rag_slim/haiku/rag/ingester/sources/s3.py b/haiku_rag_slim/haiku/rag/ingester/sources/s3.py index 8e5bb9ab..a250945a 100644 --- a/haiku_rag_slim/haiku/rag/ingester/sources/s3.py +++ b/haiku_rag_slim/haiku/rag/ingester/sources/s3.py @@ -93,17 +93,12 @@ class S3Source: if not content_type: content_type = "application/octet-stream" - extra: dict[str, str] = {} - if etag is not None: - extra["etag"] = etag - return FetchResult( uri=uri, body=body, content_type=content_type, content_hash=hashlib.md5(body, usedforsecurity=False).hexdigest(), revision=etag, - extra_metadata=extra, ) async def discover( diff --git a/haiku_rag_slim/haiku/rag/ingester/workers/pipeline.py b/haiku_rag_slim/haiku/rag/ingester/workers/pipeline.py index 5f679dd8..66c50b0f 100644 --- a/haiku_rag_slim/haiku/rag/ingester/workers/pipeline.py +++ b/haiku_rag_slim/haiku/rag/ingester/workers/pipeline.py @@ -114,7 +114,7 @@ async def run_job(client: "HaikuRAG", job: Job) -> JobResult: metadata = result.metadata or {} return JobResult( document_id=result.id, - revision=metadata.get("etag"), + revision=metadata.get("source_revision"), content_hash=metadata.get("md5"), ) except BaseException as exc: diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py index 4af3655d..8907f5a2 100644 --- a/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py +++ b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py @@ -90,6 +90,9 @@ from haiku.rag.store.upgrades.v0_45_0 import ( from haiku.rag.store.upgrades.v0_48_0 import ( upgrade_backfill_heading_hierarchy as upgrade_0_48_0_heading_hierarchy, ) +from haiku.rag.store.upgrades.v0_50_0 import ( + upgrade_canonical_metadata_keys as upgrade_0_50_0_canonical_metadata_keys, +) upgrades.append(upgrade_0_20_0_docling) upgrades.append(upgrade_0_23_1_contextualize) @@ -98,3 +101,4 @@ upgrades.append(upgrade_0_38_0_split_pages) upgrades.append(upgrade_0_40_0_document_items) upgrades.append(upgrade_0_45_0_extract_picture_bytes) upgrades.append(upgrade_0_48_0_heading_hierarchy) +upgrades.append(upgrade_0_50_0_canonical_metadata_keys) diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/v0_50_0.py b/haiku_rag_slim/haiku/rag/store/upgrades/v0_50_0.py new file mode 100644 index 00000000..044d2229 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/store/upgrades/v0_50_0.py @@ -0,0 +1,85 @@ +import json +import logging + +from haiku.rag.store.engine import Store +from haiku.rag.store.upgrades import Upgrade +from haiku.rag.utils import escape_sql_string + +logger = logging.getLogger(__name__) + +PROGRESS_INTERVAL = 50 + + +def _normalize_metadata(meta: dict) -> tuple[dict, bool]: + """Return (normalized_metadata, changed). Renames `etag` → `source_revision` + and `contentType` → `content_type`. Pre-existing canonical keys win on + conflict so repeat migrations are no-ops.""" + changed = False + out = dict(meta) + + if "etag" in out: + if "source_revision" not in out: + out["source_revision"] = out["etag"] + del out["etag"] + changed = True + + if "contentType" in out: + if "content_type" not in out: + out["content_type"] = out["contentType"] + del out["contentType"] + changed = True + + return out, changed + + +async def _apply_canonical_metadata_keys(store: Store) -> None: + """Rewrite document.metadata so revision lives under the source-agnostic + `source_revision` key and `contentType` becomes `content_type`. Documents + whose metadata already uses the canonical keys are untouched.""" + rows = ( + await store.documents_table.query().select(["id", "metadata"]).to_arrow() + ).to_pylist() + total = len(rows) + logger.info("Normalising document metadata keys across %d documents", total) + rewritten = 0 + skipped = 0 + + for idx, row in enumerate(rows, 1): + doc_id = row["id"] + raw = row.get("metadata") or "{}" + try: + meta = json.loads(raw) + except Exception: # pragma: no cover + logger.warning( + "Could not parse metadata JSON for document %s; skipping", doc_id + ) + skipped += 1 + continue + + normalized, changed = _normalize_metadata(meta) + if not changed: + continue + + safe_id = escape_sql_string(doc_id) + await store.documents_table.update( + {"metadata": json.dumps(normalized)}, + where=f"id = '{safe_id}'", + ) + rewritten += 1 + + if idx % PROGRESS_INTERVAL == 0 or idx == total: + logger.info("Progress: %d/%d (%d rewritten)", idx, total, rewritten) + + logger.info( + "Metadata key normalisation complete: %d rewritten, %d skipped of %d", + rewritten, + skipped, + total, + ) + + +upgrade_canonical_metadata_keys = Upgrade( + version="0.50.0", + apply=_apply_canonical_metadata_keys, + description="Rename document.metadata keys: etag→source_revision, contentType→content_type", +) diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 88f923c8..345b7e78 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-slim" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies" -version = "0.48.1" +version = "0.50.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } diff --git a/pyproject.toml b/pyproject.toml index c6e2d560..e68920da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling" -version = "0.48.1" +version = "0.50.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } @@ -30,7 +30,7 @@ classifiers = [ ] dependencies = [ - "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.48.1", + "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.50.0", ] [project.scripts] @@ -38,9 +38,9 @@ haiku-rag = "haiku.rag.cli:cli" [project.optional-dependencies] tui = ["textual>=8.2.4"] -s3 = ["haiku.rag-slim[s3]==0.48.1"] -cross-encoder = ["haiku.rag-slim[cross-encoder]==0.48.1"] -ingester = ["haiku.rag-slim[ingester]==0.48.1"] +s3 = ["haiku.rag-slim[s3]==0.50.0"] +cross-encoder = ["haiku.rag-slim[cross-encoder]==0.50.0"] +ingester = ["haiku.rag-slim[ingester]==0.50.0"] [build-system] requires = ["hatchling"] diff --git a/tests/ingester/test_http_source.py b/tests/ingester/test_http_source.py index 088fcbf3..9130a1a5 100644 --- a/tests/ingester/test_http_source.py +++ b/tests/ingester/test_http_source.py @@ -60,7 +60,7 @@ async def test_fetch_returns_bytes_and_md5_and_etag(): assert result.content_type == "text/markdown" # etag preferred over last-modified, surrounding quotes stripped assert result.revision == "abc123" - assert result.extra_metadata["etag"] == "abc123" + assert "etag" not in result.extra_metadata assert result.extra_metadata["last_modified"] == "Wed, 21 Oct 2025 07:28:00 GMT" diff --git a/tests/ingester/test_pipeline.py b/tests/ingester/test_pipeline.py index 6aecd747..a9d4f40c 100644 --- a/tests/ingester/test_pipeline.py +++ b/tests/ingester/test_pipeline.py @@ -44,7 +44,11 @@ async def test_upsert_calls_create_document_from_source_and_returns_metadata(): id="doc-42", content="x", uri="https://example.com/a.pdf", - metadata={"md5": "abcd", "etag": "xyz", "contentType": "application/pdf"}, + metadata={ + "md5": "abcd", + "source_revision": "xyz", + "content_type": "application/pdf", + }, ) result = await run_job( diff --git a/tests/ingester/test_revision_round_trip.py b/tests/ingester/test_revision_round_trip.py new file mode 100644 index 00000000..c546cc55 --- /dev/null +++ b/tests/ingester/test_revision_round_trip.py @@ -0,0 +1,98 @@ +"""Regression: every source's revision must round-trip from FetchResult → +document.metadata → sync_state, so the next sweep can recognise the file +as unchanged. Catches the FS-specific bug where revision was lost in the +pipeline and every periodic sweep re-enqueued every file forever. +""" + +import pytest + +from haiku.rag.client import HaikuRAG +from haiku.rag.ingester.sources.base import SourceEventKind +from haiku.rag.ingester.sources.fs import FSSource + + +@pytest.mark.asyncio +async def test_fs_ingest_writes_source_revision_to_metadata(temp_db_path, tmp_path): + file_path = tmp_path / "doc.md" + file_path.write_text("hello") + expected_revision = str(file_path.stat().st_mtime_ns) + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document_from_source(file_path) + + assert doc.metadata["source_revision"] == expected_revision + + +@pytest.mark.asyncio +async def test_fs_second_sweep_emits_unchanged_after_ingest(temp_db_path, tmp_path): + """The full round-trip: ingest a file, build a sync_state-shaped snapshot + from document.metadata, hand it to FSSource.discover() — must see + UNCHANGED, not UPSERT. This is exactly what the periodic poller does.""" + file_path = tmp_path / "doc.md" + file_path.write_text("hello") + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document_from_source(file_path) + + assert doc.uri is not None + snapshot = {doc.uri: doc.metadata["source_revision"]} + + src = FSSource(root=tmp_path) + kinds: list[SourceEventKind] = [] + async for event in src.discover(since=snapshot): + kinds.append(event.kind) + + assert kinds == [SourceEventKind.UNCHANGED] + + +@pytest.mark.asyncio +async def test_fs_second_sweep_emits_upsert_when_file_changes(temp_db_path, tmp_path): + """Counterpart to the unchanged test: a file modified after ingest still + triggers UPSERT. Ensures the round-trip doesn't accidentally over-skip.""" + file_path = tmp_path / "doc.md" + file_path.write_text("hello") + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document_from_source(file_path) + + # Bump mtime by writing different content; snapshot still has the old revision. + assert doc.uri is not None + snapshot = {doc.uri: doc.metadata["source_revision"]} + file_path.write_text("hello world") + # st_mtime_ns has nanosecond resolution; a write_text always advances it + # on any sane filesystem, but assert anyway to make the intent explicit. + assert str(file_path.stat().st_mtime_ns) != doc.metadata["source_revision"] + + src = FSSource(root=tmp_path) + kinds: list[SourceEventKind] = [] + async for event in src.discover(since=snapshot): + kinds.append(event.kind) + + assert kinds == [SourceEventKind.UPSERT] + + +@pytest.mark.asyncio +async def test_fs_head_short_circuit_skips_fetch_for_unchanged_revision( + temp_db_path, tmp_path, monkeypatch +): + """The one-shot (add-src) path: if the existing doc has source_revision + and HEAD reports the same value, fetch must not run.""" + file_path = tmp_path / "doc.md" + file_path.write_text("hello") + + async with HaikuRAG(temp_db_path, create=True) as client: + first = await client.create_document_from_source(file_path) + + fetch_calls: list[str] = [] + original_fetch = FSSource.fetch + + async def _track_fetch(self: FSSource, uri: str): # type: ignore[no-untyped-def] + fetch_calls.append(uri) + return await original_fetch(self, uri) + + monkeypatch.setattr(FSSource, "fetch", _track_fetch) + + second = await client.create_document_from_source(file_path) + + assert second.id == first.id + assert fetch_calls == [] diff --git a/tests/ingester/test_s3_source.py b/tests/ingester/test_s3_source.py index cb7cb32a..44674fb0 100644 --- a/tests/ingester/test_s3_source.py +++ b/tests/ingester/test_s3_source.py @@ -107,7 +107,7 @@ async def test_fetch_returns_bytes_md5_etag(fake_obstore_io): assert result.content_hash == hashlib.md5(body, usedforsecurity=False).hexdigest() assert result.content_type == "text/plain" assert result.revision == "abc123" - assert result.extra_metadata["etag"] == "abc123" + assert result.extra_metadata == {} head_async.assert_awaited_once() get_async.assert_awaited_once() @@ -121,7 +121,7 @@ async def test_fetch_handles_missing_etag(fake_obstore_io): src = S3Source(uri="s3://bucket/") result = await src.fetch("s3://bucket/file.txt") assert result.revision is None - assert "etag" not in result.extra_metadata + assert result.extra_metadata == {} @pytest.mark.asyncio diff --git a/tests/ingester/test_serve_integration.py b/tests/ingester/test_serve_integration.py index 7a2c5737..c1716501 100644 --- a/tests/ingester/test_serve_integration.py +++ b/tests/ingester/test_serve_integration.py @@ -52,7 +52,7 @@ async def _wait_for(predicate, *, timeout: float = 5.0, interval: float = 0.05): def _mock_client(docs_root) -> AsyncMock: """A HaikuRAG mock that returns a fresh Document for each URI it's asked - to ingest, mirroring real metadata shape (contentType + md5).""" + to ingest, mirroring real metadata shape (content_type + md5).""" client = AsyncMock(spec=HaikuRAG) counter = {"n": 0} @@ -63,7 +63,7 @@ def _mock_client(docs_root) -> AsyncMock: id=f"doc-{counter['n']}", content="x", uri=uri, - metadata={"contentType": "text/markdown", "md5": f"md5-{counter['n']}"}, + metadata={"content_type": "text/markdown", "md5": f"md5-{counter['n']}"}, ) client.create_document_from_source.side_effect = _fake_create diff --git a/tests/ingester/test_workers.py b/tests/ingester/test_workers.py index 0cd3b887..1eed49e1 100644 --- a/tests/ingester/test_workers.py +++ b/tests/ingester/test_workers.py @@ -59,7 +59,10 @@ def _pool(client, jobs, sync, **kwargs) -> WorkerPool: @pytest.mark.asyncio async def test_drain_marks_job_succeeded_and_writes_sync_state(client, jobs, sync): client.create_document_from_source.return_value = Document( - id="doc-1", content="x", uri="s3://b/k.md", metadata={"md5": "m1", "etag": "e1"} + id="doc-1", + content="x", + uri="s3://b/k.md", + metadata={"md5": "m1", "source_revision": "e1"}, ) job = await jobs.enqueue("src", "s3://b/k.md", JobOp.UPSERT, revision="e0") assert job is not None @@ -179,7 +182,7 @@ async def test_unknown_exception_caught_and_marked_dead(client, jobs, sync): @pytest.mark.asyncio async def test_workers_drain_queue_after_start(client, jobs, sync): client.create_document_from_source.return_value = Document( - id="doc", content="x", uri="u", metadata={"md5": "m", "etag": "e"} + id="doc", content="x", uri="u", metadata={"md5": "m", "source_revision": "e"} ) for i in range(5): await jobs.enqueue("src", f"u{i}", JobOp.UPSERT) diff --git a/tests/store/test_v0_50_0_migration.py b/tests/store/test_v0_50_0_migration.py new file mode 100644 index 00000000..e025ea23 --- /dev/null +++ b/tests/store/test_v0_50_0_migration.py @@ -0,0 +1,119 @@ +import json + +import pytest + +from haiku.rag.store.engine import DocumentRecord, Store + + +@pytest.mark.asyncio +class TestV0_50_0Migration: + """v0.50.0 normalises document.metadata to source-agnostic keys.""" + + async def test_renames_etag_and_content_type(self, temp_db_path): + async with Store(temp_db_path, create=True, skip_migration_check=True) as store: + await store.set_haiku_version("0.48.1") + await store.documents_table.add( + [ + DocumentRecord( + id="doc-s3", + content="x", + uri="s3://b/k", + metadata=json.dumps( + { + "etag": "abc123", + "contentType": "application/pdf", + "md5": "deadbeef", + } + ), + ), + DocumentRecord( + id="doc-fs", + content="y", + uri="file:///tmp/x.md", + metadata=json.dumps( + { + "contentType": "text/markdown", + "md5": "feedface", + } + ), + ), + ] + ) + + async with Store(temp_db_path, skip_migration_check=True) as store: + applied = await store.migrate() + assert any("0.50.0" in d for d in applied) + + rows = await store.documents_table.query().to_list() + by_id = {r["id"]: json.loads(r["metadata"]) for r in rows} + + assert by_id["doc-s3"] == { + "source_revision": "abc123", + "content_type": "application/pdf", + "md5": "deadbeef", + } + assert by_id["doc-fs"] == { + "content_type": "text/markdown", + "md5": "feedface", + } + + async def test_idempotent_on_already_migrated(self, temp_db_path): + async with Store(temp_db_path, create=True, skip_migration_check=True) as store: + await store.set_haiku_version("0.48.1") + await store.documents_table.add( + [ + DocumentRecord( + id="d", + content="x", + uri="s3://b/k", + metadata=json.dumps( + { + "source_revision": "abc", + "content_type": "text/plain", + "md5": "m", + } + ), + ) + ] + ) + + async with Store(temp_db_path, skip_migration_check=True) as store: + await store.migrate() + rows = await store.documents_table.query().to_list() + assert json.loads(rows[0]["metadata"]) == { + "source_revision": "abc", + "content_type": "text/plain", + "md5": "m", + } + + async def test_preserves_existing_canonical_keys_on_collision(self, temp_db_path): + """If both legacy and canonical keys are present, the canonical wins + and the legacy is dropped — defends against partial-migration states.""" + async with Store(temp_db_path, create=True, skip_migration_check=True) as store: + await store.set_haiku_version("0.48.1") + await store.documents_table.add( + [ + DocumentRecord( + id="d", + content="x", + uri="s3://b/k", + metadata=json.dumps( + { + "etag": "legacy", + "source_revision": "canonical", + "contentType": "text/old", + "content_type": "text/new", + } + ), + ) + ] + ) + + async with Store(temp_db_path, skip_migration_check=True) as store: + await store.migrate() + rows = await store.documents_table.query().to_list() + meta = json.loads(rows[0]["metadata"]) + assert meta == { + "source_revision": "canonical", + "content_type": "text/new", + } diff --git a/tests/test_client.py b/tests/test_client.py index 532893d7..98449f28 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -209,9 +209,9 @@ async def test_client_create_document_from_source(temp_db_path): assert doc.id is not None assert doc.content == test_content assert doc.uri == temp_path.as_uri() - assert "contentType" in doc.metadata + assert "content_type" in doc.metadata assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "text/plain" + assert doc.metadata["content_type"] == "text/plain" # Test create_document_from_source with string path doc2 = await client.create_document_from_source(source=str(temp_path)) @@ -220,7 +220,7 @@ async def test_client_create_document_from_source(temp_db_path): assert doc2.id is not None assert doc2.content == test_content assert doc2.uri == temp_path.as_uri() - assert "contentType" in doc2.metadata + assert "content_type" in doc2.metadata assert "md5" in doc2.metadata @@ -374,7 +374,7 @@ async def test_client_create_document_from_directory(temp_db_path): assert doc.id is not None assert doc.uri is not None assert "md5" in doc.metadata - assert "contentType" in doc.metadata + assert "content_type" in doc.metadata uris = [doc.uri for doc in result if doc.uri] assert any("doc1.txt" in uri for uri in uris) @@ -404,9 +404,9 @@ async def test_client_create_document_from_url(temp_db_path): assert "test content" in doc.content assert doc.uri == "https://example.com/test.html" assert doc.metadata["source_type"] == "web" - assert "contentType" in doc.metadata + assert "content_type" in doc.metadata assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "text/html" + assert doc.metadata["content_type"] == "text/html" @pytest.mark.vcr() @@ -432,9 +432,9 @@ async def test_client_create_document_from_url_with_different_content_types( assert doc.id is not None assert "Test JSON" in doc.content assert doc.uri == "https://api.example.com/data.json" - assert "contentType" in doc.metadata + assert "content_type" in doc.metadata assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "application/json" + assert doc.metadata["content_type"] == "application/json" # Test plain text content mock_text_response = AsyncMock() @@ -451,9 +451,9 @@ async def test_client_create_document_from_url_with_different_content_types( assert doc.id is not None assert doc.content == "This is plain text content from a URL." assert doc.uri == "https://example.com/readme.txt" - assert "contentType" in doc.metadata + assert "content_type" in doc.metadata assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "text/plain" + assert doc.metadata["content_type"] == "text/plain" @pytest.mark.vcr() @@ -523,7 +523,7 @@ def test_get_extension_from_content_type_or_url(): @pytest.mark.vcr() async def test_client_metadata_content_type_and_md5(temp_db_path): - """Test that contentType and md5 metadata are correctly set.""" + """Test that content_type and md5 metadata are correctly set.""" import hashlib async with HaikuRAG(temp_db_path, create=True) as client: @@ -538,7 +538,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path): doc = await client.create_document_from_source(temp_path) assert isinstance(doc, Document) - assert doc.metadata["contentType"] == "text/plain" + assert doc.metadata["content_type"] == "text/plain" assert doc.metadata["md5"] == expected_md5 mock_response = AsyncMock() @@ -552,7 +552,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path): ) assert isinstance(url_doc, Document) - assert url_doc.metadata["contentType"] == "text/plain" + assert url_doc.metadata["content_type"] == "text/plain" assert url_doc.metadata["md5"] == expected_md5 diff --git a/tests/test_s3_source.py b/tests/test_s3_source.py index 1b157566..d1938528 100644 --- a/tests/test_s3_source.py +++ b/tests/test_s3_source.py @@ -77,7 +77,7 @@ async def test_create_document_from_s3_new(fake_obstore_io, temp_db_path): doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt") assert doc.uri == "s3://my-bucket/folder/file.txt" - assert doc.metadata["etag"] == "abc123" # quotes stripped + assert doc.metadata["source_revision"] == "abc123" # quotes stripped assert doc.metadata["md5"] # real content MD5 assert doc.metadata["md5"] != "abc123" head_async.assert_awaited_once() @@ -114,7 +114,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( ): """Multipart re-upload of same content: etag changes, MD5 doesn't. - Expected: GET runs to verify, but no re-chunk; only metadata.etag updates. + Expected: GET runs to verify, but no re-chunk; only metadata.source_revision updates. """ head_async, get_async = fake_obstore_io text = b"S3 hosted content" @@ -134,7 +134,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk( assert second.id == first.id assert second.metadata["md5"] == original_md5 - assert second.metadata["etag"] == "def456-2" + assert second.metadata["source_revision"] == "def456-2" assert second.updated_at >= original_updated_at assert get_async.await_count == 2 # initial create + etag-changed compare @@ -158,7 +158,7 @@ async def test_create_document_from_s3_etag_changed_md5_changed_rechunks( assert second.id == first.id assert second.metadata["md5"] != first.metadata["md5"] - assert second.metadata["etag"] == "new999" + assert second.metadata["source_revision"] == "new999" assert "different text now" in second.content diff --git a/uv.lock b/uv.lock index c0dfdf44..16a9aa4c 100644 --- a/uv.lock +++ b/uv.lock @@ -1466,7 +1466,7 @@ wheels = [ [[package]] name = "haiku-rag" -version = "0.48.1" +version = "0.50.0" source = { editable = "." } dependencies = [ { name = "haiku-rag-slim", extra = ["cohere", "cross-encoder", "docling", "mxbai", "tui", "voyageai", "zeroentropy"] }, @@ -1558,7 +1558,7 @@ requires-dist = [ [[package]] name = "haiku-rag-slim" -version = "0.48.1" +version = "0.50.0" source = { editable = "haiku_rag_slim" } dependencies = [ { name = "docling-core" },