danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

canonical metadata keys: source_revision + content_type, bump to 0.50.0

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-05-22 15:21:38 +03:00
parent fa672c298a
commit 13cbadeb6f
No known key found for this signature in database
20 changed files with 366 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -12,6 +12,7 @@
### Changed
- `haiku-rag serve` renamed to `haiku-rag mcp` (only MCP is left). `--mcp-port` renamed to `--port`. Update any `claude_desktop_config.json` from `["serve", "--mcp", "--stdio"]` to `["mcp", "--stdio"]`.
- `document.metadata` now uses source-agnostic keys: `source_revision` (was `etag` — S3-only and never populated for FS, so periodic sweeps re-ingested every file) and `content_type` (was `contentType`, snake_case for consistency). The v0.50.0 startup migration rewrites existing documents. All four source adapters (FS, HTTP, S3, future WebDAV) now write their native revision (mtime_ns, ETag, etc.) under the same key, fixing the regression where FS sources never short-circuited on unchanged files.
- Drop `list_documents` and `get_document` from the default RAG skill's tool set; the skill now exposes only `search` and `cite`. Both tools dumped unbounded content into the agent's context (full document lists, full document bodies) and `get_document` returned no chunk_ids so its output was structurally uncitable. The analysis skill already covers these uses programmatically — `await list_documents()` and `Path('/documents/{id}/content.txt').read_text()` inside `execute_code`. The tool branches remain in `create_skill_tools` and the `skill_generator` `AVAILABLE_TOOLS` set so users can still opt in when building custom skills.
## [0.48.1] - 2026-05-21

6
docs/ingester.md
View file

@ -80,10 +80,10 @@ ingester:
```
ETags are the cheap-skip key. Each sweep lists the prefix, compares
the listed ETag against the document's stored `metadata["etag"]`, and
only fetches keys whose ETag has changed. If the bytes turn out to
the listed ETag against the document's stored `metadata["source_revision"]`,
and only fetches keys whose ETag has changed. If the bytes turn out to
match the stored MD5 (multipart re-upload landing a new ETag on the
same content), only the etag is refreshed — no re-chunk.
same content), only the revision is refreshed — no re-chunk.
`storage_options` follows the same convention as `lancedb.storage_options` —
the dict is passed straight to obstore (the Rust `object_store` library

17
haiku_rag_slim/haiku/rag/client/documents.py
View file

@ -231,10 +231,12 @@ async def _ingest_fetch_result(
)
source_metadata: dict = {
"contentType": result.content_type,
"content_type": result.content_type,
"md5": result.content_hash,
**result.extra_metadata,
}
if result.revision is not None:
source_metadata["source_revision"] = result.revision
if result.disk_path is not None:
target_path = result.disk_path
@ -384,10 +386,11 @@ async def create_document_from_source(
existing_doc = await client.get_document_by_uri(stored_uri)
# Cheap revision-based short-circuit: only worth a HEAD when we have a
# stored revision to compare against. S3 doc metadata persists "etag";
# FS/HTTP currently don't, so this branch is effectively S3-only today.
# stored revision to compare against. All sources persist their native
# revision (mtime_ns for FS, ETag for S3, ETag/Last-Modified for HTTP)
# under the canonical "source_revision" metadata key.
stored_revision = (
(existing_doc.metadata or {}).get("etag") if existing_doc else None
(existing_doc.metadata or {}).get("source_revision") if existing_doc else None
)
if existing_doc and stored_revision:
current_revision = await fetcher.head(source_str)
@ -406,14 +409,16 @@ async def create_document_from_source(
fetch_span.set_attribute("content_hash", result.content_hash)
# MD5 short-circuit: the bytes are unchanged even if the revision wasn't.
# Refresh the source-derived metadata (etag may have rolled) but skip
# Refresh the source-derived metadata (revision may have rolled) but skip
# convert/embed/store entirely.
if existing_doc and existing_doc.metadata.get("md5") == result.content_hash:
source_meta: dict = {
"contentType": result.content_type,
"content_type": result.content_type,
"md5": result.content_hash,
**result.extra_metadata,
}
if result.revision is not None:
source_meta["source_revision"] = result.revision
return await _refresh_doc_metadata(
client,
existing_doc,

7
haiku_rag_slim/haiku/rag/ingester/sources/http.py
View file

@ -14,14 +14,15 @@ from haiku.rag.ingester.sources.base import (
def _extract_revision(headers: httpx.Headers) -> tuple[str | None, dict[str, str]]:
"""Return (canonical_revision, extras). ETag is the stronger validator
and is the canonical revision when present; Last-Modified backs it up.
Last-Modified always goes into extras as separate per-source provenance
(some pipelines want both signals)."""
extra: dict[str, str] = {}
etag = (headers.get("etag") or "").strip('"').strip()
last_modified = (headers.get("last-modified") or "").strip()
if etag:
extra["etag"] = etag
if last_modified:
extra["last_modified"] = last_modified
# Prefer ETag — it's a stronger validator. Fall back to Last-Modified.
revision = etag or last_modified or None
return revision, extra

5
haiku_rag_slim/haiku/rag/ingester/sources/s3.py
View file

@ -93,17 +93,12 @@ class S3Source:
if not content_type:
content_type = "application/octet-stream"
extra: dict[str, str] = {}
if etag is not None:
extra["etag"] = etag
return FetchResult(
uri=uri,
body=body,
content_type=content_type,
content_hash=hashlib.md5(body, usedforsecurity=False).hexdigest(),
revision=etag,
extra_metadata=extra,
)
async def discover(

2
haiku_rag_slim/haiku/rag/ingester/workers/pipeline.py
View file

@ -114,7 +114,7 @@ async def run_job(client: "HaikuRAG", job: Job) -> JobResult:
metadata = result.metadata or {}
return JobResult(
document_id=result.id,
revision=metadata.get("etag"),
revision=metadata.get("source_revision"),
content_hash=metadata.get("md5"),
)
except BaseException as exc:

4
haiku_rag_slim/haiku/rag/store/upgrades/__init__.py
View file

@ -90,6 +90,9 @@ from haiku.rag.store.upgrades.v0_45_0 import (
from haiku.rag.store.upgrades.v0_48_0 import (
upgrade_backfill_heading_hierarchy as upgrade_0_48_0_heading_hierarchy,
)
from haiku.rag.store.upgrades.v0_50_0 import (
upgrade_canonical_metadata_keys as upgrade_0_50_0_canonical_metadata_keys,
)
upgrades.append(upgrade_0_20_0_docling)
upgrades.append(upgrade_0_23_1_contextualize)
@ -98,3 +101,4 @@ upgrades.append(upgrade_0_38_0_split_pages)
upgrades.append(upgrade_0_40_0_document_items)
upgrades.append(upgrade_0_45_0_extract_picture_bytes)
upgrades.append(upgrade_0_48_0_heading_hierarchy)
upgrades.append(upgrade_0_50_0_canonical_metadata_keys)

85
haiku_rag_slim/haiku/rag/store/upgrades/v0_50_0.py Normal file
View file

@ -0,0 +1,85 @@
import json
import logging
from haiku.rag.store.engine import Store
from haiku.rag.store.upgrades import Upgrade
from haiku.rag.utils import escape_sql_string
logger = logging.getLogger(__name__)
PROGRESS_INTERVAL = 50
def _normalize_metadata(meta: dict) -> tuple[dict, bool]:
"""Return (normalized_metadata, changed). Renames `etag` → `source_revision`
and `contentType` → `content_type`. Pre-existing canonical keys win on
conflict so repeat migrations are no-ops."""
changed = False
out = dict(meta)
if "etag" in out:
if "source_revision" not in out:
out["source_revision"] = out["etag"]
del out["etag"]
changed = True
if "contentType" in out:
if "content_type" not in out:
out["content_type"] = out["contentType"]
del out["contentType"]
changed = True
return out, changed
async def _apply_canonical_metadata_keys(store: Store) -> None:
"""Rewrite document.metadata so revision lives under the source-agnostic
`source_revision` key and `contentType` becomes `content_type`. Documents
whose metadata already uses the canonical keys are untouched."""
rows = (
await store.documents_table.query().select(["id", "metadata"]).to_arrow()
).to_pylist()
total = len(rows)
logger.info("Normalising document metadata keys across %d documents", total)
rewritten = 0
skipped = 0
for idx, row in enumerate(rows, 1):
doc_id = row["id"]
raw = row.get("metadata") or "{}"
try:
meta = json.loads(raw)
except Exception: # pragma: no cover
logger.warning(
"Could not parse metadata JSON for document %s; skipping", doc_id
)
skipped += 1
continue
normalized, changed = _normalize_metadata(meta)
if not changed:
continue
safe_id = escape_sql_string(doc_id)
await store.documents_table.update(
{"metadata": json.dumps(normalized)},
where=f"id = '{safe_id}'",
)
rewritten += 1
if idx % PROGRESS_INTERVAL == 0 or idx == total:
logger.info("Progress: %d/%d (%d rewritten)", idx, total, rewritten)
logger.info(
"Metadata key normalisation complete: %d rewritten, %d skipped of %d",
rewritten,
skipped,
total,
)
upgrade_canonical_metadata_keys = Upgrade(
version="0.50.0",
apply=_apply_canonical_metadata_keys,
description="Rename document.metadata keys: etag→source_revision, contentType→content_type",
)

2
haiku_rag_slim/pyproject.toml
View file

@ -2,7 +2,7 @@
name = "haiku.rag-slim"
description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies"
version = "0.48.1"
version = "0.50.0"
authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
license = { text = "MIT" }
readme = { file = "README.md", content-type = "text/markdown" }

10
pyproject.toml
View file

@ -2,7 +2,7 @@
name = "haiku.rag"
description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling"
version = "0.48.1"
version = "0.50.0"
authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
license = { text = "MIT" }
readme = { file = "README.md", content-type = "text/markdown" }
@ -30,7 +30,7 @@ classifiers = [
]
dependencies = [
"haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.48.1",
"haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.50.0",
]
[project.scripts]
@ -38,9 +38,9 @@ haiku-rag = "haiku.rag.cli:cli"
[project.optional-dependencies]
tui = ["textual>=8.2.4"]
s3 = ["haiku.rag-slim[s3]==0.48.1"]
cross-encoder = ["haiku.rag-slim[cross-encoder]==0.48.1"]
ingester = ["haiku.rag-slim[ingester]==0.48.1"]
s3 = ["haiku.rag-slim[s3]==0.50.0"]
cross-encoder = ["haiku.rag-slim[cross-encoder]==0.50.0"]
ingester = ["haiku.rag-slim[ingester]==0.50.0"]
[build-system]
requires = ["hatchling"]

2
tests/ingester/test_http_source.py
View file

@ -60,7 +60,7 @@ async def test_fetch_returns_bytes_and_md5_and_etag():
assert result.content_type == "text/markdown"
# etag preferred over last-modified, surrounding quotes stripped
assert result.revision == "abc123"
assert result.extra_metadata["etag"] == "abc123"
assert "etag" not in result.extra_metadata
assert result.extra_metadata["last_modified"] == "Wed, 21 Oct 2025 07:28:00 GMT"

6
tests/ingester/test_pipeline.py
View file

@ -44,7 +44,11 @@ async def test_upsert_calls_create_document_from_source_and_returns_metadata():
id="doc-42",
content="x",
uri="https://example.com/a.pdf",
metadata={"md5": "abcd", "etag": "xyz", "contentType": "application/pdf"},
metadata={
"md5": "abcd",
"source_revision": "xyz",
"content_type": "application/pdf",
},
)
result = await run_job(

98
tests/ingester/test_revision_round_trip.py Normal file
View file

@ -0,0 +1,98 @@
"""Regression: every source's revision must round-trip from FetchResult →
document.metadata → sync_state, so the next sweep can recognise the file
as unchanged. Catches the FS-specific bug where revision was lost in the
pipeline and every periodic sweep re-enqueued every file forever.
"""
import pytest
from haiku.rag.client import HaikuRAG
from haiku.rag.ingester.sources.base import SourceEventKind
from haiku.rag.ingester.sources.fs import FSSource
@pytest.mark.asyncio
async def test_fs_ingest_writes_source_revision_to_metadata(temp_db_path, tmp_path):
file_path = tmp_path / "doc.md"
file_path.write_text("hello")
expected_revision = str(file_path.stat().st_mtime_ns)
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document_from_source(file_path)
assert doc.metadata["source_revision"] == expected_revision
@pytest.mark.asyncio
async def test_fs_second_sweep_emits_unchanged_after_ingest(temp_db_path, tmp_path):
"""The full round-trip: ingest a file, build a sync_state-shaped snapshot
from document.metadata, hand it to FSSource.discover() — must see
UNCHANGED, not UPSERT. This is exactly what the periodic poller does."""
file_path = tmp_path / "doc.md"
file_path.write_text("hello")
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document_from_source(file_path)
assert doc.uri is not None
snapshot = {doc.uri: doc.metadata["source_revision"]}
src = FSSource(root=tmp_path)
kinds: list[SourceEventKind] = []
async for event in src.discover(since=snapshot):
kinds.append(event.kind)
assert kinds == [SourceEventKind.UNCHANGED]
@pytest.mark.asyncio
async def test_fs_second_sweep_emits_upsert_when_file_changes(temp_db_path, tmp_path):
"""Counterpart to the unchanged test: a file modified after ingest still
triggers UPSERT. Ensures the round-trip doesn't accidentally over-skip."""
file_path = tmp_path / "doc.md"
file_path.write_text("hello")
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document_from_source(file_path)
# Bump mtime by writing different content; snapshot still has the old revision.
assert doc.uri is not None
snapshot = {doc.uri: doc.metadata["source_revision"]}
file_path.write_text("hello world")
# st_mtime_ns has nanosecond resolution; a write_text always advances it
# on any sane filesystem, but assert anyway to make the intent explicit.
assert str(file_path.stat().st_mtime_ns) != doc.metadata["source_revision"]
src = FSSource(root=tmp_path)
kinds: list[SourceEventKind] = []
async for event in src.discover(since=snapshot):
kinds.append(event.kind)
assert kinds == [SourceEventKind.UPSERT]
@pytest.mark.asyncio
async def test_fs_head_short_circuit_skips_fetch_for_unchanged_revision(
temp_db_path, tmp_path, monkeypatch
):
"""The one-shot (add-src) path: if the existing doc has source_revision
and HEAD reports the same value, fetch must not run."""
file_path = tmp_path / "doc.md"
file_path.write_text("hello")
async with HaikuRAG(temp_db_path, create=True) as client:
first = await client.create_document_from_source(file_path)
fetch_calls: list[str] = []
original_fetch = FSSource.fetch
async def _track_fetch(self: FSSource, uri: str): # type: ignore[no-untyped-def]
fetch_calls.append(uri)
return await original_fetch(self, uri)
monkeypatch.setattr(FSSource, "fetch", _track_fetch)
second = await client.create_document_from_source(file_path)
assert second.id == first.id
assert fetch_calls == []

4
tests/ingester/test_s3_source.py
View file

@ -107,7 +107,7 @@ async def test_fetch_returns_bytes_md5_etag(fake_obstore_io):
assert result.content_hash == hashlib.md5(body, usedforsecurity=False).hexdigest()
assert result.content_type == "text/plain"
assert result.revision == "abc123"
assert result.extra_metadata["etag"] == "abc123"
assert result.extra_metadata == {}
head_async.assert_awaited_once()
get_async.assert_awaited_once()
@ -121,7 +121,7 @@ async def test_fetch_handles_missing_etag(fake_obstore_io):
src = S3Source(uri="s3://bucket/")
result = await src.fetch("s3://bucket/file.txt")
assert result.revision is None
assert "etag" not in result.extra_metadata
assert result.extra_metadata == {}
@pytest.mark.asyncio

4
tests/ingester/test_serve_integration.py
View file

@ -52,7 +52,7 @@ async def _wait_for(predicate, *, timeout: float = 5.0, interval: float = 0.05):
def _mock_client(docs_root) -> AsyncMock:
"""A HaikuRAG mock that returns a fresh Document for each URI it's asked
to ingest, mirroring real metadata shape (contentType + md5)."""
to ingest, mirroring real metadata shape (content_type + md5)."""
client = AsyncMock(spec=HaikuRAG)
counter = {"n": 0}
@ -63,7 +63,7 @@ def _mock_client(docs_root) -> AsyncMock:
id=f"doc-{counter['n']}",
content="x",
uri=uri,
metadata={"contentType": "text/markdown", "md5": f"md5-{counter['n']}"},
metadata={"content_type": "text/markdown", "md5": f"md5-{counter['n']}"},
)
client.create_document_from_source.side_effect = _fake_create

7
tests/ingester/test_workers.py
View file

@ -59,7 +59,10 @@ def _pool(client, jobs, sync, **kwargs) -> WorkerPool:
@pytest.mark.asyncio
async def test_drain_marks_job_succeeded_and_writes_sync_state(client, jobs, sync):
client.create_document_from_source.return_value = Document(
id="doc-1", content="x", uri="s3://b/k.md", metadata={"md5": "m1", "etag": "e1"}
id="doc-1",
content="x",
uri="s3://b/k.md",
metadata={"md5": "m1", "source_revision": "e1"},
)
job = await jobs.enqueue("src", "s3://b/k.md", JobOp.UPSERT, revision="e0")
assert job is not None
@ -179,7 +182,7 @@ async def test_unknown_exception_caught_and_marked_dead(client, jobs, sync):
@pytest.mark.asyncio
async def test_workers_drain_queue_after_start(client, jobs, sync):
client.create_document_from_source.return_value = Document(
id="doc", content="x", uri="u", metadata={"md5": "m", "etag": "e"}
id="doc", content="x", uri="u", metadata={"md5": "m", "source_revision": "e"}
)
for i in range(5):
await jobs.enqueue("src", f"u{i}", JobOp.UPSERT)

119
tests/store/test_v0_50_0_migration.py Normal file
View file

@ -0,0 +1,119 @@
import json
import pytest
from haiku.rag.store.engine import DocumentRecord, Store
@pytest.mark.asyncio
class TestV0_50_0Migration:
"""v0.50.0 normalises document.metadata to source-agnostic keys."""
async def test_renames_etag_and_content_type(self, temp_db_path):
async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
await store.set_haiku_version("0.48.1")
await store.documents_table.add(
[
DocumentRecord(
id="doc-s3",
content="x",
uri="s3://b/k",
metadata=json.dumps(
{
"etag": "abc123",
"contentType": "application/pdf",
"md5": "deadbeef",
}
),
),
DocumentRecord(
id="doc-fs",
content="y",
uri="file:///tmp/x.md",
metadata=json.dumps(
{
"contentType": "text/markdown",
"md5": "feedface",
}
),
),
]
)
async with Store(temp_db_path, skip_migration_check=True) as store:
applied = await store.migrate()
assert any("0.50.0" in d for d in applied)
rows = await store.documents_table.query().to_list()
by_id = {r["id"]: json.loads(r["metadata"]) for r in rows}
assert by_id["doc-s3"] == {
"source_revision": "abc123",
"content_type": "application/pdf",
"md5": "deadbeef",
}
assert by_id["doc-fs"] == {
"content_type": "text/markdown",
"md5": "feedface",
}
async def test_idempotent_on_already_migrated(self, temp_db_path):
async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
await store.set_haiku_version("0.48.1")
await store.documents_table.add(
[
DocumentRecord(
id="d",
content="x",
uri="s3://b/k",
metadata=json.dumps(
{
"source_revision": "abc",
"content_type": "text/plain",
"md5": "m",
}
),
)
]
)
async with Store(temp_db_path, skip_migration_check=True) as store:
await store.migrate()
rows = await store.documents_table.query().to_list()
assert json.loads(rows[0]["metadata"]) == {
"source_revision": "abc",
"content_type": "text/plain",
"md5": "m",
}
async def test_preserves_existing_canonical_keys_on_collision(self, temp_db_path):
"""If both legacy and canonical keys are present, the canonical wins
and the legacy is dropped — defends against partial-migration states."""
async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
await store.set_haiku_version("0.48.1")
await store.documents_table.add(
[
DocumentRecord(
id="d",
content="x",
uri="s3://b/k",
metadata=json.dumps(
{
"etag": "legacy",
"source_revision": "canonical",
"contentType": "text/old",
"content_type": "text/new",
}
),
)
]
)
async with Store(temp_db_path, skip_migration_check=True) as store:
await store.migrate()
rows = await store.documents_table.query().to_list()
meta = json.loads(rows[0]["metadata"])
assert meta == {
"source_revision": "canonical",
"content_type": "text/new",
}

26
tests/test_client.py
View file

@ -209,9 +209,9 @@ async def test_client_create_document_from_source(temp_db_path):
assert doc.id is not None
assert doc.content == test_content
assert doc.uri == temp_path.as_uri()
assert "contentType" in doc.metadata
assert "content_type" in doc.metadata
assert "md5" in doc.metadata
assert doc.metadata["contentType"] == "text/plain"
assert doc.metadata["content_type"] == "text/plain"
# Test create_document_from_source with string path
doc2 = await client.create_document_from_source(source=str(temp_path))
@ -220,7 +220,7 @@ async def test_client_create_document_from_source(temp_db_path):
assert doc2.id is not None
assert doc2.content == test_content
assert doc2.uri == temp_path.as_uri()
assert "contentType" in doc2.metadata
assert "content_type" in doc2.metadata
assert "md5" in doc2.metadata
@ -374,7 +374,7 @@ async def test_client_create_document_from_directory(temp_db_path):
assert doc.id is not None
assert doc.uri is not None
assert "md5" in doc.metadata
assert "contentType" in doc.metadata
assert "content_type" in doc.metadata
uris = [doc.uri for doc in result if doc.uri]
assert any("doc1.txt" in uri for uri in uris)
@ -404,9 +404,9 @@ async def test_client_create_document_from_url(temp_db_path):
assert "test content" in doc.content
assert doc.uri == "https://example.com/test.html"
assert doc.metadata["source_type"] == "web"
assert "contentType" in doc.metadata
assert "content_type" in doc.metadata
assert "md5" in doc.metadata
assert doc.metadata["contentType"] == "text/html"
assert doc.metadata["content_type"] == "text/html"
@pytest.mark.vcr()
@ -432,9 +432,9 @@ async def test_client_create_document_from_url_with_different_content_types(
assert doc.id is not None
assert "Test JSON" in doc.content
assert doc.uri == "https://api.example.com/data.json"
assert "contentType" in doc.metadata
assert "content_type" in doc.metadata
assert "md5" in doc.metadata
assert doc.metadata["contentType"] == "application/json"
assert doc.metadata["content_type"] == "application/json"
# Test plain text content
mock_text_response = AsyncMock()
@ -451,9 +451,9 @@ async def test_client_create_document_from_url_with_different_content_types(
assert doc.id is not None
assert doc.content == "This is plain text content from a URL."
assert doc.uri == "https://example.com/readme.txt"
assert "contentType" in doc.metadata
assert "content_type" in doc.metadata
assert "md5" in doc.metadata
assert doc.metadata["contentType"] == "text/plain"
assert doc.metadata["content_type"] == "text/plain"
@pytest.mark.vcr()
@ -523,7 +523,7 @@ def test_get_extension_from_content_type_or_url():
@pytest.mark.vcr()
async def test_client_metadata_content_type_and_md5(temp_db_path):
"""Test that contentType and md5 metadata are correctly set."""
"""Test that content_type and md5 metadata are correctly set."""
import hashlib
async with HaikuRAG(temp_db_path, create=True) as client:
@ -538,7 +538,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path):
doc = await client.create_document_from_source(temp_path)
assert isinstance(doc, Document)
assert doc.metadata["contentType"] == "text/plain"
assert doc.metadata["content_type"] == "text/plain"
assert doc.metadata["md5"] == expected_md5
mock_response = AsyncMock()
@ -552,7 +552,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path):
)
assert isinstance(url_doc, Document)
assert url_doc.metadata["contentType"] == "text/plain"
assert url_doc.metadata["content_type"] == "text/plain"
assert url_doc.metadata["md5"] == expected_md5

8
tests/test_s3_source.py
View file

@ -77,7 +77,7 @@ async def test_create_document_from_s3_new(fake_obstore_io, temp_db_path):
doc = await client.create_document_from_source("s3://my-bucket/folder/file.txt")
assert doc.uri == "s3://my-bucket/folder/file.txt"
assert doc.metadata["etag"] == "abc123" # quotes stripped
assert doc.metadata["source_revision"] == "abc123" # quotes stripped
assert doc.metadata["md5"] # real content MD5
assert doc.metadata["md5"] != "abc123"
head_async.assert_awaited_once()
@ -114,7 +114,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk(
):
"""Multipart re-upload of same content: etag changes, MD5 doesn't.
Expected: GET runs to verify, but no re-chunk; only metadata.etag updates.
Expected: GET runs to verify, but no re-chunk; only metadata.source_revision updates.
"""
head_async, get_async = fake_obstore_io
text = b"S3 hosted content"
@ -134,7 +134,7 @@ async def test_create_document_from_s3_etag_changed_md5_same_skips_rechunk(
assert second.id == first.id
assert second.metadata["md5"] == original_md5
assert second.metadata["etag"] == "def456-2"
assert second.metadata["source_revision"] == "def456-2"
assert second.updated_at >= original_updated_at
assert get_async.await_count == 2 # initial create + etag-changed compare
@ -158,7 +158,7 @@ async def test_create_document_from_s3_etag_changed_md5_changed_rechunks(
assert second.id == first.id
assert second.metadata["md5"] != first.metadata["md5"]
assert second.metadata["etag"] == "new999"
assert second.metadata["source_revision"] == "new999"
assert "different text now" in second.content

4
uv.lock
View file

@ -1466,7 +1466,7 @@ wheels = [
[[package]]
name = "haiku-rag"
version = "0.48.1"
version = "0.50.0"
source = { editable = "." }
dependencies = [
{ name = "haiku-rag-slim", extra = ["cohere", "cross-encoder", "docling", "mxbai", "tui", "voyageai", "zeroentropy"] },
@ -1558,7 +1558,7 @@ requires-dist = [
[[package]]
name = "haiku-rag-slim"
version = "0.48.1"
version = "0.50.0"
source = { editable = "haiku_rag_slim" }
dependencies = [
{ name = "docling-core" },